From c4274187b94ac7e6316c54ecff57a4a6328cf66c Mon Sep 17 00:00:00 2001 From: David Gustafsson Date: Tue, 28 Nov 2017 11:41:56 +0100 Subject: [PATCH] Bug 19707 - Add new operations for synchronizing Elasticsearch mappings Add new "merge", "revert" and "add" operations for synchronizing Elasticsearch mappings stored in database with mappings.yaml How to test: 1) Make sure mappings.yaml contains mappings not present in database. (Also be aware that steps below will result in changed search engine configuration, so use a test environment and/or make sure to backup database before starting) 2) Go to "Administration" and "Search Engine Configuration" 3) Append ?i_know_what_i_am_doing=1&op=add to url and perform request in browser 4) Verify that any new mappings in mappings.yaml has been added, but no current mappings has been modified or deleted 5) Edit mappings.yaml and add a new mapping for some field (new item below "mappings:" key) 6) Add a new mapping for the same field in the database (through admin ui) 7) Append ?i_know_what_i_am_doing=1&op=merge to url and perform request in browser 8) Verify that the new mapping from mappings.yaml has been added for this field, while the field added in database was preserved 10) Append ?i_know_what_i_am_doing=1&op=revert to url and perform request in browser 11) Verify that the mapping previously added the field in database has been deleted, and the mappings for this field are identical to those in mappings.yaml Sponsored-by: Gothenburg University Library --- Koha/SearchEngine/Elasticsearch.pm | 53 +++++++++++++++++++++++----- admin/searchengine/elasticsearch/mappings.pl | 19 ++++++++-- 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm index 0b33c06b3b..181a5b6f23 100644 --- a/Koha/SearchEngine/Elasticsearch.pm +++ b/Koha/SearchEngine/Elasticsearch.pm @@ -34,6 +34,8 @@ use Search::Elasticsearch; use Try::Tiny; use YAML::Syck; +use Koha::Database; + __PACKAGE__->mk_ro_accessors(qw( index )); __PACKAGE__->mk_accessors(qw( sort_fields )); @@ -245,26 +247,59 @@ sub _get_elasticsearch_mapping { return; } -sub reset_elasticsearch_mappings { - my ( $reset_fields ) = @_; +sub sync_elasticsearch_mappings { + my ($self, $options) = @_; + $options //= {}; my $mappings_yaml = C4::Context->config('elasticsearch_index_mappings'); $mappings_yaml ||= C4::Context->config('intranetdir') . '/etc/searchengine/elasticsearch/mappings.yaml'; my $indexes = LoadFile( $mappings_yaml ); while ( my ( $index_name, $fields ) = each %$indexes ) { while ( my ( $field_name, $data ) = each %$fields ) { - my $field_type = $data->{type}; - my $field_label = $data->{label}; - my $mappings = $data->{mappings}; - my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type }, { key => 'name' }); - for my $mapping ( @$mappings ) { - my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} }); - $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet} || 0, suggestible => $mapping->{suggestible} || 0, sort => $mapping->{sort} } ); + my $search_field = Koha::SearchFields->find({ 'name' => $field_name }); + if ($search_field) { + next if $options->{insert_only}; + } + else { + my $rs = Koha::SearchFields->_resultset()->create({ name => $field_name, label => $data->{label}, type => $data->{type}}); + $search_field = Koha::SearchFields->object_class->_new_from_dbic($rs); + } + if ($options->{revert_mappings}) { + # Delete all current marc_targets for field + my $rs = $search_field->_result()->search_marc_maps(); + while (my $marc_map = $rs->next) { + $search_field->_result()->remove_from_search_marc_maps($marc_map); + # Check if other search fields uses mapping, if not delete + $marc_map->delete unless (defined $marc_map->search_fields()->first); + } + } + for my $mapping ( @{$data->{mappings}} ) { + my $marc_field = Koha::SearchMarcMaps->find_or_create({ + index_name => $index_name, + marc_type => $mapping->{marc_type}, + marc_field => $mapping->{marc_field} + }); + # If merging mappings relation may already exist, remove to avoid duplicate entry + if(!($options->{revert_mappings} || $options->{insert_only})) { + $search_field->_result()->remove_from_search_marc_maps($marc_field->_result()); + } + $search_field->add_to_search_marc_maps($marc_field, { + facet => $mapping->{facet} || 0, + suggestible => $mapping->{suggestible} || 0, + sort => $mapping->{sort} + }); } } } } +sub reset_elasticsearch_mappings { + my $self = shift; + Koha::SearchFields->search->delete; + Koha::SearchMarcMaps->search->delete; + $self->sync_elasticsearch_mappings(); +} + # This overrides the accessor provided by Class::Accessor so that if # sort_fields isn't set, then it'll generate it. sub sort_fields { diff --git a/admin/searchengine/elasticsearch/mappings.pl b/admin/searchengine/elasticsearch/mappings.pl index 7d63002919..900cd3644f 100755 --- a/admin/searchengine/elasticsearch/mappings.pl +++ b/admin/searchengine/elasticsearch/mappings.pl @@ -99,12 +99,25 @@ if ( $op eq 'edit' ) { } elsif( $op eq 'reset' ) { # TODO Move this feature to the interface - my $sure = $input->param('i_know_what_i_am_doing'); - if ( $sure ) { - Koha::SearchMarcMaps->search->delete; + if ( $input->param('i_know_what_i_am_doing') ) { Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings; } } +elsif( $op eq 'add' ) { + if ( $input->param('i_know_what_i_am_doing') ) { + Koha::SearchEngine::Elasticsearch->sync_elasticsearch_mappings({ 'insert_only' => 1 }); + } +} +elsif( $op eq 'revert' ) { + if ( $input->param('i_know_what_i_am_doing') ) { + Koha::SearchEngine::Elasticsearch->sync_elasticsearch_mappings({ 'revert_mappings' => 1 }); + } +} +elsif( $op eq 'merge' ) { + if ( $input->param('i_know_what_i_am_doing') ) { + Koha::SearchEngine::Elasticsearch->sync_elasticsearch_mappings(); + } +} my @indexes; -- 2.11.0