From e23728f5a49d8257b1955a39f139b9abfa984b13 Mon Sep 17 00:00:00 2001 From: David Gustafsson Date: Tue, 28 Nov 2017 11:41:56 +0100 Subject: [PATCH] Bug 19707 - Add new operations for synchronizing Elasticsearch mappings Add new "merge", "revert" and "add" operations for synchronizing Elasticsearch mappings stored in database with mappings.yaml How to test: 1) Make sure mappings.yaml contains mappings not present in database. (Also be aware that steps below will result in changed search engine configuration, so use a test environment and/or make sure to backup database before starting) 2) Go to "Administration" and "Search Engine Configuration" 3) Append ?i_know_what_i_am_doing=1&op=add to url and perform request in browser 4) Verify that any new mappings in mappings.yaml has been added, but no current mappings has been modified or deleted 5) Edit mappings.yaml and add a new mapping for some field (new item below "mappings:" key) 6) Add a new mapping for the same field in the database (through admin ui) 7) Append ?i_know_what_i_am_doing=1&op=merge to url and perform request in browser 8) Verify that the new mapping from mappings.yaml has been added for this field, while the field added in database was preserved 10) Append ?i_know_what_i_am_doing=1&op=revert to url and perform request in browser 11) Verify that the mapping previously added the field in database has been deleted, and the mappings for this field are identical to those in mappings.yaml Sponsored-by: Gothenburg University Library --- Koha/SearchEngine/Elasticsearch.pm | 51 +++++++++++++++++++++++----- admin/searchengine/elasticsearch/mappings.pl | 19 +++++++++-- 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm index d859a39362..034a6ebf49 100644 --- a/Koha/SearchEngine/Elasticsearch.pm +++ b/Koha/SearchEngine/Elasticsearch.pm @@ -35,6 +35,7 @@ use Try::Tiny; use YAML::Syck; use Data::Dumper; # TODO remove +use Koha::Database; __PACKAGE__->mk_ro_accessors(qw( index )); __PACKAGE__->mk_accessors(qw( sort_fields )); @@ -278,24 +279,58 @@ sub _elasticsearch_mapping_for_default { }; } -sub reset_elasticsearch_mappings { +sub sync_elasticsearch_mappings { + my ($self, $options) = @_; + $options //= {}; my $mappings_yaml = C4::Context->config('intranetdir') . '/admin/searchengine/elasticsearch/mappings.yaml'; my $indexes = LoadFile( $mappings_yaml ); while ( my ( $index_name, $fields ) = each %$indexes ) { while ( my ( $field_name, $data ) = each %$fields ) { - my $field_type = $data->{type}; - my $field_label = $data->{label}; - my $mappings = $data->{mappings}; - my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type }, { key => 'name' }); - for my $mapping ( @$mappings ) { - my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} }); - $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet} || 0, suggestible => $mapping->{suggestible} || 0, sort => $mapping->{sort} } ); + my $search_field = Koha::SearchFields->find({ 'name' => $field_name }); + if ($search_field) { + next if $options->{insert_only}; + } + else { + my $rs = Koha::SearchFields->_resultset()->create({ name => $field_name, label => $data->{label}, type => $data->{type}}); + $search_field = Koha::SearchFields->object_class->_new_from_dbic($rs); + } + if ($options->{revert_mappings}) { + # Delete all current marc_targets for field + my $rs = $search_field->_result()->search_marc_maps(); + while (my $marc_map = $rs->next) { + $search_field->_result()->remove_from_search_marc_maps($marc_map); + # Check if other search fields uses mapping, if not delete + $marc_map->delete unless (defined $marc_map->search_fields()->first); + } + } + for my $mapping ( @{$data->{mappings}} ) { + my $marc_field = Koha::SearchMarcMaps->find_or_create({ + index_name => $index_name, + marc_type => $mapping->{marc_type}, + marc_field => $mapping->{marc_field} + }); + # If merging mappings relation may already exist, remove to avoid duplicate entry + if(!($options->{revert_mappings} || $options->{insert_only})) { + $search_field->_result()->remove_from_search_marc_maps($marc_field->_result()); + } + $search_field->add_to_search_marc_maps($marc_field, { + facet => $mapping->{facet} || 0, + suggestible => $mapping->{suggestible} || 0, + sort => $mapping->{sort} + }); } } } } +sub reset_elasticsearch_mappings { + my $self = shift; + Koha::SearchFields->search->delete; + Koha::SearchMarcMaps->search->delete; + $self->sync_elasticsearch_mappings(); +} + # This overrides the accessor provided by Class::Accessor so that if # sort_fields isn't set, then it'll generate it. sub sort_fields { diff --git a/admin/searchengine/elasticsearch/mappings.pl b/admin/searchengine/elasticsearch/mappings.pl index 7d63002919..900cd3644f 100755 --- a/admin/searchengine/elasticsearch/mappings.pl +++ b/admin/searchengine/elasticsearch/mappings.pl @@ -99,12 +99,25 @@ if ( $op eq 'edit' ) { } elsif( $op eq 'reset' ) { # TODO Move this feature to the interface - my $sure = $input->param('i_know_what_i_am_doing'); - if ( $sure ) { - Koha::SearchMarcMaps->search->delete; + if ( $input->param('i_know_what_i_am_doing') ) { Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings; } } +elsif( $op eq 'add' ) { + if ( $input->param('i_know_what_i_am_doing') ) { + Koha::SearchEngine::Elasticsearch->sync_elasticsearch_mappings({ 'insert_only' => 1 }); + } +} +elsif( $op eq 'revert' ) { + if ( $input->param('i_know_what_i_am_doing') ) { + Koha::SearchEngine::Elasticsearch->sync_elasticsearch_mappings({ 'revert_mappings' => 1 }); + } +} +elsif( $op eq 'merge' ) { + if ( $input->param('i_know_what_i_am_doing') ) { + Koha::SearchEngine::Elasticsearch->sync_elasticsearch_mappings(); + } +} my @indexes; -- 2.11.0