From 7050c94b91f5b76abe4aedbdcc167d5eec808222 Mon Sep 17 00:00:00 2001 From: David Gustafsson Date: Tue, 28 Nov 2017 11:41:56 +0100 Subject: [PATCH] Bug 19707 - Add new operations for synchronizing Elasticsearch mappings Add new "merge", "revert" and "add" operations for synchronizing Elasticsearch mappings stored in database with mappings.yaml How to test: 1) Make sure mappings.yaml contains mappings not present in database. (Also be aware that steps below will result in changed search engine configuration, so use a test environment and/or make sure to backup database before starting) 2) Go to "Administration" and "Search Engine Configuration" 3) Append ?i_know_what_i_am_doing=1&op=add to url and perform request in browser 4) Verify that any new mappings in mappings.yaml has been added, but no current mappings has been modified or deleted 5) Edit mappings.yaml and add a new mapping for some field (new item below "mappings:" key) 6) Add a new mapping for the same field in the database (through admin ui) 7) Append ?i_know_what_i_am_doing=1&op=merge to url and perform request in browser 8) Verify that the new mapping from mappings.yaml has been added for this field, while the field added in database was preserved 10) Append ?i_know_what_i_am_doing=1&op=revert to url and perform request in browser 11) Verify that the mapping previously added the field in database has been deleted, and the mappings for this field are identical to those in mappings.yaml Sponsored-by: Gothenburg University Library --- Koha/SearchEngine/Elasticsearch.pm | 53 ++++++++++++++++++---- admin/searchengine/elasticsearch/mappings.pl | 24 ++++++++-- .../admin/searchengine/elasticsearch/mappings.tt | 48 ++++++++++++++++++++ 3 files changed, 113 insertions(+), 12 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm index 9c114fb060..5b297016f1 100644 --- a/Koha/SearchEngine/Elasticsearch.pm +++ b/Koha/SearchEngine/Elasticsearch.pm @@ -34,6 +34,8 @@ use Search::Elasticsearch; use Try::Tiny; use YAML::Syck; +use Koha::Database; + __PACKAGE__->mk_ro_accessors(qw( index )); __PACKAGE__->mk_accessors(qw( sort_fields )); @@ -245,26 +247,59 @@ sub _get_elasticsearch_mapping { return; } -sub reset_elasticsearch_mappings { - my ( $reset_fields ) = @_; +sub sync_elasticsearch_mappings { + my ($self, $options) = @_; + $options //= {}; my $mappings_yaml = C4::Context->config('elasticsearch_index_mappings'); $mappings_yaml ||= C4::Context->config('intranetdir') . '/admin/searchengine/elasticsearch/mappings.yaml'; my $indexes = LoadFile( $mappings_yaml ); while ( my ( $index_name, $fields ) = each %$indexes ) { while ( my ( $field_name, $data ) = each %$fields ) { - my $field_type = $data->{type}; - my $field_label = $data->{label}; - my $mappings = $data->{mappings}; - my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type }, { key => 'name' }); - for my $mapping ( @$mappings ) { - my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} }); - $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet} || 0, suggestible => $mapping->{suggestible} || 0, sort => $mapping->{sort} } ); + my $search_field = Koha::SearchFields->find({ 'name' => $field_name }); + if ($search_field) { + next if $options->{insert_only}; + } + else { + my $rs = Koha::SearchFields->_resultset()->create({ name => $field_name, label => $data->{label}, type => $data->{type}}); + $search_field = Koha::SearchFields->object_class->_new_from_dbic($rs); + } + if ($options->{revert_mappings}) { + # Delete all current marc_targets for field + my $rs = $search_field->_result()->search_marc_maps(); + while (my $marc_map = $rs->next) { + $search_field->_result()->remove_from_search_marc_maps($marc_map); + # Check if other search fields uses mapping, if not delete + $marc_map->delete unless (defined $marc_map->search_fields()->first); + } + } + for my $mapping ( @{$data->{mappings}} ) { + my $marc_field = Koha::SearchMarcMaps->find_or_create({ + index_name => $index_name, + marc_type => $mapping->{marc_type}, + marc_field => $mapping->{marc_field} + }); + # If merging mappings relation may already exist, remove to avoid duplicate entry + if(!($options->{revert_mappings} || $options->{insert_only})) { + $search_field->_result()->remove_from_search_marc_maps($marc_field->_result()); + } + $search_field->add_to_search_marc_maps($marc_field, { + facet => $mapping->{facet} || 0, + suggestible => $mapping->{suggestible} || 0, + sort => $mapping->{sort} + }); } } } } +sub reset_elasticsearch_mappings { + my $self = shift; + Koha::SearchFields->search->delete; + Koha::SearchMarcMaps->search->delete; + $self->sync_elasticsearch_mappings(); +} + # This overrides the accessor provided by Class::Accessor so that if # sort_fields isn't set, then it'll generate it. sub sort_fields { diff --git a/admin/searchengine/elasticsearch/mappings.pl b/admin/searchengine/elasticsearch/mappings.pl index ecd8741533..fe2ff75a21 100755 --- a/admin/searchengine/elasticsearch/mappings.pl +++ b/admin/searchengine/elasticsearch/mappings.pl @@ -98,15 +98,33 @@ if ( $op eq 'edit' ) { } } elsif( $op eq 'reset_confirmed' ) { - Koha::SearchMarcMaps->delete; - Koha::SearchFields->delete; Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings; push @messages, { type => 'message', code => 'success_on_reset' }; } elsif( $op eq 'reset_confirm' ) { $template->param( reset_confirm => 1 ); } - +elsif( $op eq 'add_confirmed' ) { + Koha::SearchEngine::Elasticsearch->sync_elasticsearch_mappings({ 'insert_only' => 1 }); + push @messages, { type => 'message', code => 'success_on_add' }; +} +elsif( $op eq 'add_confirm' ) { + $template->param( add_confirm => 1 ); +} +elsif( $op eq 'revert_confirmed' ) { + Koha::SearchEngine::Elasticsearch->sync_elasticsearch_mappings({ 'revert_mappings' => 1 }); + push @messages, { type => 'message', code => 'success_on_revert' }; +} +elsif( $op eq 'revert_confirm' ) { + $template->param( revert_confirm => 1 ); +} +elsif( $op eq 'merge_confirmed' ) { + Koha::SearchEngine::Elasticsearch->sync_elasticsearch_mappings(); + push @messages, { type => 'message', code => 'success_on_merge' }; +} +elsif( $op eq 'merge_confirm' ) { + $template->param( merge_confirm => 1 ); +} my @indexes; diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt index c049b01040..c32ab822be 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt @@ -84,6 +84,12 @@ a.add, a.delete { Mappings updated successfully. [% CASE 'success_on_reset' %] Mappings have been reset successfully. + [% CASE 'success_on_add' %] + Mappings have been added successfully. + [% CASE 'success_on_revert' %] + Mappings have been reverted successfully. + [% CASE 'success_on_merge' %] + Mappings have been merged successfully. [% CASE %] [% m.code | html %] [% END %] @@ -124,6 +130,45 @@ a.add, a.delete { [% END %] + [% IF add_confirm %] +
+

The current mappings you see on the screen will be presevred and mappings for new fields found in mappings.yam file will be added.

+
+ + +
+
+
+ +
+
+ [% END %] + [% IF revert_confirm %] +
+

The current mappings you see on the screen will be reset to defaults if the field is also found in the mappings.yaml file.

+
+ + +
+
+
+ +
+
+ [% END %] + [% IF merge_confirm %] +
+

The current mappings you see on the screen will be preserved but new field mappings in mappings.yaml will be appended.

+
+ + +
+
+
+ +
+
+ [% END %]
    @@ -315,6 +360,9 @@ a.add, a.delete {

    + + +

-- 2.11.0