From 1b20bfc736e1cd7944938234f85c6b47d41237b3 Mon Sep 17 00:00:00 2001 From: David Gustafsson Date: Tue, 28 Nov 2017 11:41:56 +0100 Subject: [PATCH] Bug 19707 - Add new operations for synchronizing Elasticsearch mappings Add new "merge", "revert" and "add" operations for synchronizing Elasticsearch mappings stored in database with mappings.yaml How to test: 1) Make sure mappings.yaml contains mappings not present in database. (Also be aware that steps below will result in changed search engine configuration, so use a test environment and/or make sure to backup database before starting) 2) Go to "Administration" and "Search Engine Configuration" 3) Click "Add mappings" and confirm 4) Verify that any new mappings in mappings.yaml has been added, but no current mappings has been modified or deleted 5) Edit mappings.yaml and add a new mapping for some field (new item below "mappings:" key) 6) Add a new mapping for the same field in the database (through admin ui) 7) Click "Merge mappings" and confirm 8) Verify that the new mapping from mappings.yaml has been added for this field, while the field added in database was preserved 10) Click "Revert mappings" and confirm 11) Verify that the mapping previously added the field in database has been deleted, and the mappings for this field are identical to those in mappings.yaml Sponsored-by: Gothenburg University Library --- Koha/SearchEngine/Elasticsearch.pm | 39 ++++++++++++--- admin/searchengine/elasticsearch/mappings.pl | 21 ++++++++ .../searchengine/elasticsearch/mappings.tt | 48 +++++++++++++++++++ 3 files changed, 102 insertions(+), 6 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm index 24ff497e27..d44a573af0 100644 --- a/Koha/SearchEngine/Elasticsearch.pm +++ b/Koha/SearchEngine/Elasticsearch.pm @@ -45,6 +45,7 @@ use MIME::Base64; use Encode qw(encode); use Business::ISBN; use Scalar::Util qw(looks_like_number); +use Koha::Database; __PACKAGE__->mk_ro_accessors(qw( index index_name )); __PACKAGE__->mk_accessors(qw( sort_fields )); @@ -346,8 +347,9 @@ sub _load_elasticsearch_mappings { return LoadFile( $mappings_yaml ); } -sub reset_elasticsearch_mappings { - my ( $self ) = @_; +sub sync_elasticsearch_mappings { + my ($self, $options) = @_; + $options //= {}; my $indexes = $self->_load_elasticsearch_mappings(); Koha::SearchMarcMaps->delete; @@ -364,15 +366,33 @@ sub reset_elasticsearch_mappings { $sf_params{name} = $field_name; - my $search_field = Koha::SearchFields->find_or_create( \%sf_params, { key => 'name' } ); - - my $mappings = $data->{mappings}; - for my $mapping ( @$mappings ) { + my $search_field = Koha::SearchFields->find({ 'name' => $field_name }); + if ($search_field) { + next if $options->{insert_only}; + } + else { + my $rs = Koha::SearchFields->_resultset()->create(\%sf_params); + $search_field = Koha::SearchFields->object_class->_new_from_dbic($rs); + } + if ($options->{revert_mappings}) { + # Delete all current marc_targets for field + my $rs = $search_field->_result()->search_marc_maps(); + while (my $marc_map = $rs->next) { + $search_field->_result()->remove_from_search_marc_maps($marc_map); + # Check if other search fields uses mapping, if not delete + $marc_map->delete unless (defined $marc_map->search_fields()->first); + } + } + for my $mapping ( @{$data->{mappings}} ) { my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} }); + # If merging mappings relation may already exist, remove to avoid duplicate entry + if(!($options->{revert_mappings} || $options->{insert_only})) { + $search_field->_result()->remove_from_search_marc_maps($marc_field->_result()); + } $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet} || 0, suggestible => $mapping->{suggestible} || 0, @@ -388,6 +408,13 @@ sub reset_elasticsearch_mappings { # FIXME return the mappings? } +sub reset_elasticsearch_mappings { + my $self = shift; + Koha::SearchFields->search->delete; + Koha::SearchMarcMaps->search->delete; + $self->sync_elasticsearch_mappings(); +} + # This overrides the accessor provided by Class::Accessor so that if # sort_fields isn't set, then it'll generate it. sub sort_fields { diff --git a/admin/searchengine/elasticsearch/mappings.pl b/admin/searchengine/elasticsearch/mappings.pl index 57a9656f9c..53da40868e 100755 --- a/admin/searchengine/elasticsearch/mappings.pl +++ b/admin/searchengine/elasticsearch/mappings.pl @@ -177,6 +177,27 @@ elsif( $op eq 'reset_confirmed' ) { elsif( $op eq 'reset_confirm' ) { $template->param( reset_confirm => 1 ); } +elsif( $op eq 'add_confirmed' ) { + Koha::SearchEngine::Elasticsearch->sync_elasticsearch_mappings({ 'insert_only' => 1 }); + push @messages, { type => 'message', code => 'success_on_add' }; +} +elsif( $op eq 'add_confirm' ) { + $template->param( add_confirm => 1 ); +} +elsif( $op eq 'revert_confirmed' ) { + Koha::SearchEngine::Elasticsearch->sync_elasticsearch_mappings({ 'revert_mappings' => 1 }); + push @messages, { type => 'message', code => 'success_on_revert' }; +} +elsif( $op eq 'revert_confirm' ) { + $template->param( revert_confirm => 1 ); +} +elsif( $op eq 'merge_confirmed' ) { + Koha::SearchEngine::Elasticsearch->sync_elasticsearch_mappings(); + push @messages, { type => 'message', code => 'success_on_merge' }; +} +elsif( $op eq 'merge_confirm' ) { + $template->param( merge_confirm => 1 ); +} my @indexes; diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt index 6452f2a98b..290e36460a 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt @@ -99,6 +99,12 @@ a.add, a.delete { [% t("Mappings have been reset successfully.") | $raw %] [% CASE 'elasticsearch_disabled' %] [% t("Elasticsearch is currently disabled.") | $raw %] + [% CASE 'success_on_add' %] + [% t("Mappings have been added successfully.") | $raw %] + [% CASE 'success_on_revert' %] + [% t("Mappings have been reverted successfully.") | $raw %] + [% CASE 'success_on_merge' %] + [% t("Mappings have been merged successfully.") | $raw %] [% CASE %] [% m.code | html %] [% END %] @@ -146,6 +152,45 @@ a.add, a.delete { [% END %] + [% IF add_confirm %] +
+

The current mappings you see on the screen will be presevred and mappings for new fields found in mappings.yam file will be added.

+
+ + +
+
+
+ +
+
+ [% END %] + [% IF revert_confirm %] +
+

The current mappings you see on the screen will be reset to defaults if the field is also found in the mappings.yaml file.

+
+ + +
+
+
+ +
+
+ [% END %] + [% IF merge_confirm %] +
+

The current mappings you see on the screen will be preserved but new field mappings in mappings.yaml will be appended.

+
+ + +
+
+
+ +
+
+ [% END %]
    @@ -450,6 +495,9 @@ a.add, a.delete {

    + + +

    -- 2.20.1