Attachment 74819 Details for Bug 19707 – Bug 19707 - Add new operations for synchronizing Elasticsearch mappings

[patch] Bug 19707 - Add new operations for synchronizing Elasticsearch mappings

Bug-19707---Add-new-operations-for-synchronizing-E.patch (text/plain), 6.63 KB, created by David Gustafsson on 2018-04-24 17:43:36 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: David Gustafsson

Created: 2018-04-24 17:43:36 UTC

Size: 6.63 KB

patch

obsolete

>From c248ebe95136142aa8008076e523a85d72da6c4c Mon Sep 17 00:00:00 2001
>From: David Gustafsson <david.gustafsson@ub.gu.se>
>Date: Tue, 28 Nov 2017 11:41:56 +0100
>Subject: [PATCH] Bug 19707 - Add new operations for synchronizing
> Elasticsearch mappings
>
>Add new "merge", "revert" and "add" operations for synchronizing
>Elasticsearch mappings stored in database with mappings.yaml
>
>How to test:
>
>1)  Make sure mappings.yaml contains mappings not present in database.
>    (Also be aware that steps below will result in changed search engine
>    configuration, so use a test environment and/or make sure to
>    backup database before starting)
>2)  Go to "Administration" and "Search Engine Configuration"
>3)  Append ?i_know_what_i_am_doing=1&op=add to url and perform request
>    in browser
>4)  Vefify that any new mappings in mappings.yaml has been added, but
>    no current mappings has been modified
>5)  Edit mappings.yaml and add a new mapping for some field (new item
>    below "mappings:" key)
>6)  Add a new mapping for the same field in the database (through admin ui)
>7)  Append ?i_know_what_i_am_doing=1&op=merge to url and perform request
>    in browser
>8)  Verify that the new mapping from mappings.yaml has been added for this field,
>    while the field added in database was preserved
>10) Append ?i_know_what_i_am_doing=1&op=revert to url and perform request
>    in browser
>11) Vefify that the mapping previously added the field in database has
>    been deleted, and the mappings for this field are identical to those
>    in mappings.yaml
>
>Sponsored-by: Gothenburg University Library
>---
> Koha/SearchEngine/Elasticsearch.pm           | 51 +++++++++++++++++++++++-----
> admin/searchengine/elasticsearch/mappings.pl | 19 +++++++++--
> 2 files changed, 59 insertions(+), 11 deletions(-)
>
>diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm
>index d859a39362..034a6ebf49 100644
>--- a/Koha/SearchEngine/Elasticsearch.pm
>+++ b/Koha/SearchEngine/Elasticsearch.pm
>@@ -35,6 +35,7 @@ use Try::Tiny;
> use YAML::Syck;
> 
> use Data::Dumper;    # TODO remove
>+use Koha::Database;
> 
> __PACKAGE__->mk_ro_accessors(qw( index ));
> __PACKAGE__->mk_accessors(qw( sort_fields ));
>@@ -278,24 +279,58 @@ sub _elasticsearch_mapping_for_default {
>     };
> }
> 
>-sub reset_elasticsearch_mappings {
>+sub sync_elasticsearch_mappings {
>+    my ($self, $options) = @_;
>+    $options //= {};
>     my $mappings_yaml = C4::Context->config('intranetdir') . '/admin/searchengine/elasticsearch/mappings.yaml';
>     my $indexes = LoadFile( $mappings_yaml );
> 
>     while ( my ( $index_name, $fields ) = each %$indexes ) {
>         while ( my ( $field_name, $data ) = each %$fields ) {
>-            my $field_type = $data->{type};
>-            my $field_label = $data->{label};
>-            my $mappings = $data->{mappings};
>-            my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type }, { key => 'name' });
>-            for my $mapping ( @$mappings ) {
>-                my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} });
>-                $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet} || 0, suggestible => $mapping->{suggestible} || 0, sort => $mapping->{sort} } );
>+            my $search_field = Koha::SearchFields->find({ 'name' => $field_name });
>+            if ($search_field) {
>+                next if $options->{insert_only};
>+            }
>+            else {
>+                my $rs = Koha::SearchFields->_resultset()->create({ name => $field_name, label => $data->{label}, type => $data->{type}});
>+                $search_field = Koha::SearchFields->object_class->_new_from_dbic($rs);
>+            }
>+            if ($options->{revert_mappings}) {
>+                # Delete all current marc_targets for field
>+                my $rs = $search_field->_result()->search_marc_maps();
>+                while (my $marc_map = $rs->next) {
>+                    $search_field->_result()->remove_from_search_marc_maps($marc_map);
>+                    # Check if other search fields uses mapping, if not delete
>+                    $marc_map->delete unless (defined $marc_map->search_fields()->first);
>+                }
>+            }
>+            for my $mapping ( @{$data->{mappings}} ) {
>+                my $marc_field = Koha::SearchMarcMaps->find_or_create({
>+                    index_name => $index_name,
>+                    marc_type => $mapping->{marc_type},
>+                    marc_field => $mapping->{marc_field}
>+                });
>+                # If merging mappings relation may already exist, remove to avoid duplicate entry
>+                if(!($options->{revert_mappings} || $options->{insert_only})) {
>+                    $search_field->_result()->remove_from_search_marc_maps($marc_field->_result());
>+                }
>+                $search_field->add_to_search_marc_maps($marc_field, {
>+                    facet => $mapping->{facet} || 0,
>+                    suggestible => $mapping->{suggestible} || 0,
>+                    sort => $mapping->{sort}
>+                });
>             }
>         }
>     }
> }
> 
>+sub reset_elasticsearch_mappings {
>+    my $self = shift;
>+    Koha::SearchFields->search->delete;
>+    Koha::SearchMarcMaps->search->delete;
>+    $self->sync_elasticsearch_mappings();
>+}
>+
> # This overrides the accessor provided by Class::Accessor so that if
> # sort_fields isn't set, then it'll generate it.
> sub sort_fields {
>diff --git a/admin/searchengine/elasticsearch/mappings.pl b/admin/searchengine/elasticsearch/mappings.pl
>index 7d63002919..900cd3644f 100755
>--- a/admin/searchengine/elasticsearch/mappings.pl
>+++ b/admin/searchengine/elasticsearch/mappings.pl
>@@ -99,12 +99,25 @@ if ( $op eq 'edit' ) {
> }
> elsif( $op eq 'reset' ) {
>     # TODO Move this feature to the interface
>-    my $sure = $input->param('i_know_what_i_am_doing');
>-    if ( $sure ) {
>-        Koha::SearchMarcMaps->search->delete;
>+    if ( $input->param('i_know_what_i_am_doing') ) {
>         Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;
>     }
> }
>+elsif( $op eq 'add' ) {
>+    if ( $input->param('i_know_what_i_am_doing') ) {
>+        Koha::SearchEngine::Elasticsearch->sync_elasticsearch_mappings({ 'insert_only' => 1 });
>+    }
>+}
>+elsif( $op eq 'revert' ) {
>+    if ( $input->param('i_know_what_i_am_doing') ) {
>+        Koha::SearchEngine::Elasticsearch->sync_elasticsearch_mappings({ 'revert_mappings' => 1 });
>+    }
>+}
>+elsif( $op eq 'merge' ) {
>+    if ( $input->param('i_know_what_i_am_doing') ) {
>+        Koha::SearchEngine::Elasticsearch->sync_elasticsearch_mappings();
>+    }
>+}
> 
> my @indexes;
> 
>-- 
>2.11.0

Actions: View | Diff | Splinter Review

Attachments on bug 19707: 69414 | 74819 | 74820 | 75107 | 75108 | 79109 | 79112 | 82795 | 110054