Attachment 110054 Details for Bug 19707 – Bug 19707 - Add new operations for synchronizing Elasticsearch mappings

[patch] Bug 19707 - Add new operations for synchronizing Elasticsearch mappings

Bug-19707---Add-new-operations-for-synchronizing-E.patch (text/plain), 10.10 KB, created by David Gustafsson on 2020-09-14 12:37:03 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: David Gustafsson

Created: 2020-09-14 12:37:03 UTC

Size: 10.10 KB

patch

obsolete

>From 1b20bfc736e1cd7944938234f85c6b47d41237b3 Mon Sep 17 00:00:00 2001
>From: David Gustafsson <david.gustafsson@ub.gu.se>
>Date: Tue, 28 Nov 2017 11:41:56 +0100
>Subject: [PATCH] Bug 19707 - Add new operations for synchronizing
> Elasticsearch mappings
>
>Add new "merge", "revert" and "add" operations for synchronizing
>Elasticsearch mappings stored in database with mappings.yaml
>
>How to test:
>
>1)  Make sure mappings.yaml contains mappings not present in database.
>    (Also be aware that steps below will result in changed search engine
>    configuration, so use a test environment and/or make sure to
>    backup database before starting)
>2)  Go to "Administration" and "Search Engine Configuration"
>3)  Click "Add mappings" and confirm
>4)  Verify that any new mappings in mappings.yaml has been added, but
>    no current mappings has been modified or deleted
>5)  Edit mappings.yaml and add a new mapping for some field (new item
>    below "mappings:" key)
>6)  Add a new mapping for the same field in the database (through admin ui)
>7)  Click "Merge mappings" and confirm
>8)  Verify that the new mapping from mappings.yaml has been added for this field,
>    while the field added in database was preserved
>10)  Click "Revert mappings" and confirm
>11) Verify that the mapping previously added the field in database has
>    been deleted, and the mappings for this field are identical to those
>    in mappings.yaml
>
>Sponsored-by: Gothenburg University Library
>---
> Koha/SearchEngine/Elasticsearch.pm            | 39 ++++++++++++---
> admin/searchengine/elasticsearch/mappings.pl  | 21 ++++++++
> .../searchengine/elasticsearch/mappings.tt    | 48 +++++++++++++++++++
> 3 files changed, 102 insertions(+), 6 deletions(-)
>
>diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm
>index 24ff497e27..d44a573af0 100644
>--- a/Koha/SearchEngine/Elasticsearch.pm
>+++ b/Koha/SearchEngine/Elasticsearch.pm
>@@ -45,6 +45,7 @@ use MIME::Base64;
> use Encode qw(encode);
> use Business::ISBN;
> use Scalar::Util qw(looks_like_number);
>+use Koha::Database;
> 
> __PACKAGE__->mk_ro_accessors(qw( index index_name ));
> __PACKAGE__->mk_accessors(qw( sort_fields ));
>@@ -346,8 +347,9 @@ sub _load_elasticsearch_mappings {
>     return LoadFile( $mappings_yaml );
> }
> 
>-sub reset_elasticsearch_mappings {
>-    my ( $self ) = @_;
>+sub sync_elasticsearch_mappings {
>+    my ($self, $options) = @_;
>+    $options //= {};
>     my $indexes = $self->_load_elasticsearch_mappings();
> 
>     Koha::SearchMarcMaps->delete;
>@@ -364,15 +366,33 @@ sub reset_elasticsearch_mappings {
> 
>             $sf_params{name} = $field_name;
> 
>-            my $search_field = Koha::SearchFields->find_or_create( \%sf_params, { key => 'name' } );
>-
>-            my $mappings = $data->{mappings};
>-            for my $mapping ( @$mappings ) {
>+            my $search_field = Koha::SearchFields->find({ 'name' => $field_name });
>+            if ($search_field) {
>+                next if $options->{insert_only};
>+            }
>+            else {
>+                my $rs = Koha::SearchFields->_resultset()->create(\%sf_params);
>+                $search_field = Koha::SearchFields->object_class->_new_from_dbic($rs);
>+            }
>+            if ($options->{revert_mappings}) {
>+                # Delete all current marc_targets for field
>+                my $rs = $search_field->_result()->search_marc_maps();
>+                while (my $marc_map = $rs->next) {
>+                    $search_field->_result()->remove_from_search_marc_maps($marc_map);
>+                    # Check if other search fields uses mapping, if not delete
>+                    $marc_map->delete unless (defined $marc_map->search_fields()->first);
>+                }
>+            }
>+            for my $mapping ( @{$data->{mappings}} ) {
>                 my $marc_field = Koha::SearchMarcMaps->find_or_create({
>                     index_name => $index_name,
>                     marc_type => $mapping->{marc_type},
>                     marc_field => $mapping->{marc_field}
>                 });
>+                # If merging mappings relation may already exist, remove to avoid duplicate entry
>+                if(!($options->{revert_mappings} || $options->{insert_only})) {
>+                    $search_field->_result()->remove_from_search_marc_maps($marc_field->_result());
>+                }
>                 $search_field->add_to_search_marc_maps($marc_field, {
>                     facet => $mapping->{facet} || 0,
>                     suggestible => $mapping->{suggestible} || 0,
>@@ -388,6 +408,13 @@ sub reset_elasticsearch_mappings {
>     # FIXME return the mappings?
> }
> 
>+sub reset_elasticsearch_mappings {
>+    my $self = shift;
>+    Koha::SearchFields->search->delete;
>+    Koha::SearchMarcMaps->search->delete;
>+    $self->sync_elasticsearch_mappings();
>+}
>+
> # This overrides the accessor provided by Class::Accessor so that if
> # sort_fields isn't set, then it'll generate it.
> sub sort_fields {
>diff --git a/admin/searchengine/elasticsearch/mappings.pl b/admin/searchengine/elasticsearch/mappings.pl
>index 57a9656f9c..53da40868e 100755
>--- a/admin/searchengine/elasticsearch/mappings.pl
>+++ b/admin/searchengine/elasticsearch/mappings.pl
>@@ -177,6 +177,27 @@ elsif( $op eq 'reset_confirmed' ) {
> elsif( $op eq 'reset_confirm' ) {
>     $template->param( reset_confirm => 1 );
> }
>+elsif( $op eq 'add_confirmed' ) {
>+    Koha::SearchEngine::Elasticsearch->sync_elasticsearch_mappings({ 'insert_only' => 1 });
>+    push @messages, { type => 'message', code => 'success_on_add' };
>+}
>+elsif( $op eq 'add_confirm' ) {
>+    $template->param( add_confirm => 1 );
>+}
>+elsif( $op eq 'revert_confirmed' ) {
>+    Koha::SearchEngine::Elasticsearch->sync_elasticsearch_mappings({ 'revert_mappings' => 1 });
>+    push @messages, { type => 'message', code => 'success_on_revert' };
>+}
>+elsif( $op eq 'revert_confirm' ) {
>+    $template->param( revert_confirm => 1 );
>+}
>+elsif( $op eq 'merge_confirmed' ) {
>+    Koha::SearchEngine::Elasticsearch->sync_elasticsearch_mappings();
>+    push @messages, { type => 'message', code => 'success_on_merge' };
>+}
>+elsif( $op eq 'merge_confirm' ) {
>+    $template->param( merge_confirm => 1 );
>+}
> 
> my @indexes;
> 
>diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt
>index 6452f2a98b..290e36460a 100644
>--- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt
>+++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt
>@@ -99,6 +99,12 @@ a.add, a.delete {
>           [% t("Mappings have been reset successfully.") | $raw %]
>         [% CASE 'elasticsearch_disabled' %]
>           [% t("Elasticsearch is currently disabled.") | $raw %]
>+        [% CASE 'success_on_add' %]
>+          [% t("Mappings have been added successfully.") | $raw %]
>+        [% CASE 'success_on_revert' %]
>+          [% t("Mappings have been reverted successfully.") | $raw %]
>+        [% CASE 'success_on_merge' %]
>+          [% t("Mappings have been merged successfully.") | $raw %]
>         [% CASE %]
>           [% m.code | html %]
>         [% END %]
>@@ -146,6 +152,45 @@ a.add, a.delete {
>             </form>
>         </div>
>     [% END %]
>+    [% IF add_confirm %]
>+        <div class="dialog alert">
>+            <h3>The current mappings you see on the screen will be presevred and mappings for new fields found in mappings.yam file will be added.</h3>
>+            <form method="post">
>+                <input type="hidden" name="op" value="add_confirmed" />
>+                <button type="submit" class="approve"><i class="fa fa-fw fa-check"></i> Yes, add mappings</button>
>+            </form>
>+            <br>
>+            <form method="post">
>+                <button type="submit" class="deny"><i class="fa fa-fw fa-remove"></i> No, do not add mappings</button>
>+            </form>
>+        </div>
>+    [% END %]
>+    [% IF revert_confirm %]
>+        <div class="dialog alert">
>+            <h3>The current mappings you see on the screen will be reset to defaults if the field is also found in the mappings.yaml file.</h3>
>+            <form method="post">
>+                <input type="hidden" name="op" value="revert_confirmed" />
>+                <button type="submit" class="approve"><i class="fa fa-fw fa-check"></i> Yes, revert mappings</button>
>+            </form>
>+            <br>
>+            <form method="post">
>+                <button type="submit" class="deny"><i class="fa fa-fw fa-remove"></i> No, do not revert mappings</button>
>+            </form>
>+        </div>
>+    [% END %]
>+    [% IF merge_confirm %]
>+        <div class="dialog alert">
>+            <h3>The current mappings you see on the screen will be preserved but new field mappings in mappings.yaml will be appended.</h3>
>+            <form method="post">
>+                <input type="hidden" name="op" value="merge_confirmed" />
>+                <button type="submit" class="approve"><i class="fa fa-fw fa-check"></i> Yes, merge mappings</button>
>+            </form>
>+            <br>
>+            <form method="post">
>+                <button type="submit" class="deny"><i class="fa fa-fw fa-remove"></i> No, do not merge mappings</button>
>+            </form>
>+        </div>
>+    [% END %]
>     <form method="post">
>         <div id="tabs" class="toptabs" style="clear:both">
>             <ul>
>@@ -450,6 +495,9 @@ a.add, a.delete {
>         <p>
>             <button class="btn btn-default" type="submit" name="op" value="edit"><i class="fa fa-hdd-o" aria-hidden="true"></i> Save</button>
>             <button class="btn btn-default" type="submit" name="op" value="reset_confirm"><i class="fa fa-refresh" aria-hidden="true"></i> Reset Mappings</button>
>+            <button class="btn btn-default" type="submit" name="op" value="add_confirm"><i class="fa fa-plus" aria-hidden="true"></i> Add Mappings</button>
>+            <button class="btn btn-default" type="submit" name="op" value="revert_confirm"><i class="fa fa-undo" aria-hidden="true"></i> Revert Mappings</button>
>+            <button class="btn btn-default" type="submit" name="op" value="merge_confirm"></i> <i class="fa fa-plus-square" aria-hidden="true"></i> Merge Mappings</button>
>         </p>
>     </form>
> 
>-- 
>2.20.1

Actions: View | Diff | Splinter Review

Attachments on bug 19707: 69414 | 74819 | 74820 | 75107 | 75108 | 79109 | 79112 | 82795 | 110054