Attachment 71954 Details for Bug 20248 – Improve Elasticsearch mapppings UI and rebuild_elastic_search.pl.

[patch] Improve Elasticsearch mapppings UI and rebuild_elastic_search.pl.

Improve-Elasticsearch-mapppings-UI-and-rebuildelas.patch (text/plain), 7.71 KB, created by Ere Maijala on 2018-02-19 14:58:36 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Ere Maijala

Created: 2018-02-19 14:58:36 UTC

Size: 7.71 KB

patch

obsolete

>From f43b49aaffc3fb4e41aaf12363265fe6c6deb761 Mon Sep 17 00:00:00 2001
>From: Ere Maijala <ere.maijala@helsinki.fi>
>Date: Mon, 19 Feb 2018 16:54:39 +0200
>Subject: [PATCH] Improve Elasticsearch mapppings UI and
> rebuild_elastic_search.pl.
>
>https://bugs.koha-community.org/show_bug.cgi?id=20248
>---
> Koha/SearchEngine/Elasticsearch/Indexer.pm         | 22 +++++++++++++++++++
> admin/searchengine/elasticsearch/mappings.pl       | 17 +++++++++------
> .../admin/searchengine/elasticsearch/mappings.tt   | 25 +++++++++++++++++++---
> misc/search_tools/rebuild_elastic_search.pl        | 20 ++++++++++-------
> 4 files changed, 66 insertions(+), 18 deletions(-)
>
>diff --git a/Koha/SearchEngine/Elasticsearch/Indexer.pm b/Koha/SearchEngine/Elasticsearch/Indexer.pm
>index e5babee..ed8ea45 100644
>--- a/Koha/SearchEngine/Elasticsearch/Indexer.pm
>+++ b/Koha/SearchEngine/Elasticsearch/Indexer.pm
>@@ -139,6 +139,28 @@ sub delete_index_background {
>     $self->delete_index(@_);
> }
> 
>+=head2 $indexer->create_index();
>+
>+Create an index on the Elasticsearch server.
>+
>+=cut
>+
>+sub create_index {
>+    my ($self) = @_;
>+
>+    if (!$self->store) {
>+        my $params  = $self->get_elasticsearch_params();
>+        $self->store(
>+            Catmandu::Store::ElasticSearch->new(
>+                %$params,
>+                index_settings => $self->get_elasticsearch_settings(),
>+                index_mappings => $self->get_elasticsearch_mappings(),
>+            )
>+        );
>+    }
>+    $self->store->bag->commit;
>+}
>+
> =head2 $indexer->drop_index();
> 
> Drops the index from the elasticsearch server. Calling C<update_index>
>diff --git a/admin/searchengine/elasticsearch/mappings.pl b/admin/searchengine/elasticsearch/mappings.pl
>index 7d63002..243204a 100755
>--- a/admin/searchengine/elasticsearch/mappings.pl
>+++ b/admin/searchengine/elasticsearch/mappings.pl
>@@ -97,14 +97,16 @@ if ( $op eq 'edit' ) {
>         $schema->storage->txn_commit;
>     }
> }
>-elsif( $op eq 'reset' ) {
>-    # TODO Move this feature to the interface
>-    my $sure = $input->param('i_know_what_i_am_doing');
>-    if ( $sure ) {
>-        Koha::SearchMarcMaps->search->delete;
>-        Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;
>-    }
>+elsif( $op eq 'reset' || $op eq 'reset_confirmed' ) {
>+    Koha::SearchMarcMaps->delete;
>+    Koha::SearchFields->delete;
>+    Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;
>+    push @messages, { type => 'message', code => 'success_on_reset' };
> }
>+elsif( $op eq 'reset_confirm' ) {
>+    $template->param( reset_confirm => 1 );
>+}    
>+
> 
> my @indexes;
> 
>@@ -114,6 +116,7 @@ for my $index_name (qw| biblios authorities |) {
>         {   join => { search_marc_to_fields => 'search_marc_map' },
>             '+select' => [ 'search_marc_to_fields.facet', 'search_marc_to_fields.suggestible', 'search_marc_to_fields.sort', 'search_marc_map.marc_field' ],
>             '+as'     => [ 'facet',                       'suggestible',                       'sort',                       'marc_field' ],
>+            order_by => { -asc => [qw/name marc_field/] }
>         }
>     );
> 
>diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt
>index 24327c5..7cd8748 100644
>--- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt
>+++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt
>@@ -79,7 +79,9 @@ a.add, a.delete {
>           An error occurred when deleting the existing mappings. Nothing has been changed!
>           (search field [% m.values.field_name %] with mapping [% m.values.marc_field %].)
>         [% CASE 'success_on_update' %]
>-          Mapping updated successfully.
>+          Mappings updated successfully.
>+        [% CASE 'success_on_reset' %]
>+          Mappings have been reset successfully.
>         [% CASE %]
>           [% m.code %]
>         [% END %]
>@@ -88,7 +90,7 @@ a.add, a.delete {
> 
>     <h1>Search engine configuration</h1>
>     <div class="warning">
>-        Warning: Any modification in these configurations will need a total reindexation to be fully taken into account !
>+        Warning: Any modification of this configuration requires a complete index rebuild to be fully taken into account!
>     </div>
>     [% IF errors %]
>         <div class="error">
>@@ -107,6 +109,19 @@ a.add, a.delete {
>         </div>
>     [% END %]
> 
>+    [% IF reset_confirm %]
>+      <div class="dialog alert">
>+        <h3>Confirm mappings reset</h3>
>+        <form action="?" method="post">
>+          <input type="hidden" name="op" value="reset_confirmed" />
>+          <button type="submit" class="approve"><i class="fa fa-fw fa-check"></i> Yes, reset mappings</button>            
>+        </form>
>+        <br>
>+        <form action="?" method="post">
>+          <button type="submit" class="deny"><i class="fa fa-fw fa-remove"></i> No, do not reset mappings</button>
>+        </form>
>+      </div>
>+    [% ELSE %]
>     <form method="post">
>         <input type="hidden" name="op" value="edit" />
>         <div id="tabs" class="toptabs" style="clear:both">
>@@ -286,8 +301,12 @@ a.add, a.delete {
>                 </div>
>             [% END %]
>         </div>
>-        <p><button class="btn btn-default" type="submit"><i class="fa fa-hdd-o" aria-hidden="true"></i> Save</button></p>
>+            <p>
>+                <button class="btn btn-default" type="submit"><i class="fa fa-hdd-o" aria-hidden="true"></i> Save</button> 
>+                <a class="btn btn-default" href="?op=reset_confirm"><i class="fa fa-refresh" aria-hidden="true"></i> Reset Mappings</a>
>+            </p>
>     </form>
>+    [% END %]
> </div>
> 
> </div>
>diff --git a/misc/search_tools/rebuild_elastic_search.pl b/misc/search_tools/rebuild_elastic_search.pl
>index 6faa32c..63065ca 100755
>--- a/misc/search_tools/rebuild_elastic_search.pl
>+++ b/misc/search_tools/rebuild_elastic_search.pl
>@@ -92,14 +92,14 @@ use MARC::Record;
> use Modern::Perl;
> use Pod::Usage;
> 
>-use Data::Dumper; # TODO remove
>-
> my $verbose = 0;
> my $commit = 5000;
> my ($delete, $help, $man);
> my ($index_biblios, $index_authorities);
> my (@biblionumbers);
> 
>+$|=1; # flushes output
>+
> GetOptions(
>     'c|commit=i'       => \$commit,
>     'd|delete'         => \$delete,
>@@ -161,10 +161,8 @@ sub do_reindex {
> 
>     my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
>     if ($delete) {
>-
>-        # We know it's safe to not recreate the indexer because update_index
>-        # hasn't been called yet.
>         $indexer->drop_index();
>+        $indexer->create_index();
>     }
> 
>     my $count        = 0;
>@@ -173,23 +171,29 @@ sub do_reindex {
>     while ( my $record = $next->() ) {
>         my $id     = $record->id;
>         my $record = $record->record;
>-        _log( 1, "$id\n" );
>         $count++;
>+        if ( $verbose == 1 ) {
>+            _log( 1, "$count records processed\n" ) if ( $count % 1000 == 0);
>+        } else {
>+            _log( 2, "$id\n" );
>+        }
> 
>         push @id_buffer,     $id;
>         push @commit_buffer, $record;
>         if ( !( --$commit_count ) ) {
>-            _log( 2, "Committing...\n" );
>+            _log( 1, "Committing $commit records..." );
>             $indexer->update_index( \@id_buffer, \@commit_buffer );
>             $commit_count  = $commit;
>             @id_buffer     = ();
>             @commit_buffer = ();
>+            _log( 1, " done\n" );
>         }
>     }
> 
>     # There are probably uncommitted records
>+    _log( 1, "Committing final records...\n" );
>     $indexer->update_index( \@id_buffer, \@commit_buffer );
>-    _log( 1, "$count records indexed.\n" );
>+    _log( 1, "Total $count records indexed\n" );
> }
> 
> # Checks some basic stuff to ensure that it's sane before we start.
>-- 
>2.7.4

Actions: View | Diff | Splinter Review

Attachments on bug 20248: 71954 | 72801 | 72861 | 72907 | 72921 | 72922 | 72932 | 72933 | 73374 | 73796 | 73798 | 77570 | 77571 | 77572 | 77573