From 8c07caf22afeb9224fcf65a80a780e1ef3a5eb1c Mon Sep 17 00:00:00 2001 From: Ere Maijala Date: Mon, 19 Feb 2018 16:54:39 +0200 Subject: [PATCH] Bug 20248: Improve Elasticsearch mapppings UI and rebuild_elastic_search.pl. Improvements: 1) Mappings UI now has button that allows one to reset the mappings. 2) Mappings UI displays the items in alphabetical order. 3) Indexing script drops and recreates the index right away, which helps prevent ES from autocreating a bad index if someone does something while the first batch of records is being processed. 4) Indexing script has nicer output. To test: 1) Change mappings.yaml file 2) Reset mappings in UI in mappings.pl 3) Verify the mappings have been changed in UI 4) The field order is alphabetical 5) Rebuild script has an clean output Signed-off-by: Claire Hernandez --- Koha/SearchEngine/Elasticsearch/Indexer.pm | 22 +++++++++++++++++++ admin/searchengine/elasticsearch/mappings.pl | 17 +++++++++------ .../admin/searchengine/elasticsearch/mappings.tt | 25 +++++++++++++++++++--- misc/search_tools/rebuild_elastic_search.pl | 20 ++++++++++------- 4 files changed, 66 insertions(+), 18 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch/Indexer.pm b/Koha/SearchEngine/Elasticsearch/Indexer.pm index e5babee535..ed8ea458b9 100644 --- a/Koha/SearchEngine/Elasticsearch/Indexer.pm +++ b/Koha/SearchEngine/Elasticsearch/Indexer.pm @@ -139,6 +139,28 @@ sub delete_index_background { $self->delete_index(@_); } +=head2 $indexer->create_index(); + +Create an index on the Elasticsearch server. + +=cut + +sub create_index { + my ($self) = @_; + + if (!$self->store) { + my $params = $self->get_elasticsearch_params(); + $self->store( + Catmandu::Store::ElasticSearch->new( + %$params, + index_settings => $self->get_elasticsearch_settings(), + index_mappings => $self->get_elasticsearch_mappings(), + ) + ); + } + $self->store->bag->commit; +} + =head2 $indexer->drop_index(); Drops the index from the elasticsearch server. Calling C diff --git a/admin/searchengine/elasticsearch/mappings.pl b/admin/searchengine/elasticsearch/mappings.pl index 7d63002919..243204a7a5 100755 --- a/admin/searchengine/elasticsearch/mappings.pl +++ b/admin/searchengine/elasticsearch/mappings.pl @@ -97,14 +97,16 @@ if ( $op eq 'edit' ) { $schema->storage->txn_commit; } } -elsif( $op eq 'reset' ) { - # TODO Move this feature to the interface - my $sure = $input->param('i_know_what_i_am_doing'); - if ( $sure ) { - Koha::SearchMarcMaps->search->delete; - Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings; - } +elsif( $op eq 'reset' || $op eq 'reset_confirmed' ) { + Koha::SearchMarcMaps->delete; + Koha::SearchFields->delete; + Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings; + push @messages, { type => 'message', code => 'success_on_reset' }; } +elsif( $op eq 'reset_confirm' ) { + $template->param( reset_confirm => 1 ); +} + my @indexes; @@ -114,6 +116,7 @@ for my $index_name (qw| biblios authorities |) { { join => { search_marc_to_fields => 'search_marc_map' }, '+select' => [ 'search_marc_to_fields.facet', 'search_marc_to_fields.suggestible', 'search_marc_to_fields.sort', 'search_marc_map.marc_field' ], '+as' => [ 'facet', 'suggestible', 'sort', 'marc_field' ], + order_by => { -asc => [qw/name marc_field/] } } ); diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt index 24327c5996..7cd8748a21 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt @@ -79,7 +79,9 @@ a.add, a.delete { An error occurred when deleting the existing mappings. Nothing has been changed! (search field [% m.values.field_name %] with mapping [% m.values.marc_field %].) [% CASE 'success_on_update' %] - Mapping updated successfully. + Mappings updated successfully. + [% CASE 'success_on_reset' %] + Mappings have been reset successfully. [% CASE %] [% m.code %] [% END %] @@ -88,7 +90,7 @@ a.add, a.delete {

Search engine configuration

- Warning: Any modification in these configurations will need a total reindexation to be fully taken into account ! + Warning: Any modification of this configuration requires a complete index rebuild to be fully taken into account!
[% IF errors %]
@@ -107,6 +109,19 @@ a.add, a.delete {
[% END %] + [% IF reset_confirm %] +
+

Confirm mappings reset

+
+ + +
+
+
+ +
+
+ [% ELSE %]
@@ -286,8 +301,12 @@ a.add, a.delete {
[% END %] -

+

+ + Reset Mappings +

+ [% END %] diff --git a/misc/search_tools/rebuild_elastic_search.pl b/misc/search_tools/rebuild_elastic_search.pl index 6faa32c63e..63065cad79 100755 --- a/misc/search_tools/rebuild_elastic_search.pl +++ b/misc/search_tools/rebuild_elastic_search.pl @@ -92,14 +92,14 @@ use MARC::Record; use Modern::Perl; use Pod::Usage; -use Data::Dumper; # TODO remove - my $verbose = 0; my $commit = 5000; my ($delete, $help, $man); my ($index_biblios, $index_authorities); my (@biblionumbers); +$|=1; # flushes output + GetOptions( 'c|commit=i' => \$commit, 'd|delete' => \$delete, @@ -161,10 +161,8 @@ sub do_reindex { my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } ); if ($delete) { - - # We know it's safe to not recreate the indexer because update_index - # hasn't been called yet. $indexer->drop_index(); + $indexer->create_index(); } my $count = 0; @@ -173,23 +171,29 @@ sub do_reindex { while ( my $record = $next->() ) { my $id = $record->id; my $record = $record->record; - _log( 1, "$id\n" ); $count++; + if ( $verbose == 1 ) { + _log( 1, "$count records processed\n" ) if ( $count % 1000 == 0); + } else { + _log( 2, "$id\n" ); + } push @id_buffer, $id; push @commit_buffer, $record; if ( !( --$commit_count ) ) { - _log( 2, "Committing...\n" ); + _log( 1, "Committing $commit records..." ); $indexer->update_index( \@id_buffer, \@commit_buffer ); $commit_count = $commit; @id_buffer = (); @commit_buffer = (); + _log( 1, " done\n" ); } } # There are probably uncommitted records + _log( 1, "Committing final records...\n" ); $indexer->update_index( \@id_buffer, \@commit_buffer ); - _log( 1, "$count records indexed.\n" ); + _log( 1, "Total $count records indexed\n" ); } # Checks some basic stuff to ensure that it's sane before we start. -- 2.11.0