From f43b49aaffc3fb4e41aaf12363265fe6c6deb761 Mon Sep 17 00:00:00 2001 From: Ere Maijala Date: Mon, 19 Feb 2018 16:54:39 +0200 Subject: [PATCH] Improve Elasticsearch mapppings UI and rebuild_elastic_search.pl. https://bugs.koha-community.org/show_bug.cgi?id=20248 --- Koha/SearchEngine/Elasticsearch/Indexer.pm | 22 +++++++++++++++++++ admin/searchengine/elasticsearch/mappings.pl | 17 +++++++++------ .../admin/searchengine/elasticsearch/mappings.tt | 25 +++++++++++++++++++--- misc/search_tools/rebuild_elastic_search.pl | 20 ++++++++++------- 4 files changed, 66 insertions(+), 18 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch/Indexer.pm b/Koha/SearchEngine/Elasticsearch/Indexer.pm index e5babee..ed8ea45 100644 --- a/Koha/SearchEngine/Elasticsearch/Indexer.pm +++ b/Koha/SearchEngine/Elasticsearch/Indexer.pm @@ -139,6 +139,28 @@ sub delete_index_background { $self->delete_index(@_); } +=head2 $indexer->create_index(); + +Create an index on the Elasticsearch server. + +=cut + +sub create_index { + my ($self) = @_; + + if (!$self->store) { + my $params = $self->get_elasticsearch_params(); + $self->store( + Catmandu::Store::ElasticSearch->new( + %$params, + index_settings => $self->get_elasticsearch_settings(), + index_mappings => $self->get_elasticsearch_mappings(), + ) + ); + } + $self->store->bag->commit; +} + =head2 $indexer->drop_index(); Drops the index from the elasticsearch server. Calling C diff --git a/admin/searchengine/elasticsearch/mappings.pl b/admin/searchengine/elasticsearch/mappings.pl index 7d63002..243204a 100755 --- a/admin/searchengine/elasticsearch/mappings.pl +++ b/admin/searchengine/elasticsearch/mappings.pl @@ -97,14 +97,16 @@ if ( $op eq 'edit' ) { $schema->storage->txn_commit; } } -elsif( $op eq 'reset' ) { - # TODO Move this feature to the interface - my $sure = $input->param('i_know_what_i_am_doing'); - if ( $sure ) { - Koha::SearchMarcMaps->search->delete; - Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings; - } +elsif( $op eq 'reset' || $op eq 'reset_confirmed' ) { + Koha::SearchMarcMaps->delete; + Koha::SearchFields->delete; + Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings; + push @messages, { type => 'message', code => 'success_on_reset' }; } +elsif( $op eq 'reset_confirm' ) { + $template->param( reset_confirm => 1 ); +} + my @indexes; @@ -114,6 +116,7 @@ for my $index_name (qw| biblios authorities |) { { join => { search_marc_to_fields => 'search_marc_map' }, '+select' => [ 'search_marc_to_fields.facet', 'search_marc_to_fields.suggestible', 'search_marc_to_fields.sort', 'search_marc_map.marc_field' ], '+as' => [ 'facet', 'suggestible', 'sort', 'marc_field' ], + order_by => { -asc => [qw/name marc_field/] } } ); diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt index 24327c5..7cd8748 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt @@ -79,7 +79,9 @@ a.add, a.delete { An error occurred when deleting the existing mappings. Nothing has been changed! (search field [% m.values.field_name %] with mapping [% m.values.marc_field %].) [% CASE 'success_on_update' %] - Mapping updated successfully. + Mappings updated successfully. + [% CASE 'success_on_reset' %] + Mappings have been reset successfully. [% CASE %] [% m.code %] [% END %] @@ -88,7 +90,7 @@ a.add, a.delete {

Search engine configuration

- Warning: Any modification in these configurations will need a total reindexation to be fully taken into account ! + Warning: Any modification of this configuration requires a complete index rebuild to be fully taken into account!
[% IF errors %]
@@ -107,6 +109,19 @@ a.add, a.delete {
[% END %] + [% IF reset_confirm %] +
+

Confirm mappings reset

+
+ + +
+
+
+ +
+
+ [% ELSE %]
@@ -286,8 +301,12 @@ a.add, a.delete {
[% END %] -

+

+ + Reset Mappings +

+ [% END %] diff --git a/misc/search_tools/rebuild_elastic_search.pl b/misc/search_tools/rebuild_elastic_search.pl index 6faa32c..63065ca 100755 --- a/misc/search_tools/rebuild_elastic_search.pl +++ b/misc/search_tools/rebuild_elastic_search.pl @@ -92,14 +92,14 @@ use MARC::Record; use Modern::Perl; use Pod::Usage; -use Data::Dumper; # TODO remove - my $verbose = 0; my $commit = 5000; my ($delete, $help, $man); my ($index_biblios, $index_authorities); my (@biblionumbers); +$|=1; # flushes output + GetOptions( 'c|commit=i' => \$commit, 'd|delete' => \$delete, @@ -161,10 +161,8 @@ sub do_reindex { my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } ); if ($delete) { - - # We know it's safe to not recreate the indexer because update_index - # hasn't been called yet. $indexer->drop_index(); + $indexer->create_index(); } my $count = 0; @@ -173,23 +171,29 @@ sub do_reindex { while ( my $record = $next->() ) { my $id = $record->id; my $record = $record->record; - _log( 1, "$id\n" ); $count++; + if ( $verbose == 1 ) { + _log( 1, "$count records processed\n" ) if ( $count % 1000 == 0); + } else { + _log( 2, "$id\n" ); + } push @id_buffer, $id; push @commit_buffer, $record; if ( !( --$commit_count ) ) { - _log( 2, "Committing...\n" ); + _log( 1, "Committing $commit records..." ); $indexer->update_index( \@id_buffer, \@commit_buffer ); $commit_count = $commit; @id_buffer = (); @commit_buffer = (); + _log( 1, " done\n" ); } } # There are probably uncommitted records + _log( 1, "Committing final records...\n" ); $indexer->update_index( \@id_buffer, \@commit_buffer ); - _log( 1, "$count records indexed.\n" ); + _log( 1, "Total $count records indexed\n" ); } # Checks some basic stuff to ensure that it's sane before we start. -- 2.7.4