@@ -, +, @@ rebuild_elastic_search.pl. --- Koha/SearchEngine/Elasticsearch/Indexer.pm | 25 +++++++++++++++- admin/searchengine/elasticsearch/mappings.pl | 17 ++++++----- .../admin/searchengine/elasticsearch/mappings.tt | 25 ++++++++++++++-- misc/search_tools/rebuild_elastic_search.pl | 20 ++++++++----- t/db_dependent/Koha_Elasticsearch_Indexer.t | 34 ++++++++++++++++++++-- 5 files changed, 100 insertions(+), 21 deletions(-) --- a/Koha/SearchEngine/Elasticsearch/Indexer.pm +++ a/Koha/SearchEngine/Elasticsearch/Indexer.pm @@ -139,6 +139,28 @@ sub delete_index_background { $self->delete_index(@_); } +=head2 $indexer->create_index(); + +Create an index on the Elasticsearch server. + +=cut + +sub create_index { + my ($self) = @_; + + if (!$self->store) { + my $params = $self->get_elasticsearch_params(); + $self->store( + Catmandu::Store::ElasticSearch->new( + %$params, + index_settings => $self->get_elasticsearch_settings(), + index_mappings => $self->get_elasticsearch_mappings(), + ) + ); + } + $self->store->bag->commit; +} + =head2 $indexer->drop_index(); Drops the index from the elasticsearch server. Calling C @@ -161,8 +183,9 @@ sub drop_index { ) ); } - $self->store->drop(); + my $store = $self->store; $self->store(undef); + $store->drop(); } sub _sanitise_records { --- a/admin/searchengine/elasticsearch/mappings.pl +++ a/admin/searchengine/elasticsearch/mappings.pl @@ -97,15 +97,17 @@ if ( $op eq 'edit' ) { $schema->storage->txn_commit; } } -elsif( $op eq 'reset' ) { - # TODO Move this feature to the interface - my $sure = $input->param('i_know_what_i_am_doing'); - if ( $sure ) { - Koha::SearchMarcMaps->search->delete; - Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings; - } +elsif( $op eq 'reset' || $op eq 'reset_confirmed' ) { + Koha::SearchMarcMaps->delete; + Koha::SearchFields->delete; + Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings; + push @messages, { type => 'message', code => 'success_on_reset' }; +} +elsif( $op eq 'reset_confirm' ) { + $template->param( reset_confirm => 1 ); } + my @indexes; for my $index_name (qw| biblios authorities |) { @@ -114,6 +116,7 @@ for my $index_name (qw| biblios authorities |) { { join => { search_marc_to_fields => 'search_marc_map' }, '+select' => [ 'search_marc_to_fields.facet', 'search_marc_to_fields.suggestible', 'search_marc_to_fields.sort', 'search_marc_map.marc_field' ], '+as' => [ 'facet', 'suggestible', 'sort', 'marc_field' ], + order_by => { -asc => [qw/name marc_field/] } } ); --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt +++ a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt @@ -79,7 +79,9 @@ a.add, a.delete { An error occurred when deleting the existing mappings. Nothing has been changed! (search field [% m.values.field_name %] with mapping [% m.values.marc_field %].) [% CASE 'success_on_update' %] - Mapping updated successfully. + Mappings updated successfully. + [% CASE 'success_on_reset' %] + Mappings have been reset successfully. [% CASE %] [% m.code %] [% END %] @@ -88,7 +90,7 @@ a.add, a.delete {

Search engine configuration

- Warning: Any modification in these configurations will need a total reindexation to be fully taken into account ! + Warning: Any modification of this configuration requires a complete index rebuild to be fully taken into account!
[% IF errors %]
@@ -107,6 +109,19 @@ a.add, a.delete {
[% END %] + [% IF reset_confirm %] +
+

The current mappings you seen on the screen will be erased and replaced by the mappings in the mappings.yaml file.

+
+ + +
+
+
+ +
+
+ [% ELSE %]
@@ -286,8 +301,12 @@ a.add, a.delete {
[% END %] -

+

+ + Reset Mappings +

+ [% END %] --- a/misc/search_tools/rebuild_elastic_search.pl +++ a/misc/search_tools/rebuild_elastic_search.pl @@ -92,14 +92,14 @@ use MARC::Record; use Modern::Perl; use Pod::Usage; -use Data::Dumper; # TODO remove - my $verbose = 0; my $commit = 5000; my ($delete, $help, $man); my ($index_biblios, $index_authorities); my (@biblionumbers); +$|=1; # flushes output + GetOptions( 'c|commit=i' => \$commit, 'd|delete' => \$delete, @@ -161,10 +161,8 @@ sub do_reindex { my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } ); if ($delete) { - - # We know it's safe to not recreate the indexer because update_index - # hasn't been called yet. $indexer->drop_index(); + $indexer->create_index(); } my $count = 0; @@ -173,23 +171,29 @@ sub do_reindex { while ( my $record = $next->() ) { my $id = $record->id; my $record = $record->record; - _log( 1, "$id\n" ); $count++; + if ( $verbose == 1 ) { + _log( 1, "$count records processed\n" ) if ( $count % 1000 == 0); + } else { + _log( 2, "$id\n" ); + } push @id_buffer, $id; push @commit_buffer, $record; if ( !( --$commit_count ) ) { - _log( 2, "Committing...\n" ); + _log( 1, "Committing $commit records..." ); $indexer->update_index( \@id_buffer, \@commit_buffer ); $commit_count = $commit; @id_buffer = (); @commit_buffer = (); + _log( 1, " done\n" ); } } # There are probably uncommitted records + _log( 1, "Committing final records...\n" ); $indexer->update_index( \@id_buffer, \@commit_buffer ); - _log( 1, "$count records indexed.\n" ); + _log( 1, "Total $count records indexed\n" ); } # Checks some basic stuff to ensure that it's sane before we start. --- a/t/db_dependent/Koha_Elasticsearch_Indexer.t +++ a/t/db_dependent/Koha_Elasticsearch_Indexer.t @@ -17,7 +17,7 @@ use Modern::Perl; -use Test::More tests => 6; +use Test::More tests => 7; use Test::MockModule; use t::lib::Mocks; @@ -52,12 +52,42 @@ SKIP: { eval { $indexer->get_elasticsearch_params; }; - skip 'ElasticSeatch configuration not available', 1 + skip 'Elasticsearch configuration not available', 1 if $@; ok( $indexer->update_index(undef,$records), 'Update Index' ); } +subtest 'create_index() tests' => sub { + + plan tests => 3; + + my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' ); + $se->mock( 'get_elasticsearch_params', sub { + my ($self, $sub ) = @_; + + my $method = $se->original( 'get_elasticsearch_params' ); + my $params = $method->( $self ); + $params->{index_name} .= '__test'; + return $params; + }); + + my $indexer; + ok( + $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ 'index' => 'biblios' }), + 'Creating a new indexer object' + ); + ok( + $indexer->create_index(), + 'Creating an index' + ); + $indexer->drop_index(); + ok( + $indexer->drop_index(), + 'Dropping the index' + ); +}; + subtest '_convert_marc_to_json() tests' => sub { plan tests => 2; --