From 7db67285dd230753fa7c07972109f07b8d9406d2 Mon Sep 17 00:00:00 2001 From: Nick Clemens Date: Thu, 2 May 2019 11:27:27 +0000 Subject: [PATCH] Bug 22828: Elasticsearch - display errors encountered during indexing on the command line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To test: 1 - Use the Koha sample data, or insert a blank 245$b into a record (easiest way is using advanced cataloging editor 2 - Reindex elasticsearch 3 - Check the ES count on the about page 4 - Check the count in the DB (SELECT count(*) FROM biblio) 5 - They don't match! 6 - perl misc/search_tools/rebuild_elastic_search.pl -v -v 7 - No errors indicated 8 - Apply patch 9 - perl misc/search_tools/rebuild_elastic_search.pl -v 10 - You should be notified of an error 11 - perl misc/search_tools/rebuild_elastic_search.pl -v -v 12 - You should be notified of the specific biblio with an error and a (somewhat) readable reason 13 - perl misc/search_tools/rebuild_elastic_search.pl 14 - No output Signed-off-by: Ere Maijala Signed-off-by: Séverine QUEUNE Signed-off-by: Bouzid Fergani https://bugs.koha-community.org/show_bug.cgi?id=23596 --- Koha/SearchEngine/Elasticsearch/Indexer.pm | 6 +++--- misc/search_tools/rebuild_elasticsearch.pl | 30 ++++++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch/Indexer.pm b/Koha/SearchEngine/Elasticsearch/Indexer.pm index 504ccc5..86fe3da 100644 --- a/Koha/SearchEngine/Elasticsearch/Indexer.pm +++ b/Koha/SearchEngine/Elasticsearch/Indexer.pm @@ -117,15 +117,15 @@ sub update_index { }; push @body, $document; } + my $response; if (@body) { - my $response = $elasticsearch->bulk( + $response = $elasticsearch->bulk( index => $conf->{index_name}, type => 'data', # is just hard coded in Indexer.pm? body => \@body ); } - # TODO: handle response - return 1; + return $response; } =head2 set_index_status_ok diff --git a/misc/search_tools/rebuild_elasticsearch.pl b/misc/search_tools/rebuild_elasticsearch.pl index b5237cb..9bfa6e2 100755 --- a/misc/search_tools/rebuild_elasticsearch.pl +++ b/misc/search_tools/rebuild_elasticsearch.pl @@ -206,6 +206,8 @@ if ($slice_index == 0) { } } +=head1 INTERNAL METHODS + =head2 _verify_index_state _verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, 1); @@ -241,6 +243,7 @@ sub _verify_index_state { _do_reindex($callback, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); Does the actual reindexing. $callback is a function that always returns the next record. +For each index we iterate through the records, committing at specified count =cut @@ -266,7 +269,8 @@ sub _do_reindex { push @commit_buffer, $record; if ( !( --$commit_count ) ) { _log( 1, "Committing $commit records...\n" ); - $indexer->update_index( \@id_buffer, \@commit_buffer ); + my $response = $indexer->update_index( \@id_buffer, \@commit_buffer ); + _handle_response($response); $commit_count = $commit; @id_buffer = (); @commit_buffer = (); @@ -276,7 +280,8 @@ sub _do_reindex { # There are probably uncommitted records _log( 1, "Committing final records...\n" ); - $indexer->update_index( \@id_buffer, \@commit_buffer ); + my $response = $indexer->update_index( \@id_buffer, \@commit_buffer ); + _handle_response($response); _log( 1, "Total $count records indexed\n" ); } @@ -294,6 +299,27 @@ sub _sanity_check { die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf ); } +=head2 _handle_response + +Parse the return from update_index and display errors depending on verbosity of the script + +=cut + +sub _handle_response { + my ($response) = @_; + if( $response->{errors} eq 'true' ){ + _log( 1, "There were errors during indexing\n" ); + if ( $verbose > 1 ){ + foreach my $item (@{$response->{items}}){ + next unless defined $item->{index}->{error}; + print "Record #" . $item->{index}->{_id} . " " . + $item->{index}->{error}->{reason} . " (" . $item->{index}->{error}->{type} . ") : " . + $item->{index}->{error}->{caused_by}->{type} . " (" . $item->{index}->{error}->{caused_by}->{reason} . ")\n"; + } + } + } +} + =head2 _log _log($level, "Message\n"); -- 2.7.4