From cbdc6270a726c5450e264dc058bae478eb6b19c0 Mon Sep 17 00:00:00 2001 From: Nick Clemens Date: Thu, 2 May 2019 11:27:27 +0000 Subject: [PATCH] Bug 22828: Elasticsearch - display errors encountered during indexing on the command line To test: 1 - Use the Koha sample data, or insert a blank 245$b into a record (easiest way is using advanced cataloging editor 2 - Reindex elasticsearch 3 - Check the ES count on the about page 4 - Check the count in the DB (SELECT count(*) FROM biblio) 5 - They don't match! 6 - perl misc/search_tools/rebuild_elastic_search.pl -v -v 7 - No errors indicated 8 - Apply patch 9 - perl misc/search_tools/rebuild_elastic_search.pl -v 10 - You should be notified of an error 11 - perl misc/search_tools/rebuild_elastic_search.pl -v -v 12 - You should be notified of the specific biblio with an error and a (somewhat) readable reason 13 - perl misc/search_tools/rebuild_elastic_search.pl 14 - No output --- Koha/SearchEngine/Elasticsearch/Indexer.pm | 6 +-- misc/search_tools/rebuild_elastic_search.pl | 59 ++++++++++++++++++++++++----- 2 files changed, 53 insertions(+), 12 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch/Indexer.pm b/Koha/SearchEngine/Elasticsearch/Indexer.pm index 703f39f3ca..8547461a45 100644 --- a/Koha/SearchEngine/Elasticsearch/Indexer.pm +++ b/Koha/SearchEngine/Elasticsearch/Indexer.pm @@ -128,15 +128,15 @@ sub update_index { }; push @body, $document; } + my $response; if (@body) { - my $response = $elasticsearch->bulk( + $response = $elasticsearch->bulk( index => $conf->{index_name}, type => 'data', # is just hard coded in Indexer.pm? body => \@body ); } - # TODO: handle response - return 1; + return $response; } =head2 set_index_status_ok diff --git a/misc/search_tools/rebuild_elastic_search.pl b/misc/search_tools/rebuild_elastic_search.pl index 54e458ff13..57e0ee8de1 100755 --- a/misc/search_tools/rebuild_elastic_search.pl +++ b/misc/search_tools/rebuild_elastic_search.pl @@ -162,6 +162,14 @@ if ($index_authorities) { do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX); } +=head1 INTERNAL METHODS + +=head2 do_reindex + +For each index we iterate through the records, committing at specified count + +=cut + sub do_reindex { my ( $next, $index_name ) = @_; @@ -198,7 +206,8 @@ sub do_reindex { push @commit_buffer, $record; if ( !( --$commit_count ) ) { _log( 1, "Committing $commit records..." ); - $indexer->update_index( \@id_buffer, \@commit_buffer ); + my $response = $indexer->update_index( \@id_buffer, \@commit_buffer ); + _handle_response($response); $commit_count = $commit; @id_buffer = (); @commit_buffer = (); @@ -208,23 +217,55 @@ sub do_reindex { # There are probably uncommitted records _log( 1, "Committing final records...\n" ); - $indexer->update_index( \@id_buffer, \@commit_buffer ); + my $response = $indexer->update_index( \@id_buffer, \@commit_buffer ); + _handle_response($response); _log( 1, "Total $count records indexed\n" ); } -# Checks some basic stuff to ensure that it's sane before we start. +=head2 sanity_check + +Checks some basic stuff to ensure that it's sane before we start. + +=cut + sub sanity_check { # Do we have an elasticsearch block defined? my $conf = C4::Context->config('elasticsearch'); die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf ); } -# Output progress information. -# -# _log($level, $msg); -# -# Will output $msg if the verbosity setting is set to $level or more. Will -# not include a trailing newline. +=head2 _handle_response + +Parse the return from update_index and display errors depending on verbosity of the script + +=cut + +sub _handle_response { + my ($response) = @_; + if( $response->{errors} eq 'true' ){ + _log( 1, "There were errors during indexing\n" ); + if ( $verbose > 1 ){ + foreach my $item (@{$response->{items}}){ + next unless defined $item->{index}->{error}; + print "Record #" . $item->{index}->{_id} . " " . + $item->{index}->{error}->{reason} . " (" . $item->{index}->{error}->{type} . ") : " . + $item->{index}->{error}->{caused_by}->{type} . " (" . $item->{index}->{error}->{caused_by}->{reason} . ")\n"; + } + } + } +} + +=head2 _log + +Output progress information. + + _log($level, $msg); + +Will output $msg if the verbosity setting is set to $level or more. Will +not include a trailing newline. + +=cut + sub _log { my ($level, $msg) = @_; -- 2.11.0