Attachment 95530 Details for Bug 22828 – Bug 22828: Elasticsearch - display errors encountered during indexing on the command line

[patch] Bug 22828: Elasticsearch - display errors encountered during indexing on the command line

Bug-22828-Elasticsearch---display-errors-encounter.patch (text/plain), 4.48 KB, created by Séverine Queune on 2019-11-19 09:57:30 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Séverine Queune

Created: 2019-11-19 09:57:30 UTC

Size: 4.48 KB

patch

obsolete

>From 30062cb32c31ce31d6d125e584e14d57e316a78c Mon Sep 17 00:00:00 2001
>From: Nick Clemens <nick@bywatersolutions.com>
>Date: Thu, 2 May 2019 11:27:27 +0000
>Subject: [PATCH] Bug 22828: Elasticsearch - display errors encountered during
> indexing on the command line
>MIME-Version: 1.0
>Content-Type: text/plain; charset=UTF-8
>Content-Transfer-Encoding: 8bit
>
>To test:
> 1 - Use the Koha sample data, or insert a blank 245$b into a record (easiest way is using advanced cataloging editor
> 2 - Reindex elasticsearch
> 3 - Check the ES count on the about page
> 4 - Check the count in the DB (SELECT count(*) FROM biblio)
> 5 - They don't match!
> 6 - perl misc/search_tools/rebuild_elastic_search.pl -v -v
> 7 - No errors indicated
> 8 - Apply patch
> 9 - perl misc/search_tools/rebuild_elastic_search.pl -v
>10 - You should be notified of an error
>11 - perl misc/search_tools/rebuild_elastic_search.pl -v -v
>12 - You should be notified of the specific biblio with an error and a (somewhat) readable reason
>13 - perl misc/search_tools/rebuild_elastic_search.pl
>14 - No output
>
>Signed-off-by: Ere Maijala <ere.maijala@helsinki.fi>
>Signed-off-by: SÃ©verine QUEUNE <severine.queune@bulac.fr>
>---
> Koha/SearchEngine/Elasticsearch/Indexer.pm |  6 +++---
> misc/search_tools/rebuild_elasticsearch.pl | 30 ++++++++++++++++++++++++++++--
> 2 files changed, 31 insertions(+), 5 deletions(-)
>
>diff --git a/Koha/SearchEngine/Elasticsearch/Indexer.pm b/Koha/SearchEngine/Elasticsearch/Indexer.pm
>index 504ccc5..86fe3da 100644
>--- a/Koha/SearchEngine/Elasticsearch/Indexer.pm
>+++ b/Koha/SearchEngine/Elasticsearch/Indexer.pm
>@@ -117,15 +117,15 @@ sub update_index {
>         };
>         push @body, $document;
>     }
>+    my $response;
>     if (@body) {
>-        my $response = $elasticsearch->bulk(
>+        $response = $elasticsearch->bulk(
>             index => $conf->{index_name},
>             type => 'data', # is just hard coded in Indexer.pm?
>             body => \@body
>         );
>     }
>-    # TODO: handle response
>-    return 1;
>+    return $response;
> }
> 
> =head2 set_index_status_ok
>diff --git a/misc/search_tools/rebuild_elasticsearch.pl b/misc/search_tools/rebuild_elasticsearch.pl
>index b5237cb..9bfa6e2 100755
>--- a/misc/search_tools/rebuild_elasticsearch.pl
>+++ b/misc/search_tools/rebuild_elasticsearch.pl
>@@ -206,6 +206,8 @@ if ($slice_index == 0) {
>     }
> }
> 
>+=head1 INTERNAL METHODS
>+
> =head2 _verify_index_state
> 
>     _verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, 1);
>@@ -241,6 +243,7 @@ sub _verify_index_state {
>     _do_reindex($callback, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
> 
> Does the actual reindexing. $callback is a function that always returns the next record.
>+For each index we iterate through the records, committing at specified count
> 
> =cut
> 
>@@ -266,7 +269,8 @@ sub _do_reindex {
>         push @commit_buffer, $record;
>         if ( !( --$commit_count ) ) {
>             _log( 1, "Committing $commit records...\n" );
>-            $indexer->update_index( \@id_buffer, \@commit_buffer );
>+            my $response = $indexer->update_index( \@id_buffer, \@commit_buffer );
>+            _handle_response($response);
>             $commit_count  = $commit;
>             @id_buffer     = ();
>             @commit_buffer = ();
>@@ -276,7 +280,8 @@ sub _do_reindex {
> 
>     # There are probably uncommitted records
>     _log( 1, "Committing final records...\n" );
>-    $indexer->update_index( \@id_buffer, \@commit_buffer );
>+    my $response = $indexer->update_index( \@id_buffer, \@commit_buffer );
>+    _handle_response($response);
>     _log( 1, "Total $count records indexed\n" );
> }
> 
>@@ -294,6 +299,27 @@ sub _sanity_check {
>     die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
> }
> 
>+=head2 _handle_response
>+
>+Parse the return from update_index and display errors depending on verbosity of the script
>+
>+=cut
>+
>+sub _handle_response {
>+    my ($response) = @_;
>+    if( $response->{errors} eq 'true' ){
>+        _log( 1, "There were errors during indexing\n" );
>+        if ( $verbose > 1 ){
>+            foreach my $item (@{$response->{items}}){
>+                next unless defined $item->{index}->{error};
>+                print "Record #" . $item->{index}->{_id} . " " .
>+                      $item->{index}->{error}->{reason} . " (" . $item->{index}->{error}->{type} . ") : " .
>+                      $item->{index}->{error}->{caused_by}->{type} . " (" . $item->{index}->{error}->{caused_by}->{reason} . ")\n";
>+            }
>+        }
>+    }
>+}
>+
> =head2 _log
> 
>     _log($level, "Message\n");
>-- 
>2.1.4

Actions: View | Diff | Splinter Review

Attachments on bug 22828: 89226 | 89228 | 95096 | 95098 | 95530 | 99040 | 99388 | 103645 | 103646