From 7db67285dd230753fa7c07972109f07b8d9406d2 Mon Sep 17 00:00:00 2001
From: Nick Clemens <nick@bywatersolutions.com>
Date: Thu, 2 May 2019 11:27:27 +0000
Subject: [PATCH] Bug 22828: Elasticsearch - display errors encountered during
 indexing on the command line
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

To test:
 1 - Use the Koha sample data, or insert a blank 245$b into a record (easiest way is using advanced cataloging editor
 2 - Reindex elasticsearch
 3 - Check the ES count on the about page
 4 - Check the count in the DB (SELECT count(*) FROM biblio)
 5 - They don't match!
 6 - perl misc/search_tools/rebuild_elastic_search.pl -v -v
 7 - No errors indicated
 8 - Apply patch
 9 - perl misc/search_tools/rebuild_elastic_search.pl -v
10 - You should be notified of an error
11 - perl misc/search_tools/rebuild_elastic_search.pl -v -v
12 - You should be notified of the specific biblio with an error and a (somewhat) readable reason
13 - perl misc/search_tools/rebuild_elastic_search.pl
14 - No output

Signed-off-by: Ere Maijala <ere.maijala@helsinki.fi>
Signed-off-by: Séverine QUEUNE <severine.queune@bulac.fr>
Signed-off-by: Bouzid Fergani <bouzid.fergani@inlibro.com>

https://bugs.koha-community.org/show_bug.cgi?id=23596
---
 Koha/SearchEngine/Elasticsearch/Indexer.pm |  6 +++---
 misc/search_tools/rebuild_elasticsearch.pl | 30 ++++++++++++++++++++++++++++--
 2 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/Koha/SearchEngine/Elasticsearch/Indexer.pm b/Koha/SearchEngine/Elasticsearch/Indexer.pm
index 504ccc5..86fe3da 100644
--- a/Koha/SearchEngine/Elasticsearch/Indexer.pm
+++ b/Koha/SearchEngine/Elasticsearch/Indexer.pm
@@ -117,15 +117,15 @@ sub update_index {
         };
         push @body, $document;
     }
+    my $response;
     if (@body) {
-        my $response = $elasticsearch->bulk(
+        $response = $elasticsearch->bulk(
             index => $conf->{index_name},
             type => 'data', # is just hard coded in Indexer.pm?
             body => \@body
         );
     }
-    # TODO: handle response
-    return 1;
+    return $response;
 }
 
 =head2 set_index_status_ok
diff --git a/misc/search_tools/rebuild_elasticsearch.pl b/misc/search_tools/rebuild_elasticsearch.pl
index b5237cb..9bfa6e2 100755
--- a/misc/search_tools/rebuild_elasticsearch.pl
+++ b/misc/search_tools/rebuild_elasticsearch.pl
@@ -206,6 +206,8 @@ if ($slice_index == 0) {
     }
 }
 
+=head1 INTERNAL METHODS
+
 =head2 _verify_index_state
 
     _verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, 1);
@@ -241,6 +243,7 @@ sub _verify_index_state {
     _do_reindex($callback, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
 
 Does the actual reindexing. $callback is a function that always returns the next record.
+For each index we iterate through the records, committing at specified count
 
 =cut
 
@@ -266,7 +269,8 @@ sub _do_reindex {
         push @commit_buffer, $record;
         if ( !( --$commit_count ) ) {
             _log( 1, "Committing $commit records...\n" );
-            $indexer->update_index( \@id_buffer, \@commit_buffer );
+            my $response = $indexer->update_index( \@id_buffer, \@commit_buffer );
+            _handle_response($response);
             $commit_count  = $commit;
             @id_buffer     = ();
             @commit_buffer = ();
@@ -276,7 +280,8 @@ sub _do_reindex {
 
     # There are probably uncommitted records
     _log( 1, "Committing final records...\n" );
-    $indexer->update_index( \@id_buffer, \@commit_buffer );
+    my $response = $indexer->update_index( \@id_buffer, \@commit_buffer );
+    _handle_response($response);
     _log( 1, "Total $count records indexed\n" );
 }
 
@@ -294,6 +299,27 @@ sub _sanity_check {
     die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
 }
 
+=head2 _handle_response
+
+Parse the return from update_index and display errors depending on verbosity of the script
+
+=cut
+
+sub _handle_response {
+    my ($response) = @_;
+    if( $response->{errors} eq 'true' ){
+        _log( 1, "There were errors during indexing\n" );
+        if ( $verbose > 1 ){
+            foreach my $item (@{$response->{items}}){
+                next unless defined $item->{index}->{error};
+                print "Record #" . $item->{index}->{_id} . " " .
+                      $item->{index}->{error}->{reason} . " (" . $item->{index}->{error}->{type} . ") : " .
+                      $item->{index}->{error}->{caused_by}->{type} . " (" . $item->{index}->{error}->{caused_by}->{reason} . ")\n";
+            }
+        }
+    }
+}
+
 =head2 _log
 
     _log($level, "Message\n");
-- 
2.7.4