Attachment 89228 Details for Bug 22828 – Bug 22828: Elasticsearch - display errors encountered during indexing on the command line

[patch] Bug 22828: Elasticsearch - display errors encountered during indexing on the command line

Bug-22828-Elasticsearch---display-errors-encounter.patch (text/plain), 4.73 KB, created by Nick Clemens (kidclamp) on 2019-05-02 11:36:52 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Nick Clemens (kidclamp)

Created: 2019-05-02 11:36:52 UTC

Size: 4.73 KB

patch

obsolete

>From cbdc6270a726c5450e264dc058bae478eb6b19c0 Mon Sep 17 00:00:00 2001
>From: Nick Clemens <nick@bywatersolutions.com>
>Date: Thu, 2 May 2019 11:27:27 +0000
>Subject: [PATCH] Bug 22828: Elasticsearch - display errors encountered during
> indexing on the command line
>
>To test:
> 1 - Use the Koha sample data, or insert a blank 245$b into a record (easiest way is using advanced cataloging editor
> 2 - Reindex elasticsearch
> 3 - Check the ES count on the about page
> 4 - Check the count in the DB (SELECT count(*) FROM biblio)
> 5 - They don't match!
> 6 - perl misc/search_tools/rebuild_elastic_search.pl -v -v
> 7 - No errors indicated
> 8 - Apply patch
> 9 - perl misc/search_tools/rebuild_elastic_search.pl -v
>10 - You should be notified of an error
>11 - perl misc/search_tools/rebuild_elastic_search.pl -v -v
>12 - You should be notified of the specific biblio with an error and a (somewhat) readable reason
>13 - perl misc/search_tools/rebuild_elastic_search.pl
>14 - No output
>---
> Koha/SearchEngine/Elasticsearch/Indexer.pm  |  6 +--
> misc/search_tools/rebuild_elastic_search.pl | 59 ++++++++++++++++++++++++-----
> 2 files changed, 53 insertions(+), 12 deletions(-)
>
>diff --git a/Koha/SearchEngine/Elasticsearch/Indexer.pm b/Koha/SearchEngine/Elasticsearch/Indexer.pm
>index 703f39f3ca..8547461a45 100644
>--- a/Koha/SearchEngine/Elasticsearch/Indexer.pm
>+++ b/Koha/SearchEngine/Elasticsearch/Indexer.pm
>@@ -128,15 +128,15 @@ sub update_index {
>         };
>         push @body, $document;
>     }
>+    my $response;
>     if (@body) {
>-        my $response = $elasticsearch->bulk(
>+        $response = $elasticsearch->bulk(
>             index => $conf->{index_name},
>             type => 'data', # is just hard coded in Indexer.pm?
>             body => \@body
>         );
>     }
>-    # TODO: handle response
>-    return 1;
>+    return $response;
> }
> 
> =head2 set_index_status_ok
>diff --git a/misc/search_tools/rebuild_elastic_search.pl b/misc/search_tools/rebuild_elastic_search.pl
>index 54e458ff13..57e0ee8de1 100755
>--- a/misc/search_tools/rebuild_elastic_search.pl
>+++ b/misc/search_tools/rebuild_elastic_search.pl
>@@ -162,6 +162,14 @@ if ($index_authorities) {
>     do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
> }
> 
>+=head1 INTERNAL METHODS
>+
>+=head2 do_reindex
>+
>+For each index we iterate through the records, committing at specified count
>+
>+=cut
>+
> sub do_reindex {
>     my ( $next, $index_name ) = @_;
> 
>@@ -198,7 +206,8 @@ sub do_reindex {
>         push @commit_buffer, $record;
>         if ( !( --$commit_count ) ) {
>             _log( 1, "Committing $commit records..." );
>-            $indexer->update_index( \@id_buffer, \@commit_buffer );
>+            my $response = $indexer->update_index( \@id_buffer, \@commit_buffer );
>+            _handle_response($response);
>             $commit_count  = $commit;
>             @id_buffer     = ();
>             @commit_buffer = ();
>@@ -208,23 +217,55 @@ sub do_reindex {
> 
>     # There are probably uncommitted records
>     _log( 1, "Committing final records...\n" );
>-    $indexer->update_index( \@id_buffer, \@commit_buffer );
>+    my $response = $indexer->update_index( \@id_buffer, \@commit_buffer );
>+    _handle_response($response);
>     _log( 1, "Total $count records indexed\n" );
> }
> 
>-# Checks some basic stuff to ensure that it's sane before we start.
>+=head2 sanity_check
>+
>+Checks some basic stuff to ensure that it's sane before we start.
>+
>+=cut
>+
> sub sanity_check {
>     # Do we have an elasticsearch block defined?
>     my $conf = C4::Context->config('elasticsearch');
>     die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
> }
> 
>-# Output progress information.
>-#
>-#   _log($level, $msg);
>-#
>-# Will output $msg if the verbosity setting is set to $level or more. Will
>-# not include a trailing newline.
>+=head2 _handle_response
>+
>+Parse the return from update_index and display errors depending on verbosity of the script
>+
>+=cut
>+
>+sub _handle_response {
>+    my ($response) = @_;
>+    if( $response->{errors} eq 'true' ){
>+        _log( 1, "There were errors during indexing\n" );
>+        if ( $verbose > 1 ){
>+            foreach my $item (@{$response->{items}}){
>+                next unless defined $item->{index}->{error};
>+                print "Record #" . $item->{index}->{_id} . " " .
>+                      $item->{index}->{error}->{reason} . " (" . $item->{index}->{error}->{type} . ") : " .
>+                      $item->{index}->{error}->{caused_by}->{type} . " (" . $item->{index}->{error}->{caused_by}->{reason} . ")\n";
>+            }
>+        }
>+    }
>+}
>+
>+=head2 _log
>+
>+Output progress information.
>+
>+    _log($level, $msg);
>+
>+Will output $msg if the verbosity setting is set to $level or more. Will
>+not include a trailing newline.
>+
>+=cut
>+
> sub _log {
>     my ($level, $msg) = @_;
> 
>-- 
>2.11.0

Actions: View | Diff | Splinter Review

Attachments on bug 22828: 89226 | 89228 | 95096 | 95098 | 95530 | 99040 | 99388 | 103645 | 103646