From da66b668492c4aa2cfcbd97177ac2fb589ab7dc4 Mon Sep 17 00:00:00 2001
From: Nick Clemens <nick@bywatersolutions.com>
Date: Thu, 20 Jul 2017 13:30:51 +0000
Subject: [PATCH] Bug 18948 - Elasticsearch - Reindexes should use aliases to
 avoid down time while reindexing

Refactoring of rebuild_elastic_search.pl to move code into
Koha::SearchEngine::Elasticsearch::Indexer
---
 Koha/SearchEngine/Elasticsearch/Indexer.pm  | 133 +++++++++++++++++++++-------
 misc/search_tools/rebuild_elastic_search.pl |  83 ++---------------
 2 files changed, 106 insertions(+), 110 deletions(-)

diff --git a/Koha/SearchEngine/Elasticsearch/Indexer.pm b/Koha/SearchEngine/Elasticsearch/Indexer.pm
index e5babee..f9beb44 100644
--- a/Koha/SearchEngine/Elasticsearch/Indexer.pm
+++ b/Koha/SearchEngine/Elasticsearch/Indexer.pm
@@ -41,6 +41,27 @@ Koha::SearchEngine::Elasticsearch::Indexer - handles adding new records to the i
 
 =head1 FUNCTIONS
 
+=head2 Koha::SearchEngine::Elasticsearch::Indexer->({index => $index_name});
+
+$index_name should be biblios or authorities
+call our SUPER class and then set ES parameters
+
+=cut
+
+sub new {
+    my $class = shift @_;
+    my $self = $class->SUPER::new(@_);
+    my $params  = $self->get_elasticsearch_params();
+    $self->store(
+        Catmandu::Store::ElasticSearch->new(
+            %$params,
+            index_settings => $self->get_elasticsearch_settings(),
+            index_mappings => $self->get_elasticsearch_mappings(),
+        )
+    );
+    return $self;
+}
+
 =head2 $indexer->update_index($biblionums, $records);
 
 C<$biblionums> is an arrayref containing the biblionumbers for the records.
@@ -67,18 +88,8 @@ sub update_index {
     }
 
     my $from    = $self->_convert_marc_to_json($records);
-    if ( !$self->store ) {
-        my $params  = $self->get_elasticsearch_params();
-        $self->store(
-            Catmandu::Store::ElasticSearch->new(
-                %$params,
-                index_settings => $self->get_elasticsearch_settings(),
-                index_mappings => $self->get_elasticsearch_mappings(),
-            )
-        );
-    }
 
-    #print Data::Dumper::Dumper( $from->to_array );
+    print Data::Dumper::Dumper( $from->to_array );
     $self->store->bag->add_many($from);
     $self->store->bag->commit;
     return 1;
@@ -111,16 +122,6 @@ C<$biblionums> is an arrayref of biblionumbers to delete from the index.
 sub delete_index {
     my ($self, $biblionums) = @_;
 
-    if ( !$self->store ) {
-        my $params  = $self->get_elasticsearch_params();
-        $self->store(
-            Catmandu::Store::ElasticSearch->new(
-                %$params,
-                index_settings => $self->get_elasticsearch_settings(),
-                index_mappings => $self->get_elasticsearch_mappings(),
-            )
-        );
-    }
     $self->store->bag->delete($_) foreach @$biblionums;
     $self->store->bag->commit;
 }
@@ -149,20 +150,84 @@ after this will recreate it again.
 sub drop_index {
     my ($self) = @_;
 
-    if (!$self->store) {
-        # If this index doesn't exist, this will create it. Then it'll be
-        # deleted. That's not the end of the world however.
-        my $params  = $self->get_elasticsearch_params();
-        $self->store(
-            Catmandu::Store::ElasticSearch->new(
-                %$params,
-                index_settings => $self->get_elasticsearch_settings(),
-                index_mappings => $self->get_elasticsearch_mappings(),
-            )
-        );
-    }
     $self->store->drop();
-    $self->store(undef);
+    my $params  = $self->get_elasticsearch_params();
+    $self->store(
+        Catmandu::Store::ElasticSearch->new(
+            %$params,
+            index_settings => $self->get_elasticsearch_settings(),
+            index_mappings => $self->get_elasticsearch_mappings(),
+        )
+    );
+}
+
+
+=head2 $indexer->do_reindex();
+
+Performs a reindex based on records passed in
+
+=cut
+
+sub do_reindex {
+    my ($self, $next, $delete, $commit, $verbose, $biblionumbers) = @_;
+
+    _log(1, $verbose, "Indexing ".$self->index."\n");
+
+    if ( scalar @$biblionumbers ) {
+        if ( $self->index eq 'biblios' ) {
+            $next = sub {
+                my $r = shift @$biblionumbers;
+                return () unless defined $r;
+                return ( $r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
+            };
+        }
+        elsif ( $self->index eq 'authorities' ) {
+            $next = sub {
+                my $r = shift @$biblionumbers;
+                return () unless defined $r;
+                my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
+                return ($r, $a->record);
+            };
+        }
+    }
+    else {
+        my $records = $self->index eq 'biblios' ?
+            Koha::BiblioUtils->get_all_biblios_iterator() : $self->index eq 'authorities' ?
+                Koha::MetadataRecord::Authority->get_all_authorities_iterator() : die "Reindex called on undefined index";
+        $next = sub { $records->next(); };
+    }
+
+warn 
+    $self->drop_index() if ( $delete );
+
+    my $count        = 0;
+    my $commit_count = $commit;
+    my ( @id_buffer, @commit_buffer );
+    while ( my $record = $next->() ) {
+        my $id     = $record->id;
+        my $record = $record->record;
+        _log( 1, $verbose, "$id\n" );
+        $count++;
+
+        push @id_buffer,     $id;
+        push @commit_buffer, $record;
+        if ( !( --$commit_count ) ) {
+            _log( 2, $verbose, "Committing...\n" );
+            $self->update_index( \@id_buffer, \@commit_buffer );
+            $commit_count  = $commit;
+            @id_buffer     = ();
+            @commit_buffer = ();
+        }
+    }
+    # There are probably uncommitted records (beacuse we didn't hit exact commit count)
+    $self->update_index( \@id_buffer, \@commit_buffer );
+    _log( 1, $verbose, "$count records indexed.\n" );
+}
+
+
+sub _log {
+    my ($level, $verbose, $msg) = @_;
+    print $msg if ($verbose >= $level);
 }
 
 sub _sanitise_records {
diff --git a/misc/search_tools/rebuild_elastic_search.pl b/misc/search_tools/rebuild_elastic_search.pl
index 6faa32c..84a1ea0 100755
--- a/misc/search_tools/rebuild_elastic_search.pl
+++ b/misc/search_tools/rebuild_elastic_search.pl
@@ -122,74 +122,16 @@ pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
 sanity_check();
 
 my $next;
+print "$verbose verbosity";
 if ($index_biblios) {
-    _log(1, "Indexing biblios\n");
-    if (@biblionumbers) {
-        $next = sub {
-            my $r = shift @biblionumbers;
-            return () unless defined $r;
-            return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
-        };
-    } else {
-        my $records = Koha::BiblioUtils->get_all_biblios_iterator();
-        $next = sub {
-            $records->next();
-        }
-    }
-    do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
+    my $index_name = $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX;
+    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
+    $indexer->do_reindex($next,$delete,$commit,$verbose,\@biblionumbers);
 }
 if ($index_authorities) {
-    _log(1, "Indexing authorities\n");
-    if (@biblionumbers) {
-        $next = sub {
-            my $r = shift @biblionumbers;
-            return () unless defined $r;
-            my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
-            return ($r, $a->record);
-        };
-    } else {
-        my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator();
-        $next = sub {
-            $records->next();
-        }
-    }
-    do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
-}
-
-sub do_reindex {
-    my ( $next, $index_name ) = @_;
-
-    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
-    if ($delete) {
-
-        # We know it's safe to not recreate the indexer because update_index
-        # hasn't been called yet.
-        $indexer->drop_index();
-    }
-
-    my $count        = 0;
-    my $commit_count = $commit;
-    my ( @id_buffer, @commit_buffer );
-    while ( my $record = $next->() ) {
-        my $id     = $record->id;
-        my $record = $record->record;
-        _log( 1, "$id\n" );
-        $count++;
-
-        push @id_buffer,     $id;
-        push @commit_buffer, $record;
-        if ( !( --$commit_count ) ) {
-            _log( 2, "Committing...\n" );
-            $indexer->update_index( \@id_buffer, \@commit_buffer );
-            $commit_count  = $commit;
-            @id_buffer     = ();
-            @commit_buffer = ();
-        }
-    }
-
-    # There are probably uncommitted records
-    $indexer->update_index( \@id_buffer, \@commit_buffer );
-    _log( 1, "$count records indexed.\n" );
+    my $index_name = $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX;
+    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
+    $indexer->do_reindex($next,$delete,$commit,$verbose, \@biblionumbers);
 }
 
 # Checks some basic stuff to ensure that it's sane before we start.
@@ -199,14 +141,3 @@ sub sanity_check {
     die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
 }
 
-# Output progress information.
-#
-#   _log($level, $msg);
-#
-# Will output $msg if the verbosity setting is set to $level or more. Will
-# not include a trailing newline.
-sub _log {
-    my ($level, $msg) = @_;
-
-    print $msg if ($verbose >= $level);
-}
-- 
2.1.4