From 6db0c0090ed6c91791469a1b27a58b96ad7247cc Mon Sep 17 00:00:00 2001
From: Nick Clemens <nick@bywatersolutions.com>
Date: Mon, 17 Jul 2017 20:20:12 +0000
Subject: [PATCH] Bug 18948 - Elasticsearch - Reindexes should use aliases to
 avoid down time while reindexing

To test:
1  - Delete your current indices for ES (if any)
2  - Reindex your records
3  - curl 'localhost:9200/_cat/indices?v'
4  - Note indices are now of form koha_kohadev_##########
5  - The numbers there are a result of appending 'time' command to name
6  - Ensure searching works as before the patches
7  - Reindex your records with a commit setting of 1 (to slow things
        down) you may need a large numebr fo records
8  - Ensure searching works during reindex
9  - Ensure reindexing completes successfully
10 - curl 'localhost:9200/_cat/indices?v'
11 - There should only be one index each for biblios and authorities
** - Note number in above
12 - Perform a partial reindex by passing a biblionumber to the command
13 - Curl the indices again and ensure number/name has not changed
14 - Delete all indexes
    curl -XDELETE 'localhost:9200/koha_kohadev_*?pretty'
15 - Edit a known biblio:
    http://localhost:8081/cgi-bin/koha/cataloguing/addbiblio.pl?biblionumber=1
16 - Add a word to the title "potato"
17 - Save the record
18 - Search for potato
19 - You should arrive at the detail page for that record
20 - Confirm your index is of form koha_kohadev_##########
---
 Koha/SearchEngine/Elasticsearch/Indexer.pm | 104 ++++++++++++++++++++++-------
 1 file changed, 81 insertions(+), 23 deletions(-)

diff --git a/Koha/SearchEngine/Elasticsearch/Indexer.pm b/Koha/SearchEngine/Elasticsearch/Indexer.pm
index f026bda..bbd2d26 100644
--- a/Koha/SearchEngine/Elasticsearch/Indexer.pm
+++ b/Koha/SearchEngine/Elasticsearch/Indexer.pm
@@ -20,7 +20,8 @@ package Koha::SearchEngine::Elasticsearch::Indexer;
 use Carp;
 use Modern::Perl;
 use base qw(Koha::SearchEngine::Elasticsearch);
-use Data::Dumper;
+use Koha::BiblioUtils;
+use Koha::MetadataRecord::Authority;
 
 # For now just marc, but we can do anything here really
 use Catmandu::Importer::MARC;
@@ -89,7 +90,7 @@ sub update_index {
 
     my $from    = $self->_convert_marc_to_json($records);
 
-    print Data::Dumper::Dumper( $from->to_array );
+    #print Data::Dumper::Dumper( $from->to_array );$self->store->bag->add_many($from);
     $self->store->bag->add_many($from);
     $self->store->bag->commit;
     return 1;
@@ -110,7 +111,7 @@ it to be updated by a regular index cron job in the future.
 
 sub update_index_background {
     my $self = shift;
-    $self->update_index(@_);
+    $self->do_reindex(undef, undef, 0, @_);
 }
 
 =head2 $indexer->delete_index($biblionums)
@@ -142,8 +143,8 @@ sub delete_index_background {
 
 =head2 $indexer->drop_index();
 
-Drops the index from the elasticsearch server. Calling C<update_index>
-after this will recreate it again.
+Drops the index from the elasticsearch server.
+We need to create a new object once this has been done.
 
 =cut
 
@@ -151,27 +152,30 @@ sub drop_index {
     my ($self) = @_;
 
     $self->store->drop();
-    my $params  = $self->get_elasticsearch_params();
-    $self->store(
-        Catmandu::Store::ElasticSearch->new(
-            %$params,
-            index_settings => $self->get_elasticsearch_settings(),
-            index_mappings => $self->get_elasticsearch_mappings(),
-        )
-    );
 }
 
 
-=head2 $indexer->do_reindex();
+=head2 $indexer->do_reindex($delete, $commit, $verbose, $biblionumbers);
+
+Performs a reindex, full or patial is based on whether biblionumbers are passed in
+We do an additional check if an alias exists - if so and only indexing a list of bibs we use it,
+if not, or if doing a full reindex, we index into a new index and adjust the existing alias then drop the old.
+
+C<$delete> indicates if we should delete the existing index - only affects partial indexes as otherwise we use a new index in either case.
+
+C<$commit> specifies at how many records we commit the update - higher is faster but uses more memory - default is 5000.
+
+C<$verbose> specifies warning level - only for command line use currently.
 
-Performs a reindex based on records passed in
+C<$biblionums> is an arrayref containing the biblionumbers for the records.
 
 =cut
 
 sub do_reindex {
-    my ($self, $next, $delete, $commit, $verbose, $biblionumbers) = @_;
+    my ($self, $delete, $commit, $verbose, $biblionumbers) = @_;
 
     _log(1, $verbose, "Indexing ".$self->index."\n");
+    my $next;
 
     if ( scalar @$biblionumbers ) {
         if ( $self->index eq 'biblios' ) {
@@ -197,9 +201,66 @@ sub do_reindex {
         $next = sub { $records->next(); };
     }
 
-warn
-    $self->drop_index() if ( $delete );
+    my $current_index = $self->get_alias_index();
+    if ( scalar @$biblionumbers && !$delete && $current_index) {
+
+        #In this one case we index onto existing alias
+        $self->_index_records( $next, $commit, $verbose );
+
+    } else {
+
+        #In all other cases we are going to use a new index then switch our alias to this index
+        #'time' is used for providing a unique name
+        my $alias_name = $self->store->{index_name};
+
+        my $new_indexer =
+            Koha::SearchEngine::Elasticsearch::Indexer->new( {
+                index => $self->index.time 
+            });
+
+        $new_indexer->_index_records( $next, $commit, $verbose );
+
+        if ( $current_index ){                   #If the alias already exists
+            $self->update_alias({                #let's point the alias to the new index
+                old_index => $current_index,
+                new_index => $new_indexer->store->{index_name}
+            });
+            $current_index =~ s/koha_kohadev_//; #Then we need short index name to build object
+
+            my $old_index =
+                Koha::SearchEngine::Elasticsearch::Indexer->new({
+                    index => $current_index
+                });
+
+            $old_index->drop_index();            #And we drop the specific index that the alias pointed to originally
+        } else {                                 #The alias was not created
+            $self->drop_index();                 #Drop the index we created initially
+                                                 #It will have the same name as our alias so must be dropped first
+                                                 #It should be empty (we indexed into a new one)
+            $new_indexer->store->es->indices->put_alias(
+                name => $alias_name,
+                index => $new_indexer->store->{index_name}
+            );
+        }
+
+
+=head2 $indexer->index_records( $next, $commit, $verbose );
+
+Performs a reindex, full or patial is based on whether biblionumbers are passed in
+We do an additional check if an alias exists - if so and only indexing a list of bibs we use it,
+if not, or if doing a full reindex, we index into a new index and adjust the existing alias then drop the old.
+
+C<$next> is a function reference, used for iterating over records.
 
+C<$commit> specifies at how many records we commit the update - higher is faster but uses more memory - default is 5000.
+
+C<$verbose> specifies warning level - only for command line use currently.
+
+=cut
+
+sub _index_records {
+    my ( $self, $next, $commit, $verbose ) = @_;
+    $commit ||= 5000;
     my $count        = 0;
     my $commit_count = $commit;
     my ( @id_buffer, @commit_buffer );
@@ -269,8 +330,5 @@ __END__
 
 =over 4
 
-=item Chris Cormack C<< <chrisc@catalyst.net.nz> >>
-
-=item Robin Sheat C<< <robin@catalyst.net.nz> >>
-
-=back
+=item Chris Cormack C<< <chrisc@catalyst.net.nz> >>=item Robin Sheat C<< <robin@catalyst.net.nz> >>
+back
-- 
2.1.4