Attachment 70816 Details for Bug 18948 – Bug 18948 - Elasticsearch - Reindexes should use aliases to avoid down time while reindexing

[patch] Bug 18948 - Elasticsearch - Reindexes should use aliases to avoid down time while reindexing

Bug-18948---Elasticsearch---Reindexes-should-use-a.patch (text/plain), 7.60 KB, created by Nick Clemens (kidclamp) on 2018-01-23 00:55:16 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Nick Clemens (kidclamp)

Created: 2018-01-23 00:55:16 UTC

Size: 7.60 KB

patch

obsolete

>From 6db0c0090ed6c91791469a1b27a58b96ad7247cc Mon Sep 17 00:00:00 2001
>From: Nick Clemens <nick@bywatersolutions.com>
>Date: Mon, 17 Jul 2017 20:20:12 +0000
>Subject: [PATCH] Bug 18948 - Elasticsearch - Reindexes should use aliases to
> avoid down time while reindexing
>
>To test:
>1  - Delete your current indices for ES (if any)
>2  - Reindex your records
>3  - curl 'localhost:9200/_cat/indices?v'
>4  - Note indices are now of form koha_kohadev_##########
>5  - The numbers there are a result of appending 'time' command to name
>6  - Ensure searching works as before the patches
>7  - Reindex your records with a commit setting of 1 (to slow things
>        down) you may need a large numebr fo records
>8  - Ensure searching works during reindex
>9  - Ensure reindexing completes successfully
>10 - curl 'localhost:9200/_cat/indices?v'
>11 - There should only be one index each for biblios and authorities
>** - Note number in above
>12 - Perform a partial reindex by passing a biblionumber to the command
>13 - Curl the indices again and ensure number/name has not changed
>14 - Delete all indexes
>    curl -XDELETE 'localhost:9200/koha_kohadev_*?pretty'
>15 - Edit a known biblio:
>    http://localhost:8081/cgi-bin/koha/cataloguing/addbiblio.pl?biblionumber=1
>16 - Add a word to the title "potato"
>17 - Save the record
>18 - Search for potato
>19 - You should arrive at the detail page for that record
>20 - Confirm your index is of form koha_kohadev_##########
>---
> Koha/SearchEngine/Elasticsearch/Indexer.pm | 104 ++++++++++++++++++++++-------
> 1 file changed, 81 insertions(+), 23 deletions(-)
>
>diff --git a/Koha/SearchEngine/Elasticsearch/Indexer.pm b/Koha/SearchEngine/Elasticsearch/Indexer.pm
>index f026bda..bbd2d26 100644
>--- a/Koha/SearchEngine/Elasticsearch/Indexer.pm
>+++ b/Koha/SearchEngine/Elasticsearch/Indexer.pm
>@@ -20,7 +20,8 @@ package Koha::SearchEngine::Elasticsearch::Indexer;
> use Carp;
> use Modern::Perl;
> use base qw(Koha::SearchEngine::Elasticsearch);
>-use Data::Dumper;
>+use Koha::BiblioUtils;
>+use Koha::MetadataRecord::Authority;
> 
> # For now just marc, but we can do anything here really
> use Catmandu::Importer::MARC;
>@@ -89,7 +90,7 @@ sub update_index {
> 
>     my $from    = $self->_convert_marc_to_json($records);
> 
>-    print Data::Dumper::Dumper( $from->to_array );
>+    #print Data::Dumper::Dumper( $from->to_array );$self->store->bag->add_many($from);
>     $self->store->bag->add_many($from);
>     $self->store->bag->commit;
>     return 1;
>@@ -110,7 +111,7 @@ it to be updated by a regular index cron job in the future.
> 
> sub update_index_background {
>     my $self = shift;
>-    $self->update_index(@_);
>+    $self->do_reindex(undef, undef, 0, @_);
> }
> 
> =head2 $indexer->delete_index($biblionums)
>@@ -142,8 +143,8 @@ sub delete_index_background {
> 
> =head2 $indexer->drop_index();
> 
>-Drops the index from the elasticsearch server. Calling C<update_index>
>-after this will recreate it again.
>+Drops the index from the elasticsearch server.
>+We need to create a new object once this has been done.
> 
> =cut
> 
>@@ -151,27 +152,30 @@ sub drop_index {
>     my ($self) = @_;
> 
>     $self->store->drop();
>-    my $params  = $self->get_elasticsearch_params();
>-    $self->store(
>-        Catmandu::Store::ElasticSearch->new(
>-            %$params,
>-            index_settings => $self->get_elasticsearch_settings(),
>-            index_mappings => $self->get_elasticsearch_mappings(),
>-        )
>-    );
> }
> 
> 
>-=head2 $indexer->do_reindex();
>+=head2 $indexer->do_reindex($delete, $commit, $verbose, $biblionumbers);
>+
>+Performs a reindex, full or patial is based on whether biblionumbers are passed in
>+We do an additional check if an alias exists - if so and only indexing a list of bibs we use it,
>+if not, or if doing a full reindex, we index into a new index and adjust the existing alias then drop the old.
>+
>+C<$delete> indicates if we should delete the existing index - only affects partial indexes as otherwise we use a new index in either case.
>+
>+C<$commit> specifies at how many records we commit the update - higher is faster but uses more memory - default is 5000.
>+
>+C<$verbose> specifies warning level - only for command line use currently.
> 
>-Performs a reindex based on records passed in
>+C<$biblionums> is an arrayref containing the biblionumbers for the records.
> 
> =cut
> 
> sub do_reindex {
>-    my ($self, $next, $delete, $commit, $verbose, $biblionumbers) = @_;
>+    my ($self, $delete, $commit, $verbose, $biblionumbers) = @_;
> 
>     _log(1, $verbose, "Indexing ".$self->index."\n");
>+    my $next;
> 
>     if ( scalar @$biblionumbers ) {
>         if ( $self->index eq 'biblios' ) {
>@@ -197,9 +201,66 @@ sub do_reindex {
>         $next = sub { $records->next(); };
>     }
> 
>-warn
>-    $self->drop_index() if ( $delete );
>+    my $current_index = $self->get_alias_index();
>+    if ( scalar @$biblionumbers && !$delete && $current_index) {
>+
>+        #In this one case we index onto existing alias
>+        $self->_index_records( $next, $commit, $verbose );
>+
>+    } else {
>+
>+        #In all other cases we are going to use a new index then switch our alias to this index
>+        #'time' is used for providing a unique name
>+        my $alias_name = $self->store->{index_name};
>+
>+        my $new_indexer =
>+            Koha::SearchEngine::Elasticsearch::Indexer->new( {
>+                index => $self->index.time 
>+            });
>+
>+        $new_indexer->_index_records( $next, $commit, $verbose );
>+
>+        if ( $current_index ){                   #If the alias already exists
>+            $self->update_alias({                #let's point the alias to the new index
>+                old_index => $current_index,
>+                new_index => $new_indexer->store->{index_name}
>+            });
>+            $current_index =~ s/koha_kohadev_//; #Then we need short index name to build object
>+
>+            my $old_index =
>+                Koha::SearchEngine::Elasticsearch::Indexer->new({
>+                    index => $current_index
>+                });
>+
>+            $old_index->drop_index();            #And we drop the specific index that the alias pointed to originally
>+        } else {                                 #The alias was not created
>+            $self->drop_index();                 #Drop the index we created initially
>+                                                 #It will have the same name as our alias so must be dropped first
>+                                                 #It should be empty (we indexed into a new one)
>+            $new_indexer->store->es->indices->put_alias(
>+                name => $alias_name,
>+                index => $new_indexer->store->{index_name}
>+            );
>+        }
>+
>+
>+=head2 $indexer->index_records( $next, $commit, $verbose );
>+
>+Performs a reindex, full or patial is based on whether biblionumbers are passed in
>+We do an additional check if an alias exists - if so and only indexing a list of bibs we use it,
>+if not, or if doing a full reindex, we index into a new index and adjust the existing alias then drop the old.
>+
>+C<$next> is a function reference, used for iterating over records.
> 
>+C<$commit> specifies at how many records we commit the update - higher is faster but uses more memory - default is 5000.
>+
>+C<$verbose> specifies warning level - only for command line use currently.
>+
>+=cut
>+
>+sub _index_records {
>+    my ( $self, $next, $commit, $verbose ) = @_;
>+    $commit ||= 5000;
>     my $count        = 0;
>     my $commit_count = $commit;
>     my ( @id_buffer, @commit_buffer );
>@@ -269,8 +330,5 @@ __END__
> 
> =over 4
> 
>-=item Chris Cormack C<< <chrisc@catalyst.net.nz> >>
>-
>-=item Robin Sheat C<< <robin@catalyst.net.nz> >>
>-
>-=back
>+=item Chris Cormack C<< <chrisc@catalyst.net.nz> >>=item Robin Sheat C<< <robin@catalyst.net.nz> >>
>+back
>-- 
>2.1.4

Actions: View | Diff | Splinter Review

Attachments on bug 18948: 65086 | 65147 | 65159 | 70815 | 70816