From 6db0c0090ed6c91791469a1b27a58b96ad7247cc Mon Sep 17 00:00:00 2001 From: Nick Clemens Date: Mon, 17 Jul 2017 20:20:12 +0000 Subject: [PATCH] Bug 18948 - Elasticsearch - Reindexes should use aliases to avoid down time while reindexing To test: 1 - Delete your current indices for ES (if any) 2 - Reindex your records 3 - curl 'localhost:9200/_cat/indices?v' 4 - Note indices are now of form koha_kohadev_########## 5 - The numbers there are a result of appending 'time' command to name 6 - Ensure searching works as before the patches 7 - Reindex your records with a commit setting of 1 (to slow things down) you may need a large numebr fo records 8 - Ensure searching works during reindex 9 - Ensure reindexing completes successfully 10 - curl 'localhost:9200/_cat/indices?v' 11 - There should only be one index each for biblios and authorities ** - Note number in above 12 - Perform a partial reindex by passing a biblionumber to the command 13 - Curl the indices again and ensure number/name has not changed 14 - Delete all indexes curl -XDELETE 'localhost:9200/koha_kohadev_*?pretty' 15 - Edit a known biblio: http://localhost:8081/cgi-bin/koha/cataloguing/addbiblio.pl?biblionumber=1 16 - Add a word to the title "potato" 17 - Save the record 18 - Search for potato 19 - You should arrive at the detail page for that record 20 - Confirm your index is of form koha_kohadev_########## --- Koha/SearchEngine/Elasticsearch/Indexer.pm | 104 ++++++++++++++++++++++------- 1 file changed, 81 insertions(+), 23 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch/Indexer.pm b/Koha/SearchEngine/Elasticsearch/Indexer.pm index f026bda..bbd2d26 100644 --- a/Koha/SearchEngine/Elasticsearch/Indexer.pm +++ b/Koha/SearchEngine/Elasticsearch/Indexer.pm @@ -20,7 +20,8 @@ package Koha::SearchEngine::Elasticsearch::Indexer; use Carp; use Modern::Perl; use base qw(Koha::SearchEngine::Elasticsearch); -use Data::Dumper; +use Koha::BiblioUtils; +use Koha::MetadataRecord::Authority; # For now just marc, but we can do anything here really use Catmandu::Importer::MARC; @@ -89,7 +90,7 @@ sub update_index { my $from = $self->_convert_marc_to_json($records); - print Data::Dumper::Dumper( $from->to_array ); + #print Data::Dumper::Dumper( $from->to_array );$self->store->bag->add_many($from); $self->store->bag->add_many($from); $self->store->bag->commit; return 1; @@ -110,7 +111,7 @@ it to be updated by a regular index cron job in the future. sub update_index_background { my $self = shift; - $self->update_index(@_); + $self->do_reindex(undef, undef, 0, @_); } =head2 $indexer->delete_index($biblionums) @@ -142,8 +143,8 @@ sub delete_index_background { =head2 $indexer->drop_index(); -Drops the index from the elasticsearch server. Calling C -after this will recreate it again. +Drops the index from the elasticsearch server. +We need to create a new object once this has been done. =cut @@ -151,27 +152,30 @@ sub drop_index { my ($self) = @_; $self->store->drop(); - my $params = $self->get_elasticsearch_params(); - $self->store( - Catmandu::Store::ElasticSearch->new( - %$params, - index_settings => $self->get_elasticsearch_settings(), - index_mappings => $self->get_elasticsearch_mappings(), - ) - ); } -=head2 $indexer->do_reindex(); +=head2 $indexer->do_reindex($delete, $commit, $verbose, $biblionumbers); + +Performs a reindex, full or patial is based on whether biblionumbers are passed in +We do an additional check if an alias exists - if so and only indexing a list of bibs we use it, +if not, or if doing a full reindex, we index into a new index and adjust the existing alias then drop the old. + +C<$delete> indicates if we should delete the existing index - only affects partial indexes as otherwise we use a new index in either case. + +C<$commit> specifies at how many records we commit the update - higher is faster but uses more memory - default is 5000. + +C<$verbose> specifies warning level - only for command line use currently. -Performs a reindex based on records passed in +C<$biblionums> is an arrayref containing the biblionumbers for the records. =cut sub do_reindex { - my ($self, $next, $delete, $commit, $verbose, $biblionumbers) = @_; + my ($self, $delete, $commit, $verbose, $biblionumbers) = @_; _log(1, $verbose, "Indexing ".$self->index."\n"); + my $next; if ( scalar @$biblionumbers ) { if ( $self->index eq 'biblios' ) { @@ -197,9 +201,66 @@ sub do_reindex { $next = sub { $records->next(); }; } -warn - $self->drop_index() if ( $delete ); + my $current_index = $self->get_alias_index(); + if ( scalar @$biblionumbers && !$delete && $current_index) { + + #In this one case we index onto existing alias + $self->_index_records( $next, $commit, $verbose ); + + } else { + + #In all other cases we are going to use a new index then switch our alias to this index + #'time' is used for providing a unique name + my $alias_name = $self->store->{index_name}; + + my $new_indexer = + Koha::SearchEngine::Elasticsearch::Indexer->new( { + index => $self->index.time + }); + + $new_indexer->_index_records( $next, $commit, $verbose ); + + if ( $current_index ){ #If the alias already exists + $self->update_alias({ #let's point the alias to the new index + old_index => $current_index, + new_index => $new_indexer->store->{index_name} + }); + $current_index =~ s/koha_kohadev_//; #Then we need short index name to build object + + my $old_index = + Koha::SearchEngine::Elasticsearch::Indexer->new({ + index => $current_index + }); + + $old_index->drop_index(); #And we drop the specific index that the alias pointed to originally + } else { #The alias was not created + $self->drop_index(); #Drop the index we created initially + #It will have the same name as our alias so must be dropped first + #It should be empty (we indexed into a new one) + $new_indexer->store->es->indices->put_alias( + name => $alias_name, + index => $new_indexer->store->{index_name} + ); + } + + +=head2 $indexer->index_records( $next, $commit, $verbose ); + +Performs a reindex, full or patial is based on whether biblionumbers are passed in +We do an additional check if an alias exists - if so and only indexing a list of bibs we use it, +if not, or if doing a full reindex, we index into a new index and adjust the existing alias then drop the old. + +C<$next> is a function reference, used for iterating over records. +C<$commit> specifies at how many records we commit the update - higher is faster but uses more memory - default is 5000. + +C<$verbose> specifies warning level - only for command line use currently. + +=cut + +sub _index_records { + my ( $self, $next, $commit, $verbose ) = @_; + $commit ||= 5000; my $count = 0; my $commit_count = $commit; my ( @id_buffer, @commit_buffer ); @@ -269,8 +330,5 @@ __END__ =over 4 -=item Chris Cormack C<< >> - -=item Robin Sheat C<< >> - -=back +=item Chris Cormack C<< >>=item Robin Sheat C<< >> +back -- 2.1.4