From d76ac1735d769a84665d4e3abd1879ec12e5d98a Mon Sep 17 00:00:00 2001 From: Nick Clemens Date: Mon, 17 Jul 2017 20:20:12 +0000 Subject: [PATCH] Bug 18948 - Elasticsearch - Reindexes should use aliases to avoid down time while reindexing To test: 1 - Delete your current indices for ES (if any) 2 - Reindex your records 3 - curl 'localhost:9200/_cat/indices?v' 4 - Note indices are now of form koha_kohadev_########## 5 - The numbers there are a result of appending 'time' command to name 6 - Ensure searching works as before the patches 7 - Reindex your records with a commit setting of 1 (to slow things down) you may need a large numebr fo records 8 - Ensure searching works during reindex 9 - Ensure reindexing completes successfully 10 - curl 'localhost:9200/_cat/indices?v' 11 - There should only be one index each for biblios and authorities ** - Note number in above 12 - Perform a partial reindex by passing a biblionumber to the command 13 - Curl the indices again and ensure number/name has not changed 14 - Delete all indexes curl -XDELETE 'localhost:9200/koha_kohadev_*?pretty' 15 - Edit a known biblio: http://localhost:8081/cgi-bin/koha/cataloguing/addbiblio.pl?biblionumber=1 16 - Add a word to the title "potato" 17 - Save the record 18 - Search for potato 19 - You should arrive at the detail page for that record 20 - Confirm your index is of form koha_kohadev_########## --- C4/Biblio.pm | 6 +- Koha/SearchEngine/Elasticsearch.pm | 51 ++++++++++++- Koha/SearchEngine/Elasticsearch/Indexer.pm | 112 +++++++++++++++++++++++----- misc/search_tools/rebuild_elastic_search.pl | 14 ++-- 4 files changed, 148 insertions(+), 35 deletions(-) diff --git a/C4/Biblio.pm b/C4/Biblio.pm index 0ee689d..dca648d 100644 --- a/C4/Biblio.pm +++ b/C4/Biblio.pm @@ -2829,11 +2829,7 @@ sub ModZebra { } ); if ( $op eq 'specialUpdate' ) { - unless ($record) { - $record = GetMarcBiblio($biblionumber, 1); - } - my $records = [$record]; - $indexer->update_index_background( [$biblionumber], [$record] ); + $indexer->update_index_background( [$biblionumber] ); } elsif ( $op eq 'recordDelete' ) { $indexer->delete_index_background( [$biblionumber] ); diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm index 43bb0a8..cccc8ac 100644 --- a/Koha/SearchEngine/Elasticsearch.pm +++ b/Koha/SearchEngine/Elasticsearch.pm @@ -37,7 +37,7 @@ __PACKAGE__->mk_ro_accessors(qw( index )); __PACKAGE__->mk_accessors(qw( sort_fields )); # Constants to refer to the standard index names -Readonly our $BIBLIOS_INDEX => 'biblios'; +Readonly our $BIBLIOS_INDEX => 'biblios'; #These are going to be the suffixes for our aliases - most access to indexes will be via alias Readonly our $AUTHORITIES_INDEX => 'authorities'; =head1 NAME @@ -236,6 +236,48 @@ sub get_elasticsearch_mappings { return $mappings; } +=head2 get_alias_index + +Get the ES specific index names that an alias points to + +Returns an index name or undef + +=cut + +sub get_alias_index { + my ($self) = @_; + my $alias = $self->store->{index_name}; + if ( $self->store->es->indices->exists_alias( name => $alias ) ) { #use the embedded es object to access ES methods directly and check if alias points to anything + my $current_index_info = $self->store->es->indices->get_alias( name => $alias ); #we get a hashref of info + my @current_indices = keys %$current_index_info; #the keys contain the names of indexes pointed to be the alias + return $current_indices[0]; #For now we are maintaining a 1:1 ratio, pointing to several is possible + } else { + return undef; + } +} + +=head2 update_alias_indices + +Takes our ES alias name and two index names and swaps the old for the new +return + +=cut + +sub update_alias { + my $self = shift; + my ( $params ) = @_; + my $alias = $self->get_elasticsearch_params()->{index_name}; #FIXME this should probably be an object property/method + die "Must supply old_index and new_index parameters" unless ( $params->{old_index} && $params->{new_index} ); + return $self->store->es->indices->update_aliases( #this is atomic so we won't delete unless this works + body => { + actions => [ + { add => { alias => $alias, index => $params->{new_index} } }, + { remove => { alias => $alias, index => $params->{old_index} } } + ] + } + ); +} + =head2 _elasticsearch_mapping_for_* Get the ES mappings for the given data type or a special mapping case @@ -418,10 +460,15 @@ These are of a form suited to Catmandu's MARC fixers. sub _foreach_mapping { my ( $self, $sub ) = @_; + my $index_type; + + $self->index =~ /biblio/ ? $index_type = 'biblios' : + $self->index =~ /authorities/ ? $index_type = 'authorities' : return; + # TODO use a caching framework here my $search_fields = Koha::Database->schema->resultset('SearchField')->search( { - 'search_marc_map.index_name' => $self->index, + 'search_marc_map.index_name' => $index_type, }, { join => { search_marc_to_fields => 'search_marc_map' }, '+select' => [ diff --git a/Koha/SearchEngine/Elasticsearch/Indexer.pm b/Koha/SearchEngine/Elasticsearch/Indexer.pm index f9beb44..9fedb2e 100644 --- a/Koha/SearchEngine/Elasticsearch/Indexer.pm +++ b/Koha/SearchEngine/Elasticsearch/Indexer.pm @@ -20,7 +20,9 @@ package Koha::SearchEngine::Elasticsearch::Indexer; use Carp; use Modern::Perl; use base qw(Koha::SearchEngine::Elasticsearch); -use Data::Dumper; + +use Koha::BiblioUtils; +use Koha::MetadataRecord::Authority; # For now just marc, but we can do anything here really use Catmandu::Importer::MARC; @@ -89,7 +91,7 @@ sub update_index { my $from = $self->_convert_marc_to_json($records); - print Data::Dumper::Dumper( $from->to_array ); + #print Data::Dumper::Dumper( $from->to_array ); $self->store->bag->add_many($from); $self->store->bag->commit; return 1; @@ -110,7 +112,8 @@ it to be updated by a regular index cron job in the future. sub update_index_background { my $self = shift; - $self->update_index(@_); + warn Data::Dumper::Dumper( @_ ); + $self->do_reindex(undef, undef, 0, @_); } =head2 $indexer->delete_index($biblionums) @@ -142,8 +145,8 @@ sub delete_index_background { =head2 $indexer->drop_index(); -Drops the index from the elasticsearch server. Calling C -after this will recreate it again. +Drops the index from the elasticsearch server. +We need to create a new object once this has been done. =cut @@ -151,27 +154,30 @@ sub drop_index { my ($self) = @_; $self->store->drop(); - my $params = $self->get_elasticsearch_params(); - $self->store( - Catmandu::Store::ElasticSearch->new( - %$params, - index_settings => $self->get_elasticsearch_settings(), - index_mappings => $self->get_elasticsearch_mappings(), - ) - ); } -=head2 $indexer->do_reindex(); +=head2 $indexer->do_reindex($delete, $commit, $verbose, $biblionumbers); + +Performs a reindex, full or patial is based on whether biblionumbers are passed in +We do an additional check if an alias exists - if so and only indexing a list of bibs we use it, +if not, or if doing a full reindex, we index into a new index and adjust the existing alias then drop the old. -Performs a reindex based on records passed in +C<$delete> indicates if we should delete the existing index - only affects partial indexes as otherwise we use a new index in either case. + +C<$commit> specifies at how many records we commit the update - higher is faster but uses more memory - default is 5000. + +C<$verbose> specifies warning level - only for command line use currently. + +C<$biblionums> is an arrayref containing the biblionumbers for the records. =cut sub do_reindex { - my ($self, $next, $delete, $commit, $verbose, $biblionumbers) = @_; + my ($self, $delete, $commit, $verbose, $biblionumbers) = @_; _log(1, $verbose, "Indexing ".$self->index."\n"); + my $next; if ( scalar @$biblionumbers ) { if ( $self->index eq 'biblios' ) { @@ -197,8 +203,69 @@ sub do_reindex { $next = sub { $records->next(); }; } -warn - $self->drop_index() if ( $delete ); + my $current_index = $self->get_alias_index(); + + if ( scalar @$biblionumbers && !$delete && $current_index) { + + #In this one case we index onto existing alias + $self->_index_records( $next, $commit, $verbose ); + + } else { + + #In all other cases we are going to use a new index then switch our alias to this index + #'time' is used for providing a unique name + my $alias_name = $self->store->{index_name}; + + my $new_indexer = + Koha::SearchEngine::Elasticsearch::Indexer->new( { + index => $self->index.time + }); + + $new_indexer->_index_records( $next, $commit, $verbose ); + + if ( $current_index ){ #If the alias already exists + $self->update_alias({ #let's point the alias to the new index + old_index => $current_index, + new_index => $new_indexer->store->{index_name} + }); + $current_index =~ s/koha_kohadev_//; #Then we need short index name to build object + + my $old_index = + Koha::SearchEngine::Elasticsearch::Indexer->new({ + index => $current_index + }); + + $old_index->drop_index(); #And we drop the specific index that the alias pointed to originally + } else { #The alias was not created + $self->drop_index(); #Drop the index we created initially + #It will have the same name as our alias so must be dropped first + #It should be empty (we indexed into a new one) + $new_indexer->store->es->indices->put_alias( + name => $alias_name, + index => $new_indexer->store->{index_name} + ); + } + + } +} + +=head2 $indexer->index_records( $next, $commit, $verbose ); + +Performs a reindex, full or patial is based on whether biblionumbers are passed in +We do an additional check if an alias exists - if so and only indexing a list of bibs we use it, +if not, or if doing a full reindex, we index into a new index and adjust the existing alias then drop the old. + +C<$next> is a function reference, used for iterating over records. + +C<$commit> specifies at how many records we commit the update - higher is faster but uses more memory - default is 5000. + +C<$verbose> specifies warning level - only for command line use currently. + +=cut + +sub _index_records { + my ( $self, $next, $commit, $verbose ) = @_; + $commit ||= 5000; my $count = 0; my $commit_count = $commit; @@ -219,11 +286,16 @@ warn @commit_buffer = (); } } - # There are probably uncommitted records (beacuse we didn't hit exact commit count) + # There are probably uncommitted records (because we didn't hit exact commit count) $self->update_index( \@id_buffer, \@commit_buffer ); _log( 1, $verbose, "$count records indexed.\n" ); } +=head2 $indexer->_log( $level, $verbose, $msg ) + +Internal function, print msg to screen if verbosity > level + +=cut sub _log { my ($level, $verbose, $msg) = @_; @@ -273,4 +345,6 @@ __END__ =item Robin Sheat C<< >> +=item Nick Clemens C<< >> + =back diff --git a/misc/search_tools/rebuild_elastic_search.pl b/misc/search_tools/rebuild_elastic_search.pl index 84a1ea0..e714b7e 100755 --- a/misc/search_tools/rebuild_elastic_search.pl +++ b/misc/search_tools/rebuild_elastic_search.pl @@ -92,8 +92,6 @@ use MARC::Record; use Modern::Perl; use Pod::Usage; -use Data::Dumper; # TODO remove - my $verbose = 0; my $commit = 5000; my ($delete, $help, $man); @@ -103,9 +101,9 @@ my (@biblionumbers); GetOptions( 'c|commit=i' => \$commit, 'd|delete' => \$delete, - 'a|authorities' => \$index_authorities, - 'b|biblios' => \$index_biblios, - 'bn|bnumber=i' => \@biblionumbers, + 'a|authorities' => \$index_authorities, + 'b|biblios' => \$index_biblios, + 'bn|bnumber=i' => \@biblionumbers, 'v|verbose+' => \$verbose, 'h|help' => \$help, 'man' => \$man, @@ -121,17 +119,15 @@ pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; sanity_check(); -my $next; -print "$verbose verbosity"; if ($index_biblios) { my $index_name = $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX; my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name }); - $indexer->do_reindex($next,$delete,$commit,$verbose,\@biblionumbers); + $indexer->do_reindex($delete,$commit,$verbose,\@biblionumbers); } if ($index_authorities) { my $index_name = $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX; my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name }); - $indexer->do_reindex($next,$delete,$commit,$verbose, \@biblionumbers); + $indexer->do_reindex($delete,$commit,$verbose, \@biblionumbers); } # Checks some basic stuff to ensure that it's sane before we start. -- 2.1.4