From b5f0de3ed0473396894bd1b9f741ba34b6d206a0 Mon Sep 17 00:00:00 2001 From: Nick Clemens Date: Mon, 17 Jul 2017 20:20:12 +0000 Subject: [PATCH] Bug 18948 - Elasticsearch - Reindexes should use aliases to avoid down time while reindexing To test: 1 - Delete your current indices for ES (if any) 2 - Reindex your records 3 - curl -XGET 'localhost:9200/koha_kohadev?pretty' 4 - Note indices are now of form koha_kohadev_########## 5 - The numbers there are a result of appending 'time' command to name 6 - Ensure searching works as before the patches 7 - Reindex your records with a commit setting of 1 (to slow things down) you may need a large numebr fo records 8 - Ensure searching works during reindex 9 - Ensure reindexing completes successfully 10 - curl -XGET 'localhost:9200/koha_kohadev?pretty' 11 - There should only be one index each for biblios and authorities ** - Note number in above 12 - Perform a partial reindex by passing a biblionumber to the command 13 - Curl the indices again and ensure number/name has not changed --- Koha/SearchEngine/Elasticsearch.pm | 61 ++++++++++++++++++++++++++++- misc/search_tools/rebuild_elastic_search.pl | 45 +++++++++++++++------ 2 files changed, 91 insertions(+), 15 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm index 43bb0a8..32e795a 100644 --- a/Koha/SearchEngine/Elasticsearch.pm +++ b/Koha/SearchEngine/Elasticsearch.pm @@ -37,7 +37,7 @@ __PACKAGE__->mk_ro_accessors(qw( index )); __PACKAGE__->mk_accessors(qw( sort_fields )); # Constants to refer to the standard index names -Readonly our $BIBLIOS_INDEX => 'biblios'; +Readonly our $BIBLIOS_INDEX => 'biblios'; #These are going to be the suffixes for our aliases - most access to indexes will be via alias Readonly our $AUTHORITIES_INDEX => 'authorities'; =head1 NAME @@ -236,6 +236,58 @@ sub get_elasticsearch_mappings { return $mappings; } +=head2 get_alias_index + +Get the ES specific index names that an alias points to + +Returns an index name or undef + +=cut + +sub get_alias_index { + my ($self) = @_; + my $alias = $self->get_elasticsearch_params()->{index_name}; #FIXME this should probably be an object property/method + if ( !$self->store ) { + my $params = $self->get_elasticsearch_params(); + $self->store( + Catmandu::Store::ElasticSearch->new( + %$params, + index_settings => $self->get_elasticsearch_settings(), + index_mappings => $self->get_elasticsearch_mappings(), + ) + ); + } + if ( $self->store->es->indices->exists_alias( name => $alias ) ) { #use the embedded es object to access ES methods directly and check if alias points to anything + my $current_index_info = $self->store->es->indices->get_alias( name => $alias ); #we get a hashref of info + my @current_indices = keys %$current_index_info; #the keys contain the names of indexes pointed to be the alias + return $current_indices[0]; #For now we are maintaining a 1:1 ratio, pointing to several is possible + } else { + return undef; + } +} + +=head2 update_alias_indices + +Takes our ES alias name and two index names and swaps the old for the new +return + +=cut + +sub update_alias { + my $self = shift; + my ( $params ) = @_; + my $alias = $self->get_elasticsearch_params()->{index_name}; #FIXME this should probably be an object property/method + die "Must supply old_index and new_index parameters" unless ( $params->{old_index} && $params->{new_index} ); + return $self->store->es->indices->update_aliases( #this is atomic so we won't delete unless this works + body => { + actions => [ + { add => { alias => $alias, index => $params->{new_index} } }, + { remove => { alias => $alias, index => $params->{old_index} } } + ] + } + ); +} + =head2 _elasticsearch_mapping_for_* Get the ES mappings for the given data type or a special mapping case @@ -418,10 +470,15 @@ These are of a form suited to Catmandu's MARC fixers. sub _foreach_mapping { my ( $self, $sub ) = @_; + my $index_type; + + $self->index =~ /biblio/ ? $index_type = 'biblios' : + $self->index =~ /authorities/ ? $index_type = 'authorities' : return; + # TODO use a caching framework here my $search_fields = Koha::Database->schema->resultset('SearchField')->search( { - 'search_marc_map.index_name' => $self->index, + 'search_marc_map.index_name' => $index_type, }, { join => { search_marc_to_fields => 'search_marc_map' }, '+select' => [ diff --git a/misc/search_tools/rebuild_elastic_search.pl b/misc/search_tools/rebuild_elastic_search.pl index 6faa32c..1d430c5 100755 --- a/misc/search_tools/rebuild_elastic_search.pl +++ b/misc/search_tools/rebuild_elastic_search.pl @@ -158,14 +158,41 @@ if ($index_authorities) { sub do_reindex { my ( $next, $index_name ) = @_; - my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } ); - if ($delete) { - # We know it's safe to not recreate the indexer because update_index - # hasn't been called yet. - $indexer->drop_index(); + if (scalar @biblionumbers && !$delete) { #case where we index onto existing alias + _index_records( $next, $indexer ); + } else { + my $new_index_name = $index_name.time; #In the case of full reindex or deletion we are indexing into a new index + #and then updating the alias - time gives us a unique name + my $new_indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $new_index_name } ); + _index_records( $next, $new_indexer ); + if ( my $current_index = $indexer->get_alias_index() ){ + $indexer->update_alias({ + old_index => $current_index, + new_index => $new_indexer->get_elasticsearch_params()->{index_name} + }); + $current_index =~ s/koha_kohadev_//; #need short index name to build object + $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $current_index } ); + $indexer->drop_index(); + } else { + $indexer->store->es->indices->put_alias( + name => $indexer->get_elasticsearch_params()->{index_name}, + index => $new_indexer->get_elasticsearch_params()->{index_name} + ); + } } +} + +# Checks some basic stuff to ensure that it's sane before we start. +sub sanity_check { + # Do we have an elasticsearch block defined? + my $conf = C4::Context->config('elasticsearch'); + die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf ); +} + +sub _index_records { + my ( $next, $indexer ) = @_; my $count = 0; my $commit_count = $commit; @@ -186,19 +213,11 @@ sub do_reindex { @commit_buffer = (); } } - # There are probably uncommitted records $indexer->update_index( \@id_buffer, \@commit_buffer ); _log( 1, "$count records indexed.\n" ); } -# Checks some basic stuff to ensure that it's sane before we start. -sub sanity_check { - # Do we have an elasticsearch block defined? - my $conf = C4::Context->config('elasticsearch'); - die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf ); -} - # Output progress information. # # _log($level, $msg); -- 2.1.4