From da66b668492c4aa2cfcbd97177ac2fb589ab7dc4 Mon Sep 17 00:00:00 2001 From: Nick Clemens Date: Thu, 20 Jul 2017 13:30:51 +0000 Subject: [PATCH] Bug 18948 - Elasticsearch - Reindexes should use aliases to avoid down time while reindexing Refactoring of rebuild_elastic_search.pl to move code into Koha::SearchEngine::Elasticsearch::Indexer --- Koha/SearchEngine/Elasticsearch/Indexer.pm | 133 +++++++++++++++++++++------- misc/search_tools/rebuild_elastic_search.pl | 83 ++--------------- 2 files changed, 106 insertions(+), 110 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch/Indexer.pm b/Koha/SearchEngine/Elasticsearch/Indexer.pm index e5babee..f9beb44 100644 --- a/Koha/SearchEngine/Elasticsearch/Indexer.pm +++ b/Koha/SearchEngine/Elasticsearch/Indexer.pm @@ -41,6 +41,27 @@ Koha::SearchEngine::Elasticsearch::Indexer - handles adding new records to the i =head1 FUNCTIONS +=head2 Koha::SearchEngine::Elasticsearch::Indexer->({index => $index_name}); + +$index_name should be biblios or authorities +call our SUPER class and then set ES parameters + +=cut + +sub new { + my $class = shift @_; + my $self = $class->SUPER::new(@_); + my $params = $self->get_elasticsearch_params(); + $self->store( + Catmandu::Store::ElasticSearch->new( + %$params, + index_settings => $self->get_elasticsearch_settings(), + index_mappings => $self->get_elasticsearch_mappings(), + ) + ); + return $self; +} + =head2 $indexer->update_index($biblionums, $records); C<$biblionums> is an arrayref containing the biblionumbers for the records. @@ -67,18 +88,8 @@ sub update_index { } my $from = $self->_convert_marc_to_json($records); - if ( !$self->store ) { - my $params = $self->get_elasticsearch_params(); - $self->store( - Catmandu::Store::ElasticSearch->new( - %$params, - index_settings => $self->get_elasticsearch_settings(), - index_mappings => $self->get_elasticsearch_mappings(), - ) - ); - } - #print Data::Dumper::Dumper( $from->to_array ); + print Data::Dumper::Dumper( $from->to_array ); $self->store->bag->add_many($from); $self->store->bag->commit; return 1; @@ -111,16 +122,6 @@ C<$biblionums> is an arrayref of biblionumbers to delete from the index. sub delete_index { my ($self, $biblionums) = @_; - if ( !$self->store ) { - my $params = $self->get_elasticsearch_params(); - $self->store( - Catmandu::Store::ElasticSearch->new( - %$params, - index_settings => $self->get_elasticsearch_settings(), - index_mappings => $self->get_elasticsearch_mappings(), - ) - ); - } $self->store->bag->delete($_) foreach @$biblionums; $self->store->bag->commit; } @@ -149,20 +150,84 @@ after this will recreate it again. sub drop_index { my ($self) = @_; - if (!$self->store) { - # If this index doesn't exist, this will create it. Then it'll be - # deleted. That's not the end of the world however. - my $params = $self->get_elasticsearch_params(); - $self->store( - Catmandu::Store::ElasticSearch->new( - %$params, - index_settings => $self->get_elasticsearch_settings(), - index_mappings => $self->get_elasticsearch_mappings(), - ) - ); - } $self->store->drop(); - $self->store(undef); + my $params = $self->get_elasticsearch_params(); + $self->store( + Catmandu::Store::ElasticSearch->new( + %$params, + index_settings => $self->get_elasticsearch_settings(), + index_mappings => $self->get_elasticsearch_mappings(), + ) + ); +} + + +=head2 $indexer->do_reindex(); + +Performs a reindex based on records passed in + +=cut + +sub do_reindex { + my ($self, $next, $delete, $commit, $verbose, $biblionumbers) = @_; + + _log(1, $verbose, "Indexing ".$self->index."\n"); + + if ( scalar @$biblionumbers ) { + if ( $self->index eq 'biblios' ) { + $next = sub { + my $r = shift @$biblionumbers; + return () unless defined $r; + return ( $r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); + }; + } + elsif ( $self->index eq 'authorities' ) { + $next = sub { + my $r = shift @$biblionumbers; + return () unless defined $r; + my $a = Koha::MetadataRecord::Authority->get_from_authid($r); + return ($r, $a->record); + }; + } + } + else { + my $records = $self->index eq 'biblios' ? + Koha::BiblioUtils->get_all_biblios_iterator() : $self->index eq 'authorities' ? + Koha::MetadataRecord::Authority->get_all_authorities_iterator() : die "Reindex called on undefined index"; + $next = sub { $records->next(); }; + } + +warn + $self->drop_index() if ( $delete ); + + my $count = 0; + my $commit_count = $commit; + my ( @id_buffer, @commit_buffer ); + while ( my $record = $next->() ) { + my $id = $record->id; + my $record = $record->record; + _log( 1, $verbose, "$id\n" ); + $count++; + + push @id_buffer, $id; + push @commit_buffer, $record; + if ( !( --$commit_count ) ) { + _log( 2, $verbose, "Committing...\n" ); + $self->update_index( \@id_buffer, \@commit_buffer ); + $commit_count = $commit; + @id_buffer = (); + @commit_buffer = (); + } + } + # There are probably uncommitted records (beacuse we didn't hit exact commit count) + $self->update_index( \@id_buffer, \@commit_buffer ); + _log( 1, $verbose, "$count records indexed.\n" ); +} + + +sub _log { + my ($level, $verbose, $msg) = @_; + print $msg if ($verbose >= $level); } sub _sanitise_records { diff --git a/misc/search_tools/rebuild_elastic_search.pl b/misc/search_tools/rebuild_elastic_search.pl index 6faa32c..84a1ea0 100755 --- a/misc/search_tools/rebuild_elastic_search.pl +++ b/misc/search_tools/rebuild_elastic_search.pl @@ -122,74 +122,16 @@ pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; sanity_check(); my $next; +print "$verbose verbosity"; if ($index_biblios) { - _log(1, "Indexing biblios\n"); - if (@biblionumbers) { - $next = sub { - my $r = shift @biblionumbers; - return () unless defined $r; - return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); - }; - } else { - my $records = Koha::BiblioUtils->get_all_biblios_iterator(); - $next = sub { - $records->next(); - } - } - do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); + my $index_name = $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX; + my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name }); + $indexer->do_reindex($next,$delete,$commit,$verbose,\@biblionumbers); } if ($index_authorities) { - _log(1, "Indexing authorities\n"); - if (@biblionumbers) { - $next = sub { - my $r = shift @biblionumbers; - return () unless defined $r; - my $a = Koha::MetadataRecord::Authority->get_from_authid($r); - return ($r, $a->record); - }; - } else { - my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(); - $next = sub { - $records->next(); - } - } - do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX); -} - -sub do_reindex { - my ( $next, $index_name ) = @_; - - my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } ); - if ($delete) { - - # We know it's safe to not recreate the indexer because update_index - # hasn't been called yet. - $indexer->drop_index(); - } - - my $count = 0; - my $commit_count = $commit; - my ( @id_buffer, @commit_buffer ); - while ( my $record = $next->() ) { - my $id = $record->id; - my $record = $record->record; - _log( 1, "$id\n" ); - $count++; - - push @id_buffer, $id; - push @commit_buffer, $record; - if ( !( --$commit_count ) ) { - _log( 2, "Committing...\n" ); - $indexer->update_index( \@id_buffer, \@commit_buffer ); - $commit_count = $commit; - @id_buffer = (); - @commit_buffer = (); - } - } - - # There are probably uncommitted records - $indexer->update_index( \@id_buffer, \@commit_buffer ); - _log( 1, "$count records indexed.\n" ); + my $index_name = $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX; + my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name }); + $indexer->do_reindex($next,$delete,$commit,$verbose, \@biblionumbers); } # Checks some basic stuff to ensure that it's sane before we start. @@ -199,14 +141,3 @@ sub sanity_check { die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf ); } -# Output progress information. -# -# _log($level, $msg); -# -# Will output $msg if the verbosity setting is set to $level or more. Will -# not include a trailing newline. -sub _log { - my ($level, $msg) = @_; - - print $msg if ($verbose >= $level); -} -- 2.1.4