Bugzilla – Attachment 70815 Details for
Bug 18948
Reindexing should use aliases to avoid down time
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 18948 - Elasticsearch - Reindexes should use aliases to avoid down time while reindexing
Bug-18948---Elasticsearch---Reindexes-should-use-a.patch (text/plain), 9.42 KB, created by
Nick Clemens (kidclamp)
on 2018-01-23 00:55:13 UTC
(
hide
)
Description:
Bug 18948 - Elasticsearch - Reindexes should use aliases to avoid down time while reindexing
Filename:
MIME Type:
Creator:
Nick Clemens (kidclamp)
Created:
2018-01-23 00:55:13 UTC
Size:
9.42 KB
patch
obsolete
>From c05d9fb535ec0a2405b3d4e249d2d7b401f03837 Mon Sep 17 00:00:00 2001 >From: Nick Clemens <nick@bywatersolutions.com> >Date: Thu, 20 Jul 2017 13:30:51 +0000 >Subject: [PATCH] Bug 18948 - Elasticsearch - Reindexes should use aliases to > avoid down time while reindexing > >Refactoring of rebuild_elastic_search.pl to move code into >Koha::SearchEngine::Elasticsearch::Indexer >--- > Koha/SearchEngine/Elasticsearch/Indexer.pm | 133 +++++++++++++++++++++------- > misc/search_tools/rebuild_elastic_search.pl | 83 ++--------------- > 2 files changed, 106 insertions(+), 110 deletions(-) > >diff --git a/Koha/SearchEngine/Elasticsearch/Indexer.pm b/Koha/SearchEngine/Elasticsearch/Indexer.pm >index e5babee..f026bda 100644 >--- a/Koha/SearchEngine/Elasticsearch/Indexer.pm >+++ b/Koha/SearchEngine/Elasticsearch/Indexer.pm >@@ -41,6 +41,27 @@ Koha::SearchEngine::Elasticsearch::Indexer - handles adding new records to the i > > =head1 FUNCTIONS > >+=head2 Koha::SearchEngine::Elasticsearch::Indexer->({index => $index_name}); >+ >+$index_name should be biblios or authorities >+call our SUPER class and then set ES parameters >+ >+=cut >+ >+sub new { >+ my $class = shift @_; >+ my $self = $class->SUPER::new(@_); >+ my $params = $self->get_elasticsearch_params(); >+ $self->store( >+ Catmandu::Store::ElasticSearch->new( >+ %$params, >+ index_settings => $self->get_elasticsearch_settings(), >+ index_mappings => $self->get_elasticsearch_mappings(), >+ ) >+ ); >+ return $self; >+} >+ > =head2 $indexer->update_index($biblionums, $records); > > C<$biblionums> is an arrayref containing the biblionumbers for the records. >@@ -67,18 +88,8 @@ sub update_index { > } > > my $from = $self->_convert_marc_to_json($records); >- if ( !$self->store ) { >- my $params = $self->get_elasticsearch_params(); >- $self->store( >- Catmandu::Store::ElasticSearch->new( >- %$params, >- index_settings => $self->get_elasticsearch_settings(), >- index_mappings => $self->get_elasticsearch_mappings(), >- ) >- ); >- } > >- #print Data::Dumper::Dumper( $from->to_array ); >+ print Data::Dumper::Dumper( $from->to_array ); > $self->store->bag->add_many($from); > $self->store->bag->commit; > return 1; >@@ -111,16 +122,6 @@ C<$biblionums> is an arrayref of biblionumbers to delete from the index. > sub delete_index { > my ($self, $biblionums) = @_; > >- if ( !$self->store ) { >- my $params = $self->get_elasticsearch_params(); >- $self->store( >- Catmandu::Store::ElasticSearch->new( >- %$params, >- index_settings => $self->get_elasticsearch_settings(), >- index_mappings => $self->get_elasticsearch_mappings(), >- ) >- ); >- } > $self->store->bag->delete($_) foreach @$biblionums; > $self->store->bag->commit; > } >@@ -149,20 +150,84 @@ after this will recreate it again. > sub drop_index { > my ($self) = @_; > >- if (!$self->store) { >- # If this index doesn't exist, this will create it. Then it'll be >- # deleted. That's not the end of the world however. >- my $params = $self->get_elasticsearch_params(); >- $self->store( >- Catmandu::Store::ElasticSearch->new( >- %$params, >- index_settings => $self->get_elasticsearch_settings(), >- index_mappings => $self->get_elasticsearch_mappings(), >- ) >- ); >- } > $self->store->drop(); >- $self->store(undef); >+ my $params = $self->get_elasticsearch_params(); >+ $self->store( >+ Catmandu::Store::ElasticSearch->new( >+ %$params, >+ index_settings => $self->get_elasticsearch_settings(), >+ index_mappings => $self->get_elasticsearch_mappings(), >+ ) >+ ); >+} >+ >+ >+=head2 $indexer->do_reindex(); >+ >+Performs a reindex based on records passed in >+ >+=cut >+ >+sub do_reindex { >+ my ($self, $next, $delete, $commit, $verbose, $biblionumbers) = @_; >+ >+ _log(1, $verbose, "Indexing ".$self->index."\n"); >+ >+ if ( scalar @$biblionumbers ) { >+ if ( $self->index eq 'biblios' ) { >+ $next = sub { >+ my $r = shift @$biblionumbers; >+ return () unless defined $r; >+ return ( $r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); >+ }; >+ } >+ elsif ( $self->index eq 'authorities' ) { >+ $next = sub { >+ my $r = shift @$biblionumbers; >+ return () unless defined $r; >+ my $a = Koha::MetadataRecord::Authority->get_from_authid($r); >+ return ($r, $a->record); >+ }; >+ } >+ } >+ else { >+ my $records = $self->index eq 'biblios' ? >+ Koha::BiblioUtils->get_all_biblios_iterator() : $self->index eq 'authorities' ? >+ Koha::MetadataRecord::Authority->get_all_authorities_iterator() : die "Reindex called on undefined index"; >+ $next = sub { $records->next(); }; >+ } >+ >+warn >+ $self->drop_index() if ( $delete ); >+ >+ my $count = 0; >+ my $commit_count = $commit; >+ my ( @id_buffer, @commit_buffer ); >+ while ( my $record = $next->() ) { >+ my $id = $record->id; >+ my $record = $record->record; >+ _log( 1, $verbose, "$id\n" ); >+ $count++; >+ >+ push @id_buffer, $id; >+ push @commit_buffer, $record; >+ if ( !( --$commit_count ) ) { >+ _log( 2, $verbose, "Committing...\n" ); >+ $self->update_index( \@id_buffer, \@commit_buffer ); >+ $commit_count = $commit; >+ @id_buffer = (); >+ @commit_buffer = (); >+ } >+ } >+ # There are probably uncommitted records (beacuse we didn't hit exact commit count) >+ $self->update_index( \@id_buffer, \@commit_buffer ); >+ _log( 1, $verbose, "$count records indexed.\n" ); >+} >+ >+ >+sub _log { >+ my ($level, $verbose, $msg) = @_; >+ print $msg if ($verbose >= $level); > } > > sub _sanitise_records { >diff --git a/misc/search_tools/rebuild_elastic_search.pl b/misc/search_tools/rebuild_elastic_search.pl >index 6faa32c..84a1ea0 100755 >--- a/misc/search_tools/rebuild_elastic_search.pl >+++ b/misc/search_tools/rebuild_elastic_search.pl >@@ -122,74 +122,16 @@ pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; > sanity_check(); > > my $next; >+print "$verbose verbosity"; > if ($index_biblios) { >- _log(1, "Indexing biblios\n"); >- if (@biblionumbers) { >- $next = sub { >- my $r = shift @biblionumbers; >- return () unless defined $r; >- return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); >- }; >- } else { >- my $records = Koha::BiblioUtils->get_all_biblios_iterator(); >- $next = sub { >- $records->next(); >- } >- } >- do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); >+ my $index_name = $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX; >+ my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name }); >+ $indexer->do_reindex($next,$delete,$commit,$verbose,\@biblionumbers); > } > if ($index_authorities) { >- _log(1, "Indexing authorities\n"); >- if (@biblionumbers) { >- $next = sub { >- my $r = shift @biblionumbers; >- return () unless defined $r; >- my $a = Koha::MetadataRecord::Authority->get_from_authid($r); >- return ($r, $a->record); >- }; >- } else { >- my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(); >- $next = sub { >- $records->next(); >- } >- } >- do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX); >-} >- >-sub do_reindex { >- my ( $next, $index_name ) = @_; >- >- my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } ); >- if ($delete) { >- >- # We know it's safe to not recreate the indexer because update_index >- # hasn't been called yet. >- $indexer->drop_index(); >- } >- >- my $count = 0; >- my $commit_count = $commit; >- my ( @id_buffer, @commit_buffer ); >- while ( my $record = $next->() ) { >- my $id = $record->id; >- my $record = $record->record; >- _log( 1, "$id\n" ); >- $count++; >- >- push @id_buffer, $id; >- push @commit_buffer, $record; >- if ( !( --$commit_count ) ) { >- _log( 2, "Committing...\n" ); >- $indexer->update_index( \@id_buffer, \@commit_buffer ); >- $commit_count = $commit; >- @id_buffer = (); >- @commit_buffer = (); >- } >- } >- >- # There are probably uncommitted records >- $indexer->update_index( \@id_buffer, \@commit_buffer ); >- _log( 1, "$count records indexed.\n" ); >+ my $index_name = $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX; >+ my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name }); >+ $indexer->do_reindex($next,$delete,$commit,$verbose, \@biblionumbers); > } > > # Checks some basic stuff to ensure that it's sane before we start. >@@ -199,14 +141,3 @@ sub sanity_check { > die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf ); > } > >-# Output progress information. >-# >-# _log($level, $msg); >-# >-# Will output $msg if the verbosity setting is set to $level or more. Will >-# not include a trailing newline. >-sub _log { >- my ($level, $msg) = @_; >- >- print $msg if ($verbose >= $level); >-} >-- >2.1.4
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 18948
:
65086
|
65147
|
65159
| 70815 |
70816