From 7b9413f8e1f601961dca9d8d87248292e6d25e12 Mon Sep 17 00:00:00 2001 From: Ere Maijala Date: Fri, 23 Nov 2018 10:43:34 +0200 Subject: [PATCH] Bug 21872: Add slice parameter to rebuild_elastic_search.pl The slice parameter allows one to define a slice of the records to index for parallel processing. --- Koha/BiblioUtils.pm | 29 ++++++++- Koha/MetadataRecord/Authority.pm | 91 +++++++++++++++++++---------- misc/search_tools/rebuild_elastic_search.pl | 42 ++++++++----- 3 files changed, 116 insertions(+), 46 deletions(-) diff --git a/Koha/BiblioUtils.pm b/Koha/BiblioUtils.pm index 73fc12a..82f7858 100644 --- a/Koha/BiblioUtils.pm +++ b/Koha/BiblioUtils.pm @@ -101,29 +101,54 @@ sub get_from_biblionumber { =head2 get_all_biblios_iterator - my $it = Koha::BiblioUtils->get_all_biblios_iterator(); + my $it = Koha::BiblioUtils->get_all_biblios_iterator(%options); This will provide an iterator object that will, one by one, provide the Koha::BiblioUtils of each biblio. This will include the item data. The iterator is a Koha::MetadataIterator object. +Possible options are: + +=over 4 + +=item C + +slice may be defined as a hash of two values: index and count. index +is the slice number to process and count is total number of slices. +With this information the iterator returns just the given slice of +records instead of all. + =cut sub get_all_biblios_iterator { + my ($self, %options) = @_; + + my ($slice_modulo, $slice_count); + if ($options{slice}) { + $slice_count = $options{slice}->{count}; + $slice_modulo = $options{slice}->{index}; + $slice_modulo = 0 if ($slice_modulo == $slice_count); + } + my $database = Koha::Database->new(); my $schema = $database->schema(); my $rs = $schema->resultset('Biblio')->search( {}, { columns => [qw/ biblionumber /] } ); + my $next_func = sub { - # Warn and skip bad records, otherwise we break the loop while (1) { my $row = $rs->next(); return if !$row; + # Check if biblionumber matches our slice definition + if (defined $slice_count) { + next unless $row->biblionumber % $slice_count == $slice_modulo; + } my $marc = C4::Biblio::GetMarcBiblio({ biblionumber => $row->biblionumber, embed_items => 1 }); + # Warn and skip bad records, otherwise we break the loop my $next = eval { __PACKAGE__->new($marc, $row->biblionumber); }; diff --git a/Koha/MetadataRecord/Authority.pm b/Koha/MetadataRecord/Authority.pm index f9eedf8..0ffe4f3 100644 --- a/Koha/MetadataRecord/Authority.pm +++ b/Koha/MetadataRecord/Authority.pm @@ -150,52 +150,81 @@ sub authorized_heading { =head2 get_all_authorities_iterator - my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(); + my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%options); This will provide an iterator object that will, one by one, provide the Koha::MetadataRecord::Authority of each authority. The iterator is a Koha::MetadataIterator object. +Possible options are: + +=over 4 + +=item C + +slice may be defined as a hash of two values: index and count. index +is the slice number to process and count is total number of slices. +With this information the iterator returns just the given slice of +records instead of all. + =cut sub get_all_authorities_iterator { + my ($self, %options) = @_; + + my ($slice_modulo, $slice_count); + if ($options{slice}) { + $slice_count = $options{slice}->{count}; + $slice_modulo = $options{slice}->{index}; + $slice_modulo = 0 if ($slice_modulo == $slice_count); + } + my $database = Koha::Database->new(); my $schema = $database->schema(); my $rs = $schema->resultset('AuthHeader')->search( { marcxml => { '!=', undef } }, { columns => [qw/ authid authtypecode marcxml /] } ); + my $next_func = sub { - my $row = $rs->next(); - return if !$row; - my $authid = $row->authid; - my $authtypecode = $row->authtypecode; - my $marcxml = $row->marcxml; - - my $record = eval { - MARC::Record->new_from_xml( - StripNonXmlChars($marcxml), - 'UTF-8', - ( - C4::Context->preference("marcflavour") eq "UNIMARC" - ? "UNIMARCAUTH" - : C4::Context->preference("marcflavour") - ) - ); - }; - confess $@ if ($@); - $record->encoding('UTF-8'); - - # I'm not sure why we don't use the authtypecode from the database, - # but this is how the original code does it. - require C4::AuthoritiesMarc; - $authtypecode = C4::AuthoritiesMarc::GuessAuthTypeCode($record); - - my $auth = __PACKAGE__->new( $record, { authid => $authid, id => $authid, authtypecode => $authtypecode } ); - - return $auth; - }; - return Koha::MetadataIterator->new($next_func); + while (1) { + my $row = $rs->next(); + return if !$row; + + # Check if authid matches our slice definition + if (defined $slice_count) { + next unless $row->authid % $slice_count == $slice_modulo; + } + + my $authid = $row->authid; + my $authtypecode = $row->authtypecode; + my $marcxml = $row->marcxml; + + my $record = eval { + MARC::Record->new_from_xml( + StripNonXmlChars($marcxml), + 'UTF-8', + ( + C4::Context->preference("marcflavour") eq "UNIMARC" + ? "UNIMARCAUTH" + : C4::Context->preference("marcflavour") + ) + ); + }; + confess $@ if ($@); + $record->encoding('UTF-8'); + + # I'm not sure why we don't use the authtypecode from the database, + # but this is how the original code does it. + require C4::AuthoritiesMarc; + $authtypecode = C4::AuthoritiesMarc::GuessAuthTypeCode($record); + + my $auth = __PACKAGE__->new( $record, { authid => $authid, id => $authid, authtypecode => $authtypecode } ); + + return $auth; + } + }; + return Koha::MetadataIterator->new($next_func); } 1; diff --git a/misc/search_tools/rebuild_elastic_search.pl b/misc/search_tools/rebuild_elastic_search.pl index caa9a3d..cc2af11 100755 --- a/misc/search_tools/rebuild_elastic_search.pl +++ b/misc/search_tools/rebuild_elastic_search.pl @@ -64,6 +64,14 @@ Only index the supplied biblionumber, mostly for testing purposes. May be repeated. This also applies to authorities via authid, so if you're using it, you probably only want to do one or the other at a time. +=item B<-s|--slice> + +Only index a slice of the records. A slice is defined as two integers separated +by a comma. The first one is the slice to process and the second one is total +number of slices. e.g. --slice=1,3 processes every third record starting from +the first and --slice=2,3 processes every third record starting from the +second. + =item B<-v|--verbose> By default, this program only emits warnings and errors. This makes it talk @@ -94,21 +102,22 @@ use Pod::Usage; my $verbose = 0; my $commit = 5000; -my ($delete, $help, $man); +my ($delete, $help, $man, $slice); my ($index_biblios, $index_authorities); my (@biblionumbers); $|=1; # flushes output GetOptions( - 'c|commit=i' => \$commit, - 'd|delete' => \$delete, + 'c|commit=i' => \$commit, + 'd|delete' => \$delete, 'a|authorities' => \$index_authorities, - 'b|biblios' => \$index_biblios, - 'bn|bnumber=i' => \@biblionumbers, - 'v|verbose+' => \$verbose, - 'h|help' => \$help, - 'man' => \$man, + 'b|biblios' => \$index_biblios, + 'bn|bnumber=i' => \@biblionumbers, + 's|slice=s' => \$slice, + 'v|verbose+' => \$verbose, + 'h|help' => \$help, + 'man' => \$man, ); # Default is to do both @@ -121,6 +130,13 @@ pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; sanity_check(); +my %iterator_options; +if ($slice) { + my ($slice_index, $slice_count) = $slice =~ /^(\d+),(\d+)$/; + _log(1, "Processing slice $slice_index of $slice_count\n"); + $iterator_options{slice} = { index => $slice_index, count => $slice_count }; +} + my $next; if ($index_biblios) { _log(1, "Indexing biblios\n"); @@ -131,7 +147,7 @@ if ($index_biblios) { return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); }; } else { - my $records = Koha::BiblioUtils->get_all_biblios_iterator(); + my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options); $next = sub { $records->next(); } @@ -148,7 +164,7 @@ if ($index_authorities) { return ($r, $a->record); }; } else { - my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(); + my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options); $next = sub { $records->next(); } @@ -191,12 +207,12 @@ sub do_reindex { push @id_buffer, $id; push @commit_buffer, $record; if ( !( --$commit_count ) ) { - _log( 1, "Committing $commit records..." ); + _log( 1, "Committing $commit records...\n" ); $indexer->update_index( \@id_buffer, \@commit_buffer ); $commit_count = $commit; @id_buffer = (); @commit_buffer = (); - _log( 1, " done\n" ); + _log( 1, "Commit complete\n" ); } } @@ -222,5 +238,5 @@ sub sanity_check { sub _log { my ($level, $msg) = @_; - print $msg if ($verbose >= $level); + print "[$$] $msg" if ($verbose >= $level); } -- 2.7.4