From 19fc86a345cd2784134b537acce5abebf5e0e4d6 Mon Sep 17 00:00:00 2001 From: Ere Maijala Date: Fri, 23 Nov 2018 10:43:34 +0200 Subject: [PATCH] Bug 21872: Add slice parameter to rebuild_elastic_search.pl The slice parameter allows one to define a slice of the records to index for parallel processing. --- Koha/BiblioUtils.pm | 28 +++++++++++++++++-- Koha/MetadataRecord/Authority.pm | 36 +++++++++++++++++++++++-- misc/search_tools/rebuild_elastic_search.pl | 42 ++++++++++++++++++++--------- 3 files changed, 89 insertions(+), 17 deletions(-) diff --git a/Koha/BiblioUtils.pm b/Koha/BiblioUtils.pm index 73fc12a..b934b2c 100644 --- a/Koha/BiblioUtils.pm +++ b/Koha/BiblioUtils.pm @@ -101,20 +101,44 @@ sub get_from_biblionumber { =head2 get_all_biblios_iterator - my $it = Koha::BiblioUtils->get_all_biblios_iterator(); + my $it = Koha::BiblioUtils->get_all_biblios_iterator(%options); This will provide an iterator object that will, one by one, provide the Koha::BiblioUtils of each biblio. This will include the item data. The iterator is a Koha::MetadataIterator object. +Possible options are: + +=over 4 + +=item C + +slice may be defined as a hash of two values: index and count. index +is the slice number to process and count is total number of slices. +With this information the iterator returns just the given slice of +records instead of all. + =cut sub get_all_biblios_iterator { + my ($self, %options) = @_; + + my $search_terms = {}; + my ($slice_modulo, $slice_count); + if ($options{slice}) { + $slice_count = $options{slice}->{count}; + $slice_modulo = $options{slice}->{index}; + $slice_modulo = 0 if ($slice_modulo == $slice_count); + + $search_terms = \[ ' mod(biblionumber, ?) = ?', $slice_count, $slice_modulo]; + } + my $database = Koha::Database->new(); my $schema = $database->schema(); my $rs = - $schema->resultset('Biblio')->search( {}, + $schema->resultset('Biblio')->search( + $search_terms, { columns => [qw/ biblionumber /] } ); my $next_func = sub { # Warn and skip bad records, otherwise we break the loop diff --git a/Koha/MetadataRecord/Authority.pm b/Koha/MetadataRecord/Authority.pm index f9eedf8..3adac20 100644 --- a/Koha/MetadataRecord/Authority.pm +++ b/Koha/MetadataRecord/Authority.pm @@ -150,20 +150,52 @@ sub authorized_heading { =head2 get_all_authorities_iterator - my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(); + my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%options); This will provide an iterator object that will, one by one, provide the Koha::MetadataRecord::Authority of each authority. The iterator is a Koha::MetadataIterator object. +Possible options are: + +=over 4 + +=item C + +slice may be defined as a hash of two values: index and count. index +is the slice number to process and count is total number of slices. +With this information the iterator returns just the given slice of +records instead of all. + =cut sub get_all_authorities_iterator { + my ($self, %options) = @_; + + my $search_terms = { + marcxml => { '!=', undef } + }; + my ($slice_modulo, $slice_count); + if ($options{slice}) { + $slice_count = $options{slice}->{count}; + $slice_modulo = $options{slice}->{index}; + $slice_modulo = 0 if ($slice_modulo == $slice_count); + + $search_terms->{authid} = \[ ' mod ? = ?', $slice_count, $slice_modulo]; + $search_terms = { + '-and' => [ + $search_terms, + \[ ' mod(authid, ?) = ?', $slice_count, $slice_modulo] + ] + }; + } + my $database = Koha::Database->new(); my $schema = $database->schema(); my $rs = - $schema->resultset('AuthHeader')->search( { marcxml => { '!=', undef } }, + $schema->resultset('AuthHeader')->search( + $search_terms, { columns => [qw/ authid authtypecode marcxml /] } ); my $next_func = sub { my $row = $rs->next(); diff --git a/misc/search_tools/rebuild_elastic_search.pl b/misc/search_tools/rebuild_elastic_search.pl index caa9a3d..df7bfef 100755 --- a/misc/search_tools/rebuild_elastic_search.pl +++ b/misc/search_tools/rebuild_elastic_search.pl @@ -64,6 +64,14 @@ Only index the supplied biblionumber, mostly for testing purposes. May be repeated. This also applies to authorities via authid, so if you're using it, you probably only want to do one or the other at a time. +=item B<-s|--slice> + +Only index a slice of the records. A slice is defined as two integers separated +by a comma. The first one is the slice to process and the second one is total +number of slices. e.g. --slice=1,3 processes every third record starting from +the first and --slice=2,3 processes every third record starting from the +second. + =item B<-v|--verbose> By default, this program only emits warnings and errors. This makes it talk @@ -94,21 +102,22 @@ use Pod::Usage; my $verbose = 0; my $commit = 5000; -my ($delete, $help, $man); +my ($delete, $help, $man, $slice); my ($index_biblios, $index_authorities); my (@biblionumbers); $|=1; # flushes output GetOptions( - 'c|commit=i' => \$commit, - 'd|delete' => \$delete, + 'c|commit=i' => \$commit, + 'd|delete' => \$delete, 'a|authorities' => \$index_authorities, - 'b|biblios' => \$index_biblios, - 'bn|bnumber=i' => \@biblionumbers, - 'v|verbose+' => \$verbose, - 'h|help' => \$help, - 'man' => \$man, + 'b|biblios' => \$index_biblios, + 'bn|bnumber=i' => \@biblionumbers, + 's|slice=s' => \$slice, + 'v|verbose+' => \$verbose, + 'h|help' => \$help, + 'man' => \$man, ); # Default is to do both @@ -121,6 +130,13 @@ pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; sanity_check(); +my %iterator_options; +if ($slice) { + my ($slice_index, $slice_count) = $slice =~ /^(\d+),(\d+)$/; + _log(1, "Processing slice $slice_index of $slice_count\n"); + $iterator_options{slice} = { index => $slice_index, count => $slice_count }; +} + my $next; if ($index_biblios) { _log(1, "Indexing biblios\n"); @@ -131,7 +147,7 @@ if ($index_biblios) { return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); }; } else { - my $records = Koha::BiblioUtils->get_all_biblios_iterator(); + my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options); $next = sub { $records->next(); } @@ -148,7 +164,7 @@ if ($index_authorities) { return ($r, $a->record); }; } else { - my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(); + my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options); $next = sub { $records->next(); } @@ -191,12 +207,12 @@ sub do_reindex { push @id_buffer, $id; push @commit_buffer, $record; if ( !( --$commit_count ) ) { - _log( 1, "Committing $commit records..." ); + _log( 1, "Committing $commit records...\n" ); $indexer->update_index( \@id_buffer, \@commit_buffer ); $commit_count = $commit; @id_buffer = (); @commit_buffer = (); - _log( 1, " done\n" ); + _log( 1, "Commit complete\n" ); } } @@ -222,5 +238,5 @@ sub sanity_check { sub _log { my ($level, $msg) = @_; - print $msg if ($verbose >= $level); + print "[$$] $msg" if ($verbose >= $level); } -- 2.7.4