@@ -, +, @@ multi-threaded --- Koha/BiblioUtils.pm | 20 +++++++-- misc/record_batches.pl | 68 +++++++++++++++++++++++++++++ misc/search_tools/rebuild_elastic_search.pl | 29 ++++++++++-- 3 files changed, 110 insertions(+), 7 deletions(-) create mode 100644 misc/record_batches.pl --- a/Koha/BiblioUtils.pm +++ a/Koha/BiblioUtils.pm @@ -111,11 +111,25 @@ The iterator is a Koha::MetadataIterator object. =cut sub get_all_biblios_iterator { + my ($self, $params) = @_; my $database = Koha::Database->new(); my $schema = $database->schema(); - my $rs = - $schema->resultset('Biblio')->search( {}, - { columns => [qw/ biblionumber /] } ); + $params //= {}; + + my $conditions = {}; + + if ($params->{start_biblionumber}) { + $conditions->{biblionumber}->{'>='} = $params->{start_biblionumber}; + } + + if ($params->{end_biblionumber}) { + $conditions->{biblionumber}->{'<='} = $params->{end_biblionumber}; + } + + my $rs = $schema->resultset('Biblio')->search( + $conditions, + { columns => [qw/ biblionumber /] } + ); my $next_func = sub { # Warn and skip bad records, otherwise we break the loop while (1) { --- a/misc/record_batches.pl +++ a/misc/record_batches.pl @@ -0,0 +1,68 @@ +#!/usr/bin/perl + +use Modern::Perl; +use C4::Context; +use Getopt::Long; +use Pod::Usage; + +my $gt_biblionumber; +my $batch_size; +my $help; +GetOptions( + 'gt_biblionumber=i' => \$gt_biblionumber, + 'batch_size=i' => \$batch_size, + 'h|help|?' => \$help +); +if ($help) { + pod2usage(1); +} + +my $dbh = C4::Context->dbh; +my $step = $batch_size || 25000; +my $offset = 0; +my $prev_biblionumber; +my ($max_biblionumber) = @{ $dbh->selectcol_arrayref(qq{SELECT MAX(`biblionumber`) FROM `biblio`}) }; +my @ranges; +my $biblionumber; + +while (1) { + my $query = $gt_biblionumber ? + qq{SELECT `biblionumber` FROM `biblio` WHERE `biblionumber` >= $gt_biblionumber ORDER BY `biblionumber` ASC LIMIT 1 OFFSET $offset} : + qq{SELECT `biblionumber` FROM `biblio` ORDER BY `biblionumber` ASC LIMIT 1 OFFSET $offset}; + my ($biblionumber) = @{ $dbh->selectcol_arrayref($query) }; + if (!$biblionumber) { + push @ranges, [$prev_biblionumber, $max_biblionumber]; + last; + } + elsif ($prev_biblionumber) { + push @ranges, [$prev_biblionumber, $biblionumber - 1]; + } + $prev_biblionumber = $biblionumber; + $offset += $step; +} + +foreach my $range (@ranges) { + print join(' ', @{$range}) . "\n"; # @TODO: Option for argument and range separtor? +} + +=head1 NAME + +record batches - This script outputs ranges of biblionumbers + +=head1 SYNOPSIS + +record_batches.pl [-h|--help] [--gt_biblionumber=BIBLIONUMBER] + +=head1 OPTIONS + +=over + +=item B<-h|--help> + +Print a brief help message. + +=item B<--gt_biblionumber> + + --gt_biblionumber=BIBLIONUMBER Output batches with biblionumber >= BIBLIONUMBER + +=item B<--batch_size> --- a/misc/search_tools/rebuild_elastic_search.pl +++ a/misc/search_tools/rebuild_elastic_search.pl @@ -64,6 +64,15 @@ Only index the supplied biblionumber, mostly for testing purposes. May be repeated. This also applies to authorities via authid, so if you're using it, you probably only want to do one or the other at a time. +=item B<-sbn|--start-bnumber> + +Only index biblios with biblionumber greater or equal than supplied +biblionumber. + +=item B<-ebn|--end-bnumber> + +Only index biblios with biblionumber less or equal to supplied biblionumber. + =item B<-v|--verbose> By default, this program only emits warnings and errors. This makes it talk @@ -94,9 +103,16 @@ use Pod::Usage; my $verbose = 0; my $commit = 5000; -my ($delete, $help, $man); -my ($index_biblios, $index_authorities); -my (@biblionumbers); +my ( + $delete, + $help, + $man, + $index_biblios, + $index_authorities, + @biblionumbers, + $start_biblionumber, + $end_biblionumber +); $|=1; # flushes output @@ -106,6 +122,8 @@ GetOptions( 'a|authorities' => \$index_authorities, 'b|biblios' => \$index_biblios, 'bn|bnumber=i' => \@biblionumbers, + 'sbn|start-bnumber=i' => \$start_biblionumber, + 'ebn|end-bnumber=i' => \$end_biblionumber, 'v|verbose+' => \$verbose, 'h|help' => \$help, 'man' => \$man, @@ -131,7 +149,10 @@ if ($index_biblios) { return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); }; } else { - my $records = Koha::BiblioUtils->get_all_biblios_iterator(); + my $records = Koha::BiblioUtils->get_all_biblios_iterator({ + 'start_biblionumber' => $start_biblionumber, + 'end_biblionumber' => $end_biblionumber, + }); $next = sub { $records->next(); } --