From 83ea1a9bbe4a234ee7d02e0d0eb65b3eedbea649 Mon Sep 17 00:00:00 2001 From: David Gustafsson Date: Thu, 22 Nov 2018 17:55:49 +0100 Subject: [PATCH] Bug 21872: Elasticsearch indexing faster by making it multi-threaded Add record_batches script for generating biblionumber bathes and add --start-bnumber and --end-bnumber to rebuild_elastic_search.pl script. --- Koha/BiblioUtils.pm | 20 +++++++-- misc/record_batches.pl | 68 +++++++++++++++++++++++++++++ misc/search_tools/rebuild_elastic_search.pl | 29 ++++++++++-- 3 files changed, 110 insertions(+), 7 deletions(-) create mode 100644 misc/record_batches.pl diff --git a/Koha/BiblioUtils.pm b/Koha/BiblioUtils.pm index 73fc12aa4b..2676428939 100644 --- a/Koha/BiblioUtils.pm +++ b/Koha/BiblioUtils.pm @@ -111,11 +111,25 @@ The iterator is a Koha::MetadataIterator object. =cut sub get_all_biblios_iterator { + my ($self, $params) = @_; my $database = Koha::Database->new(); my $schema = $database->schema(); - my $rs = - $schema->resultset('Biblio')->search( {}, - { columns => [qw/ biblionumber /] } ); + $params //= {}; + + my $conditions = {}; + + if ($params->{start_biblionumber}) { + $conditions->{biblionumber}->{'>='} = $params->{start_biblionumber}; + } + + if ($params->{end_biblionumber}) { + $conditions->{biblionumber}->{'<='} = $params->{end_biblionumber}; + } + + my $rs = $schema->resultset('Biblio')->search( + $conditions, + { columns => [qw/ biblionumber /] } + ); my $next_func = sub { # Warn and skip bad records, otherwise we break the loop while (1) { diff --git a/misc/record_batches.pl b/misc/record_batches.pl new file mode 100644 index 0000000000..e003c8ef5c --- /dev/null +++ b/misc/record_batches.pl @@ -0,0 +1,68 @@ +#!/usr/bin/perl + +use Modern::Perl; +use C4::Context; +use Getopt::Long; +use Pod::Usage; + +my $gt_biblionumber; +my $batch_size; +my $help; +GetOptions( + 'gt_biblionumber=i' => \$gt_biblionumber, + 'batch_size=i' => \$batch_size, + 'h|help|?' => \$help +); +if ($help) { + pod2usage(1); +} + +my $dbh = C4::Context->dbh; +my $step = $batch_size || 25000; +my $offset = 0; +my $prev_biblionumber; +my ($max_biblionumber) = @{ $dbh->selectcol_arrayref(qq{SELECT MAX(`biblionumber`) FROM `biblio`}) }; +my @ranges; +my $biblionumber; + +while (1) { + my $query = $gt_biblionumber ? + qq{SELECT `biblionumber` FROM `biblio` WHERE `biblionumber` >= $gt_biblionumber ORDER BY `biblionumber` ASC LIMIT 1 OFFSET $offset} : + qq{SELECT `biblionumber` FROM `biblio` ORDER BY `biblionumber` ASC LIMIT 1 OFFSET $offset}; + my ($biblionumber) = @{ $dbh->selectcol_arrayref($query) }; + if (!$biblionumber) { + push @ranges, [$prev_biblionumber, $max_biblionumber]; + last; + } + elsif ($prev_biblionumber) { + push @ranges, [$prev_biblionumber, $biblionumber - 1]; + } + $prev_biblionumber = $biblionumber; + $offset += $step; +} + +foreach my $range (@ranges) { + print join(' ', @{$range}) . "\n"; # @TODO: Option for argument and range separtor? +} + +=head1 NAME + +record batches - This script outputs ranges of biblionumbers + +=head1 SYNOPSIS + +record_batches.pl [-h|--help] [--gt_biblionumber=BIBLIONUMBER] + +=head1 OPTIONS + +=over + +=item B<-h|--help> + +Print a brief help message. + +=item B<--gt_biblionumber> + + --gt_biblionumber=BIBLIONUMBER Output batches with biblionumber >= BIBLIONUMBER + +=item B<--batch_size> diff --git a/misc/search_tools/rebuild_elastic_search.pl b/misc/search_tools/rebuild_elastic_search.pl index 63065cad79..1b75749954 100755 --- a/misc/search_tools/rebuild_elastic_search.pl +++ b/misc/search_tools/rebuild_elastic_search.pl @@ -64,6 +64,15 @@ Only index the supplied biblionumber, mostly for testing purposes. May be repeated. This also applies to authorities via authid, so if you're using it, you probably only want to do one or the other at a time. +=item B<-sbn|--start-bnumber> + +Only index biblios with biblionumber greater or equal than supplied +biblionumber. + +=item B<-ebn|--end-bnumber> + +Only index biblios with biblionumber less or equal to supplied biblionumber. + =item B<-v|--verbose> By default, this program only emits warnings and errors. This makes it talk @@ -94,9 +103,16 @@ use Pod::Usage; my $verbose = 0; my $commit = 5000; -my ($delete, $help, $man); -my ($index_biblios, $index_authorities); -my (@biblionumbers); +my ( + $delete, + $help, + $man, + $index_biblios, + $index_authorities, + @biblionumbers, + $start_biblionumber, + $end_biblionumber +); $|=1; # flushes output @@ -106,6 +122,8 @@ GetOptions( 'a|authorities' => \$index_authorities, 'b|biblios' => \$index_biblios, 'bn|bnumber=i' => \@biblionumbers, + 'sbn|start-bnumber=i' => \$start_biblionumber, + 'ebn|end-bnumber=i' => \$end_biblionumber, 'v|verbose+' => \$verbose, 'h|help' => \$help, 'man' => \$man, @@ -131,7 +149,10 @@ if ($index_biblios) { return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); }; } else { - my $records = Koha::BiblioUtils->get_all_biblios_iterator(); + my $records = Koha::BiblioUtils->get_all_biblios_iterator({ + 'start_biblionumber' => $start_biblionumber, + 'end_biblionumber' => $end_biblionumber, + }); $next = sub { $records->next(); } -- 2.11.0