From ecdfe980b599ab7a279096f63cd0d1f3f5493cba Mon Sep 17 00:00:00 2001 From: David Gustafsson Date: Thu, 22 Nov 2018 22:33:37 +0100 Subject: [PATCH] Bug 21872: Add parallel_rebuild_biblios.pl script Add parallel_rebuild_biblios.pl script for rebuild biblios index in parallel. Adjust some option names and script pods. --- misc/record_batches.pl | 60 +++++++----- misc/search_tools/parallel_rebuild_biblios.pl | 132 ++++++++++++++++++++++++++ misc/search_tools/rebuild_elastic_search.pl | 8 +- 3 files changed, 171 insertions(+), 29 deletions(-) create mode 100755 misc/search_tools/parallel_rebuild_biblios.pl diff --git a/misc/record_batches.pl b/misc/record_batches.pl index c157e65a77..d85136f0b1 100755 --- a/misc/record_batches.pl +++ b/misc/record_batches.pl @@ -1,5 +1,31 @@ #!/usr/bin/perl +=head1 NAME + +record_batches.pl - Outputs biblio record batches as ranges of biblionumbers. + +=head1 SYNOPSIS + +record_batches.pl [-h|--help] [--startbiblionumber=BIBLIONUMBER] + +=head1 OPTIONS + +=over + +=item B<-h|--help> + +Print a brief help message. + +=item B<--startbiblionumber> + + --startbiblionumber=BIBLIONUMBER Output batches with biblionumber >= BIBLIONUMBER + +=item B<--batchsize> + +=back + +=cut + use Modern::Perl; use C4::Context; use Getopt::Long; @@ -25,6 +51,13 @@ if (!$start_biblionumber) { $start_biblionumber = $min_biblionumber; } +# TODO: Could be options +my $range_separator = " "; +my $batch_separator = "\n"; +sub print_range { + print join($range_separator, @_) . $batch_separator; +} + while (1) { my $offset = $batch_size - 1; my $query = @@ -35,15 +68,14 @@ while (1) { my ($max_biblionumber) = @{ $dbh->selectcol_arrayref(qq{SELECT MAX(`biblionumber`) FROM `biblio`}) }; if($max_biblionumber) { if($max_biblionumber >= $start_biblionumber) { - # @TODO: Option for argument and range separator? - print join(' ', ($start_biblionumber, $max_biblionumber)) . "\n"; + print_range($start_biblionumber, $max_biblionumber); } } } last; } else { - print join(' ', ($start_biblionumber, $end_biblionumber)) . "\n"; + print_range($start_biblionumber, $end_biblionumber); } if ($next_biblionumber) { $start_biblionumber = $next_biblionumber; @@ -52,25 +84,3 @@ while (1) { last; } } - -=head1 NAME - -record batches - This script outputs ranges of biblionumbers - -=head1 SYNOPSIS - -record_batches.pl [-h|--help] [--startbiblionumber=BIBLIONUMBER] - -=head1 OPTIONS - -=over - -=item B<-h|--help> - -Print a brief help message. - -=item B<--startbiblionumber> - - --startbiblionumber=BIBLIONUMBER Output batches with biblionumber >= BIBLIONUMBER - -=item B<--batchsize> diff --git a/misc/search_tools/parallel_rebuild_biblios.pl b/misc/search_tools/parallel_rebuild_biblios.pl new file mode 100755 index 0000000000..3b2c82930a --- /dev/null +++ b/misc/search_tools/parallel_rebuild_biblios.pl @@ -0,0 +1,132 @@ +#!/usr/bin/perl + +=head1 NAME + +parallel_rebuild_bibios.pl - inserts biblios records from a Koha database into Elasticsearch in parallel. + +=head1 SYNOPSIS + +B +[B<-h|--help>] +[B<-v|--verbose>] +[B<-j|--jobs>=C] +[B<-b|--batchsize>=C] +[B<-c|--commit>=C] +[B<--startbiblionumber>=C] +[B<-d|--delete>] + +=head1 OPTIONS + +=over + +=item B<-h|--help> + +Print a brief help message. + +=item B<-v|--verbose> + +Passed to rebuild_elastic_search.pl. Increase verbosity. + +=item B<-j|--jobs>=C + +Run up to N jobs in parallel. 0 means as many as possible. Default is to run +one job per CPU core. This is passed directly to parrallel as B<-j>C. Se C +for more information about variants of C. + +=item B<-b|--batchsize>=C + +Specify batch size for batches of biblios passed to rebuild_elastic_search.pl. + +=item B<-c|--commit>=C + +Passed to rebuild_elastic_search.pl. Specify how many records will be batched up before batch committed to Elasticsearch. + +=item B<--startbiblionumber>=C + +Specify starting biblionumber, only biblios with equal or higher biblionumber will be indexed. + +=item B<-d|--delete> + +Delete the index and recreate it before indexing. + +=back + +=cut + +use Modern::Perl; +use Getopt::Long; +use Pod::Usage; +use Koha::SearchEngine::Elasticsearch::Indexer; +use C4::Context; + + +#TODO: Is the the correct way of getting koha root? +my $koha_root = C4::Context->config('intranetdir'); + +my ( + $help, + $man, + $verbose, + $jobs, + $batch_size, + $commit_size, + $start_biblionumber, + $delete +); + +# Default option values +$batch_size = 30000; +GetOptions( + 'h|help' => \$help, + 'man' => \$man, + 'v|verbose' => \$verbose, + 'j|jobs=i' => \$jobs, + 'b|batchsize=i' => \$batch_size, + 'c|commit=i' => \$commit_size, + 'startbiblionumber=i' => \$start_biblionumber, + 'd|delete' => \$delete +); + +pod2usage(1) if $help; +pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; + +if ($delete) { + my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ + index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX + }); + $indexer->drop_index(); + $indexer->create_index(); +} + +# record_batches.pl options +my @record_batches_opts; +if ($start_biblionumber) { + push @record_batches_opts, "--startbiblionumber=$start_biblionumber"; +} +push @record_batches_opts, "--batchsize=$batch_size"; +my $record_batches_opts = join(' ', @record_batches_opts); + +# rebuild_elastic_search.pl options +my @rebuild_elastic_opts = ('-b'); +if ($verbose) { + push @rebuild_elastic_opts, "-v"; +} +if ($commit_size) { + push @rebuild_elastic_opts, "--commit=$commit_size"; +} +my $rebuild_elastic_opts = join(' ', @rebuild_elastic_opts); + +# parallel options +my @parallel_opts; +if ($jobs) { + push @parallel_opts, "-j$jobs"; +} +my $parallel_opts = join(' ', @parallel_opts); + +my $command = join('', ( + "$koha_root/misc/record_batches.pl $record_batches_opts | ", + "parallel $parallel_opts --colsep ' ' ", + "$koha_root/misc/search_tools/rebuild_elastic_search.pl $rebuild_elastic_opts --startbiblionumber={1} --endbiblionumber={2}" +)); + +exec $command; diff --git a/misc/search_tools/rebuild_elastic_search.pl b/misc/search_tools/rebuild_elastic_search.pl index 1b75749954..46a1fa58df 100755 --- a/misc/search_tools/rebuild_elastic_search.pl +++ b/misc/search_tools/rebuild_elastic_search.pl @@ -64,12 +64,12 @@ Only index the supplied biblionumber, mostly for testing purposes. May be repeated. This also applies to authorities via authid, so if you're using it, you probably only want to do one or the other at a time. -=item B<-sbn|--start-bnumber> +=item B<-sbn|--startbiblionumber> Only index biblios with biblionumber greater or equal than supplied biblionumber. -=item B<-ebn|--end-bnumber> +=item B<-ebn|--endbiblionumber> Only index biblios with biblionumber less or equal to supplied biblionumber. @@ -122,8 +122,8 @@ GetOptions( 'a|authorities' => \$index_authorities, 'b|biblios' => \$index_biblios, 'bn|bnumber=i' => \@biblionumbers, - 'sbn|start-bnumber=i' => \$start_biblionumber, - 'ebn|end-bnumber=i' => \$end_biblionumber, + 'sbn|startbiblionumber=i' => \$start_biblionumber, + 'ebn|endbiblionumber=i' => \$end_biblionumber, 'v|verbose+' => \$verbose, 'h|help' => \$help, 'man' => \$man, -- 2.11.0