@@ -, +, @@ --- misc/record_batches.pl | 60 +++++++----- misc/search_tools/parallel_rebuild_biblios.pl | 132 ++++++++++++++++++++++++++ misc/search_tools/rebuild_elastic_search.pl | 8 +- 3 files changed, 171 insertions(+), 29 deletions(-) create mode 100755 misc/search_tools/parallel_rebuild_biblios.pl --- a/misc/record_batches.pl +++ a/misc/record_batches.pl @@ -1,5 +1,31 @@ #!/usr/bin/perl +=head1 NAME + +record_batches.pl - Outputs biblio record batches as ranges of biblionumbers. + +=head1 SYNOPSIS + +record_batches.pl [-h|--help] [--startbiblionumber=BIBLIONUMBER] + +=head1 OPTIONS + +=over + +=item B<-h|--help> + +Print a brief help message. + +=item B<--startbiblionumber> + + --startbiblionumber=BIBLIONUMBER Output batches with biblionumber >= BIBLIONUMBER + +=item B<--batchsize> + +=back + +=cut + use Modern::Perl; use C4::Context; use Getopt::Long; @@ -25,6 +51,13 @@ if (!$start_biblionumber) { $start_biblionumber = $min_biblionumber; } +# TODO: Could be options +my $range_separator = " "; +my $batch_separator = "\n"; +sub print_range { + print join($range_separator, @_) . $batch_separator; +} + while (1) { my $offset = $batch_size - 1; my $query = @@ -35,15 +68,14 @@ while (1) { my ($max_biblionumber) = @{ $dbh->selectcol_arrayref(qq{SELECT MAX(`biblionumber`) FROM `biblio`}) }; if($max_biblionumber) { if($max_biblionumber >= $start_biblionumber) { - # @TODO: Option for argument and range separator? - print join(' ', ($start_biblionumber, $max_biblionumber)) . "\n"; + print_range($start_biblionumber, $max_biblionumber); } } } last; } else { - print join(' ', ($start_biblionumber, $end_biblionumber)) . "\n"; + print_range($start_biblionumber, $end_biblionumber); } if ($next_biblionumber) { $start_biblionumber = $next_biblionumber; @@ -52,25 +84,3 @@ while (1) { last; } } - -=head1 NAME - -record batches - This script outputs ranges of biblionumbers - -=head1 SYNOPSIS - -record_batches.pl [-h|--help] [--startbiblionumber=BIBLIONUMBER] - -=head1 OPTIONS - -=over - -=item B<-h|--help> - -Print a brief help message. - -=item B<--startbiblionumber> - - --startbiblionumber=BIBLIONUMBER Output batches with biblionumber >= BIBLIONUMBER - -=item B<--batchsize> --- a/misc/search_tools/parallel_rebuild_biblios.pl +++ a/misc/search_tools/parallel_rebuild_biblios.pl @@ -0,0 +1,132 @@ +#!/usr/bin/perl + +=head1 NAME + +parallel_rebuild_bibios.pl - inserts biblios records from a Koha database into Elasticsearch in parallel. + +=head1 SYNOPSIS + +B +[B<-h|--help>] +[B<-v|--verbose>] +[B<-j|--jobs>=C] +[B<-b|--batchsize>=C] +[B<-c|--commit>=C] +[B<--startbiblionumber>=C] +[B<-d|--delete>] + +=head1 OPTIONS + +=over + +=item B<-h|--help> + +Print a brief help message. + +=item B<-v|--verbose> + +Passed to rebuild_elastic_search.pl. Increase verbosity. + +=item B<-j|--jobs>=C + +Run up to N jobs in parallel. 0 means as many as possible. Default is to run +one job per CPU core. This is passed directly to parrallel as B<-j>C. Se C +for more information about variants of C. + +=item B<-b|--batchsize>=C + +Specify batch size for batches of biblios passed to rebuild_elastic_search.pl. + +=item B<-c|--commit>=C + +Passed to rebuild_elastic_search.pl. Specify how many records will be batched up before batch committed to Elasticsearch. + +=item B<--startbiblionumber>=C + +Specify starting biblionumber, only biblios with equal or higher biblionumber will be indexed. + +=item B<-d|--delete> + +Delete the index and recreate it before indexing. + +=back + +=cut + +use Modern::Perl; +use Getopt::Long; +use Pod::Usage; +use Koha::SearchEngine::Elasticsearch::Indexer; +use C4::Context; + + +#TODO: Is the the correct way of getting koha root? +my $koha_root = C4::Context->config('intranetdir'); + +my ( + $help, + $man, + $verbose, + $jobs, + $batch_size, + $commit_size, + $start_biblionumber, + $delete +); + +# Default option values +$batch_size = 30000; +GetOptions( + 'h|help' => \$help, + 'man' => \$man, + 'v|verbose' => \$verbose, + 'j|jobs=i' => \$jobs, + 'b|batchsize=i' => \$batch_size, + 'c|commit=i' => \$commit_size, + 'startbiblionumber=i' => \$start_biblionumber, + 'd|delete' => \$delete +); + +pod2usage(1) if $help; +pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; + +if ($delete) { + my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ + index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX + }); + $indexer->drop_index(); + $indexer->create_index(); +} + +# record_batches.pl options +my @record_batches_opts; +if ($start_biblionumber) { + push @record_batches_opts, "--startbiblionumber=$start_biblionumber"; +} +push @record_batches_opts, "--batchsize=$batch_size"; +my $record_batches_opts = join(' ', @record_batches_opts); + +# rebuild_elastic_search.pl options +my @rebuild_elastic_opts = ('-b'); +if ($verbose) { + push @rebuild_elastic_opts, "-v"; +} +if ($commit_size) { + push @rebuild_elastic_opts, "--commit=$commit_size"; +} +my $rebuild_elastic_opts = join(' ', @rebuild_elastic_opts); + +# parallel options +my @parallel_opts; +if ($jobs) { + push @parallel_opts, "-j$jobs"; +} +my $parallel_opts = join(' ', @parallel_opts); + +my $command = join('', ( + "$koha_root/misc/record_batches.pl $record_batches_opts | ", + "parallel $parallel_opts --colsep ' ' ", + "$koha_root/misc/search_tools/rebuild_elastic_search.pl $rebuild_elastic_opts --startbiblionumber={1} --endbiblionumber={2}" +)); + +exec $command; --- a/misc/search_tools/rebuild_elastic_search.pl +++ a/misc/search_tools/rebuild_elastic_search.pl @@ -64,12 +64,12 @@ Only index the supplied biblionumber, mostly for testing purposes. May be repeated. This also applies to authorities via authid, so if you're using it, you probably only want to do one or the other at a time. -=item B<-sbn|--start-bnumber> +=item B<-sbn|--startbiblionumber> Only index biblios with biblionumber greater or equal than supplied biblionumber. -=item B<-ebn|--end-bnumber> +=item B<-ebn|--endbiblionumber> Only index biblios with biblionumber less or equal to supplied biblionumber. @@ -122,8 +122,8 @@ GetOptions( 'a|authorities' => \$index_authorities, 'b|biblios' => \$index_biblios, 'bn|bnumber=i' => \@biblionumbers, - 'sbn|start-bnumber=i' => \$start_biblionumber, - 'ebn|end-bnumber=i' => \$end_biblionumber, + 'sbn|startbiblionumber=i' => \$start_biblionumber, + 'ebn|endbiblionumber=i' => \$end_biblionumber, 'v|verbose+' => \$verbose, 'h|help' => \$help, 'man' => \$man, --