Bugzilla – Attachment 82586 Details for
Bug 21872
Elasticsearch indexing faster by making it multi-threaded
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 21872: Add parallel_rebuild_biblios.pl script
Bug-21872-Add-parallelrebuildbibliospl-script.patch (text/plain), 7.50 KB, created by
David Gustafsson
on 2018-11-22 21:37:00 UTC
(
hide
)
Description:
Bug 21872: Add parallel_rebuild_biblios.pl script
Filename:
MIME Type:
Creator:
David Gustafsson
Created:
2018-11-22 21:37:00 UTC
Size:
7.50 KB
patch
obsolete
>From ecdfe980b599ab7a279096f63cd0d1f3f5493cba Mon Sep 17 00:00:00 2001 >From: David Gustafsson <david.gustafsson@ub.gu.se> >Date: Thu, 22 Nov 2018 22:33:37 +0100 >Subject: [PATCH] Bug 21872: Add parallel_rebuild_biblios.pl script > >Add parallel_rebuild_biblios.pl script for rebuild biblios index in >parallel. Adjust some option names and script pods. >--- > misc/record_batches.pl | 60 +++++++----- > misc/search_tools/parallel_rebuild_biblios.pl | 132 ++++++++++++++++++++++++++ > misc/search_tools/rebuild_elastic_search.pl | 8 +- > 3 files changed, 171 insertions(+), 29 deletions(-) > create mode 100755 misc/search_tools/parallel_rebuild_biblios.pl > >diff --git a/misc/record_batches.pl b/misc/record_batches.pl >index c157e65a77..d85136f0b1 100755 >--- a/misc/record_batches.pl >+++ b/misc/record_batches.pl >@@ -1,5 +1,31 @@ > #!/usr/bin/perl > >+=head1 NAME >+ >+record_batches.pl - Outputs biblio record batches as ranges of biblionumbers. >+ >+=head1 SYNOPSIS >+ >+record_batches.pl [-h|--help] [--startbiblionumber=BIBLIONUMBER] >+ >+=head1 OPTIONS >+ >+=over >+ >+=item B<-h|--help> >+ >+Print a brief help message. >+ >+=item B<--startbiblionumber> >+ >+ --startbiblionumber=BIBLIONUMBER Output batches with biblionumber >= BIBLIONUMBER >+ >+=item B<--batchsize> >+ >+=back >+ >+=cut >+ > use Modern::Perl; > use C4::Context; > use Getopt::Long; >@@ -25,6 +51,13 @@ if (!$start_biblionumber) { > $start_biblionumber = $min_biblionumber; > } > >+# TODO: Could be options >+my $range_separator = " "; >+my $batch_separator = "\n"; >+sub print_range { >+ print join($range_separator, @_) . $batch_separator; >+} >+ > while (1) { > my $offset = $batch_size - 1; > my $query = >@@ -35,15 +68,14 @@ while (1) { > my ($max_biblionumber) = @{ $dbh->selectcol_arrayref(qq{SELECT MAX(`biblionumber`) FROM `biblio`}) }; > if($max_biblionumber) { > if($max_biblionumber >= $start_biblionumber) { >- # @TODO: Option for argument and range separator? >- print join(' ', ($start_biblionumber, $max_biblionumber)) . "\n"; >+ print_range($start_biblionumber, $max_biblionumber); > } > } > } > last; > } > else { >- print join(' ', ($start_biblionumber, $end_biblionumber)) . "\n"; >+ print_range($start_biblionumber, $end_biblionumber); > } > if ($next_biblionumber) { > $start_biblionumber = $next_biblionumber; >@@ -52,25 +84,3 @@ while (1) { > last; > } > } >- >-=head1 NAME >- >-record batches - This script outputs ranges of biblionumbers >- >-=head1 SYNOPSIS >- >-record_batches.pl [-h|--help] [--startbiblionumber=BIBLIONUMBER] >- >-=head1 OPTIONS >- >-=over >- >-=item B<-h|--help> >- >-Print a brief help message. >- >-=item B<--startbiblionumber> >- >- --startbiblionumber=BIBLIONUMBER Output batches with biblionumber >= BIBLIONUMBER >- >-=item B<--batchsize> >diff --git a/misc/search_tools/parallel_rebuild_biblios.pl b/misc/search_tools/parallel_rebuild_biblios.pl >new file mode 100755 >index 0000000000..3b2c82930a >--- /dev/null >+++ b/misc/search_tools/parallel_rebuild_biblios.pl >@@ -0,0 +1,132 @@ >+#!/usr/bin/perl >+ >+=head1 NAME >+ >+parallel_rebuild_bibios.pl - inserts biblios records from a Koha database into Elasticsearch in parallel. >+ >+=head1 SYNOPSIS >+ >+B<parallel_rebuild_biblios.pl> >+[B<-h|--help>] >+[B<-v|--verbose>] >+[B<-j|--jobs>=C<jobs>] >+[B<-b|--batchsize>=C<size>] >+[B<-c|--commit>=C<size>] >+[B<--startbiblionumber>=C<biblionumber>] >+[B<-d|--delete>] >+ >+=head1 OPTIONS >+ >+=over >+ >+=item B<-h|--help> >+ >+Print a brief help message. >+ >+=item B<-v|--verbose> >+ >+Passed to rebuild_elastic_search.pl. Increase verbosity. >+ >+=item B<-j|--jobs>=C<N> >+ >+Run up to N jobs in parallel. 0 means as many as possible. Default is to run >+one job per CPU core. This is passed directly to parrallel as B<-j>C<N>. Se C<man parrallel> >+for more information about variants of C<N>. >+ >+=item B<-b|--batchsize>=C<size> >+ >+Specify batch size for batches of biblios passed to rebuild_elastic_search.pl. >+ >+=item B<-c|--commit>=C<size> >+ >+Passed to rebuild_elastic_search.pl. Specify how many records will be batched up before batch committed to Elasticsearch. >+ >+=item B<--startbiblionumber>=C<biblionumber> >+ >+Specify starting biblionumber, only biblios with equal or higher biblionumber will be indexed. >+ >+=item B<-d|--delete> >+ >+Delete the index and recreate it before indexing. >+ >+=back >+ >+=cut >+ >+use Modern::Perl; >+use Getopt::Long; >+use Pod::Usage; >+use Koha::SearchEngine::Elasticsearch::Indexer; >+use C4::Context; >+ >+ >+#TODO: Is the the correct way of getting koha root? >+my $koha_root = C4::Context->config('intranetdir'); >+ >+my ( >+ $help, >+ $man, >+ $verbose, >+ $jobs, >+ $batch_size, >+ $commit_size, >+ $start_biblionumber, >+ $delete >+); >+ >+# Default option values >+$batch_size = 30000; >+GetOptions( >+ 'h|help' => \$help, >+ 'man' => \$man, >+ 'v|verbose' => \$verbose, >+ 'j|jobs=i' => \$jobs, >+ 'b|batchsize=i' => \$batch_size, >+ 'c|commit=i' => \$commit_size, >+ 'startbiblionumber=i' => \$start_biblionumber, >+ 'd|delete' => \$delete >+); >+ >+pod2usage(1) if $help; >+pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; >+ >+if ($delete) { >+ my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ >+ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX >+ }); >+ $indexer->drop_index(); >+ $indexer->create_index(); >+} >+ >+# record_batches.pl options >+my @record_batches_opts; >+if ($start_biblionumber) { >+ push @record_batches_opts, "--startbiblionumber=$start_biblionumber"; >+} >+push @record_batches_opts, "--batchsize=$batch_size"; >+my $record_batches_opts = join(' ', @record_batches_opts); >+ >+# rebuild_elastic_search.pl options >+my @rebuild_elastic_opts = ('-b'); >+if ($verbose) { >+ push @rebuild_elastic_opts, "-v"; >+} >+if ($commit_size) { >+ push @rebuild_elastic_opts, "--commit=$commit_size"; >+} >+my $rebuild_elastic_opts = join(' ', @rebuild_elastic_opts); >+ >+# parallel options >+my @parallel_opts; >+if ($jobs) { >+ push @parallel_opts, "-j$jobs"; >+} >+my $parallel_opts = join(' ', @parallel_opts); >+ >+my $command = join('', ( >+ "$koha_root/misc/record_batches.pl $record_batches_opts | ", >+ "parallel $parallel_opts --colsep ' ' ", >+ "$koha_root/misc/search_tools/rebuild_elastic_search.pl $rebuild_elastic_opts --startbiblionumber={1} --endbiblionumber={2}" >+)); >+ >+exec $command; >diff --git a/misc/search_tools/rebuild_elastic_search.pl b/misc/search_tools/rebuild_elastic_search.pl >index 1b75749954..46a1fa58df 100755 >--- a/misc/search_tools/rebuild_elastic_search.pl >+++ b/misc/search_tools/rebuild_elastic_search.pl >@@ -64,12 +64,12 @@ Only index the supplied biblionumber, mostly for testing purposes. May be > repeated. This also applies to authorities via authid, so if you're using it, > you probably only want to do one or the other at a time. > >-=item B<-sbn|--start-bnumber> >+=item B<-sbn|--startbiblionumber> > > Only index biblios with biblionumber greater or equal than supplied > biblionumber. > >-=item B<-ebn|--end-bnumber> >+=item B<-ebn|--endbiblionumber> > > Only index biblios with biblionumber less or equal to supplied biblionumber. > >@@ -122,8 +122,8 @@ GetOptions( > 'a|authorities' => \$index_authorities, > 'b|biblios' => \$index_biblios, > 'bn|bnumber=i' => \@biblionumbers, >- 'sbn|start-bnumber=i' => \$start_biblionumber, >- 'ebn|end-bnumber=i' => \$end_biblionumber, >+ 'sbn|startbiblionumber=i' => \$start_biblionumber, >+ 'ebn|endbiblionumber=i' => \$end_biblionumber, > 'v|verbose+' => \$verbose, > 'h|help' => \$help, > 'man' => \$man, >-- >2.11.0
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 21872
:
82582
|
82583
|
82584
|
82585
|
82586
|
82598
|
82599
|
82780
|
85119
|
85121
|
85975
|
85976
|
85977
|
85978
|
85979
|
86053
|
86054
|
86055
|
86056
|
86057
|
89109
|
89110
|
89111
|
89112
|
89113