From 792a533938eeed2451b521553cd3475b87225ce9 Mon Sep 17 00:00:00 2001 From: Axel Amghar Date: Mon, 6 May 2019 17:25:23 +0200 Subject: [PATCH] Bug 20384 - Elastic rebuild script improvements - options --offset & --length This patch add the options --offset & --length to the script: misc/search_tools/rebuild_elastic_search.pl To test : - apply the patch - verify that the script by default is working, launch : perl misc/search_tools/rebuild_elastic_search.pl -v -d -c 100 - launch with the option --offset - then with the option --length - launch with both options : perl misc/search_tools/rebuild_elastic_search.pl -v -d -c 100 --offset 1000 --length 200 - verify that the total records indexed is 200 launch perl misc/search_tools/rebuild_elastic_search.pl --man for help --- Koha/BiblioUtils.pm | 9 +++++-- Koha/MetadataRecord/Authority.pm | 8 ++++-- misc/search_tools/rebuild_elastic_search.pl | 38 ++++++++++++++++++++++++----- 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/Koha/BiblioUtils.pm b/Koha/BiblioUtils.pm index 73fc12a..ee276a9 100644 --- a/Koha/BiblioUtils.pm +++ b/Koha/BiblioUtils.pm @@ -111,11 +111,16 @@ The iterator is a Koha::MetadataIterator object. =cut sub get_all_biblios_iterator { + my ($class, $offset, $length) = @_; my $database = Koha::Database->new(); my $schema = $database->schema(); + my $rs = - $schema->resultset('Biblio')->search( {}, - { columns => [qw/ biblionumber /] } ); + $schema->resultset('Biblio')->search( {},{ + columns => [qw/ biblionumber / ] , + offset => $offset , + rows => $length }); + my $next_func = sub { # Warn and skip bad records, otherwise we break the loop while (1) { diff --git a/Koha/MetadataRecord/Authority.pm b/Koha/MetadataRecord/Authority.pm index 062d19c..182a915 100644 --- a/Koha/MetadataRecord/Authority.pm +++ b/Koha/MetadataRecord/Authority.pm @@ -161,11 +161,15 @@ The iterator is a Koha::MetadataIterator object. =cut sub get_all_authorities_iterator { + my ($class, $offset, $length) = @_; my $database = Koha::Database->new(); my $schema = $database->schema(); my $rs = - $schema->resultset('AuthHeader')->search( { marcxml => { '!=', undef } }, - { columns => [qw/ authid authtypecode marcxml /] } ); + $schema->resultset('AuthHeader')->search( { marcxml => { '!=', undef } },{ + columns => [qw/ authid authtypecode marcxml /], + offset => $offset, + rows => $length }); + my $next_func = sub { my $row = $rs->next(); return if !$row; diff --git a/misc/search_tools/rebuild_elastic_search.pl b/misc/search_tools/rebuild_elastic_search.pl index a233cf5..b879654 100755 --- a/misc/search_tools/rebuild_elastic_search.pl +++ b/misc/search_tools/rebuild_elastic_search.pl @@ -28,6 +28,12 @@ rebuild_elastic_search.pl - inserts records from a Koha database into Elasticsea B [B<-c|--commit>=C] [B<-v|--verbose>] +[B<-d|--delete>] +[B<-a|--authorities>] +[B<-b|--biblios>] +[B<-bn|--bnnumber>] +[B<--length>] +[B<--offset>] [B<-h|--help>] [B<--man>] @@ -64,6 +70,16 @@ Only index the supplied biblionumber, mostly for testing purposes. May be repeated. This also applies to authorities via authid, so if you're using it, you probably only want to do one or the other at a time. +=item B<--offset> + +Specify the row number for the first row to be indexed. +With this option you have to choose between biblios (-b) or authorities (-a). + +=item B<--length> + +Specify the number of rows to be indexed. +With this option you have to choose between biblios (-b) or authorities (-a). + =item B<-v|--verbose> By default, this program only emits warnings and errors. This makes it talk @@ -98,16 +114,19 @@ my $commit = 5000; my ($delete, $help, $man); my ($index_biblios, $index_authorities); my (@biblionumbers); +my ($offset, $length); $|=1; # flushes output GetOptions( 'c|commit=i' => \$commit, 'd|delete' => \$delete, - 'a|authorities' => \$index_authorities, - 'b|biblios' => \$index_biblios, - 'bn|bnumber=i' => \@biblionumbers, + 'a|authorities' => \$index_authorities, + 'b|biblios' => \$index_biblios, + 'bn|bnumber=i' => \@biblionumbers, 'v|verbose+' => \$verbose, + 'offset=i' => \$offset, + 'length=i' => \$length, 'h|help' => \$help, 'man' => \$man, ); @@ -120,6 +139,13 @@ unless ($index_authorities || $index_biblios) { pod2usage(1) if $help; pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; +if ($offset || $length){ + if ( $index_biblios && $index_authorities){ + printf "With options --offset or --length, you have to choose between biblios (-b) or authorities (-a)." . "\n"; + pod2usage(1); + } +} + sanity_check(); my $next; @@ -131,8 +157,8 @@ if ($index_biblios) { return () unless defined $r; return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); }; - } else { - my $records = Koha::BiblioUtils->get_all_biblios_iterator(); + } else { + my $records = Koha::BiblioUtils->get_all_biblios_iterator($offset, $length); $next = sub { $records->next(); } @@ -149,7 +175,7 @@ if ($index_authorities) { return ($r, $a->record); }; } else { - my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(); + my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator($offset, $length); $next = sub { $records->next(); } -- 2.7.4