From 018323269cb501603d068cbac96bf7e7c780ed0d Mon Sep 17 00:00:00 2001
From: Axel Amghar <axel.amghar@biblibre.com>
Date: Mon, 6 May 2019 17:25:23 +0200
Subject: [PATCH] Bug 20384 - Elastic rebuild script improvements - options
 --offset, --length & --file

This patch add the options --offset, --length & --file to the script:
misc/search_tools/rebuild_elastic_search.pl

To test :
- apply the patch
- verify that the script by default is working, launch :
	perl misc/search_tools/rebuild_elastic_search.pl -v -d -c 100

- launch with the option --offset
- then with the option --length
- launch with both options :
	perl misc/search_tools/rebuild_elastic_search.pl -v -d -c 100 --offset 1000 --length 200
- verify that the total records indexed is 200

launch perl misc/search_tools/rebuild_elastic_search.pl --man
for help

Option --file :

To test:
- after testing offset and length, apply this patch
- launch perl misc/search_tools/rebuild_elastic_search.pl -v -d -c 100 --file your_file.csv
your file need to contain a column of biblionumber from biblio or authid from auth_header (only one column)
You can go in Koha -> reports -> Create from sql
SELECT biblionumber FROM biblio ;
and then download in csv to have your csv file.
- You can test with others query like :
select biblionumber from biblio WHERE author LIKE 'a%';
Total number of results (411);
- verify that you have the rigth number of records indexed
---
 Koha/BiblioUtils.pm                        |  19 ++--
 Koha/MetadataRecord/Authority.pm           |  21 ++--
 misc/search_tools/rebuild_elasticsearch.pl | 161 ++++++++++++++++++++++-------
 3 files changed, 151 insertions(+), 50 deletions(-)

diff --git a/Koha/BiblioUtils.pm b/Koha/BiblioUtils.pm
index 02c10f58aa..b361f4b813 100644
--- a/Koha/BiblioUtils.pm
+++ b/Koha/BiblioUtils.pm
@@ -101,7 +101,7 @@ sub get_from_biblionumber {
 
 =head2 get_all_biblios_iterator
 
-    my $it = Koha::BiblioUtils->get_all_biblios_iterator(%options);
+    my $it = Koha::BiblioUtils->get_all_biblios_iterator(\%options, $offset, $length);
 
 This will provide an iterator object that will, one by one, provide the
 Koha::BiblioUtils of each biblio. This will include the item data.
@@ -124,13 +124,14 @@ records instead of all.
 =cut
 
 sub get_all_biblios_iterator {
-    my ($self, %options) = @_;
+    my ($self, $options, $offset, $length) = @_;
 
     my $search_terms = {};
     my ($slice_modulo, $slice_count);
-    if ($options{slice}) {
-        $slice_count = $options{slice}->{count};
-        $slice_modulo = $options{slice}->{index};
+    if ($options->{slice}) {
+        $slice_count = $options->{slice}->{count};
+        $slice_modulo = $options->{slice}->{index};
+
         $search_terms = \[ 'mod(biblionumber, ?) = ?', $slice_count, $slice_modulo ];
     }
 
@@ -138,8 +139,12 @@ sub get_all_biblios_iterator {
     my $schema   = $database->schema();
     my $rs =
       $schema->resultset('Biblio')->search(
-        $search_terms,
-        { columns => [qw/ biblionumber /] } );
+        $search_terms, {
+        columns => [qw/ biblionumber /] ,
+        offset => $offset ,
+        rows => $length
+      });
+
     my $next_func = sub {
         # Warn and skip bad records, otherwise we break the loop
         while (1) {
diff --git a/Koha/MetadataRecord/Authority.pm b/Koha/MetadataRecord/Authority.pm
index c3e1cf8edd..eb75724af4 100644
--- a/Koha/MetadataRecord/Authority.pm
+++ b/Koha/MetadataRecord/Authority.pm
@@ -151,7 +151,7 @@ sub authorized_heading {
 
 =head2 get_all_authorities_iterator
 
-    my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%options);
+    my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(\%options, $offset, $length);
 
 This will provide an iterator object that will, one by one, provide the
 Koha::MetadataRecord::Authority of each authority.
@@ -174,15 +174,15 @@ records instead of all.
 =cut
 
 sub get_all_authorities_iterator {
-    my ($self, %options) = @_;
+    my ($self, $options, $offset, $length) = @_;
 
     my $search_terms = {
         marcxml => { '!=', undef }
     };
     my ($slice_modulo, $slice_count);
-    if ($options{slice}) {
-        $slice_count = $options{slice}->{count};
-        $slice_modulo = $options{slice}->{index};
+    if ($options->{slice}) {
+        $slice_count = $options->{slice}->{count};
+        $slice_modulo = $options->{slice}->{index};
         $search_terms = {
             '-and' => [
                 %{$search_terms},
@@ -193,10 +193,15 @@ sub get_all_authorities_iterator {
 
     my $database = Koha::Database->new();
     my $schema   = $database->schema();
+
     my $rs =
-      $schema->resultset('AuthHeader')->search(
-        $search_terms,
-        { columns => [qw/ authid authtypecode marcxml /] } );
+        $schema->resultset('AuthHeader')->search(
+            $search_terms,
+            { columns => [qw/ authid authtypecode marcxml /],
+            offset => $offset,
+            rows => $length }
+        );
+
     my $next_func = sub {
         my $row = $rs->next();
         return if !$row;
diff --git a/misc/search_tools/rebuild_elasticsearch.pl b/misc/search_tools/rebuild_elasticsearch.pl
index 6cff2eae2e..58ce4dce8d 100755
--- a/misc/search_tools/rebuild_elasticsearch.pl
+++ b/misc/search_tools/rebuild_elasticsearch.pl
@@ -35,6 +35,13 @@ B<rebuild_elasticsearch.pl>
 [B<-ai|--authid>]
 [B<-p|--processes>]
 [B<-v|--verbose>]
+[B<-d|--delete>]
+[B<-a|--authorities>]
+[B<-b|--biblios>]
+[B<-f|--file>]
+[B<-bn|--bnnumber>]
+[B<--length>]
+[B<--offset>]
 [B<-h|--help>]
 [B<--man>]
 
@@ -70,21 +77,37 @@ specifying neither and so both get indexed.
 Index the biblios only. Combining this with B<-a> is the same as
 specifying neither and so both get indexed.
 
+=item B<-f|--file>
+
+Read a csv file created with a sql query, the csv must contain a column of biblionumber from biblios
+or a column of authid from auth_header, then index biblios or authorities, but not both.
+With this option you can't have options B<--length>, B<--offset>, and B<-bn>.
+
 =item B<-bn|--bnumber>
 
 Only index the supplied biblionumber, mostly for testing purposes. May be
-repeated.
+repeated. Cannot be combined with options B<--offset> and B<--length>.
 
 =item B<-ai|--authid>
 
 Only index the supplied authority id, mostly for testing purposes. May be
-repeated.
+repeated. Cannot be combined with options B<--offset> and B<--length>.
 
 =item B<-p|--processes>
 
 Number of processes to use for indexing. This can be used to do more indexing
 work in parallel on multicore systems. By default, a single process is used.
 
+=item B<--offset>
+
+Specify the row number for the first row to be indexed.
+With this option you have to choose between biblios B<-b> or authorities B<-a>.
+
+=item B<--length>
+
+Specify the number of rows to be indexed.
+With this option you have to choose between biblios B<-b> or authorities B<-a>.
+
 =item B<-v|--verbose>
 
 By default, this program only emits warnings and errors. This makes it talk
@@ -116,12 +139,14 @@ use MARC::Field;
 use MARC::Record;
 use Modern::Perl;
 use Pod::Usage;
+use Text::CSV;
 
 my $verbose = 0;
 my $commit = 5000;
 my ($delete, $reset, $help, $man, $processes);
 my ($index_biblios, $index_authorities);
 my (@biblionumbers,@authids);
+my ($offset, $length, $file);
 
 $|=1; # flushes output
 
@@ -135,10 +160,17 @@ GetOptions(
     'ai|authid=i'  => \@authids,
     'p|processes=i' => \$processes,
     'v|verbose+'    => \$verbose,
+    'offset=i'      => \$offset,
+    'length=i'      => \$length,
+    'f|file=s'      => \$file,
     'h|help'        => \$help,
     'man'           => \$man,
+
 );
 
+pod2usage(1) if $help;
+pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
+
 # Default is to do both
 unless ($index_authorities || $index_biblios) {
     $index_authorities = $index_biblios = 1;
@@ -148,18 +180,30 @@ if ($processes && ( @biblionumbers || @authids) ) {
     die "Argument p|processes cannot be combined with bn|bnumber or ai|authid";
 }
 
-pod2usage(1) if $help;
-pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
+if ($file){
+    if ($offset || $length || @biblionumbers || @authids){
+        die "With options --file, you can't have options --length, --offset, -bn, and -ai" . "\n";
+    }
+}
 
-_sanity_check();
+if (@biblionumbers || @authids){
+    if ($length || $offset){
+        die "With option -bn or -ai, you can't have options --length, --offset" . "\n";
+    }
+}
 
 if ($reset){
     Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;
     $delete = 1;
 }
 
-_verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete) if ($index_biblios);
-_verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete) if ($index_authorities);
+if ($offset || $length){
+    if ( $index_biblios && $index_authorities){
+        die "With options --offset or --length, you have to choose between biblios (-b) or authorities (-a)." . "\n";
+    }
+}
+
+_sanity_check();
 
 my $slice_index = 0;
 my $slice_count = ( $processes //= 1 );
@@ -184,38 +228,85 @@ if ($slice_count > 1) {
 }
 
 my $next;
-if ($index_biblios) {
-    _log(1, "Indexing biblios\n");
-    if (@biblionumbers) {
-        $next = sub {
-            my $r = shift @biblionumbers;
-            return () unless defined $r;
-            return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
-        };
-    } else {
-        my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options);
-        $next = sub {
-            $records->next();
+if ($file){
+    my @bibnums;
+    my $i = 0;
+    my $input = $file;
+    my $parser = Text::CSV->new();
+    open ( my $csv, "<", $input );
+
+    while (<$csv>) {
+        if ($parser->parse($_)){
+            my @columns = $parser->fields();
+            @bibnums[$i] = @columns;
+        $i++;
         }
     }
-    _do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
-}
-if ($index_authorities) {
-    _log(1, "Indexing authorities\n");
-    if (@authids) {
-        $next = sub {
-            my $r = shift @authids;
-            return () unless defined $r;
-            my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
-            return ($r, $a);
-        };
-    } else {
-        my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options);
-        $next = sub {
-            $records->next();
+    close $csv;
+
+    if (@bibnums){
+        my $type = shift @bibnums;
+        if ($type eq "biblionumber"){
+            _verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete);
+            _log(1, "Indexing biblios\n");
+            $next = sub {
+                my $bibnum = shift @bibnums;
+                return () unless defined $bibnum;
+                return ($bibnum, Koha::BiblioUtils->get_from_biblionumber($bibnum, item_data => 1 ));
+            };
+            _do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
+
+        } elsif ($type eq "authid"){
+            _verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete);
+            _log(1, "Indexing authorities\n");
+            $next = sub {
+                my $authid = shift @bibnums;
+                return () unless defined $authid;
+                return ($authid,Koha::MetadataRecord::Authority->get_from_authid($authid));
+            };
+            _do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
+        }
+    }
+
+} else {
+    if ($index_biblios) {
+        _verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete);
+        _log(1, "Indexing biblios\n");
+
+        if (@biblionumbers) {
+            $next = sub {
+                my $r = shift @biblionumbers;
+                return () unless defined $r;
+                return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
+            };
+        }  else {
+            my $records = Koha::BiblioUtils->get_all_biblios_iterator(\%iterator_options,$offset, $length);
+            $next = sub {
+                $records->next();
+            }
+        }
+        _do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
+    }
+    if ($index_authorities) {
+        _verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete);
+        _log(1, "Indexing authorities\n");
+
+        if (@authids) {
+            $next = sub {
+                my $r = shift @authids;
+                return () unless defined $r;
+                my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
+                return ($r, $a);
+            };
+
+        } else {
+            my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(\%iterator_options,$offset, $length);
+            $next = sub {
+                $records->next();
+            }
         }
+        _do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
     }
-    _do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
 }
 
 if ($slice_index == 0) {
-- 
2.11.0