From 7023db4782040d39f6a746c7f2571130884efec6 Mon Sep 17 00:00:00 2001 From: Axel Amghar Date: Mon, 6 May 2019 17:25:23 +0200 Subject: [PATCH] Bug 20384 - Elastic rebuild script improvements - options --offset, --length & --file This patch add the options --offset, --length & --file to the script: misc/search_tools/rebuild_elastic_search.pl To test : - apply the patch - verify that the script by default is working, launch : perl misc/search_tools/rebuild_elastic_search.pl -v -d -c 100 - launch with the option --offset - then with the option --length - launch with both options : perl misc/search_tools/rebuild_elastic_search.pl -v -d -c 100 --offset 1000 --length 200 - verify that the total records indexed is 200 launch perl misc/search_tools/rebuild_elastic_search.pl --man for help Option --file : To test: - after testing offset and length, apply this patch - launch perl misc/search_tools/rebuild_elastic_search.pl -v -d -c 100 --file your_file.csv your file need to contain a column of biblionumber from biblio or authid from auth_header (only one column) You can go in Koha -> reports -> Create from sql SELECT biblionumber FROM biblio ; and then download in csv to have your csv file. - You can test with others query like : select biblionumber from biblio WHERE author LIKE 'a%'; Total number of results (411); - verify that you have the rigth number of records indexed --- Koha/BiblioUtils.pm | 19 ++-- Koha/MetadataRecord/Authority.pm | 21 ++-- Koha/SearchEngine/Elasticsearch/Indexer.pm | 2 +- misc/search_tools/rebuild_elasticsearch.pl | 162 ++++++++++++++++++++++------- 4 files changed, 153 insertions(+), 51 deletions(-) diff --git a/Koha/BiblioUtils.pm b/Koha/BiblioUtils.pm index fd6969d..aed83b5 100644 --- a/Koha/BiblioUtils.pm +++ b/Koha/BiblioUtils.pm @@ -101,7 +101,7 @@ sub get_from_biblionumber { =head2 get_all_biblios_iterator - my $it = Koha::BiblioUtils->get_all_biblios_iterator(%options); + my $it = Koha::BiblioUtils->get_all_biblios_iterator(\%options, $offset, $length); This will provide an iterator object that will, one by one, provide the Koha::BiblioUtils of each biblio. This will include the item data. @@ -124,13 +124,14 @@ records instead of all. =cut sub get_all_biblios_iterator { - my ($self, %options) = @_; + my ($self, $options, $offset, $length) = @_; my $search_terms = {}; my ($slice_modulo, $slice_count); - if ($options{slice}) { - $slice_count = $options{slice}->{count}; - $slice_modulo = $options{slice}->{index}; + if ($options->{slice}) { + $slice_count = $options->{slice}->{count}; + $slice_modulo = $options->{slice}->{index}; + $search_terms = \[ 'mod(biblionumber, ?) = ?', $slice_count, $slice_modulo ]; } @@ -138,8 +139,12 @@ sub get_all_biblios_iterator { my $schema = $database->schema(); my $rs = $schema->resultset('Biblio')->search( - $search_terms, - { columns => [qw/ biblionumber /] } ); + $search_terms, { + columns => [qw/ biblionumber /] , + offset => $offset , + rows => $length + }); + my $next_func = sub { # Warn and skip bad records, otherwise we break the loop while (1) { diff --git a/Koha/MetadataRecord/Authority.pm b/Koha/MetadataRecord/Authority.pm index 17245f0..a4c3fd0 100644 --- a/Koha/MetadataRecord/Authority.pm +++ b/Koha/MetadataRecord/Authority.pm @@ -151,7 +151,7 @@ sub authorized_heading { =head2 get_all_authorities_iterator - my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%options); + my $it = Koha::MetadataRecord::Authority->get_all_authorities_iterator(\%options, $offset, $length); This will provide an iterator object that will, one by one, provide the Koha::MetadataRecord::Authority of each authority. @@ -174,15 +174,15 @@ records instead of all. =cut sub get_all_authorities_iterator { - my ($self, %options) = @_; + my ($self, $options, $offset, $length) = @_; my $search_terms = { marcxml => { '!=', undef } }; my ($slice_modulo, $slice_count); - if ($options{slice}) { - $slice_count = $options{slice}->{count}; - $slice_modulo = $options{slice}->{index}; + if ($options->{slice}) { + $slice_count = $options->{slice}->{count}; + $slice_modulo = $options->{slice}->{index}; $search_terms = { '-and' => [ %{$search_terms}, @@ -193,10 +193,15 @@ sub get_all_authorities_iterator { my $database = Koha::Database->new(); my $schema = $database->schema(); + my $rs = - $schema->resultset('AuthHeader')->search( - $search_terms, - { columns => [qw/ authid authtypecode marcxml /] } ); + $schema->resultset('AuthHeader')->search( + $search_terms, + { columns => [qw/ authid authtypecode marcxml /], + offset => $offset, + rows => $length } + ); + my $next_func = sub { my $row = $rs->next(); return if !$row; diff --git a/Koha/SearchEngine/Elasticsearch/Indexer.pm b/Koha/SearchEngine/Elasticsearch/Indexer.pm index 703f39f..33d0ced 100644 --- a/Koha/SearchEngine/Elasticsearch/Indexer.pm +++ b/Koha/SearchEngine/Elasticsearch/Indexer.pm @@ -399,7 +399,7 @@ sub _sanitise_records { if ( $rec ) { $rec->delete_fields($rec->field('999')); # Make sure biblionumber is a string. Elasticsearch would consider int and string different IDs. - $rec->append_fields(MARC::Field->new('999','','','c' => "" . $bibnum, 'd' => "" . $bibnum)); + $rec->append_fields(MARC::Field->new('999','','','c' => "" . $bibnum, 'd' => "" . $bibnum)) if ($bibnum); } } } diff --git a/misc/search_tools/rebuild_elasticsearch.pl b/misc/search_tools/rebuild_elasticsearch.pl index 1199b63..29b85fa 100755 --- a/misc/search_tools/rebuild_elasticsearch.pl +++ b/misc/search_tools/rebuild_elasticsearch.pl @@ -28,6 +28,13 @@ rebuild_elasticsearch.pl - inserts records from a Koha database into Elasticsear B [B<-c|--commit>=C] [B<-v|--verbose>] +[B<-d|--delete>] +[B<-a|--authorities>] +[B<-b|--biblios>] +[B<-f|--file>] +[B<-bn|--bnnumber>] +[B<--length>] +[B<--offset>] [B<-h|--help>] [B<--man>] @@ -58,17 +65,34 @@ specifying neither and so both get indexed. Index the biblios only. Combining this with B<-a> is the same as specifying neither and so both get indexed. +=item B<-f|--file> + +Read a csv file created with a sql query, the csv must contain a column of biblionumber from biblios +or a column of authid from auth_header, then index biblios or authorities, but not both. +With this option you can't have options B<--length>, B<--offset>, and B<-bn>. + =item B<-bn|--bnumber> Only index the supplied biblionumber, mostly for testing purposes. May be repeated. This also applies to authorities via authid, so if you're using it, -you probably only want to do one or the other at a time. +you probably only want to do one or the other at a time, +with this option you can't have options B<--offset> and B<--length>. =item B<-p|--processes> Number of processes to use for indexing. This can be used to do more indexing work in parallel on multicore systems. By default, a single process is used. +=item B<--offset> + +Specify the row number for the first row to be indexed. +With this option you have to choose between biblios B<-b> or authorities B<-a>. + +=item B<--length> + +Specify the number of rows to be indexed. +With this option you have to choose between biblios B<-b> or authorities B<-a>. + =item B<-v|--verbose> By default, this program only emits warnings and errors. This makes it talk @@ -99,12 +123,14 @@ use MARC::Field; use MARC::Record; use Modern::Perl; use Pod::Usage; +use Text::CSV; my $verbose = 0; my $commit = 5000; my ($delete, $help, $man, $processes); my ($index_biblios, $index_authorities); -my (@record_numbers); +my @record_numbers; +my ($offset, $length, $file); $|=1; # flushes output @@ -116,10 +142,17 @@ GetOptions( 'bn|bnumber=i' => \@record_numbers, 'p|processes=i' => \$processes, 'v|verbose+' => \$verbose, + 'offset=i' => \$offset, + 'length=i' => \$length, + 'f|file=s' => \$file, 'h|help' => \$help, 'man' => \$man, + ); +pod2usage(1) if $help; +pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; + # Default is to do both unless ($index_authorities || $index_biblios) { $index_authorities = $index_biblios = 1; @@ -129,13 +162,25 @@ if ($processes && @record_numbers) { die "Argument p|processes cannot be combined with bn|bnumber"; } -pod2usage(1) if $help; -pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; +if ($file){ + if ($offset || $length || @record_numbers){ + die "With options --file, you can't have options --length, --offset, and -bn" . "\n"; + } +} -_sanity_check(); +if (@record_numbers){ + if ($length || $offset){ + die "With option -bn, you can't have options --length, --offset" . "\n"; + } +} -_verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete) if ($index_biblios); -_verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete) if ($index_authorities); +if ($offset || $length){ + if ( $index_biblios && $index_authorities){ + die "With options --offset or --length, you have to choose between biblios (-b) or authorities (-a)." . "\n"; + } +} + +_sanity_check(); my $slice_index = 0; my $slice_count = ( $processes //= 1 ); @@ -160,38 +205,85 @@ if ($slice_count > 1) { } my $next; -if ($index_biblios) { - _log(1, "Indexing biblios\n"); - if (@record_numbers) { - $next = sub { - my $r = shift @record_numbers; - return () unless defined $r; - return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); - }; - } else { - my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options); - $next = sub { - $records->next(); +if ($file){ + my @bibnums; + my $i = 0; + my $input = $file; + my $parser = Text::CSV->new(); + open ( my $csv, "<", $input ); + + while (<$csv>) { + if ($parser->parse($_)){ + my @columns = $parser->fields(); + @bibnums[$i] = @columns; + $i++; } } - _do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); -} -if ($index_authorities) { - _log(1, "Indexing authorities\n"); - if (@record_numbers) { - $next = sub { - my $r = shift @record_numbers; - return () unless defined $r; - my $a = Koha::MetadataRecord::Authority->get_from_authid($r); - return ($r, $a->record); - }; - } else { - my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options); - $next = sub { - $records->next(); + close $csv; + + if (@bibnums){ + my $type = shift @bibnums; + if ($type eq "biblionumber"){ + _verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete); + _log(1, "Indexing biblios\n"); + $next = sub { + my $bibnum = shift @bibnums; + return () unless defined $bibnum; + return ($bibnum, Koha::BiblioUtils->get_from_biblionumber($bibnum, item_data => 1 )); + }; + _do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); + + } elsif ($type eq "authid"){ + _verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete); + _log(1, "Indexing authorities\n"); + $next = sub { + my $authid = shift @bibnums; + return () unless defined $authid; + return ($authid,Koha::MetadataRecord::Authority->get_from_authid($authid)); + }; + _do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX); + } + } + +} else { + if ($index_biblios) { + _verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete); + _log(1, "Indexing biblios\n"); + + if (@record_numbers) { + my @bibnums = @record_numbers; + $next = sub { + my $r = shift @bibnums; + return () unless defined $r; + return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); + }; + } else { + my $records = Koha::BiblioUtils->get_all_biblios_iterator(\%iterator_options,$offset, $length); + $next = sub { + $records->next(); + } + } + _do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); + } + if ($index_authorities) { + _verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete); + _log(1, "Indexing authorities\n"); + + if (@record_numbers) { + my @authnums = @record_numbers; + $next = sub { + my $r = shift @authnums; + return () unless defined $r; + return ($r,Koha::MetadataRecord::Authority->get_from_authid($r)); + }; + } else { + my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(\%iterator_options,$offset, $length); + $next = sub { + $records->next(); + } } + _do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX); } - _do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX); } if ($slice_index == 0) { -- 2.7.4