From 31e0ebe669d7780b973a90c6d2f9a4b6910a85b7 Mon Sep 17 00:00:00 2001 From: Axel Amghar Date: Tue, 14 May 2019 10:04:05 +0200 Subject: [PATCH] Bug 20384 - Elastic rebuild script improvements : option --file To test: - after testing offset and length, apply this patch - launch perl misc/search_tools/rebuild_elastic_search.pl -v -d -c 100 --file your_file.csv your file need to contain a column of biblionumber from biblio or authid from auth_header (only one column) You can go in Koha -> reports -> Create from sql SELECT biblionumber FROM biblio ; and then download in csv to have your csv file. - You can test with others query like : select biblionumber from biblio WHERE author LIKE 'a%'; Total number of results (411); - verify that you have the rigth number of records indexed --- Koha/SearchEngine/Elasticsearch/Indexer.pm | 2 +- misc/search_tools/rebuild_elastic_search.pl | 135 +++++++++++++++++++++------- 2 files changed, 102 insertions(+), 35 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch/Indexer.pm b/Koha/SearchEngine/Elasticsearch/Indexer.pm index 703f39f..33d0ced 100644 --- a/Koha/SearchEngine/Elasticsearch/Indexer.pm +++ b/Koha/SearchEngine/Elasticsearch/Indexer.pm @@ -399,7 +399,7 @@ sub _sanitise_records { if ( $rec ) { $rec->delete_fields($rec->field('999')); # Make sure biblionumber is a string. Elasticsearch would consider int and string different IDs. - $rec->append_fields(MARC::Field->new('999','','','c' => "" . $bibnum, 'd' => "" . $bibnum)); + $rec->append_fields(MARC::Field->new('999','','','c' => "" . $bibnum, 'd' => "" . $bibnum)) if ($bibnum); } } } diff --git a/misc/search_tools/rebuild_elastic_search.pl b/misc/search_tools/rebuild_elastic_search.pl index b879654..568d495 100755 --- a/misc/search_tools/rebuild_elastic_search.pl +++ b/misc/search_tools/rebuild_elastic_search.pl @@ -31,6 +31,7 @@ B [B<-d|--delete>] [B<-a|--authorities>] [B<-b|--biblios>] +[B<-f|--file>] [B<-bn|--bnnumber>] [B<--length>] [B<--offset>] @@ -64,21 +65,28 @@ specifying neither and so both get indexed. Index the biblios only. Combining this with B<-a> is the same as specifying neither and so both get indexed. +=item B<-f|--file> + +Read a csv file created with a sql query, the csv must contain a column of biblionumber from biblios +or a column of authid from auth_header, then index biblios or authorities, but not both. +With this option you can't have options B<--length>, B<--offset>, and B<-bn>. + =item B<-bn|--bnumber> Only index the supplied biblionumber, mostly for testing purposes. May be repeated. This also applies to authorities via authid, so if you're using it, -you probably only want to do one or the other at a time. +you probably only want to do one or the other at a time, +with this option you can't have options B<--offset> and B<--length>. =item B<--offset> Specify the row number for the first row to be indexed. -With this option you have to choose between biblios (-b) or authorities (-a). +With this option you have to choose between biblios B<-b> or authorities B<-a>. =item B<--length> Specify the number of rows to be indexed. -With this option you have to choose between biblios (-b) or authorities (-a). +With this option you have to choose between biblios B<-b> or authorities B<-a>. =item B<-v|--verbose> @@ -108,13 +116,14 @@ use MARC::Field; use MARC::Record; use Modern::Perl; use Pod::Usage; +use Text::CSV; my $verbose = 0; my $commit = 5000; my ($delete, $help, $man); my ($index_biblios, $index_authorities); my (@biblionumbers); -my ($offset, $length); +my ($offset, $length, $file); $|=1; # flushes output @@ -127,17 +136,32 @@ GetOptions( 'v|verbose+' => \$verbose, 'offset=i' => \$offset, 'length=i' => \$length, + 'f|file=s' => \$file, 'h|help' => \$help, 'man' => \$man, ); +pod2usage(1) if $help; +pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; + # Default is to do both unless ($index_authorities || $index_biblios) { $index_authorities = $index_biblios = 1; } -pod2usage(1) if $help; -pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; +if ($file){ + if ($offset || $length || @biblionumbers){ + printf "With options --file, you can't have options --length, --offset, and -bn" . "\n"; + pod2usage(1); + } +} + +if (@biblionumbers){ + if ($length || $offset){ + printf "With option -bn, you can't have options --length, --offset" . "\n"; + pod2usage(1); + } +} if ($offset || $length){ if ( $index_biblios && $index_authorities){ @@ -149,40 +173,83 @@ if ($offset || $length){ sanity_check(); my $next; -if ($index_biblios) { - _log(1, "Indexing biblios\n"); - if (@biblionumbers) { - $next = sub { - my $r = shift @biblionumbers; - return () unless defined $r; - return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); - }; - } else { - my $records = Koha::BiblioUtils->get_all_biblios_iterator($offset, $length); - $next = sub { - $records->next(); + +if ($file){ + my @bibnums; + my $i = 0; + my $input = $file; + my $parser = Text::CSV->new(); + open ( my $csv, "<", $input ); + + while (<$csv>) { + if ($parser->parse($_)){ + my @columns = $parser->fields(); + @bibnums[$i] = @columns; + $i++; } } - do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); -} -if ($index_authorities) { - _log(1, "Indexing authorities\n"); - if (@biblionumbers) { - $next = sub { - my $r = shift @biblionumbers; - return () unless defined $r; - my $a = Koha::MetadataRecord::Authority->get_from_authid($r); - return ($r, $a->record); - }; - } else { - my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator($offset, $length); - $next = sub { - $records->next(); + close $csv; + + if (@bibnums){ + my $type = shift @bibnums; + if ($type eq "biblionumber"){ + _log(1, "Indexing biblios\n"); + $next = sub { + my $bibnum = shift @bibnums; + return () unless defined $bibnum; + return ($bibnum, Koha::BiblioUtils->get_from_biblionumber($bibnum, item_data => 1 )); + }; + do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); + + } elsif ($type eq "authid"){ + _log(1, "Indexing authorities\n"); + $next = sub { + my $authid = shift @bibnums; + return () unless defined $authid; + return ($authid,Koha::MetadataRecord::Authority->get_from_authid($authid)); + }; + do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX); } } - do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX); + +} else { + if ($index_biblios) { + _log(1, "Indexing biblios\n"); + + if (@biblionumbers) { + $next = sub { + my $r = shift @biblionumbers; + return () unless defined $r; + return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); + }; + } else { + my $records = Koha::BiblioUtils->get_all_biblios_iterator($offset, $length); + $next = sub { + $records->next(); + } + } + do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); + } + if ($index_authorities) { + _log(1, "Indexing authorities\n"); + + if (@biblionumbers) { + $next = sub { + my $r = shift @biblionumbers; + return () unless defined $r; + return ($r,Koha::MetadataRecord::Authority->get_from_authid($r)); + }; + } else { + my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator($offset, $length); + $next = sub { + $records->next(); + } + } + do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX); + } } + sub do_reindex { my ( $next, $index_name ) = @_; -- 2.7.4