From 77bda1b07f1afcb949fc6ba42cd91ee6c66573c0 Mon Sep 17 00:00:00 2001 From: Paul Poulain Date: Wed, 1 Feb 2012 15:15:02 +0100 Subject: [PATCH 1/2] Bug 7246 add offset/length and where options to rebuild_zebra This patch reimplement a feature that is on biblibre/master for Koha-community/master It adds 4 parameters: * offset = the offset of record. Say 1000 to start rebuilding at the 1000th record of your database * length = how many records to export. Say 400 to export only 400 records * where = add a where clause to rebuild only a given itemtype, or anything you want to filter on * l = how many items should be export with biblios. This is a usefull option when you have records with so many items that they can result in a record higher than 99999bytes, that zebra don't like at all Another improvement resulting from offset & length limit is the rebuild_zebra_sliced.zsh that will be submitted in another patch. rebuild_zebra_sliced will slice your all database in small chunks, and, if something went wrong for a given slice, will slice the slice, and repeat, until you reach a slice size of 1, showing which record is wrong in your database. --- misc/migration_tools/rebuild_zebra.pl | 33 +++++++++++++++++++++++++++++---- 1 files changed, 29 insertions(+), 4 deletions(-) diff --git a/misc/migration_tools/rebuild_zebra.pl b/misc/migration_tools/rebuild_zebra.pl index 31e8125..94b37b8 100755 --- a/misc/migration_tools/rebuild_zebra.pl +++ b/misc/migration_tools/rebuild_zebra.pl @@ -34,6 +34,10 @@ my $want_help; my $as_xml; my $process_zebraqueue; my $do_not_clear_zebraqueue; +my $item_limit; +my $length; +my $where; +my $offset; my $verbose_logging = 0; my $zebraidx_log_opt = " -v none,fatal,warn "; my $result = GetOptions( @@ -51,7 +55,11 @@ my $result = GetOptions( 'x' => \$as_xml, 'y' => \$do_not_clear_zebraqueue, 'z' => \$process_zebraqueue, - 'v+' => \$verbose_logging, + 'l:i' => \$item_limit, + 'where:s' => \$where, + 'length:i' => \$length, + 'offset:i' => \$offset, + 'v+' => \$verbose_logging, ); @@ -294,13 +302,21 @@ sub select_all_records { } sub select_all_authorities { - my $sth = $dbh->prepare("SELECT authid FROM auth_header"); + my $strsth=qq{SELECT authid FROM auth_header}; + $strsth.=qq{ WHERE $where } if ($where); + $strsth.=qq{ LIMIT $length } if ($length && !$offset); + $strsth.=qq{ LIMIT $offset,$length } if ($length && $offset); + my $sth = $dbh->prepare($strsth); $sth->execute(); return $sth; } sub select_all_biblios { - my $sth = $dbh->prepare("SELECT biblionumber FROM biblioitems ORDER BY biblionumber"); + my $strsth = qq{ SELECT biblionumber FROM biblioitems }; + $strsth.=qq{ WHERE $where } if ($where); + $strsth.=qq{ LIMIT $length } if ($length && !$offset); + $strsth.=qq{ LIMIT $offset,$length } if ($offset); + my $sth = $dbh->prepare($strsth); $sth->execute(); return $sth; } @@ -635,11 +651,20 @@ Parameters: the same records - specify -y to override this. Cannot be used with -z. + -l set a maximum number of exported items per biblio. + Doesn't work with -nosanitize. -v increase the amount of logging. Normally only warnings and errors from the indexing are shown. Use log level 2 (-v -v) to include all Zebra logs. - -munge-config Deprecated option to try + --length 1234 how many biblio you want to export + --offset 1243 offset you want to start to + example: --offset 500 --length=500 will result in a LIMIT 500,1000 (exporting 1000 records, starting by the 500th one) + note that the numbers are NOT related to biblionumber, that's the intended behaviour. + --where let you specify a WHERE query, like itemtype='BOOK' + or something like that + + --munge-config Deprecated option to try to fix Zebra config files. --help or -h show this message. _USAGE_ -- 1.7.0.4