From 67faa31d44bb0a19cd8d1ae527553f0fddc3c02a Mon Sep 17 00:00:00 2001 From: Fridolyn SOMERS Date: Fri, 14 Jun 2013 17:36:16 +0200 Subject: [PATCH] Bug 10482 - add items limit and sort to rebuild zebra When indexing biblio records, the items are added into MARC format during database export (MARCXML or ISO2709). For some records, there is a huge number of items, mainly serials creating an item for each issue. This patch adds an option in rebuild_zebra.pl to limit the number of items added to record and an option to define a sort of those items. This will speed-up some indexing and avoid a known bug : when Zebra has a record with size too big for ISO2709 format this record returned as search result is broken (some items fields contains odd characters). Also corrects the fact that items where not added with noxml option : in get_raw_marc_record() : - if noxml option was missing, GetMarcBiblio($record_number, 1) whas used : 1 means "add all items" - if noxml option was missing, record whas build with biblioitems.marc whith no item fields With this patch the MARC::Record object is build with marc/marcxml and then items are added by add_items_to_biblio(). Test plan : - Look at help : misc/migration_tools/rebuild_zebra.pl -h - Create a biblio record with more than 2 items : for example biblionumber=123 - Rebuild this biblio with a limit of 2 items : misc/migration_tools/rebuild_zebra.pl -b -v --items-limit 2 --where "biblionumber=123" - Look at this biblio in search results page at intranet => There should be only 2 items - Rebuild this biblio without limit : misc/migration_tools/rebuild_zebra.pl -b -v --where "biblionumber=123" - Look at this biblio in search results page at intranet => There should be all items - Create a biblio record with 2 items with different items.homebranch : for example biblionumber=123 and branches "Centerville" (code CPL) and "Union" (code UPL) - Rebuild this biblio with a sort by home branch ascending and a limit of one item : misc/migration_tools/rebuild_zebra.pl -b -v --where "biblionumber=123" --items-sort="homebranch ASC" --items-limit 1 - Look at this biblio in search results page at intranet => There should be only the item with home branch "Centerville" - Rebuild this biblio with a sort by home branch descending and a limit of one item : misc/migration_tools/rebuild_zebra.pl -b -v --where "biblionumber=123" --items-sort="homebranch DESC" --items-limit 1 - Look at this biblio in search results page at intranet => There should be only the item with home branch "Union" - Rebuild this biblio with --noxml option : misc/migration_tools/rebuild_zebra.pl -b -v --where "biblionumber=123" --noxml => There should be all items --- misc/migration_tools/rebuild_zebra.pl | 43 ++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/misc/migration_tools/rebuild_zebra.pl b/misc/migration_tools/rebuild_zebra.pl index a46526d..a9f07b9 100755 --- a/misc/migration_tools/rebuild_zebra.pl +++ b/misc/migration_tools/rebuild_zebra.pl @@ -41,6 +41,8 @@ my $where; my $offset; my $run_as_root; my $run_user = (getpwuid($<))[0]; +my $items_sort; +my $items_limit; my $verbose_logging = 0; my $zebraidx_log_opt = " -v none,fatal,warn "; @@ -65,6 +67,8 @@ my $result = GetOptions( 'offset:i' => \$offset, 'v+' => \$verbose_logging, 'run-as-root' => \$run_as_root, + 'items-sort:s' => \$items_sort, + 'items-limit:i' => \$items_limit, ); if (not $result or $want_help) { @@ -103,6 +107,12 @@ if ($process_zebraqueue and $do_not_clear_zebraqueue) { die $msg; } +if (($items_limit or $items_sort) and !$biblios) { + my $msg = "Cannot specify --items_limit or --items_sort without -b\n"; + $msg .= "Please do '$0 --help' to see usage.\n"; + die $msg; +} + if ($reset) { $noshadow = 1; } @@ -563,7 +573,7 @@ sub get_raw_marc_record { $fetch_sth->finish(); return unless $marc; } else { - eval { $marc = GetMarcBiblio($record_number, 1); }; + eval { $marc = GetMarcBiblio($record_number); }; if ($@ || !$marc) { # here we do warn since catching an exception # means that the bib was found but failed @@ -572,6 +582,7 @@ sub get_raw_marc_record { return; } } + add_items_to_biblio($marc, $record_number); } else { eval { $marc = GetAuthority($record_number); }; if ($@) { @@ -582,6 +593,32 @@ sub get_raw_marc_record { return $marc; } +sub add_items_to_biblio { + my $record = shift; + my $biblionumber = shift; + return unless $record && $biblionumber; + + my $items_query = q{ + SELECT itemnumber + FROM items + WHERE biblionumber = ? + }; + if ($items_sort) { + $items_query .= q{ ORDER BY } . $items_sort; + } + $items_query .= q{ LIMIT } . $items_limit if ($items_limit); + my $items_sth = $dbh->prepare($items_query); + $items_sth->execute($biblionumber); + + my @itemnumbers; + while ( my $itemnumber = $items_sth->fetchrow_array ) { + push @itemnumbers, $itemnumber; + } + + C4::Biblio::EmbedItemsInMarcBiblio( $record, $biblionumber, \@itemnumbers ) + if (@itemnumbers); +} + sub fix_leader { # FIXME - this routine is suspect # It blanks the Leader/00-05 and Leader/12-16 to @@ -745,6 +782,10 @@ Parameters: --where let you specify a WHERE query, like itemtype='BOOK' or something like that + --items-sort items sort column, from items table, and order. + example: --items-sort "dateaccessioned DESC" + --items-limit 50 max number of items per biblio + --munge-config Deprecated option to try to fix Zebra config files. -- 1.7.10.4