From ae74dca5d9ba13cda5a29c1a517dc34b6dab144f Mon Sep 17 00:00:00 2001 From: Henri-Damien LAURENT Date: Fri, 3 Sep 2010 12:16:52 +0200 Subject: [PATCH 1/9] Bug 5579: remove items from MARC bib This is a squash of four patches by Henri-Damien Laurent starting work on removing the copy of item record information in the 9XX field of bibliographic records. The reason for doing this is primarily to improve performance, in particular, the expense of having to add/modify the bib record whenever an item changes. Now, whenever an item changes, the bib record is put in the queue to be reindexed; when the bib is indexed, the 9XX fields are inserted into the version of the bib that Zebra indexes. Since rebuild_zebra.pl runs in a separate process, the processing of the bib record will not delay (e.g.) circulation. As part of upgrading to 3.4, the following batch script should be run: misc/maintenance/remove_items_from_biblioitems.pl --run This should be followed by a complete reindexing of the bib records, e.g., misc/migration_tools/rebuild_zebra.pl -b -r Signed-off-by: Galen Charlton --- C4/Biblio.pm | 24 +--------- C4/Items.pm | 28 ++++------- misc/maintenance/remove_items_from_biblioitems.pl | 50 +++++++++++++++++++++ misc/migration_tools/rebuild_zebra.pl | 31 ++++++++++++- 4 files changed, 89 insertions(+), 44 deletions(-) create mode 100644 misc/maintenance/remove_items_from_biblioitems.pl diff --git a/C4/Biblio.pm b/C4/Biblio.pm index a0e06f6..9cc39e2 100644 --- a/C4/Biblio.pm +++ b/C4/Biblio.pm @@ -311,29 +311,6 @@ sub ModBiblio { $record->delete_field($field); } - # parse each item, and, for an unknown reason, re-encode each subfield - # if you don't do that, the record will have encoding mixed - # and the biblio will be re-encoded. - # strange, I (Paul P.) searched more than 1 day to understand what happends - # but could only solve the problem this way... - my @fields = $oldRecord->field($itemtag); - foreach my $fielditem (@fields) { - my $field; - foreach ( $fielditem->subfields() ) { - # re-encode the subfield only if it isn't already in utf-8. - my ($tag, $value) = @$_; - $tag = Encode::encode('utf-8', $tag) unless utf8::is_utf8($tag); - $value = Encode::encode('utf-8', $value) unless utf8::is_utf8($value); - - if ($field) { - $field->add_subfields( $tag => $value ); - } else { - $field = MARC::Field->new( "$itemtag", '', '', $tag => $value ); - } - } - $record->append_fields($field); - } - foreach my $field ($record->fields()) { if (! $field->is_control_field()) { if (scalar($field->subfields()) == 0) { @@ -1065,6 +1042,7 @@ sub GetMarcBiblio { if ($marcxml) { $record = eval { MARC::Record::new_from_xml( $marcxml, "utf8", C4::Context->preference('marcflavour') ) }; if ($@) { warn " problem with :$biblionumber : $@ \n$marcxml"; } + return unless $record; # $record = MARC::Record::new_from_usmarc( $marc) if $marc; return $record; diff --git a/C4/Items.pm b/C4/Items.pm index 939e682..741ea4d 100644 --- a/C4/Items.pm +++ b/C4/Items.pm @@ -263,6 +263,7 @@ sub AddItem { # create MARC tag representing item and add to bib my $new_item_marc = _marc_from_item_hash($item, $frameworkcode, $unlinked_item_subfields); _add_item_field_to_biblio($new_item_marc, $item->{'biblionumber'}, $frameworkcode ); + #_add_item_field_to_biblio($new_item_marc, $item->{'biblionumber'}, $frameworkcode ); logaction("CATALOGUING", "ADD", $itemnumber, "item") if C4::Context->preference("CataloguingLog"); @@ -368,7 +369,7 @@ sub AddItemBatchFromMarc { } # update the MARC biblio - $biblionumber = ModBiblioMarc( $record, $biblionumber, $frameworkcode ); + # $biblionumber = ModBiblioMarc( $record, $biblionumber, $frameworkcode ); return (\@itemnumbers, \@errors); } @@ -513,7 +514,7 @@ sub ModItem { my $new_item_marc = _marc_from_item_hash($whole_item, $frameworkcode, $unlinked_item_subfields) or die "FAILED _marc_from_item_hash($whole_item, $frameworkcode)"; - _replace_item_field_in_biblio($new_item_marc, $biblionumber, $itemnumber, $frameworkcode); + #_replace_item_field_in_biblio($new_item_marc, $biblionumber, $itemnumber, $frameworkcode); ($new_item_marc eq '0') and die "$new_item_marc is '0', not hashref"; # logaction line would crash anyway logaction("CATALOGUING", "MODIFY", $itemnumber, $new_item_marc->as_formatted) if C4::Context->preference("CataloguingLog"); } @@ -576,23 +577,13 @@ sub DelItem { # get the MARC record my $record = GetMarcBiblio($biblionumber); - my $frameworkcode = GetFrameworkCode($biblionumber); + ModZebra( $biblionumber, "specialUpdate", "biblioserver", undef, undef ); # backup the record my $copy2deleted = $dbh->prepare("UPDATE deleteditems SET marc=? WHERE itemnumber=?"); $copy2deleted->execute( $record->as_usmarc(), $itemnumber ); #search item field code - my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",$frameworkcode); - my @fields = $record->field($itemtag); - - # delete the item specified - foreach my $field (@fields) { - if ( $field->subfield($itemsubfield) eq $itemnumber ) { - $record->delete_field($field); - } - } - &ModBiblioMarc( $record, $biblionumber, $frameworkcode ); logaction("CATALOGUING", "DELETE", $itemnumber, "item") if C4::Context->preference("CataloguingLog"); } @@ -2066,16 +2057,16 @@ sub MoveItemFromBiblio { } # Saving the modification - ModBiblioMarc($record, $frombiblio, $frameworkcode); + #ModBiblioMarc($record, $frombiblio, $frameworkcode); # Getting the record we want to move the item to - $record = GetMarcBiblio($tobiblio); + #$record = GetMarcBiblio($tobiblio); # Inserting the previously saved item - $record->insert_fields_ordered($item); + #$record->insert_fields_ordered($item); # Saving the modification - ModBiblioMarc($record, $tobiblio, $frameworkcode); + #ModBiblioMarc($record, $tobiblio, $frameworkcode); } else { return undef; @@ -2159,6 +2150,7 @@ sub _koha_modify_item { $error.="ERROR in _koha_modify_item $query".$dbh->errstr; warn $error; } + ModZebra( $item->{biblionumber}, "specialUpdate", "biblioserver", undef, undef ); return ($item->{'itemnumber'},$error); } @@ -2302,7 +2294,7 @@ sub _replace_item_field_in_biblio { } # save the record - ModBiblioMarc($completeRecord, $biblionumber, $frameworkcode); + #ModBiblioMarc($completeRecord, $biblionumber, $frameworkcode); } =head2 _repack_item_errors diff --git a/misc/maintenance/remove_items_from_biblioitems.pl b/misc/maintenance/remove_items_from_biblioitems.pl new file mode 100644 index 0000000..1a50f62 --- /dev/null +++ b/misc/maintenance/remove_items_from_biblioitems.pl @@ -0,0 +1,50 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use C4::Context; +use C4::Biblio; +use Getopt::Long; + +my ($wherestring,$run,$want_help); +my $result = GetOptions( + 'where:s' => \$wherestring, + '--run' => \$run, + 'help|h' => \$want_help, +); +if ( not $result or $want_help ) { + print_usage(); + exit 0; +} + + +my $dbh=C4::Context->dbh; +my $querysth=qq{SELECT biblionumber from biblioitems }; +$querysth.=" WHERE $wherestring " if ($wherestring); +my $query=$dbh->prepare($querysth); + +$query->execute; +while (my $biblionumber=$query->fetchrow){ + my $record=GetMarcBiblio($biblionumber); + + if ($record){ + ModBiblio($record,$biblionumber,GetFrameworkCode($biblionumber)) ; + } + else { + print "error in $biblionumber : can't parse biblio"; + } +} +sub print_usage { + print <<_USAGE_; +$0: removes items from selected biblios + + +Parameters: + -where use this to limit modifications to some biblios + --run run the command + --help or -h show this message. +_USAGE_ +} + +# diff --git a/misc/migration_tools/rebuild_zebra.pl b/misc/migration_tools/rebuild_zebra.pl index 682c280..74c73a1 100755 --- a/misc/migration_tools/rebuild_zebra.pl +++ b/misc/migration_tools/rebuild_zebra.pl @@ -9,6 +9,7 @@ use File::Temp qw/ tempdir /; use File::Path; use C4::Biblio; use C4::AuthoritiesMarc; +use C4::Items; # # script that checks zebradir structure & create directories & mandatory files if needed @@ -310,6 +311,7 @@ sub export_marc_records_from_sth { my $num_exported = 0; open (OUT, ">:utf8 ", "$directory/exported_records") or die $!; my $i = 0; + my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",''); while (my ($record_number) = $sth->fetchrow_array) { print "." if ( $verbose_logging ); print "\r$i" unless ($i++ %100 or !$verbose_logging); @@ -317,6 +319,29 @@ sub export_marc_records_from_sth { my $marcxml = $record_type eq 'biblio' ? GetXmlBiblio( $record_number ) : GetAuthorityXML( $record_number ); + if ($record_type eq 'biblio'){ + #CALL sub ProcessItems + my @items=GetItemsInfo($record_number,'intra',30); + if (@items){ + my $record=MARC::Record->new; + my @itemsrecord; + foreach my $item (@items){ + my $record=Item2Marc($item, $record_number); + push @itemsrecord, $record->field($itemtag); + #if xml then print itemfield as xml + # and update marcxml + # else push field + } + $record->insert_fields_ordered(@itemsrecord); + my $itemsxml=$record->as_xml_record(); + my $searchstring='\n'; + my $index=index($itemsxml,'\n',0); + $itemsxml=substr($itemsxml,$index+length($searchstring)); + $searchstring=''; + $marcxml=substr($marcxml,0,index($marcxml,$searchstring)); + $marcxml.=$itemsxml; + } + } if ( $marcxml ) { print OUT $marcxml if $marcxml; $num_exported++; @@ -330,7 +355,7 @@ sub export_marc_records_from_sth { # strung together with no single root element. zebraidx doesn't seem # to care, though, at least if you're using the GRS-1 filter. It does # care if you're using the DOM filter, which requires valid XML file(s). - print OUT ($as_xml) ? $marc->as_xml_record() : $marc->as_usmarc(); + print OUT ($as_xml) ? $marc->as_xml_record(C4::Context->preference('marcflavour')) : $marc->as_usmarc(); $num_exported++; } } @@ -358,7 +383,7 @@ sub export_marc_records_from_list { # strung together with no single root element. zebraidx doesn't seem # to care, though, at least if you're using the GRS-1 filter. It does # care if you're using the DOM filter, which requires valid XML file(s). - print OUT ($as_xml) ? $marc->as_xml_record() : $marc->as_usmarc(); + print OUT ($as_xml) ? $marc->as_xml_record(C4::Context->preference('marcflavour')) : $marc->as_usmarc(); $num_exported++; } } @@ -387,7 +412,7 @@ sub generate_deleted_marc_records { fix_unimarc_100($marc); } - print OUT ($as_xml) ? $marc->as_xml_record() : $marc->as_usmarc(); + print OUT ($as_xml) ? $marc->as_xml_record(C4::Context->preference("marcflavour")) : $marc->as_usmarc(); $num_exported++; } print "\nRecords exported: $num_exported\n" if ( $verbose_logging ); -- 1.7.1 From a2f0dd1a7310424da5de830db02c018c7cb510c8 Mon Sep 17 00:00:00 2001 From: Galen Charlton Date: Sat, 2 Apr 2011 21:09:59 -0400 Subject: [PATCH 2/9] bug 5579: improvements to remove_items_from_biblioitems.pl * add copyright statement * make --run switch work; this is now required to cause the update to run * improve help text * make remove_items_from_biblioitems.pl executable Signed-off-by: Galen Charlton --- misc/maintenance/remove_items_from_biblioitems.pl | 61 ++++++++++++++------- 1 files changed, 41 insertions(+), 20 deletions(-) mode change 100644 => 100755 misc/maintenance/remove_items_from_biblioitems.pl diff --git a/misc/maintenance/remove_items_from_biblioitems.pl b/misc/maintenance/remove_items_from_biblioitems.pl old mode 100644 new mode 100755 index 1a50f62..0778b29 --- a/misc/maintenance/remove_items_from_biblioitems.pl +++ b/misc/maintenance/remove_items_from_biblioitems.pl @@ -1,5 +1,23 @@ #!/usr/bin/perl +# Copyright 2010 BibLibre +# Copyright 2011 Equinox Software, Inc. +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + use strict; use warnings; @@ -7,44 +25,47 @@ use C4::Context; use C4::Biblio; use Getopt::Long; -my ($wherestring,$run,$want_help); -my $result = GetOptions( - 'where:s' => \$wherestring, +my ($wherestring, $run, $want_help); +my $result = GetOptions( + 'where:s' => \$wherestring, '--run' => \$run, - 'help|h' => \$want_help, + 'help|h' => \$want_help, ); -if ( not $result or $want_help ) { + +if ( not $result or not $run or $want_help ) { print_usage(); exit 0; } - -my $dbh=C4::Context->dbh; -my $querysth=qq{SELECT biblionumber from biblioitems }; -$querysth.=" WHERE $wherestring " if ($wherestring); -my $query=$dbh->prepare($querysth); - +my $dbh = C4::Context->dbh; +my $querysth = qq{SELECT biblionumber from biblioitems }; +$querysth .= " WHERE $wherestring " if ($wherestring); +my $query = $dbh->prepare($querysth); $query->execute; -while (my $biblionumber=$query->fetchrow){ - my $record=GetMarcBiblio($biblionumber); +while (my $biblionumber = $query->fetchrow){ + my $record = GetMarcBiblio($biblionumber); - if ($record){ - ModBiblio($record,$biblionumber,GetFrameworkCode($biblionumber)) ; + if ($record) { + ModBiblio($record, $biblionumber, GetFrameworkCode($biblionumber)) ; } else { print "error in $biblionumber : can't parse biblio"; } } + sub print_usage { print <<_USAGE_; $0: removes items from selected biblios +This utility is meant to be run as part of the upgrade to +Koha 3.4. It removes the 9XX fields in the bib records that +store a (now) duplicate copy of the item record information. After +running this utility, a full reindexing of the bibliographic records +should be run using rebuild_zebra.pl -b -r. Parameters: - -where use this to limit modifications to some biblios - --run run the command - --help or -h show this message. + -where use this to limit modifications to selected biblios + --run perform the update + --help or -h show this message _USAGE_ } - -# -- 1.7.1 From a13a59a8407bee8af620c613f258578cacc48ec2 Mon Sep 17 00:00:00 2001 From: Galen Charlton Date: Thu, 3 Mar 2011 18:24:19 -0500 Subject: [PATCH 3/9] bug 5579: fix moving items from bib to another * Must signal both bibs to be reindexed Signed-off-by: Galen Charlton --- C4/Items.pm | 55 +++++++------------------------------------------------ 1 files changed, 7 insertions(+), 48 deletions(-) diff --git a/C4/Items.pm b/C4/Items.pm index 741ea4d..50e3760 100644 --- a/C4/Items.pm +++ b/C4/Items.pm @@ -2021,59 +2021,18 @@ sub MoveItemFromBiblio { $sth = $dbh->prepare("UPDATE items SET biblioitemnumber = ?, biblionumber = ? WHERE itemnumber = ? AND biblionumber = ?"); my $return = $sth->execute($tobiblioitem, $tobiblio, $itemnumber, $frombiblio); if ($return == 1) { - - # Getting framework - my $frameworkcode = GetFrameworkCode($frombiblio); - - # Getting marc field for itemnumber - my ($itemtag, $itemsubfield) = GetMarcFromKohaField('items.itemnumber', $frameworkcode); - - # Getting the record we want to move the item from - my $record = GetMarcBiblio($frombiblio); - - # The item we want to move - my $item; - - # For each item - foreach my $fielditem ($record->field($itemtag)){ - # If it is the item we want to move - if ($fielditem->subfield($itemsubfield) == $itemnumber) { - # We save it - $item = $fielditem; - # Then delete it from the record - $record->delete_field($fielditem) - } - } - - # If we found an item (should always true, except in case of database-marcxml inconsistency) - if ($item) { - + ModZebra( $tobiblio, "specialUpdate", "biblioserver", undef, undef ); + ModZebra( $frombiblio, "specialUpdate", "biblioserver", undef, undef ); # Checking if the item we want to move is in an order - my $order = GetOrderFromItemnumber($itemnumber); + my $order = GetOrderFromItemnumber($itemnumber); if ($order) { - # Replacing the biblionumber within the order if necessary - $order->{'biblionumber'} = $tobiblio; + # Replacing the biblionumber within the order if necessary + $order->{'biblionumber'} = $tobiblio; ModOrder($order); } - - # Saving the modification - #ModBiblioMarc($record, $frombiblio, $frameworkcode); - - # Getting the record we want to move the item to - #$record = GetMarcBiblio($tobiblio); - - # Inserting the previously saved item - #$record->insert_fields_ordered($item); - - # Saving the modification - #ModBiblioMarc($record, $tobiblio, $frameworkcode); - - } else { - return undef; + return $tobiblio; } - } else { - return undef; - } + return; } =head2 DelItemCheck -- 1.7.1 From 2154a9fc4f77b4225f81cc3aa8988f4f72e8e441 Mon Sep 17 00:00:00 2001 From: Galen Charlton Date: Thu, 3 Mar 2011 18:29:32 -0500 Subject: [PATCH 4/9] bug 5579: correctly signal when to reindex bibs * when item changes * when new item is added Signed-off-by: Galen Charlton --- C4/Items.pm | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/C4/Items.pm b/C4/Items.pm index 50e3760..a866789 100644 --- a/C4/Items.pm +++ b/C4/Items.pm @@ -261,9 +261,9 @@ sub AddItem { $item->{'itemnumber'} = $itemnumber; # create MARC tag representing item and add to bib - my $new_item_marc = _marc_from_item_hash($item, $frameworkcode, $unlinked_item_subfields); - _add_item_field_to_biblio($new_item_marc, $item->{'biblionumber'}, $frameworkcode ); + #my $new_item_marc = _marc_from_item_hash($item, $frameworkcode, $unlinked_item_subfields); #_add_item_field_to_biblio($new_item_marc, $item->{'biblionumber'}, $frameworkcode ); + ModZebra( $item->{biblionumber}, "specialUpdate", "biblioserver", undef, undef ); logaction("CATALOGUING", "ADD", $itemnumber, "item") if C4::Context->preference("CataloguingLog"); @@ -507,6 +507,7 @@ sub ModItem { # update biblio MARC XML my $whole_item = GetItem($itemnumber) or die "FAILED GetItem($itemnumber)"; + ModZebra( $whole_item->{biblionumber}, "specialUpdate", "biblioserver", undef, undef ); unless (defined $unlinked_item_subfields) { $unlinked_item_subfields = _parse_unlinked_item_subfields_from_xml($whole_item->{'more_subfields_xml'}); @@ -2109,7 +2110,6 @@ sub _koha_modify_item { $error.="ERROR in _koha_modify_item $query".$dbh->errstr; warn $error; } - ModZebra( $item->{biblionumber}, "specialUpdate", "biblioserver", undef, undef ); return ($item->{'itemnumber'},$error); } -- 1.7.1 From f52dc33e4b122dcbffc2144a638ee15327566fcc Mon Sep 17 00:00:00 2001 From: Galen Charlton Date: Thu, 3 Mar 2011 18:41:42 -0500 Subject: [PATCH 5/9] bug 5579: new routine to embed items in bib Adds a new routine, C4::Biblio::EmbedItemsInMarcBiblio, to embed the items in the bib record when necessary: * cataloging/additem.pl * rebuild_zebra.pl Signed-off-by: Galen Charlton --- C4/Biblio.pm | 33 +++++++++++++++++++++++++++++++++ cataloguing/additem.pl | 3 ++- misc/migration_tools/rebuild_zebra.pl | 24 +++++++++++------------- 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/C4/Biblio.pm b/C4/Biblio.pm index 9cc39e2..c8f3a14 100644 --- a/C4/Biblio.pm +++ b/C4/Biblio.pm @@ -2,6 +2,7 @@ package C4::Biblio; # Copyright 2000-2002 Katipo Communications # Copyright 2010 BibLibre +# Copyright 2011 Equinox Software, Inc. # # This file is part of Koha. # @@ -35,6 +36,7 @@ use C4::ClassSource; use C4::Charset; require C4::Heading; require C4::Serials; +require C4::Items; use vars qw($VERSION @ISA @EXPORT); @@ -2530,6 +2532,37 @@ sub GetNoZebraIndexes { return %indexes; } +=head2 EmbedItemsInMarcBiblio + + EmbedItemsInMarcBiblio($marc, $biblionumber); + +Given a MARC::Record object containing a bib record, +modify it to include the items attached to it as 9XX +per the bib's MARC framework. + +=cut + +sub EmbedItemsInMarcBiblio { + my ($marc, $biblionumber) = @_; + + my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField('items.itemnumber', GetFrameworkCode($biblionumber)); + # delete any fields already in the record that use the item tag + foreach my $field ( $marc->field($itemtag) ) { + $marc->delete_field($field); + } + + # ... and embed the current items + my $dbh = C4::Context->dbh; + my $sth = $dbh->prepare("SELECT itemnumber FROM items WHERE biblionumber = ?"); + $sth->execute($biblionumber); + my @item_fields; + while (my ($itemnumber) = $sth->fetchrow_array) { + my $item_marc = C4::Items::GetMarcItem($biblionumber, $itemnumber); + push @item_fields, $item_marc->field($itemtag); + } + $marc->insert_fields_ordered(@item_fields); +} + =head1 INTERNAL FUNCTIONS =head2 _DelBiblioNoZebra($biblionumber,$record,$server); diff --git a/cataloguing/additem.pl b/cataloguing/additem.pl index b2b1192..2c312f5 100755 --- a/cataloguing/additem.pl +++ b/cataloguing/additem.pl @@ -494,13 +494,14 @@ if ($op eq "additem") { # now, build existiing item list my $temp = GetMarcBiblio( $biblionumber ); -my @fields = $temp->fields(); #my @fields = $record->fields(); my %witness; #---- stores the list of subfields used at least once, with the "meaning" of the code my @big_array; #---- finds where items.itemnumber is stored my ( $itemtagfield, $itemtagsubfield) = &GetMarcFromKohaField("items.itemnumber", $frameworkcode); my ($branchtagfield, $branchtagsubfield) = &GetMarcFromKohaField("items.homebranch", $frameworkcode); +C4::Biblio::EmbedItemsInMarcBiblio($temp, $biblionumber); +my @fields = $temp->fields(); foreach my $field (@fields) { next if ( $field->tag() < 10 ); diff --git a/misc/migration_tools/rebuild_zebra.pl b/misc/migration_tools/rebuild_zebra.pl index 74c73a1..5be01cf 100755 --- a/misc/migration_tools/rebuild_zebra.pl +++ b/misc/migration_tools/rebuild_zebra.pl @@ -320,26 +320,22 @@ sub export_marc_records_from_sth { ? GetXmlBiblio( $record_number ) : GetAuthorityXML( $record_number ); if ($record_type eq 'biblio'){ - #CALL sub ProcessItems - my @items=GetItemsInfo($record_number,'intra',30); + my @items = GetItemsInfo($record_number, 'intra'); if (@items){ - my $record=MARC::Record->new; + my $record = MARC::Record->new; my @itemsrecord; foreach my $item (@items){ - my $record=Item2Marc($item, $record_number); + my $record = Item2Marc($item, $record_number); push @itemsrecord, $record->field($itemtag); - #if xml then print itemfield as xml - # and update marcxml - # else push field } $record->insert_fields_ordered(@itemsrecord); my $itemsxml=$record->as_xml_record(); - my $searchstring='\n'; - my $index=index($itemsxml,'\n',0); - $itemsxml=substr($itemsxml,$index+length($searchstring)); - $searchstring=''; - $marcxml=substr($marcxml,0,index($marcxml,$searchstring)); - $marcxml.=$itemsxml; + my $searchstring = '\n'; + my $index = index($itemsxml, '\n', 0); + $itemsxml = substr($itemsxml, $index + length($searchstring)); + $searchstring = ''; + $marcxml = substr($marcxml, 0, index($marcxml, $searchstring)); + $marcxml .= $itemsxml; } } if ( $marcxml ) { @@ -469,6 +465,8 @@ sub get_raw_marc_record { return; } } + # ITEM + C4::Biblio::EmbedItemsInMarcBiblio($marc, $record_number); } else { eval { $marc = GetAuthority($record_number); }; if ($@) { -- 1.7.1 From de9ec160241aa3db59d9b841eb4033ae2cb75bfc Mon Sep 17 00:00:00 2001 From: Galen Charlton Date: Thu, 3 Mar 2011 19:09:35 -0500 Subject: [PATCH 6/9] bug 5579: reduce processing to log item changes * no need to record full bib MARC when logging change to an item record * IDEA: set up a separate ItemLog syspref Signed-off-by: Galen Charlton --- C4/Items.pm | 11 ++--------- 1 files changed, 2 insertions(+), 9 deletions(-) diff --git a/C4/Items.pm b/C4/Items.pm index a866789..672ed95 100644 --- a/C4/Items.pm +++ b/C4/Items.pm @@ -33,6 +33,7 @@ use C4::Branch; require C4::Reserves; use C4::Charset; use C4::Acquisition; +use Data::Dumper; use vars qw($VERSION @ISA @EXPORT); @@ -509,15 +510,7 @@ sub ModItem { my $whole_item = GetItem($itemnumber) or die "FAILED GetItem($itemnumber)"; ModZebra( $whole_item->{biblionumber}, "specialUpdate", "biblioserver", undef, undef ); - unless (defined $unlinked_item_subfields) { - $unlinked_item_subfields = _parse_unlinked_item_subfields_from_xml($whole_item->{'more_subfields_xml'}); - } - my $new_item_marc = _marc_from_item_hash($whole_item, $frameworkcode, $unlinked_item_subfields) - or die "FAILED _marc_from_item_hash($whole_item, $frameworkcode)"; - - #_replace_item_field_in_biblio($new_item_marc, $biblionumber, $itemnumber, $frameworkcode); - ($new_item_marc eq '0') and die "$new_item_marc is '0', not hashref"; # logaction line would crash anyway - logaction("CATALOGUING", "MODIFY", $itemnumber, $new_item_marc->as_formatted) if C4::Context->preference("CataloguingLog"); + logaction("CATALOGUING", "MODIFY", $itemnumber, Dumper($item)) if C4::Context->preference("CataloguingLog"); } =head2 ModItemTransfer -- 1.7.1 From c421420806a6dc6029b570aab39f03d79bdbadd6 Mon Sep 17 00:00:00 2001 From: Galen Charlton Date: Sat, 2 Apr 2011 20:48:40 -0400 Subject: [PATCH 7/9] bug 5579: remove sync_items_in_marc_bib.pl Now that item data is inserted into the MARC bib record at the point of indexing, sync_items_in_marc_bib.pl is no longer needed. Signed-off-by: Galen Charlton --- misc/maintenance/sync_items_in_marc_bib.pl | 151 ---------------------------- 1 files changed, 0 insertions(+), 151 deletions(-) delete mode 100755 misc/maintenance/sync_items_in_marc_bib.pl diff --git a/misc/maintenance/sync_items_in_marc_bib.pl b/misc/maintenance/sync_items_in_marc_bib.pl deleted file mode 100755 index d3eb2ef..0000000 --- a/misc/maintenance/sync_items_in_marc_bib.pl +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/perl - -use strict; -#use warnings; FIXME - Bug 2505 -BEGIN { - # find Koha's Perl modules - # test carefully before changing this - use FindBin; - eval { require "$FindBin::Bin/../kohalib.pl" }; -} - -use C4::Context; -use C4::Biblio; -use C4::Items; -use Getopt::Long; - -$| = 1; - -# command-line parameters -my $want_help = 0; -my $do_update = 0; -my $wherestrings; - -my $result = GetOptions( - 'run-update' => \$do_update, - 'where=s@' => \$wherestrings, - 'h|help' => \$want_help, -); - -if (not $result or $want_help or not $do_update) { - print_usage(); - exit 0; -} - -my $num_bibs_processed = 0; -my $num_bibs_modified = 0; -my $num_marc_items_deleted = 0; -my $num_marc_items_added = 0; -my $num_bad_bibs = 0; -my $dbh = C4::Context->dbh; -$dbh->{AutoCommit} = 0; - -our ($itemtag, $itemsubfield) = GetMarcFromKohaField("items.itemnumber", ''); -our ($item_sth) = $dbh->prepare("SELECT itemnumber FROM items WHERE biblionumber = ?"); - -process_bibs(); -$dbh->commit(); - -exit 0; - -sub process_bibs { - my $sql = "SELECT biblionumber FROM biblio JOIN biblioitems USING (biblionumber)"; - $sql.="WHERE ". join(" AND ",@$wherestrings) if ($wherestrings); - $sql.="ORDER BY biblionumber ASC"; - my $sth = $dbh->prepare($sql); - eval{$sth->execute();}; - if ($@){ die "error $@";}; - while (my ($biblionumber) = $sth->fetchrow_array()) { - $num_bibs_processed++; - process_bib($biblionumber); - - if (($num_bibs_processed % 100) == 0) { - print_progress_and_commit($num_bibs_processed); - } - } - - $dbh->commit; - - print <<_SUMMARY_; - -Embedded item synchronization report ------------------------------------- -Number of bibs checked: $num_bibs_processed -Number of bibs modified: $num_bibs_modified -Number of item fields removed from bibs: $num_marc_items_deleted -Number of item fields added to bibs: $num_marc_items_added -Number of bibs with errors: $num_bad_bibs -_SUMMARY_ -} - -sub process_bib { - my $biblionumber = shift; - - my $bib = GetMarcBiblio($biblionumber); - unless (defined $bib) { - print "\nCould not retrieve bib $biblionumber from the database - record is corrupt.\n"; - $num_bad_bibs++; - return; - } - - my $bib_modified = 0; - - # delete any item tags - foreach my $field ($bib->field($itemtag)) { - unless ($bib->delete_field($field)) { - warn "Could not delete item in $itemtag for biblionumber $biblionumber"; - next; - } - $num_marc_items_deleted++; - $bib_modified = 1; - } - - # add back items from items table - $item_sth->execute($biblionumber); - while (my $itemnumber = $item_sth->fetchrow_array) { - my $marc_item = C4::Items::GetMarcItem($biblionumber, $itemnumber); - unless ($marc_item) { - warn "FAILED C4::Items::GetMarcItem for biblionumber=$biblionumber, itemnumber=$itemnumber"; - next; - } - foreach my $item_field ($marc_item->field($itemtag)) { - $bib->insert_fields_ordered($item_field); - $num_marc_items_added++; - $bib_modified = 1; - } - } - - if ($bib_modified) { - ModBiblioMarc($bib, $biblionumber, GetFrameworkCode($biblionumber)); - $num_bibs_modified++; - } - -} - -sub print_progress_and_commit { - my $recs = shift; - $dbh->commit(); - print "... processed $recs records\n"; -} - -sub print_usage { - print <<_USAGE_; -$0: synchronize item data embedded in MARC bibs - -Replaces the item data embedded in the MARC bib -records (for indexing) with the authoritative -item data as stored in the items table. - -If Zebra is used, run rebuild_zebra.pl -b -r after -running this script. - -NOTE: this script should be run only if there is -reason to suspect that the embedded item tags are -not in sync with the items table. - -Parameters: - --run-update run the synchronization - --where condition selects the biblios on a criterium (Repeatable) - --help or -h show this message. -_USAGE_ -} -- 1.7.1 From 7097149f3bff3ac0a276918d24c5663635c5458e Mon Sep 17 00:00:00 2001 From: Galen Charlton Date: Sat, 2 Apr 2011 21:22:02 -0400 Subject: [PATCH 8/9] bug 5579: remove unused routines Remove the following routines which used to handle embedding item data in the bibliographic record: C4::Items::_replace_item_field_in_biblio C4::Items::_add_item_field_to_biblio Signed-off-by: Galen Charlton --- C4/Items.pm | 71 ++-------------------------------------------------------- 1 files changed, 3 insertions(+), 68 deletions(-) diff --git a/C4/Items.pm b/C4/Items.pm index 672ed95..ceb831f 100644 --- a/C4/Items.pm +++ b/C4/Items.pm @@ -261,9 +261,6 @@ sub AddItem { my ( $itemnumber, $error ) = _koha_new_item( $item, $item->{barcode} ); $item->{'itemnumber'} = $itemnumber; - # create MARC tag representing item and add to bib - #my $new_item_marc = _marc_from_item_hash($item, $frameworkcode, $unlinked_item_subfields); - #_add_item_field_to_biblio($new_item_marc, $item->{'biblionumber'}, $frameworkcode ); ModZebra( $item->{biblionumber}, "specialUpdate", "biblioserver", undef, undef ); logaction("CATALOGUING", "ADD", $itemnumber, "item") if C4::Context->preference("CataloguingLog"); @@ -506,9 +503,9 @@ sub ModItem { # update items table _koha_modify_item($item); - # update biblio MARC XML - my $whole_item = GetItem($itemnumber) or die "FAILED GetItem($itemnumber)"; - ModZebra( $whole_item->{biblionumber}, "specialUpdate", "biblioserver", undef, undef ); + # request that bib be reindexed so that searching on current + # item status is possible + ModZebra( $biblionumber, "specialUpdate", "biblioserver", undef, undef ); logaction("CATALOGUING", "MODIFY", $itemnumber, Dumper($item)) if C4::Context->preference("CataloguingLog"); } @@ -2187,68 +2184,6 @@ sub _marc_from_item_hash { return $item_marc; } -=head2 _add_item_field_to_biblio - - _add_item_field_to_biblio($item_marc, $biblionumber, $frameworkcode); - -Adds the fields from a MARC record containing the -representation of a Koha item record to the MARC -biblio record. The input C<$item_marc> record -is expect to contain just one field, the embedded -item information field. - -=cut - -sub _add_item_field_to_biblio { - my ($item_marc, $biblionumber, $frameworkcode) = @_; - - my $biblio_marc = GetMarcBiblio($biblionumber); - foreach my $field ($item_marc->fields()) { - $biblio_marc->append_fields($field); - } - - ModBiblioMarc($biblio_marc, $biblionumber, $frameworkcode); -} - -=head2 _replace_item_field_in_biblio - - &_replace_item_field_in_biblio($item_marc, $biblionumber, $itemnumber, $frameworkcode) - -Given a MARC::Record C<$item_marc> containing one tag with the MARC -representation of the item, examine the biblio MARC -for the corresponding tag for that item and -replace it with the tag from C<$item_marc>. - -=cut - -sub _replace_item_field_in_biblio { - my ($ItemRecord, $biblionumber, $itemnumber, $frameworkcode) = @_; - my $dbh = C4::Context->dbh; - - # get complete MARC record & replace the item field by the new one - my $completeRecord = GetMarcBiblio($biblionumber); - my ($itemtag,$itemsubfield) = GetMarcFromKohaField("items.itemnumber",$frameworkcode); - my $itemField = $ItemRecord->field($itemtag); - my @items = $completeRecord->field($itemtag); - my $found = 0; - foreach (@items) { - if ($_->subfield($itemsubfield) eq $itemnumber) { - $_->replace_with($itemField); - $found = 1; - } - } - - unless ($found) { - # If we haven't found the matching field, - # just add it. However, this means that - # there is likely a bug. - $completeRecord->append_fields($itemField); - } - - # save the record - #ModBiblioMarc($completeRecord, $biblionumber, $frameworkcode); -} - =head2 _repack_item_errors Add an error message hash generated by C -- 1.7.1 From ba219f74251db5604d37fac1fca792c635d37746 Mon Sep 17 00:00:00 2001 From: Galen Charlton Date: Sat, 2 Apr 2011 22:19:39 -0400 Subject: [PATCH 9/9] bug 5579: teach MARC export to embed items when needed Signed-off-by: Galen Charlton --- tools/export.pl | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/export.pl b/tools/export.pl index fc0c7be..d858f44 100755 --- a/tools/export.pl +++ b/tools/export.pl @@ -133,13 +133,14 @@ if ($op eq "export") { next; } next if not defined $record; - if ( $dont_export_items || $strip_nonlocal_items || $limit_ind_branch) { + C4::Biblio::EmbedItemsInMarcBiblio($record, $biblionumber) unless $dont_export_items; + if ($strip_nonlocal_items || $limit_ind_branch) { my ( $homebranchfield, $homebranchsubfield ) = GetMarcFromKohaField( 'items.homebranch', '' ); for my $itemfield ($record->field($homebranchfield)){ # if stripping nonlocal items, use loggedinuser's branch if they didn't select one $branch = C4::Context->userenv->{'branch'} unless $branch; - $record->delete_field($itemfield) if($dont_export_items || ($itemfield->subfield($homebranchsubfield) ne $branch) ) ; + $record->delete_field($itemfield) if($itemfield->subfield($homebranchsubfield) ne $branch) ; } } -- 1.7.1