From 2d46b8f3ee57e3c6c5030864e57d3a88a4202fc7 Mon Sep 17 00:00:00 2001 From: Olli-Antti Kivilahti Date: Fri, 4 Nov 2016 11:57:12 +0200 Subject: [PATCH] Bug 15399 - MARCXML records larger than 1 MB (1048576 bytes) are not searchable. fka. KD-1512 - Remove items from the search index for serial mothers Make stuff searchable again. --- misc/migration_tools/rebuild_zebra.pl | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/misc/migration_tools/rebuild_zebra.pl b/misc/migration_tools/rebuild_zebra.pl index 410f2e7..22a6201 100755 --- a/misc/migration_tools/rebuild_zebra.pl +++ b/misc/migration_tools/rebuild_zebra.pl @@ -680,6 +680,7 @@ sub get_raw_marc_record { warn "error retrieving biblio $record_number"; return; } + _protectZebraFromTooManyItems($marc); } } else { eval { $marc = GetAuthority($record_number); }; @@ -788,6 +789,36 @@ sub do_indexing { } +=head2 _protectZebraFromTooManyItems + + _protectZebraFromTooManyItems($marcRecord); + +Finds the repeatable items-field inside the given MARC::Record and drops all but the first handful of new items. +Zebra cannot index a record if it is over 1MB in size, and this limit can be reached by injecting the Item-data +to the MARC-Record. +For ex. serials have one MARC record with possibly thousands of items. + +=cut + +my ($itemsBarcodeFieldCode, $itemsBarcodeSubfieldCode, $serialFieldCode, $serialSubfieldCode); +sub _protectZebraFromTooManyItems { + use Modern::Perl; #WTF this script doesn't even have warnings on. Goddamn + my ($marc) = @_; + + my ($itemsBarcodeFieldCode, $itemsBarcodeSubfieldCode) = C4::Biblio::GetMarcFromKohaField('items.barcode','') unless $itemsBarcodeFieldCode; + my ($serialFieldCode, $serialSubfieldCode) = C4::Biblio::GetMarcFromKohaField('biblio.serial','') unless $serialFieldCode; + if ($marc->subfield($serialFieldCode, $serialSubfieldCode)) { #Is a serial, has a subscription basically + my $maxIndexedItems = 500; #in MARC mode we should be able to create a ISO-format MARC21 record with this many items. Record size limit is 99999 characters. + my @fields = reverse $marc->field($itemsBarcodeFieldCode); #field actually returns fields + my $fCnt = scalar(@fields)-1; + + if (scalar(@fields) > $maxIndexedItems) { + my @deletable = @fields[$maxIndexedItems..$fCnt]; + $marc->delete_fields(@deletable); #Trim the rest of the items. + } + } +} + sub _flock { # test if flock is present; if so, use it; if not, return true # op refers to the official flock operations including LOCK_EX, -- 1.9.1