From 241556f4bb9b9672729c560a3837902834d78db6 Mon Sep 17 00:00:00 2001 From: Marcel de Rooy Date: Fri, 5 Sep 2014 17:40:10 +0200 Subject: [PATCH] Bug 12872: Exclude limited number of fields from Zebra indexing Content-Type: text/plain; charset=utf-8 This patch adds the use of preference ZebraExclude to rebuild_zebra.pl. If that pref does not exist or is empty, behavior is unchanged. The pref allows you to exclude fields, specific subfields, and even field groups like 9XX from passing them to Zebra. The magic field XXX allows you to skip a whole record while indexing; this is of course only practical with a preceding specific condition. The pref may contain multiple lines with their own conditions. A line may start with a regex like /regex/:fieldlist. The regex is applied to the marcxml. If it matches, the field excludes are processed. Test plan: The plan includes a few rebuilds. It may be practical to run this test on a smaller test database. [1a] Edit ZebraExclude syspref. Set it to 590, 952e [1b] Edit a few records/items for these fields. [1c] Rebuild the zebra index. [1d] Check that info in 590 and 952e cannot be found in Zebra. [2a] Edit ZebraExclude syspref. Set it to /_skip_/: XXX [2b] Enter the _skip_ string into a few biblios (not in the same field). [2c] Rebuild the zebra index. [2d] Check that you cannot find these records in Zebra. [3a] Edit ZebraExclude syspref. Set it to 9XX [3b] Edit a few 9XX fields including some items (952). [3c] Rebuild the zebra index. [3d] Check that you cannot find 9XX info in Zebra. [4 ] Enter some other value in the pref. Edit, rebuild and check. Signed-off-by: Paola Rossi --- misc/migration_tools/ZebraExclude.pm | 156 +++++++++++++++++++++++++++++++++ misc/migration_tools/rebuild_zebra.pl | 19 +++-- 2 files changed, 168 insertions(+), 7 deletions(-) create mode 100644 misc/migration_tools/ZebraExclude.pm diff --git a/misc/migration_tools/ZebraExclude.pm b/misc/migration_tools/ZebraExclude.pm new file mode 100644 index 0000000..6e8b6e1 --- /dev/null +++ b/misc/migration_tools/ZebraExclude.pm @@ -0,0 +1,156 @@ +#!/usr/bin/perl + +# Copyright 2014 Rijksmuseum +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; + +sub load_zebra_exclude { + my $pref = C4::Context->preference('ZebraExclude'); + #this pref consists of lines /condition/:field list + #the condition is a regex applied to the marcxml (and is optional) + #field list contains XXX (whole record), 9XX (tab), 700 (field) or + #650$ac (one or more subfields) + return if !$pref; + + my $ret; #return hashref + #add the biblionumber and biblioitemnumber tag for Koha to returned hash + #we are asking for the default framework just as GetMarcBiblio does + #the returned subfield is ignored; we will not exclude these tags + ( $ret->{biblio} ) = GetMarcFromKohaField( "biblio.biblionumber", ''); + ( $ret->{biblioitem} ) = + GetMarcFromKohaField( "biblioitems.biblioitemnumber", ''); + + my @lines= split "\n", $pref; + $ret->{list} = []; + foreach my $li ( @lines ) { + my ( $regex, $list ); + if( $li =~ /\/(.*)(?{list}}, [ $fh, $regex ] if $fh; + } + return $ret; +} + +sub exclude_fields { + my ($marc, $exclude) = @_; + + foreach my $rf ( @{$exclude->{list}} ) { + my $rx= $rf->[1]; + #check the condition if present; if not, continue + #if the condition fails, skip this exclude step + if( $rx && $marc->as_xml !~ /$rx/ ) { + next; + } + + #check wildcards like XXX or 9XX + my $wildcards = $rf->[0]->{wildcard}; + foreach my $w ( @$wildcards ) { + if( $w eq 'XXX' ) { + #skip the whole record + return; + } else { + $w =~ s/XX/\.\./; + $marc->delete_fields( _skip999($marc->field($w), $exclude) ); + } + } + + #check whole fields + my $wholefields= $rf->[0]->{field}; + foreach( @$wholefields ) { + $marc->delete_fields( $marc->field($_) ); + } + + #check subfields + my $subs= $rf->[0]->{subfield}; + foreach my $ref ( @$subs ) { + my ($tag, $sub)= @$ref; + foreach my $fld ( $marc->field($tag) ) { + $fld->delete_subfield( code => qr/$sub/ ); + if( !$fld->subfields() ) { + #if it was the last subfield, delete whole field + $marc->delete_fields( $fld ); + } + } + } + } + return 1; +} + +sub _test_condition { + my $regex= shift; + if( $regex ) { + #test the regex + eval { 'whatever string' =~ /$regex/ }; + if( $@ ) { + warn "Zebra exclude regex $regex fails!\n"; + return; + } + } + return $regex; +} + +sub _parse_fieldlist { + my ( $list, $check999 ) = @_; + + my $ret={}; + my @list = split ",", $list; + foreach( @list ) { + next if !/^\s*(\d{3}|\dXX|XXX)\s*[\$:-]?\s*([A-Za-z0-9]*)\s*$/; + my $tag= $1; + my $sub= $2; + if ( $tag =~ /XX/ ) { + #we ignore sub here + #the test for 999 tags in this case is done later on + push @{$ret->{wildcard}}, $tag; + } elsif( defined $sub && $sub ne '' ) { #could be e.g. '0' + next if !_comp999( $tag, $check999 ); + push @{$ret->{subfield}}, [ $tag, $sub ]; + } else { + next if !_comp999( $tag, $check999 ); + push @{$ret->{field}}, $tag; + } + } + if( keys %$ret ) { # add possible missing hash keys + $ret->{wildcard} = [] if !exists $ret->{wildcard}; + $ret->{subfield} = [] if !exists $ret->{subfield}; + $ret->{field} = [] if !exists $ret->{field}; + return $ret; + } + return; +} + +sub _skip999 { + my $excl= pop @_; + #pass list of fields, removes the Koha biblio and biblioitem field + #in order to not delete them + #default tag is 999, but may have changed.. + grep { _comp999($_->tag, $excl) } @_; +} + +sub _comp999 { + my ($tag, $excl) = @_; + return $tag ne $excl->{biblio} && $tag ne $excl->{biblioitem}; +} + +1; diff --git a/misc/migration_tools/rebuild_zebra.pl b/misc/migration_tools/rebuild_zebra.pl index aa31cb2..f88c308 100755 --- a/misc/migration_tools/rebuild_zebra.pl +++ b/misc/migration_tools/rebuild_zebra.pl @@ -13,6 +13,7 @@ use C4::AuthoritiesMarc; use C4::Items; use Koha::RecordProcessor; use XML::LibXML; +use misc::migration_tools::ZebraExclude; use constant LOCK_FILENAME => 'rebuild..LCK'; @@ -155,6 +156,8 @@ my $kohadir = C4::Context->config('intranetdir'); my $bib_index_mode = C4::Context->config('zebra_bib_index_mode') // 'dom'; my $auth_index_mode = C4::Context->config('zebra_auth_index_mode') // 'dom'; +my $zebra_excl= load_zebra_exclude(); + my $dbh = C4::Context->dbh; my ($biblionumbertagfield,$biblionumbertagsubfield) = &GetMarcFromKohaField("biblio.biblionumber",""); my ($biblioitemnumbertagfield,$biblioitemnumbertagsubfield) = &GetMarcFromKohaField("biblioitems.biblioitemnumber",""); @@ -635,18 +638,20 @@ sub get_corrected_marc_record { my $marc = get_raw_marc_record($record_type, $record_number, $noxml); if (defined $marc) { + return if $zebra_excl && !defined( exclude_fields($marc,$zebra_excl) ); fix_leader($marc); - if ($record_type eq 'authority') { - fix_authority_id($marc, $record_number); - } elsif ($record_type eq 'biblio' && C4::Context->preference('IncludeSeeFromInSearches')) { - my $normalizer = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } ); - $marc = $normalizer->process($marc); - } if (C4::Context->preference("marcflavour") eq "UNIMARC") { fix_unimarc_100($marc); } + if ($record_type eq 'authority') { + fix_authority_id($marc, $record_number); + } elsif ($record_type eq 'biblio') { + if (C4::Context->preference('IncludeSeeFromInSearches')) { + my $normalizer = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } ); + $marc = $normalizer->process($marc); + } + } } - return $marc; } -- 1.7.7.6