@@ -, +, @@ indexing --- misc/migration_tools/ZebraExclude.pm | 156 +++++++++++++++++++++++++++++++++ misc/migration_tools/rebuild_zebra.pl | 19 +++-- 2 files changed, 168 insertions(+), 7 deletions(-) create mode 100644 misc/migration_tools/ZebraExclude.pm --- a/misc/migration_tools/ZebraExclude.pm +++ a/misc/migration_tools/ZebraExclude.pm @@ -0,0 +1,156 @@ +#!/usr/bin/perl + +# Copyright 2014 Rijksmuseum +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; + +sub load_zebra_exclude { + my $pref = C4::Context->preference('ZebraExclude'); + #this pref consists of lines /condition/:field list + #the condition is a regex applied to the marcxml (and is optional) + #field list contains XXX (whole record), 9XX (tab), 700 (field) or + #650$ac (one or more subfields) + return if !$pref; + + my $ret; #return hashref + #add the biblionumber and biblioitemnumber tag for Koha to returned hash + #we are asking for the default framework just as GetMarcBiblio does + #the returned subfield is ignored; we will not exclude these tags + ( $ret->{biblio} ) = GetMarcFromKohaField( "biblio.biblionumber", ''); + ( $ret->{biblioitem} ) = + GetMarcFromKohaField( "biblioitems.biblioitemnumber", ''); + + my @lines= split "\n", $pref; + $ret->{list} = []; + foreach my $li ( @lines ) { + my ( $regex, $list ); + if( $li =~ /\/(.*)(?{list}}, [ $fh, $regex ] if $fh; + } + return $ret; +} + +sub exclude_fields { + my ($marc, $exclude) = @_; + + foreach my $rf ( @{$exclude->{list}} ) { + my $rx= $rf->[1]; + #check the condition if present; if not, continue + #if the condition fails, skip this exclude step + if( $rx && $marc->as_xml !~ /$rx/ ) { + next; + } + + #check wildcards like XXX or 9XX + my $wildcards = $rf->[0]->{wildcard}; + foreach my $w ( @$wildcards ) { + if( $w eq 'XXX' ) { + #skip the whole record + return; + } else { + $w =~ s/XX/\.\./; + $marc->delete_fields( _skip999($marc->field($w), $exclude) ); + } + } + + #check whole fields + my $wholefields= $rf->[0]->{field}; + foreach( @$wholefields ) { + $marc->delete_fields( $marc->field($_) ); + } + + #check subfields + my $subs= $rf->[0]->{subfield}; + foreach my $ref ( @$subs ) { + my ($tag, $sub)= @$ref; + foreach my $fld ( $marc->field($tag) ) { + $fld->delete_subfield( code => qr/$sub/ ); + if( !$fld->subfields() ) { + #if it was the last subfield, delete whole field + $marc->delete_fields( $fld ); + } + } + } + } + return 1; +} + +sub _test_condition { + my $regex= shift; + if( $regex ) { + #test the regex + eval { 'whatever string' =~ /$regex/ }; + if( $@ ) { + warn "Zebra exclude regex $regex fails!\n"; + return; + } + } + return $regex; +} + +sub _parse_fieldlist { + my ( $list, $check999 ) = @_; + + my $ret={}; + my @list = split ",", $list; + foreach( @list ) { + next if !/^\s*(\d{3}|\dXX|XXX)\s*[\$:-]?\s*([A-Za-z0-9]*)\s*$/; + my $tag= $1; + my $sub= $2; + if ( $tag =~ /XX/ ) { + #we ignore sub here + #the test for 999 tags in this case is done later on + push @{$ret->{wildcard}}, $tag; + } elsif( defined $sub && $sub ne '' ) { #could be e.g. '0' + next if !_comp999( $tag, $check999 ); + push @{$ret->{subfield}}, [ $tag, $sub ]; + } else { + next if !_comp999( $tag, $check999 ); + push @{$ret->{field}}, $tag; + } + } + if( keys %$ret ) { # add possible missing hash keys + $ret->{wildcard} = [] if !exists $ret->{wildcard}; + $ret->{subfield} = [] if !exists $ret->{subfield}; + $ret->{field} = [] if !exists $ret->{field}; + return $ret; + } + return; +} + +sub _skip999 { + my $excl= pop @_; + #pass list of fields, removes the Koha biblio and biblioitem field + #in order to not delete them + #default tag is 999, but may have changed.. + grep { _comp999($_->tag, $excl) } @_; +} + +sub _comp999 { + my ($tag, $excl) = @_; + return $tag ne $excl->{biblio} && $tag ne $excl->{biblioitem}; +} + +1; --- a/misc/migration_tools/rebuild_zebra.pl +++ a/misc/migration_tools/rebuild_zebra.pl @@ -13,6 +13,7 @@ use C4::AuthoritiesMarc; use C4::Items; use Koha::RecordProcessor; use XML::LibXML; +use misc::migration_tools::ZebraExclude; use constant LOCK_FILENAME => 'rebuild..LCK'; @@ -155,6 +156,8 @@ my $kohadir = C4::Context->config('intranetdir'); my $bib_index_mode = C4::Context->config('zebra_bib_index_mode') || 'grs1'; my $auth_index_mode = C4::Context->config('zebra_auth_index_mode') || 'dom'; +my $zebra_excl= load_zebra_exclude(); + my $dbh = C4::Context->dbh; my ($biblionumbertagfield,$biblionumbertagsubfield) = &GetMarcFromKohaField("biblio.biblionumber",""); my ($biblioitemnumbertagfield,$biblioitemnumbertagsubfield) = &GetMarcFromKohaField("biblioitems.biblioitemnumber",""); @@ -635,18 +638,20 @@ sub get_corrected_marc_record { my $marc = get_raw_marc_record($record_type, $record_number, $noxml); if (defined $marc) { + return if $zebra_excl && !defined( exclude_fields($marc,$zebra_excl) ); fix_leader($marc); - if ($record_type eq 'authority') { - fix_authority_id($marc, $record_number); - } elsif ($record_type eq 'biblio' && C4::Context->preference('IncludeSeeFromInSearches')) { - my $normalizer = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } ); - $marc = $normalizer->process($marc); - } if (C4::Context->preference("marcflavour") eq "UNIMARC") { fix_unimarc_100($marc); } + if ($record_type eq 'authority') { + fix_authority_id($marc, $record_number); + } elsif ($record_type eq 'biblio') { + if (C4::Context->preference('IncludeSeeFromInSearches')) { + my $normalizer = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } ); + $marc = $normalizer->process($marc); + } + } } - return $marc; } --