Bugzilla – Attachment 31527 Details for
Bug 12872
Exclude some fields from Zebra indexing
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 12872: Exclude limited number of fields from Zebra indexing
Bug-12872-Exclude-limited-number-of-fields-from-Ze.patch (text/plain), 9.06 KB, created by
Marcel de Rooy
on 2014-09-11 13:28:13 UTC
(
hide
)
Description:
Bug 12872: Exclude limited number of fields from Zebra indexing
Filename:
MIME Type:
Creator:
Marcel de Rooy
Created:
2014-09-11 13:28:13 UTC
Size:
9.06 KB
patch
obsolete
>From 7eadbe7ee2d354aeca1b871b3e83768e54896310 Mon Sep 17 00:00:00 2001 >From: Marcel de Rooy <m.de.rooy@rijksmuseum.nl> >Date: Fri, 5 Sep 2014 17:40:10 +0200 >Subject: [PATCH] Bug 12872: Exclude limited number of fields from Zebra > indexing >Content-Type: text/plain; charset=utf-8 > >This patch adds the use of preference ZebraExclude to rebuild_zebra.pl. >If that pref does not exist or is empty, behavior is unchanged. > >The pref allows you to exclude fields, specific subfields, and even >field groups like 9XX from passing them to Zebra. The magic field XXX allows >you to skip a whole record while indexing; this is of course only >practical with a preceding specific condition. > >The pref may contain multiple lines with their own conditions. A line may >start with a regex like /regex/:fieldlist. The regex is applied to the >marcxml. If it matches, the field excludes are processed. > >Test plan: >The plan includes a few rebuilds. It may be practical to run this test on >a smaller test database. > >[1a] Edit ZebraExclude syspref. Set it to 590, 952e >[1b] Edit a few records/items for these fields. >[1c] Rebuild the zebra index. >[1d] Check that info in 590 and 952e cannot be found in Zebra. > >[2a] Edit ZebraExclude syspref. Set it to /_skip_/: XXX >[2b] Enter the _skip_ string into a few biblios (not in the same field). >[2c] Rebuild the zebra index. >[2d] Check that you cannot find these records in Zebra. > >[3a] Edit ZebraExclude syspref. Set it to 9XX >[3b] Edit a few 9XX fields including some items (952). >[3c] Rebuild the zebra index. >[3d] Check that you cannot find 9XX info in Zebra. > >[4 ] Enter some other value in the pref. Edit, rebuild and check. >--- > misc/migration_tools/ZebraExclude.pm | 156 +++++++++++++++++++++++++++++++++ > misc/migration_tools/rebuild_zebra.pl | 19 +++-- > 2 files changed, 168 insertions(+), 7 deletions(-) > create mode 100644 misc/migration_tools/ZebraExclude.pm > >diff --git a/misc/migration_tools/ZebraExclude.pm b/misc/migration_tools/ZebraExclude.pm >new file mode 100644 >index 0000000..6e8b6e1 >--- /dev/null >+++ b/misc/migration_tools/ZebraExclude.pm >@@ -0,0 +1,156 @@ >+#!/usr/bin/perl >+ >+# Copyright 2014 Rijksmuseum >+# >+# This file is part of Koha. >+# >+# Koha is free software; you can redistribute it and/or modify it under the >+# terms of the GNU General Public License as published by the Free Software >+# Foundation; either version 3 of the License, or (at your option) any later >+# version. >+# >+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY >+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR >+# A PARTICULAR PURPOSE. See the GNU General Public License for more details. >+# >+# You should have received a copy of the GNU General Public License along >+# with Koha; if not, write to the Free Software Foundation, Inc., >+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. >+ >+use Modern::Perl; >+ >+sub load_zebra_exclude { >+ my $pref = C4::Context->preference('ZebraExclude'); >+ #this pref consists of lines /condition/:field list >+ #the condition is a regex applied to the marcxml (and is optional) >+ #field list contains XXX (whole record), 9XX (tab), 700 (field) or >+ #650$ac (one or more subfields) >+ return if !$pref; >+ >+ my $ret; #return hashref >+ #add the biblionumber and biblioitemnumber tag for Koha to returned hash >+ #we are asking for the default framework just as GetMarcBiblio does >+ #the returned subfield is ignored; we will not exclude these tags >+ ( $ret->{biblio} ) = GetMarcFromKohaField( "biblio.biblionumber", ''); >+ ( $ret->{biblioitem} ) = >+ GetMarcFromKohaField( "biblioitems.biblioitemnumber", ''); >+ >+ my @lines= split "\n", $pref; >+ $ret->{list} = []; >+ foreach my $li ( @lines ) { >+ my ( $regex, $list ); >+ if( $li =~ /\/(.*)(?<!\\)\/\s*:\s*/ ) { >+ $regex= $1; >+ $list= $'; >+ } else { >+ $list= $li; >+ } >+ $regex = _test_condition( $regex ); >+ my $fh = _parse_fieldlist( $list , $ret ); >+ push @{$ret->{list}}, [ $fh, $regex ] if $fh; >+ } >+ return $ret; >+} >+ >+sub exclude_fields { >+ my ($marc, $exclude) = @_; >+ >+ foreach my $rf ( @{$exclude->{list}} ) { >+ my $rx= $rf->[1]; >+ #check the condition if present; if not, continue >+ #if the condition fails, skip this exclude step >+ if( $rx && $marc->as_xml !~ /$rx/ ) { >+ next; >+ } >+ >+ #check wildcards like XXX or 9XX >+ my $wildcards = $rf->[0]->{wildcard}; >+ foreach my $w ( @$wildcards ) { >+ if( $w eq 'XXX' ) { >+ #skip the whole record >+ return; >+ } else { >+ $w =~ s/XX/\.\./; >+ $marc->delete_fields( _skip999($marc->field($w), $exclude) ); >+ } >+ } >+ >+ #check whole fields >+ my $wholefields= $rf->[0]->{field}; >+ foreach( @$wholefields ) { >+ $marc->delete_fields( $marc->field($_) ); >+ } >+ >+ #check subfields >+ my $subs= $rf->[0]->{subfield}; >+ foreach my $ref ( @$subs ) { >+ my ($tag, $sub)= @$ref; >+ foreach my $fld ( $marc->field($tag) ) { >+ $fld->delete_subfield( code => qr/$sub/ ); >+ if( !$fld->subfields() ) { >+ #if it was the last subfield, delete whole field >+ $marc->delete_fields( $fld ); >+ } >+ } >+ } >+ } >+ return 1; >+} >+ >+sub _test_condition { >+ my $regex= shift; >+ if( $regex ) { >+ #test the regex >+ eval { 'whatever string' =~ /$regex/ }; >+ if( $@ ) { >+ warn "Zebra exclude regex $regex fails!\n"; >+ return; >+ } >+ } >+ return $regex; >+} >+ >+sub _parse_fieldlist { >+ my ( $list, $check999 ) = @_; >+ >+ my $ret={}; >+ my @list = split ",", $list; >+ foreach( @list ) { >+ next if !/^\s*(\d{3}|\dXX|XXX)\s*[\$:-]?\s*([A-Za-z0-9]*)\s*$/; >+ my $tag= $1; >+ my $sub= $2; >+ if ( $tag =~ /XX/ ) { >+ #we ignore sub here >+ #the test for 999 tags in this case is done later on >+ push @{$ret->{wildcard}}, $tag; >+ } elsif( defined $sub && $sub ne '' ) { #could be e.g. '0' >+ next if !_comp999( $tag, $check999 ); >+ push @{$ret->{subfield}}, [ $tag, $sub ]; >+ } else { >+ next if !_comp999( $tag, $check999 ); >+ push @{$ret->{field}}, $tag; >+ } >+ } >+ if( keys %$ret ) { # add possible missing hash keys >+ $ret->{wildcard} = [] if !exists $ret->{wildcard}; >+ $ret->{subfield} = [] if !exists $ret->{subfield}; >+ $ret->{field} = [] if !exists $ret->{field}; >+ return $ret; >+ } >+ return; >+} >+ >+sub _skip999 { >+ my $excl= pop @_; >+ #pass list of fields, removes the Koha biblio and biblioitem field >+ #in order to not delete them >+ #default tag is 999, but may have changed.. >+ grep { _comp999($_->tag, $excl) } @_; >+} >+ >+sub _comp999 { >+ my ($tag, $excl) = @_; >+ return $tag ne $excl->{biblio} && $tag ne $excl->{biblioitem}; >+} >+ >+1; >diff --git a/misc/migration_tools/rebuild_zebra.pl b/misc/migration_tools/rebuild_zebra.pl >index 682e4be..5bebccf 100755 >--- a/misc/migration_tools/rebuild_zebra.pl >+++ b/misc/migration_tools/rebuild_zebra.pl >@@ -13,6 +13,7 @@ use C4::AuthoritiesMarc; > use C4::Items; > use Koha::RecordProcessor; > use XML::LibXML; >+use misc::migration_tools::ZebraExclude; > > use constant LOCK_FILENAME => 'rebuild..LCK'; > >@@ -155,6 +156,8 @@ my $kohadir = C4::Context->config('intranetdir'); > my $bib_index_mode = C4::Context->config('zebra_bib_index_mode') || 'grs1'; > my $auth_index_mode = C4::Context->config('zebra_auth_index_mode') || 'dom'; > >+my $zebra_excl= load_zebra_exclude(); >+ > my $dbh = C4::Context->dbh; > my ($biblionumbertagfield,$biblionumbertagsubfield) = &GetMarcFromKohaField("biblio.biblionumber",""); > my ($biblioitemnumbertagfield,$biblioitemnumbertagsubfield) = &GetMarcFromKohaField("biblioitems.biblioitemnumber",""); >@@ -635,18 +638,20 @@ sub get_corrected_marc_record { > my $marc = get_raw_marc_record($record_type, $record_number, $noxml); > > if (defined $marc) { >+ return if $zebra_excl && !defined( exclude_fields($marc,$zebra_excl) ); > fix_leader($marc); >- if ($record_type eq 'authority') { >- fix_authority_id($marc, $record_number); >- } elsif ($record_type eq 'biblio' && C4::Context->preference('IncludeSeeFromInSearches')) { >- my $normalizer = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } ); >- $marc = $normalizer->process($marc); >- } > if (C4::Context->preference("marcflavour") eq "UNIMARC") { > fix_unimarc_100($marc); > } >+ if ($record_type eq 'authority') { >+ fix_authority_id($marc, $record_number); >+ } elsif ($record_type eq 'biblio') { >+ if (C4::Context->preference('IncludeSeeFromInSearches')) { >+ my $normalizer = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } ); >+ $marc = $normalizer->process($marc); >+ } >+ } > } >- > return $marc; > } > >-- >1.7.7.6
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 12872
:
31366
|
31367
|
31368
|
31527
|
31528
|
31529
|
32578
|
32584
|
32585
|
32586
|
32892
|
32893
|
32894