From c6dd241f88a9529201db8eee907af7af1d11df16 Mon Sep 17 00:00:00 2001 From: Julian Maurice Date: Tue, 29 May 2012 09:54:57 +0200 Subject: [PATCH] Bug 8994: Improve FindDuplicateAuthority With two new sysprefs: - FindDuplicateOptions, - FindDuplicateMaxResults, you can modify the behaviour of this sub in order to fit your needs. You can now search on another index than 'he' and compare subfields of search results. For example, if FindDuplicateOptions is 'he,wrdl/2..ab | See/4..ab', FindDuplicate will search on index 'he' and try to compare $a and $b subfields for all 2XX fields of search results with $a and $b for 2XX fields of newly created authority. If there is no results it will try to search on 'See' index and compare 4XX$a$b of search results with 2XX$a$b of created authority. FindDuplicateMaxResults define how many results should be fetched for comparison. The '/2..ab' part can be omitted from FindDuplicateOptions. In this case, no comparison is made on subfields and the first search result is returned as duplicate. To keep the current behaviour, fill FindDuplicateOptions with 'he,wrdl' (default value in updatedatabase.pl) Note: search on 'at' index is still done in all cases, so only the authorities of the same type are returned. --- C4/AuthoritiesMarc.pm | 142 +++++++++++++++++---- installer/data/mysql/atomicupdate/bug_8994.sql | 4 + installer/data/mysql/sysprefs.sql | 2 + .../en/modules/admin/preferences/authorities.pref | 19 +++ 4 files changed, 140 insertions(+), 27 deletions(-) create mode 100644 installer/data/mysql/atomicupdate/bug_8994.sql diff --git a/C4/AuthoritiesMarc.pm b/C4/AuthoritiesMarc.pm index f617437..98b99f6 100644 --- a/C4/AuthoritiesMarc.pm +++ b/C4/AuthoritiesMarc.pm @@ -842,42 +842,130 @@ Comments : an improvement would be to return All the records that match. =cut +sub _find_duplicate_authority_try { + my ($try, $authorityfield, $authtypecode) = @_; + + my $filtervalues = qr([\001-\040\Q!'"`#$%&*+,-./:;<=>?@(){[}_|~\E\]]); + + my $op = 'and'; + if (C4::Context->preference('UseQueryParser')) { + my $QParser = C4::Context->queryparser; + if ($QParser) { + $op = '&&'; + } else { + $op = 'and'; + } + } + + # build a request for SearchAuthorities + my $query = "at=$authtypecode"; + + my $duplicaterecord; + if ($try->{field} and $try->{subfields}) { + foreach my $s (@{ $try->{subfields} }) { + my $v = $authorityfield->subfield($s); + if ($v) { + $v =~ s/$filtervalues/ /g; + $v =~ s/^\s*//; + $v =~ s/\s*$//; + $query .= qq/ $op $try->{index}="$v"/; + } + } + my $maxresults = C4::Context->preference('FindDuplicateAuthorityMaxResults') || 10; + my ($error, $results, $total_hits) = + C4::Search::SimpleSearch($query, 0, $maxresults, ["authorityserver"]); + return unless (!defined $error and @{$results}); + + foreach my $result (@$results) { + my $marcrecord = + C4::Search::new_record_from_zebra( 'authorityserver', $result ); + my @marcfields = $marcrecord->field($try->{field}); + next if (scalar @marcfields == 0); + my $stillmatch = 1; + foreach my $marcfield (@marcfields) { + foreach my $subfield (@{ $try->{subfields} }) { + my $authoritysubfield = $authorityfield->subfield($subfield); + my $marcsubfield = $marcfield->subfield($subfield); + if ( (defined $authoritysubfield xor defined $marcsubfield) + or (defined $authoritysubfield + and $authoritysubfield ne $marcsubfield) + ) { + $stillmatch = 0; + last; + } + } + last unless $stillmatch; + } + if ($stillmatch) { + $duplicaterecord = $marcrecord; + last; + } + } + } else { + foreach my $s ($authorityfield->subfields()) { + next unless ($s->[0] =~ /[A-z]/); + $s->[1] =~ s/$filtervalues/ /g; + $query .= qq/ $op $try->{index}="$s->[1]"/; + } + my ($error, $results, $total_hits) = + C4::Search::SimpleSearch( $query, 0, 1, ["authorityserver"] ); + if (!defined $error and @{$results}) { + $duplicaterecord = + C4::Search::new_record_from_zebra( 'authorityserver', + $results->[0] ); + } + } + + return $duplicaterecord; +} + sub FindDuplicateAuthority { + my ($record, $authtypecode) = @_; + + return unless $record; + + my $opts = C4::Context->preference('FindDuplicateAuthorityOptions'); + return unless $opts; - my ($record,$authtypecode)=@_; -# warn "IN for ".$record->as_formatted; my $dbh = C4::Context->dbh; -# warn "".$record->as_formatted; - my $sth = $dbh->prepare("select auth_tag_to_report from auth_types where authtypecode=?"); + my $sth = $dbh->prepare(" + SELECT auth_tag_to_report FROM auth_types WHERE authtypecode = ? + "); $sth->execute($authtypecode); my ($auth_tag_to_report) = $sth->fetchrow; $sth->finish; -# warn "record :".$record->as_formatted." auth_tag_to_report :$auth_tag_to_report"; - # build a request for SearchAuthorities - my $QParser; - $QParser = C4::Context->queryparser if (C4::Context->preference('UseQueryParser')); - my $op; - if ($QParser) { - $op = '&&'; - } else { - $op = 'and'; - } - my $query='at:'.$authtypecode.' '; - my $filtervalues=qr([\001-\040\Q!'"`#$%&*+,-./:;<=>?@(){[}_|~\E\]]); - if ($record->field($auth_tag_to_report)) { - foreach ($record->field($auth_tag_to_report)->subfields()) { - $_->[1]=~s/$filtervalues/ /g; $query.= " $op he:\"".$_->[1]."\"" if ($_->[0]=~/[A-z]/); + + my $authorityfield = $record->field($auth_tag_to_report); + return unless $authorityfield; + + my @tries; + $opts =~ s/ //g; + foreach my $try (split /\|/, $opts) { + my ($index, $matchcheck) = split m#/#, $try; + my ($field, @subfields); + if ($matchcheck and length($matchcheck) > 3) { + $field = substr $matchcheck, 0, 3; + @subfields = split(//, substr($matchcheck, 3)); } + push @tries, { + index => $index, + field => $field, + subfields => \@subfields, + }; } - my ($error, $results, $total_hits) = C4::Search::SimpleSearch( $query, 0, 1, [ "authorityserver" ] ); - # there is at least 1 result => return the 1st one - if (!defined $error && @{$results} ) { - my $marcrecord = C4::Search::new_record_from_zebra( - 'authorityserver', - $results->[0] - ); - return $marcrecord->field('001')->data,BuildSummary($marcrecord,$marcrecord->field('001')->data,$authtypecode); + + my $duplicaterecord; + foreach my $try (@tries) { + $duplicaterecord = _find_duplicate_authority_try($try, $authorityfield, + $authtypecode); + last if $duplicaterecord; + } + + if ($duplicaterecord) { + my $authid = $duplicaterecord->field('001')->data; + return $authid, BuildSummary($duplicaterecord, $authid, $authtypecode); } + # no result, returns nothing return; } diff --git a/installer/data/mysql/atomicupdate/bug_8994.sql b/installer/data/mysql/atomicupdate/bug_8994.sql new file mode 100644 index 0000000..67ffb5d --- /dev/null +++ b/installer/data/mysql/atomicupdate/bug_8994.sql @@ -0,0 +1,4 @@ +INSERT INTO systempreferences (variable, value, explanation, options, type) +VALUES +('FindDuplicateAuthorityOptions', 'he,wrdl', 'Options to determine how to search for duplicate authorities. General form: "index1/XXXcc#index2" where indexX is the index where we want to search, and XXXcc the field(s) and subfield(s) we want to check for each search results (if omitted, the first result is returned as duplicate)', NULL, 'Free'), +('FindDuplicateAuthorityMaxResults', '10', 'Maximum number of results fetched when searching a duplicate', NULL, 'Integer'); diff --git a/installer/data/mysql/sysprefs.sql b/installer/data/mysql/sysprefs.sql index ac79f44..b225e15 100644 --- a/installer/data/mysql/sysprefs.sql +++ b/installer/data/mysql/sysprefs.sql @@ -131,6 +131,8 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` ('FacetLabelTruncationLength','20',NULL,'Specify the facet max length in OPAC','Integer'), ('FacetMaxCount','20',NULL,'Specify the max facet count for each category','Integer'), ('FilterBeforeOverdueReport','0','','Do not run overdue report until filter selected','YesNo'), +('FindDuplicateAuthorityOptions','he,wrdl',NULL,'Options to determine how to search for duplicate authorities. General form: "index1/XXXcc|index2" where indexX is the index where we want to search, and XXXcc the field(s) and subfield(s) we want to check for each search results (if omitted, the first result is returned as duplicate)','Free'), +('FindDuplicateAuthorityMaxResults','10',NULL'Maximum number of results fetched when searching a duplicate','Free'), ('FineNotifyAtCheckin','0',NULL,'If ON notify librarians of overdue fines on the items they are checking in.','YesNo'), ('finesCalendar','noFinesWhenClosed','ignoreCalendar|noFinesWhenClosed','Specify whether to use the Calendar in calculating duedates and fines','Choice'), ('FinesIncludeGracePeriod','1',NULL,'If enabled, fines calculations will include the grace period.','YesNo'), diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/authorities.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/authorities.pref index 2762411..908d841 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/authorities.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/authorities.pref @@ -49,6 +49,25 @@ Authorities: defautl: "afrey50 ba0" type: textarea class: code + - + - pref: FindDuplicateAuthorityOptions + class: long + - '
Options to determine how to search for duplicate authorities when creating new authorities. General form: "index1/XXXcc | index2" where indexX is the index where we want to search, and XXXcc the field(s) and subfield(s) we want to check for each search results (if "XXXcc" part is omitted, the first result is returned as duplicate)' + - '
Examples:' + - '
- he,wrdl: Search in "he" index, do not check subfields data in search results, the first result is returned as a duplicate (default).' + - '
- he,wrdl/2..ab: Search in "he" index, and compare 2XX$a and 2XX$b of search results with XXX$a and XXX$b of created authority (XXX is the "Authority field to copy"). The first matching authority is returned as duplicate.' + - '
- he,wrdl | See,wrdl/4..ab: Search in "he" index and do not check subfields. If there is no results, search in "See" index and compare 4XX$a and 4XX$b of search results with XXX$a and XXX$b of created authority.' + - '
Notes:' + - '
- only authorities of the same type are returned by the search' + - '
- the query to find authorities is build with subfields in syspref. For example he,wrdl/2..ab will search he,wrdl="2XX$a" and "he,wrdl="2XX$b"' + - '
- if compared subfields are repeatable, only the first of each field is checked' + - '
- spaces are insignificant' + - '
- if value is empty, it disable authority duplicate searching.' + - + - 'Maximum number of results fetched when searching a duplicate:' + - pref: FindDuplicateAuthorityMaxResults + class: integer + - "It's only used when FindDuplicateAuthorityOptions contains fields and subfields to check against (default: 10)" Linker: - - Use the -- 1.9.1