From 6aa934ef5c4e8b5f41314b5eb8899d618e5473d3 Mon Sep 17 00:00:00 2001 From: Nicholas van Oudtshoorn Date: Thu, 14 Jul 2016 12:47:57 +0800 Subject: [PATCH] Create a new Linker, BestMatch, that will link to the identical record or fuzzily link to a record that is identical apart from whitespace and punctuation Don't limit authority search to 20 - make it super large to catch large false positives --- C4/Heading.pm | 9 +- C4/Linker/BestMatch.pm | 135 +++++++++++++++++++++ installer/data/mysql/sysprefs.sql | 2 +- installer/data/mysql/updatedatabase.pl | 2 +- .../en/modules/admin/preferences/authorities.pref | 1 + 5 files changed, 145 insertions(+), 4 deletions(-) create mode 100644 C4/Linker/BestMatch.pm diff --git a/C4/Heading.pm b/C4/Heading.pm index e323b5b..d7cd69e 100644 --- a/C4/Heading.pm +++ b/C4/Heading.pm @@ -150,7 +150,9 @@ SearchAuthorities will return only authids. sub authorities { my $self = shift; my $skipmetadata = shift; - my ( $results, $total ) = _search( $self, 'match-heading', $skipmetadata ); + my $maxlength = shift || undef; + + my ( $results, $total ) = _search( $self, 'match-heading', $skipmetadata, $maxlength ); return $results; } @@ -180,12 +182,15 @@ sub _search { my $self = shift; my $index = shift || undef; my $skipmetadata = shift || undef; + my $maxlength = shift || undef; my @marclist; my @and_or; my @excluding = []; my @operator; my @value; + $maxlength=20 unless $maxlength; + if ($index) { push @marclist, $index; push @and_or, 'and'; @@ -203,7 +208,7 @@ sub _search { require C4::AuthoritiesMarc; return C4::AuthoritiesMarc::SearchAuthorities( \@marclist, \@and_or, \@excluding, \@operator, - \@value, 0, 20, $self->{'auth_type'}, + \@value, 0, $maxlength, $self->{'auth_type'}, 'AuthidAsc', $skipmetadata ); } diff --git a/C4/Linker/BestMatch.pm b/C4/Linker/BestMatch.pm new file mode 100644 index 0000000..6553c7b --- /dev/null +++ b/C4/Linker/BestMatch.pm @@ -0,0 +1,135 @@ +package C4::Linker::BestMatch; + +# Copyright 2011 C & P Bibliography Services +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use strict; +use warnings; +use Carp; +use C4::Heading; +use C4::AuthoritiesMarc; +use C4::Linker::Default; # Use Default for flipping + +use base qw(C4::Linker); + +sub new { + my $class = shift; + my $param = shift; + + my $self = $class->SUPER::new($param); + $self->{'default_linker'} = C4::Linker::Default->new($param); + bless $self, $class; + return $self; +} + +sub get_link { + my $self = shift; + my $heading = shift; + my $search_form = $heading->search_form(); + my $authid; + my $fuzzy = 0; + + my $fieldasstring = lc $heading->field()->as_string('abcdefghijklmnopqrstuvwxyz'); + # Remove subdivisions - they mess up our search here + $fieldasstring =~ s/(generalsubdiv|formsubdiv|chronologicalsubdiv|geographicalsubdiv|\s|,|-|;)//g; + $fieldasstring =~ s/&/and/g; # Normalise to and + + if ( $self->{'cache'}->{$fieldasstring}->{'cached'} ) { + $authid = $self->{'cache'}->{$fieldasstring}->{'authid'}; + $fuzzy = $self->{'cache'}->{$fieldasstring}->{'fuzzy'}; + #warn "Cached result for $fieldasstring as $authid (Fuzzy: $fuzzy)"; + } else { + #look for matching authorities + my $authorities = $heading->authorities(1,9999999); # $skipmetadata = true + if ($#{$authorities} >= 0) { + my %authcodes = ( + 'PERSO_NAME','100', + 'CORPO_NAME', '110', + 'MEETI_NAME', '111', + 'UNIF_TITLE', '130', + 'CHRON_TERM', '148', + 'TOPIC_TERM', '150', + 'GEOGR_NAME', '151', + 'GENRE/FORM', '155', + 'GEN_SUBDIV', '180', + 'GEO_SUBDIV', '181', + 'CHRON_SUBD', '182', + 'FORM_SUBD', '185' + ); + for (my $n = 0; $n <= $#{$authorities}; $n = $n + 1) { + my $auth = GetAuthority($authorities->[$n]->{'authid'}); + my $auth_field = $auth->field($authcodes{$heading->{'auth_type'}}); + my $authfieldstring = lc $auth->field($authcodes{$heading->{'auth_type'}})->as_string('abcdefghijklmnopqrstuvwxyz'); + $authfieldstring =~ s/(generalsubdiv|formsubdiv|chronologicalsubdiv|geographicalsubdiv|\s|,|-|;)//g; + $authfieldstring =~ s/&/and/g; + if ($fieldasstring eq $authfieldstring) { + $authid = $authorities->[$n]->{'authid'}; + $n = $#{$authorities} + 1; # break out of the loop + } + } + if (! defined $authid) { + warn "No authority could be perfectly matched for " . $heading->field()->as_string('abcdefghijklmnopqrstuvwxyz') . "\n"; + $fieldasstring =~ s/(\d|\.)//g; # Try alos removing full-stops and numbers + for (my $n = 0; $n <= $#{$authorities}; $n = $n + 1) { + my $auth = GetAuthority($authorities->[$n]->{'authid'}); + my $auth_field = $auth->field($authcodes{$heading->{'auth_type'}}); + my $authfieldstring = lc $auth->field($authcodes{$heading->{'auth_type'}})->as_string('abcdefghijklmnopqrstuvwxyz'); + $authfieldstring =~ s/(generalsubdiv|formsubdiv|chronologicalsubdiv|geographicalsubdiv|\s|\d|,|-|;|\.)//g; + if ($fieldasstring eq $authfieldstring) { + warn "Fuzzy matching $fieldasstring with $authfieldstring\n"; + $authid = $authorities->[$n]->{'authid'}; + $fuzzy = 1; + $n = $#{$authorities} + 1; # break out of the loop + } + } + } + if (! defined $authid) { + warn "No match found\n"; + } + } else { + warn "No match found\n"; + } + $self->{'cache'}->{$fieldasstring}->{'cached'} = 1; + $self->{'cache'}->{$fieldasstring}->{'authid'} = $authid; + $self->{'cache'}->{$fieldasstring}->{'fuzzy'} = $fuzzy; + } + + return $self->SUPER::_handle_auth_limit($authid), $fuzzy; +} + +sub update_cache { + my $self = shift; + my $heading = shift; + my $authid = shift; + $self->{'default_linker'}->update_cache( $heading, $authid ); +} + +sub flip_heading { + my $self = shift; + my $heading = shift; + + return $self->{'default_linker'}->flip($heading); +} + +1; +__END__ + +=head1 NAME + +C4::Linker::BestMatch - match against the authority record that is identical or near identical (fuzzy) + +=cut diff --git a/installer/data/mysql/sysprefs.sql b/installer/data/mysql/sysprefs.sql index c9a7c7e..461ca39 100644 --- a/installer/data/mysql/sysprefs.sql +++ b/installer/data/mysql/sysprefs.sql @@ -221,7 +221,7 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` ('LibraryThingForLibrariesID','','','See:http://librarything.com/forlibraries/','free'), ('LibraryThingForLibrariesTabbedView','0','','Put LibraryThingForLibraries Content in Tabs.','YesNo'), ('LinkerKeepStale','0',NULL,'If ON the authority linker will keep existing authority links for headings where it is unable to find a match.','YesNo'), -('LinkerModule','Default','Default|FirstMatch|LastMatch','Chooses which linker module to use (see documentation).','Choice'), +('LinkerModule','Default','Default|FirstMatch|LastMatch|BestMatch','Chooses which linker module to use (see documentation).','Choice'), ('LinkerOptions','','','A pipe-separated list of options for the linker.','free'), ('LinkerRelink','1',NULL,'If ON the authority linker will relink headings that have previously been linked every time it runs.','YesNo'), ('LocalCoverImages','0','1','Display local cover images on intranet details pages.','YesNo'), diff --git a/installer/data/mysql/updatedatabase.pl b/installer/data/mysql/updatedatabase.pl index c84fc3b..aad6391 100755 --- a/installer/data/mysql/updatedatabase.pl +++ b/installer/data/mysql/updatedatabase.pl @@ -4756,7 +4756,7 @@ if (C4::Context->preference("Version") < TransformToNum($DBversion)) { $DBversion = "3.07.00.021"; if (C4::Context->preference("Version") < TransformToNum($DBversion)) { $dbh->do( - "INSERT INTO systempreferences (variable,value,explanation,options,type) VALUES('LinkerModule','Default','Chooses which linker module to use (see documentation).','Default|FirstMatchLastMatch','Choice');" + "INSERT INTO systempreferences (variable,value,explanation,options,type) VALUES('LinkerModule','Default','Chooses which linker module to use (see documentation).','Default|FirstMatch|LastMatch|BestMatch','Choice');" ); $dbh->do( "INSERT INTO systempreferences (variable,value,explanation,options,type) VALUES('LinkerOptions','','A pipe-separated list of options for the linker.','','free');" diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/authorities.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/authorities.pref index 2762411..6a1c5de 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/authorities.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/authorities.pref @@ -58,6 +58,7 @@ Authorities: Default: Default FirstMatch: "First Match" LastMatch: "Last Match" + BestMatch: "Best Match" - linker module for matching headings to authority records. - - Set the following options for the authority linker -- 2.7.4