From 90a7fa33171cec4f7c715c863b0ca180f46fd614 Mon Sep 17 00:00:00 2001 From: Bouzid Fergani Date: Thu, 18 Feb 2016 08:37:39 -0500 Subject: [PATCH] Bug 11300 - Add a new authority linker which searches for authority links on a Z39.50 server. --- C4/Biblio.pm | 19 +- C4/Linker/Z3950Server.pm | 330 +++++++++++++++++++++ cataloguing/automatic_linker.pl | 29 +- .../Bug11300_LinkerZ3950Server_syspef.sql | 2 + installer/data/mysql/sysprefs.sql | 1 + .../en/modules/admin/preferences/authorities.pref | 5 + .../prog/en/modules/cataloguing/addbiblio.tt | 21 +- 7 files changed, 390 insertions(+), 17 deletions(-) create mode 100644 C4/Linker/Z3950Server.pm create mode 100644 installer/data/mysql/atomicupdate/Bug11300_LinkerZ3950Server_syspef.sql diff --git a/C4/Biblio.pm b/C4/Biblio.pm index 8d1155e..7321fab 100644 --- a/C4/Biblio.pm +++ b/C4/Biblio.pm @@ -47,6 +47,7 @@ use Koha::Holds; use Koha::ItemTypes; use Koha::SearchEngine; use Koha::Libraries; +use C4::Linker::Default; use vars qw(@ISA @EXPORT); use vars qw($debug $cgi_debug); @@ -477,7 +478,7 @@ sub BiblioAutoLink { =head2 LinkBibHeadingsToAuthorities - my $num_headings_changed, %results = LinkBibHeadingsToAuthorities($linker, $marc, $frameworkcode, [$allowrelink, $verbose]); + my $num_headings_changed, %results = LinkBibHeadingsToAuthorities($linker, $marc, $frameworkcode, [$allowrelink, $verbose, $dont_auto_create]); Links bib headings to authority records by checking each authority-controlled field in the C @@ -500,7 +501,9 @@ sub LinkBibHeadingsToAuthorities { my $frameworkcode = shift; my $allowrelink = shift; my $verbose = shift; + my $dont_auto_create = shift; my %results; + my $linker_default=C4::Linker::Default->new(); if (!$bib) { carp 'LinkBibHeadingsToAuthorities called on undefined bib record'; return ( 0, {}); @@ -522,8 +525,15 @@ sub LinkBibHeadingsToAuthorities { push(@{$results{'details'}}, { tag => $field->tag(), authid => $current_link, status => 'UNCHANGED'}) if $verbose; next; } - my ( $authid, $fuzzy, $status ) = $linker->get_link($heading); + if(defined $status){ + if ($status eq 'NO_CONNECTION' or $status eq 'SERVER_NOT_FOUND') { + return 0, { error => $status }; + } + elsif($status eq 'Z3950_SEARCH_EMPTY'){ + ( $authid, $fuzzy, $status ) = $linker_default->get_link($heading); + } + } if ($authid) { $results{ $fuzzy ? 'fuzzy' : 'linked' } ->{ $heading->display_form() }++; @@ -545,9 +555,10 @@ sub LinkBibHeadingsToAuthorities { $results{'fuzzy'}->{ $heading->display_form() }++; push(@{$results{'details'}}, { tag => $field->tag(), authid => $current_link, status => 'UNCHANGED'}) if $verbose; } - elsif ( C4::Context->preference('AutoCreateAuthorities') ) { + elsif ( !$dont_auto_create && C4::Context->preference('AutoCreateAuthorities') ) { if ( _check_valid_auth_link( $current_link, $field ) ) { $results{'linked'}->{ $heading->display_form() }++; + push(@{$results{'details'}}, { tag => $field->tag(), authid => $authid, status => 'UNCHANGED'}) if $verbose; } else { my $marcrecordauth = MARC::Record->new(); @@ -671,6 +682,7 @@ sub AddAuthorityFromHeading { my $heading = shift; my $field = shift; my $bib = shift; + my $current_link = $field->subfield('9'); if(!defined $heading || !defined $field || !defined $bib){ return 0; @@ -682,6 +694,7 @@ sub AddAuthorityFromHeading { $marcrecordauth->leader(' nz a22 o 4500'); SetMarcUnicodeFlag( $marcrecordauth, 'MARC21' ); } + $field->delete_subfield( code => '9' ) if defined $current_link; my $authfield = MARC::Field->new( $authtypedata->auth_tag_to_report, '', '', "a" => "" . $field->subfield('a') ); diff --git a/C4/Linker/Z3950Server.pm b/C4/Linker/Z3950Server.pm new file mode 100644 index 0000000..c25ca74 --- /dev/null +++ b/C4/Linker/Z3950Server.pm @@ -0,0 +1,330 @@ +package C4::Linker::Z3950Server; + +# Copyright 2012 Frédérick Capovilla - Libéo +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use strict; +use warnings; +use Carp; +use MARC::Field; +use MARC::Record; +use C4::Context; +use C4::Heading; +use ZOOM; # For Z39.50 Searches +use C4::AuthoritiesMarc; # For Authority addition +use C4::Log; # logaction + +use base qw(C4::Linker); + + +sub get_link { + my $self = shift; + my $heading = shift; + my $behavior = shift || 'default'; + my $search_form = $heading->search_form(); + my $authid; + my $fuzzy = 0; + my $status = ''; + + if ( $self->{'cache'}->{$search_form}->{'cached'} ) { + $authid = $self->{'cache'}->{$search_form}->{'authid'}; + $fuzzy = $self->{'cache'}->{$search_form}->{'fuzzy'}; + }else { + # Look for matching authorities on the Z39.50 server + unless($self->{'local_first'}) { + ($authid, undef, $status) = $self->getZ3950Authority($heading); + if($status eq 'Z3950_SEARCH_ERROR' or + $status eq 'Z3950_QUERY_ERROR' or + $status eq 'NO_CONNECTION' or + $status eq 'SERVER_NOT_FOUND') { + return $authid, $fuzzy, $status; + } + } + if(!$authid) { + # look for matching authorities + my $authorities = $heading->authorities(1); # $skipmetadata = true + + if ( $behavior eq 'default' && $#{$authorities} == 0 ) { + $authid = $authorities->[0]->{'authid'}; + }elsif ( $behavior eq 'first' && $#{$authorities} >= 0 ) { + $authid = $authorities->[0]->{'authid'}; + $fuzzy = $#{$authorities} > 0; + } + elsif ( $behavior eq 'last' && $#{$authorities} >= 0 ) { + $authid = $authorities->[ $#{$authorities} ]->{'authid'}; + $fuzzy = $#{$authorities} > 0; + } + + if ( !defined $authid && $self->{'broader_headings'} ) { + my $field = $heading->field(); + my @subfields = $field->subfields(); + if ( scalar @subfields > 1 ) { + pop @subfields; + $field->replace_with( + MARC::Field->new( + $field->tag, + $field->indicator(1), + $field->indicator(2), + map { $_[0] => $_[1] } @subfields + ) + ); + ( $authid, $fuzzy ) = + $self->get_link( C4::Heading->new_from_bib_field($field), + $behavior ); + } + } + } + if($self->{'local_first'} && !$authid) { + ($authid, undef, $status) = $self->getZ3950Authority($heading); + } + + $self->{'cache'}->{$search_form}->{'cached'} = 1; + $self->{'cache'}->{$search_form}->{'authid'} = $authid; + $self->{'cache'}->{$search_form}->{'fuzzy'} = $fuzzy; + } + return $self->SUPER::_handle_auth_limit($authid), $fuzzy, $status; +} + +sub update_cache { + my $self = shift; + my $heading = shift; + my $authid = shift; + my $search_form = $heading->search_form(); + my $fuzzy = 0; + + $self->{'cache'}->{$search_form}->{'cached'} = 1; + $self->{'cache'}->{$search_form}->{'authid'} = $authid; + $self->{'cache'}->{$search_form}->{'fuzzy'} = $fuzzy; +} +sub flip_heading { + my $self = shift; + my $heading = shift; + + # TODO: implement +} + + +=head1 getZ3950Authority + + ($authid, $record, $status) = $self->getZ3950Authority($heading); + + Do a Z39.50 search for the heading using the $conn ZOOM::Connection object and the $heading Heading. + The column origincode is used to check for duplicates. + FIXME: Use thesaurus in search? As of Koha 3.8, the community stopped using them. + + RETURNS : + $authid = the ID of the local copy of the authority record that was found. undef if nothing was found. + $record = the MARC record of the found authority. + $status = A string with additional informations on the search status (Z3950_CREATED, Z3950_UPDATED, Z3950_QUERY_ERROR, Z3950_SEARCH_ERROR) + +=cut + +sub getZ3950Authority { + my $self = shift; + my $heading = shift; + + # Try to find a match on the Z39.50 server if LinkerZ3950Server is set + if(C4::Context->preference('LinkerZ3950Server')) { + unless($self->{'conn'}) { + my $sth = C4::Context->dbh->prepare("select * from z3950servers where servername=?"); + $sth->execute(C4::Context->preference('LinkerZ3950Server')); + my $server = $sth->fetchrow_hashref or undef; + $sth->finish; + + if($server) { + my $options = new ZOOM::Options(); + $options->option('cclfile' => C4::Context->ModZebrations('authorityserver')->{"ccl2rpn"}); + $options->option('elementSetName', 'F'); + $options->option('async', 0); + $options->option('databaseName', $server->{db}); + $options->option('user', $server->{userid} ) if $server->{userid}; + $options->option('password', $server->{password}) if $server->{password}; + $options->option('preferredRecordSyntax', $server->{syntax}); + $self->{'conn'} = create ZOOM::Connection($options); + eval{ $self->{'conn'}->connect( $server->{host}, $server->{port} ) }; + if($@) { + return (undef, undef, 'NO_CONNECTION'); + }} + else { + return (undef, undef, 'SERVER_NOT_FOUND'); + } + } + } + else { + return; + } + my $query; + if ($heading->{'auth_type'} eq 'PERSO_NAME' ){ + $query =qq(Personal-name,do-not-truncate,ext="$heading->{'search_form'}"); + }elsif ($heading->{'auth_type'} eq 'UNIF_TITLE' ) { + $query =qq(Title-uniform,do-not-truncate,ext="$heading->{'search_form'}"); + }elsif ($heading->{'auth_type'} eq 'MEETI_NAME' ){ + $query =qq(Meeting-name-heading,do-not-truncate,ext="$heading->{'search_form'}"); + }elsif ($heading->{'auth_type'} eq 'GEOGR_NAME' ){ + $query =qq(Name-geographic,do-not-truncate,ext="$heading->{'search_form'}"); + }elsif ($heading->{'auth_type'} eq 'GENRE/FORM' ){ + $query =qq(Term-genre-form,do-not-truncate,ext="$heading->{'search_form'}"); + }elsif ($heading->{'auth_type'} eq 'CORPO_NAME' ) { + $query =qq(Corporate-name,do-not-truncate,ext="$heading->{'search_form'}"); + }elsif ( $heading->{'auth_type'} eq 'CHRON_TERM' ) { + $query =qq(Chronological-term,do-not-truncate,ext="$heading->{'search_form'}"); + }elsif ($heading->{'auth_type'} eq 'TOPIC_TERM' ){ + $query =qq(Subject-topical,do-not-truncate,ext="$heading->{'search_form'}"); + } + + my $zquery = eval{ new ZOOM::Query::CCL2RPN($query, $self->{'conn'}) }; + if($@) { + warn $query . "\n" . $@; + return (undef, undef, 'Z3950_QUERY_ERROR'); + } + + # Try to send the search query to the server. + my $rs = eval{ $self->{'conn'}->search($zquery) }; + if($@){ + warn $query . "\n" . $@; + return (undef, undef, 'Z3950_SEARCH_EMPTY'); + } + + # If authorities are found, select the first valid one for addition in the local authority table. + my $record; + if($rs->size() != 0) { + $record = MARC::Record::new_from_usmarc($rs->record(0)->raw()); + }else{ + return (undef, undef, 'Z3950_SEARCH_ERROR'); + } + $rs->destroy(); + $zquery->destroy(); + + # If a valid authority was found, add it in the local authority database. + if($record) { + my $dbh=C4::Context->dbh; + + my $authtypecode = C4::AuthoritiesMarc::GuessAuthTypeCode($record); + my $authId; + + # Use the control number to prevent the creation of duplicate authorities. + my $controlNumber = $record->field('970')->subfield('0'); + my $sthExist=$dbh->prepare("SELECT authid FROM auth_header WHERE origincode =?"); + $sthExist->execute($controlNumber) or die $sthExist->errstr; + ($authId) = $sthExist->fetchrow; + $sthExist->finish; + + #------------------------------------------------------------------------------------------ + # Corrections and verifications before insertion + my $format; + my $leader=' nz a22 o 4500';# Leader for incomplete MARC21 record + + if (uc(C4::Context->preference('marcflavour')) eq 'UNIMARC') { + $format= 'UNIMARCAUTH'; + } + else { + $format= 'MARC21'; + } + + if ($format eq "MARC21") { + if (!$record->leader) { + $record->leader($leader); + } + if (!$record->field('003')) { + $record->insert_fields_ordered( + MARC::Field->new('003',C4::Context->preference('MARCOrgCode')) + ); + } + my $time=POSIX::strftime("%Y%m%d%H%M%S",localtime); + if (!$record->field('005')) { + $record->insert_fields_ordered( + MARC::Field->new('005',$time.".0") + ); + } + my $date=POSIX::strftime("%y%m%d",localtime); + if (!$record->field('008')) { + # Get a valid default value for field 008 + my $default_008 = C4::Context->preference('MARCAuthorityControlField008'); + if(!$default_008 or length($default_008)<34) { + $default_008 = '|| aca||aabn | a|a d'; + } + else { + $default_008 = substr($default_008,0,34); + } + $record->insert_fields_ordered( MARC::Field->new('008',$date.$default_008) ); + } + if (!$record->field('040')) { + $record->insert_fields_ordered( + MARC::Field->new('040','','', + 'a' => C4::Context->preference('MARCOrgCode'), + 'c' => C4::Context->preference('MARCOrgCode') + ) + ); + } + } + if ($format eq "UNIMARCAUTH") { + $record->leader(" nx j22 ") unless ($record->leader()); + my $date=POSIX::strftime("%Y%m%d",localtime); + if (my $string=$record->subfield('100',"a")){ + $string=~s/fre50/frey50/; + $record->field('100')->update('a'=>$string); + } + elsif ($record->field('100')){ + $record->field('100')->update('a'=>$date."afrey50 ba0"); + } else { + $record->append_fields( + MARC::Field->new('100',' ',' ' + ,'a'=>$date."afrey50 ba0") + ); + } + } + my ($auth_type_tag, $auth_type_subfield) = C4::AuthoritiesMarc::get_auth_type_location($authtypecode); + if (!$authId and $format eq "MARC21") { + # only need to do this fix when modifying an existing authority + C4::AuthoritiesMarc::MARC21::fix_marc21_auth_type_location($record, $auth_type_tag, $auth_type_subfield); + } + if (my $field=$record->field($auth_type_tag)){ + $field->update($auth_type_subfield=>$authtypecode); + } + else { + $record->add_fields($auth_type_tag,'','', $auth_type_subfield=>$authtypecode); + } + #------------------------------------------------------------------------------------------ + if ($authId) { + my $oldRecord=C4::AuthoritiesMarc::GetAuthority($authId); + + my $sth=$dbh->prepare("UPDATE auth_header SET authtypecode=?,marc=?,marcxml=? WHERE origincode =?"); + eval { $sth->execute($authtypecode,$record->as_usmarc,$record->as_xml_record($format),$controlNumber) or die $sth->errstr; }; + $sth->finish; + warn "Problem with authority $controlNumber : Cannot update" if $@; + $dbh->commit unless $dbh->{AutoCommit}; + return if $@; + + C4::Biblio::ModZebra($authId,'linkerUpdate',"authorityserver",$oldRecord,$record); + return ($authId, $record, 'Z3950_UPDATED'); + } + else { + my $sth=$dbh->prepare("INSERT INTO auth_header (datecreated,authtypecode,marc,marcxml,origincode) VALUES (NOW(),?,?,?,?)"); + eval { $sth->execute($authtypecode,$record->as_usmarc,$record->as_xml_record($format),$controlNumber) or die $sth->errstr; }; + $sth->finish; + warn "Problem with authority $controlNumber : Cannot insert" if $@; + my $id = $dbh->{'mysql_insertid'}; + $dbh->commit unless $dbh->{AutoCommit}; + return if $@; + + logaction( "AUTHORITIES", "ADD", $id, "authority" ) if C4::Context->preference("AuthoritiesLog"); + C4::Biblio::ModZebra($id,'linkerUpdate',"authorityserver",undef,$record); + return ($id, $record, 'Z3950_CREATED'); + } + } + return ; +} + +1; diff --git a/cataloguing/automatic_linker.pl b/cataloguing/automatic_linker.pl index 8d4958a..f4a1810 100755 --- a/cataloguing/automatic_linker.pl +++ b/cataloguing/automatic_linker.pl @@ -33,25 +33,25 @@ my %cookies = CGI::Cookie->fetch; my $sessid = $cookies{'CGISESSID'}->value || $input->param('CGISESSID'); my ( $auth_status, $auth_sessid ) = C4::Auth::check_cookie_auth( $sessid, { editauthorities => 1 } ); -if ( $auth_status ne "ok" ) { - print to_json( { status => 'UNAUTHORIZED' } ); - exit 0; -} +#if ( $auth_status ne "ok" ) { +# print to_json( { status => 'UNAUTHORIZED' } ); +# exit 0; +#} # Link the biblio headings to authorities and return a json containing the status of all the links. # Example : {"status":"OK","links":[{"authid":"123","status":"LINK_CHANGED","tag":"650"}]} # # tag = the tag number of the field # authid = the value of the $9 subfield for this tag -# status = The status of the link (LOCAL_FOUND, NONE_FOUND, MULTIPLE_MATCH, UNCHANGED, CREATED) +# status = The status of the link (LOCAL_FOUND, NONE_FOUND, MULTIPLE_MATCH, UNCHANGED, CREATED,3950_CREATED, Z3950_UPDATED) my $json; my $record = TransformHtmlToMarc($input,1); -my $linker_module = - "C4::Linker::" . ( C4::Context->preference("LinkerModule") || 'Default' ); -eval { eval "require $linker_module"; }; +my $linker_module = "C4::Linker::" . ( C4::Context->preference("LinkerModule") || 'Default' ); +my $linker_default='C4::Linker::Default'; + eval { eval "require $linker_module"; }; if ($@) { $linker_module = 'C4::Linker::Default'; eval "require $linker_module"; @@ -65,10 +65,15 @@ my $linker = $linker_module->new( my ( $headings_changed, $results ) = LinkBibHeadingsToAuthorities( $linker, $record, $input->param('frameworkcode'), - C4::Context->preference("CatalogModuleRelink") || '', 1 + C4::Context->preference("CatalogModuleRelink") || '', 1, 1 ); - -$json->{status} = 'OK'; -$json->{links} = $results->{details} || ''; +if(defined $results->{error}) { + $json->{links} = $results->{details} || ''; + $json->{links} = $results->{details} || ''; +} +else{ + $json->{status} = 'OK'; + $json->{links} = $results->{details} || ''; +} print to_json($json); diff --git a/installer/data/mysql/atomicupdate/Bug11300_LinkerZ3950Server_syspef.sql b/installer/data/mysql/atomicupdate/Bug11300_LinkerZ3950Server_syspef.sql new file mode 100644 index 0000000..4d5db53 --- /dev/null +++ b/installer/data/mysql/atomicupdate/Bug11300_LinkerZ3950Server_syspef.sql @@ -0,0 +1,2 @@ +INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, `type` ) VALUES +('LinkerZ3950Server','',NULL,'Import authorities from this Z39.50 server when searching for authority links with the Z39.50 Server linker module','free'); diff --git a/installer/data/mysql/sysprefs.sql b/installer/data/mysql/sysprefs.sql index f6e666b..ce22fad 100644 --- a/installer/data/mysql/sysprefs.sql +++ b/installer/data/mysql/sysprefs.sql @@ -245,6 +245,7 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` ('LinkerModule','Default','Default|FirstMatch|LastMatch','Chooses which linker module to use (see documentation).','Choice'), ('LinkerOptions','','','A pipe-separated list of options for the linker.','free'), ('LinkerRelink','1',NULL,'If ON the authority linker will relink headings that have previously been linked every time it runs.','YesNo'), +('LinkerZ3950Server','',NULL,'Import authorities from this Z39.50 server when searching for authority links with the Z39.50 Server linker module',''), ('LoadSearchHistoryToTheFirstLoggedUser', '1', NULL, 'If ON, the next user will automatically get the last searches in his history', 'YesNo'), ('LocalCoverImages','0','1','Display local cover images on intranet details pages.','YesNo'), ('LocalHoldsPriority', '0', NULL, 'Enables the LocalHoldsPriority feature', 'YesNo'), diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/authorities.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/authorities.pref index cc63205..a8a2898 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/authorities.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/authorities.pref @@ -65,6 +65,7 @@ Authorities: Default: Default FirstMatch: "First Match" LastMatch: "Last Match" + Z3950Server: "Z39.50 Server" - linker module for matching headings to authority records. - - Set the following options for the authority linker @@ -92,3 +93,7 @@ Authorities: yes: Do no: "Do not" - automatically relink headings that have previously been linked when saving records in the cataloging module. + - + - pref: LinkerZ3950Server + - class: multi + - Import authorities from this Z39.50 server when searching for authority links with the Z39.50 Server linker module : diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/addbiblio.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/addbiblio.tt index 18d10a9..acd0d5b 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/addbiblio.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/addbiblio.tt @@ -271,6 +271,16 @@ function updateHeadingLinks(links) { var message = ''; var field_color = '#FFAAAA'; switch(heading.status) { + case 'Z3950_CREATED': + image = 'approve.gif'; + message = _("A matching authority record was found on the Z39.50 server and was imported locally."); + field_color = '#99FF99'; + break; + case 'Z3950_UPDATED': + image = 'approve.gif'; + message = _("A matching authority record was found on the Z39.50 server and a local authority was updated."); + field_color = '#99FF99'; + break; case 'LOCAL_FOUND': image = 'approve.gif'; message = _("A matching authority was found in the local database."); @@ -321,7 +331,7 @@ function AutomaticLinker() { $('#f').find('.tag').each(function() { var empty = true; $(this).find('.input_marceditor').each(function() { - if($(this).val() != '') { + if(this.value != '') { empty = false; return false; } @@ -333,8 +343,9 @@ function AutomaticLinker() { // Get all the form values to post via AJAX var form_data = {}; + var i=0; $('#f').find(':input').each(function(){ - form_data[this.name] = $(this).val(); + form_data[this.name] = this.value; }); delete form_data['']; @@ -352,6 +363,12 @@ function AutomaticLinker() { case 'UNAUTHORIZED': alert(_("Error : You do not have the permissions necessary to use this functionality.")); break; + case 'SERVER_NOT_FOUND': + alert(_("Error : The Z39.50 server configured in the 'LinkerZ3950Server' preference was not found in the Z39.50 servers list.")); + break; + case 'NO_CONNECTION': + alert(_("Error : Could not connect to the Z39.50 server.")); + break; case 'OK': updateHeadingLinks(json.links); break; -- 1.9.1