From 30a3cf15ac06f9ef065c676423fdefac9ebd559f Mon Sep 17 00:00:00 2001 From: Matthias Meusburger Date: Thu, 20 Jul 2023 16:29:49 +0200 Subject: [PATCH] Bug 35659: OAI Harvester Test plan: - Run updatedatabase.pl - Apply the DBIC update patch - Setup an OAI repository in Administration -> OAI repositories (admin/oaiservers.pl) If the repository dataformat is in oai_dc, you can use the sample XSLT files: For Unimarc: OAIDCtoUNIMARCXML.xsl for bibliographic records AuthOAIDCtoUNIMARCXML.xsl for authorities For Marc21: OAIDCtoMARC21XML.xsl for bibliographic records AuthOAIDCtoMARC21XML.xsl for authorities If the repository dataformat is in marc-xml, you can use OAIMarcxml2KohaMarcxml.xsl for both Unimarc and Marc21. - Some repositories to test with: Bibliographic records, oai_dc: Endpoint: http://staroai.theses.fr/OAIHandler Set: ddc:260 XSLT: /koha-tmpl/intranet-tmpl/prog/en/xslt/OAIDCtoUNIMARCXML.xsl or XSLT: /koha-tmpl/intranet-tmpl/prog/en/xslt/OAIDCtoMARC21XML.xsl Authorities, oai_dc: Endpoint: https://www.idref.fr/OAI/oai.jsp Set: a XSLT: /koha-tmpl/intranet-tmpl/prog/en/xslt/AuthOAIDCtoUNIMARCXML.xsl or XSLT: /koha-tmpl/intranet-tmpl/prog/en/xslt/AuthOAIDCtoMARC21XML.xsl Authorities, marc-xml, Unimarc: Endpoint: https://www.idref.fr/OAI/oai.jsp Set: a XSLT: /koha-tmpl/intranet-tmpl/prog/en/xslt/OAIMarcxml2KohaMarcxml.xsl - Note: marc-xml harvesting will not work with HTTP::OAI::Harvester version 3.xx Version 4.xx is needed. See Bug 17704 for updating Koha default version to 4.xx - If you want to receive a report email, fill the OAI-PMH:HarvestEmailReport syspref. The model used (OAI_HARVEST_REPORT) can be edited in tools -> Notices and slips. - start a harvest with the following command: misc/cronjobs/harvest_oai.pl This script starts an OAI Harvest This script has the following parameters : -h --help: this message -v --verbose (enable output to stdout) -r --repository: id of the OAI repository -d --days: number of days to harvest from (optional) -l --list: list the OAI repositories -f --force: force harvesting (ignore records datestamps) (optional) Example: misc/cronjobs/harvest_oai.pl -r 1 -d 10 -v -f - Check the logs in the Log Viewer (tools/viewlog.pl) (CronjobLog syspref must be activated) Sponsored-by: KohaLa Signed-off-by: John Doe Signed-off-by: Michal --- Koha/Import/Oaipmh/Authorities.pm | 51 +++ Koha/Import/Oaipmh/Authority.pm | 41 ++ Koha/Import/Oaipmh/Biblio.pm | 40 ++ Koha/Import/Oaipmh/Biblios.pm | 51 +++ Koha/OAI/Client/Harvester.pm | 372 ++++++++++++++++++ Koha/OaiServer.pm | 45 +++ Koha/OaiServers.pm | 57 +++ admin/oaiservers.pl | 122 ++++++ .../data/mysql/atomicupdate/Bug-35659.pl | 88 +++++ .../mysql/en/mandatory/sample_notices.yml | 23 ++ installer/data/mysql/kohastructure.sql | 58 +++ installer/data/mysql/mandatory/sysprefs.sql | 1 + .../data/mysql/mandatory/userpermissions.sql | 2 +- .../prog/en/includes/admin-menu.inc | 1 + .../prog/en/includes/oai-admin-search.inc | 43 ++ .../prog/en/includes/permissions.inc | 2 +- .../prog/en/modules/admin/admin-home.tt | 2 + .../prog/en/modules/admin/oaiservers.tt | 229 +++++++++++ .../admin/preferences/web_services.pref | 5 + .../prog/en/xslt/AuthOAIDCtoMARC21XML.xsl | 35 ++ .../prog/en/xslt/AuthOAIDCtoUNIMARCXML.xsl | 42 ++ .../prog/en/xslt/OAIDCtoMARC21XML.xsl | 367 +++++++++++++++++ .../prog/en/xslt/OAIDCtoUNIMARCXML.xsl | 341 ++++++++++++++++ .../prog/en/xslt/OAIMarcxml2KohaMarcxml.xsl | 23 ++ misc/cronjobs/harvest_oai.pl | 110 ++++++ t/db_dependent/Koha/OAIHarvester.t | 78 ++++ t/db_dependent/Koha/OAIServers.t | 95 +++++ 27 files changed, 2322 insertions(+), 2 deletions(-) create mode 100644 Koha/Import/Oaipmh/Authorities.pm create mode 100644 Koha/Import/Oaipmh/Authority.pm create mode 100644 Koha/Import/Oaipmh/Biblio.pm create mode 100644 Koha/Import/Oaipmh/Biblios.pm create mode 100644 Koha/OAI/Client/Harvester.pm create mode 100644 Koha/OaiServer.pm create mode 100644 Koha/OaiServers.pm create mode 100755 admin/oaiservers.pl create mode 100755 installer/data/mysql/atomicupdate/Bug-35659.pl create mode 100644 koha-tmpl/intranet-tmpl/prog/en/includes/oai-admin-search.inc create mode 100644 koha-tmpl/intranet-tmpl/prog/en/modules/admin/oaiservers.tt create mode 100644 koha-tmpl/intranet-tmpl/prog/en/xslt/AuthOAIDCtoMARC21XML.xsl create mode 100644 koha-tmpl/intranet-tmpl/prog/en/xslt/AuthOAIDCtoUNIMARCXML.xsl create mode 100644 koha-tmpl/intranet-tmpl/prog/en/xslt/OAIDCtoMARC21XML.xsl create mode 100644 koha-tmpl/intranet-tmpl/prog/en/xslt/OAIDCtoUNIMARCXML.xsl create mode 100644 koha-tmpl/intranet-tmpl/prog/en/xslt/OAIMarcxml2KohaMarcxml.xsl create mode 100755 misc/cronjobs/harvest_oai.pl create mode 100755 t/db_dependent/Koha/OAIHarvester.t create mode 100755 t/db_dependent/Koha/OAIServers.t diff --git a/Koha/Import/Oaipmh/Authorities.pm b/Koha/Import/Oaipmh/Authorities.pm new file mode 100644 index 0000000000..90372a8609 --- /dev/null +++ b/Koha/Import/Oaipmh/Authorities.pm @@ -0,0 +1,51 @@ +package Koha::Import::Oaipmh::Authorities; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License + +use Modern::Perl; + +use Koha::Import::Oaipmh::Authority; + +use base qw(Koha::Objects); + +=head1 NAME + +Koha::Import::Oaipmh::Authorities + +This object represents a collection of OAI-PMH records being imported as authorities + +=head1 API + +=head2 Methods + + +=head3 type + +=cut + +sub _type { + return 'ImportOaipmhAuthority'; +} + +=head3 object_class + +Koha::Object class + +=cut + +sub object_class { + return 'Koha::Import::Oaipmh::Authority'; +} +1; diff --git a/Koha/Import/Oaipmh/Authority.pm b/Koha/Import/Oaipmh/Authority.pm new file mode 100644 index 0000000000..fe90c432a2 --- /dev/null +++ b/Koha/Import/Oaipmh/Authority.pm @@ -0,0 +1,41 @@ +package Koha::Import::Oaipmh::Authority; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License + +use Modern::Perl; + +use base qw(Koha::Object); + +=head1 NAME + +Koha::Import::Oaipmh::Authority + +This object represents an OAI-PMH record being imported as an authority + +=head1 API + +=head2 Class Methods + +=cut + +=head3 _type + +=cut + +sub _type { + return 'ImportOaipmhAuthority'; +} + +1; diff --git a/Koha/Import/Oaipmh/Biblio.pm b/Koha/Import/Oaipmh/Biblio.pm new file mode 100644 index 0000000000..d59e5a6449 --- /dev/null +++ b/Koha/Import/Oaipmh/Biblio.pm @@ -0,0 +1,40 @@ +package Koha::Import::Oaipmh::Biblio; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License + +use Modern::Perl; + +use base qw(Koha::Object); + +=head1 NAME + +Koha::Import::Oaipmh::Biblio + +This object represents an OAI-PMH record being imported as a bibliographic record + +=head1 API + +=head2 Methods + + +=head3 _type + +=cut + +sub _type { + return 'ImportOaipmhBiblio'; +} + +1; diff --git a/Koha/Import/Oaipmh/Biblios.pm b/Koha/Import/Oaipmh/Biblios.pm new file mode 100644 index 0000000000..9ae111e96b --- /dev/null +++ b/Koha/Import/Oaipmh/Biblios.pm @@ -0,0 +1,51 @@ +package Koha::Import::Oaipmh::Biblios; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License + +use Modern::Perl; + +use Koha::Import::Oaipmh::Biblio; + +use base qw(Koha::Objects); + +=head1 NAME + +Koha::Import::Oaipmh::Biblios + +This object represents a collection of OAI-PMH records being imported as bibliographic records + +=head1 API + +=head2 Methods + + +=head3 type + +=cut + +sub _type { + return 'ImportOaipmhBiblio'; +} + +=head3 object_class + +Koha::Object class + +=cut + +sub object_class { + return 'Koha::Import::Oaipmh::Biblio'; +} +1; diff --git a/Koha/OAI/Client/Harvester.pm b/Koha/OAI/Client/Harvester.pm new file mode 100644 index 0000000000..f800fb1dc1 --- /dev/null +++ b/Koha/OAI/Client/Harvester.pm @@ -0,0 +1,372 @@ +# Copyright BibLibre 2023 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +package Koha::OAI::Client::Harvester; + +=head1 NAME + +Koha::OAI::Client::Harvester - OAI Harvester + +=head1 DESCRIPTION + +Koha::OAI::Client::Harvester contains OAI-PMH harvester main functions + +=cut + +use utf8; +use open qw( :std :utf8 ); + +use C4::Biblio qw( AddBiblio GetFrameworkCode ModBiblio DelBiblio ); +use C4::AuthoritiesMarc qw (AddAuthority GuessAuthTypeCode ModAuthority DelAuthority ); +use C4::Log qw( cronlogaction ); +use HTTP::OAI; +use HTTP::OAI::Metadata::OAI_DC; +use Koha::DateUtils qw( dt_from_string ); +use Koha::OaiServers; +use Koha::Import::Oaipmh::Biblio; +use Koha::Import::Oaipmh::Biblios; +use Koha::Import::Oaipmh::Authority; +use Koha::Import::Oaipmh::Authorities; +use Koha::XSLT::Base; +use MARC::File::XML; +use MARC::Record; +use Modern::Perl; +use DateTime; +use DateTime::Format::Strptime; +use Try::Tiny qw( catch try ); +use Date::Calc qw( + Add_Delta_Days + Today +); + +my $strp = DateTime::Format::Strptime->new( + pattern => '%FT%TZ', + time_zone => 'floating', +); + +our $verbose = 0; +our $server; +our $force = 0; +our $days = 0; +our $xslt_engine = Koha::XSLT::Base->new; + +=head2 new + +$harvester = Koha::OAI::Client::Harvester->new( { server => $server, verbose => $verbose, days => $days, force => $force } ); + +New instance of Koha::OAI::Client::Harvester + +C<$server> An OAI repository (Koha::OaiServer) + +C<$verbose> print log messages to stdout when enabled + +C<$days> number of days to harvest from (optional) + +C<$force> force harvesting (ignore records datestamps) + +=cut + +sub new { + + my ( $self, $args ) = @_; + + $server = $args->{'server'}; + $verbose = $args->{'verbose'}; + $days = $args->{'days'}; + $force = $args->{'force'}; + return $self; +} + +=head2 init + +Starts harvesting + +=cut + +sub init { + my ( $self, $args ) = @_; + + printlog("Starting OAI Harvest from repository"); + + my $h = HTTP::OAI::Harvester->new( + repository => HTTP::OAI::Identify->new( + baseURL => $server->endpoint, + version => '2.0', + ) + ); + + printlog( "Using HTTP::OAI::Harvester version " . $HTTP::OAI::Harvester::VERSION ); + + printlog( "Using endpoint " + . $server->endpoint + . ", set " + . $server->oai_set + . ", dataformat " + . $server->dataformat + . ", recordtype " + . $server->recordtype ); + + try { + my $lmdf = $h->ListMetadataFormats(); + my $lmdflog = ""; + for ( $lmdf->metadataFormat ) { + $lmdflog .= $_->metadataPrefix . " "; + } + printlog("Available metadata formats for this repository: $lmdflog"); + } catch { + printlog("ListMetadataFormats failed"); + }; + + if ( $server->add_xslt ) { + printlog( "Using XSLT file " . $server->add_xslt ); + } else { + printlog("Not using an XSLT file"); + } + + my $start_date = ''; + my $today_date = ''; + + if ($days) { + + # Change this to yyyy-mm-dd + my $dt_today = dt_from_string(); + my ( $nowyear, $nowmonth, $nowday ) = Today(); + my @date = Add_Delta_Days( $nowyear, $nowmonth, $nowday, -$days ); + my $dt_start = DateTime->new( year => $date[0], month => $date[1], day => $date[2] ); + $start_date = $dt_start->ymd(); + printlog("Harvesting from $start_date"); + } + + printlog("Asking for records"); + my $response = $h->ListRecords( + metadataPrefix => $server->dataformat, + set => $server->oai_set, + from => $start_date, + ); + if ( $response->is_error ) { + printlog( "Error requesting ListRecords: " . $response->code . " " . $response->message ); + exit; + } + printlog( "Request URL: " . $response->requestURL ); + + my %stats; + my @statuses = qw(added updated deleted in_error skipped total); + foreach my $status (@statuses) { + $stats{$status} = 0; + } + + printlog("Starting processing results"); + while ( my $oai_record = $response->next ) { + $stats{'total'}++; + my $status = $self->processRecord($oai_record); + $stats{$status}++; + } + + my $results = ''; + foreach my $status (@statuses) { + $results .= $stats{$status} . " $status\n"; + } + printlog( "Harvest results:\n" . $results ); + + if ( C4::Context->preference("OAI-PMH:HarvestEmailReport") ) { + + my $letter = C4::Letters::GetPreparedLetter( + ( + module => 'catalogue', + letter_code => 'OAI_HARVEST_REPORT', + substitute => { + servername => $server->servername, + endpoint => $server->endpoint, + set => $server->oai_set, + dataformat => $server->dataformat, + recordtype => $server->recordtype, + added => $stats{'added'}, + updated => $stats{'updated'}, + deleted => $stats{'deleted'}, + skipped => $stats{'skipped'}, + in_error => $stats{'in_error'}, + total => $stats{'total'} + }, + ) + ) or printlog("OAI_HARVEST_REPORT letter not found"); + + my $success = C4::Letters::EnqueueLetter( + { + letter => $letter, + to_address => C4::Context->preference("OAI-PMH:HarvestEmailReport"), message_transport_type => 'email' + } + ); + if ($success) { + printlog("Email report enqueued"); + } else { + printlog("Unable to enqueue report email"); + } + } + + printlog("Ending OAI Harvest from repository"); +} + +=head2 processRecord + +This method processes an incoming OAI record + +=cut + +sub processRecord { + my $self = shift; + my $oai_record = shift; + my $status = ''; + unless ( $oai_record->identifier ) { + printlog("No identifier found"); + $status = 'skipped'; + return $status; + } + + my $to_delete; + my $outputUnimarc; + my $marcrecord; + if ( $oai_record->status && $oai_record->status eq "deleted" ) { + $to_delete = 1; + } else { + if ( $server->add_xslt ) { + $outputUnimarc = $xslt_engine->transform( $oai_record->metadata->dom, $server->add_xslt ); + my $err = $xslt_engine->err; + if ($err) { + printlog("XSLT::Base error: $err"); + $status = 'in_error'; + return $status; + } + } else { + $outputUnimarc = $oai_record->metadata->dom; + } + $marcrecord = MARC::Record->new_from_xml( $outputUnimarc, 'UTF-8' ); + } + my $imported_record; + if ( $server->recordtype eq "biblio" ) { + $imported_record = Koha::Import::Oaipmh::Biblios->find( + { + repository => $server->endpoint, + identifier => $oai_record->identifier, + recordtype => $server->recordtype, + } + ); + } else { + $imported_record = Koha::Import::Oaipmh::Authorities->find( + { + repository => $server->endpoint, + identifier => $oai_record->identifier, + recordtype => $server->recordtype, + } + ); + } + + if ($imported_record) { + if ($to_delete) { + if ( $server->recordtype eq "biblio" ) { + my $biblionumber = $imported_record->biblionumber; + my $error = DelBiblio($biblionumber); + if ($error) { + printlog( + "Record " . $oai_record->identifier . " not deleted, biblionumber: $biblionumber ($error)" ); + $status = 'in_error'; + } else { + $imported_record->delete; + printlog( "Record " . $oai_record->identifier . " deleted, biblionumber: $biblionumber" ); + $status = 'deleted'; + } + } else { + my $authid = $imported_record->authid; + DelAuthority( { authid => $authid } ); + $imported_record->delete; + printlog( "Record " . $oai_record->identifier . " deleted, authid: $authid" ); + $status = 'deleted'; + } + } else { + my $existing_dt = $strp->parse_datetime( $imported_record->datestamp ); + my $incoming_dt = $strp->parse_datetime( $oai_record->datestamp ); + + if ( $force || !$incoming_dt || !$existing_dt || $incoming_dt > $existing_dt ) { + if ( $server->recordtype eq "biblio" ) { + my $biblionumber = $imported_record->biblionumber; + my $result = ModBiblio( $marcrecord, $biblionumber, GetFrameworkCode($biblionumber) ); + printlog( "Record " . $oai_record->identifier . " updated, biblionumber: $biblionumber" ); + } else { + my $authid = $imported_record->authid; + my $result = ModAuthority( $authid, $marcrecord, GuessAuthTypeCode($marcrecord) ); + printlog( "Record " . $oai_record->identifier . " updated, authid: $authid" ); + } + $imported_record->update( + { + datestamp => $imported_record->datestamp, + } + ); + $status = 'updated'; + } else { + printlog( "Record " + . $oai_record->identifier + . " skipped (incoming record was not newer than existing record)" ); + $status = 'skipped'; + } + } + } elsif ( !$to_delete ) { + if ( $server->recordtype eq "biblio" ) { + my ( $biblionumber, $biblioitemnumber ) = AddBiblio($marcrecord); + printlog( $oai_record->identifier . " added, biblionumber: $biblionumber" ); + Koha::Import::Oaipmh::Biblio->new( + { + repository => $server->endpoint, + identifier => $oai_record->identifier, + biblionumber => $biblionumber, + recordtype => $server->recordtype, + datestamp => $oai_record->datestamp, + } + )->store; + } else { + my $authid = AddAuthority( $marcrecord, "", GuessAuthTypeCode($marcrecord) ); + printlog( $oai_record->identifier . " added, authid: $authid" ); + Koha::Import::Oaipmh::Authority->new( + { + repository => $server->endpoint, + identifier => $oai_record->identifier, + authid => $authid, + recordtype => $server->recordtype, + datestamp => $oai_record->datestamp, + } + )->store; + } + $status = 'added'; + } else { + printlog( "Record " . $oai_record->identifier . " skipped (record not present or already deleted)" ); + $status = 'skipped'; + } + return $status; +} + +=head2 printlog + +This method adds a cronlog and prints to stdout if verbose is enabled + +=cut + +sub printlog { + my $message = shift; + $message = $server->servername . ": " . $message; + print $message . "\n" if ($verbose); + cronlogaction( { info => $message } ); +} + +1; diff --git a/Koha/OaiServer.pm b/Koha/OaiServer.pm new file mode 100644 index 0000000000..fa97c1f0b7 --- /dev/null +++ b/Koha/OaiServer.pm @@ -0,0 +1,45 @@ +package Koha::OaiServer; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; + + +use Koha::Database; + +use base qw(Koha::Object); + +=head1 NAME + +Koha::OaiServer - Koha OaiServer Object class + +=head1 API + +=head2 Class Methods + +=cut + +=head3 _type + +Return type of Object relating to Schema ResultSet + +=cut + +sub _type { + return 'Oaiserver'; +} + +1; diff --git a/Koha/OaiServers.pm b/Koha/OaiServers.pm new file mode 100644 index 0000000000..b752419c5d --- /dev/null +++ b/Koha/OaiServers.pm @@ -0,0 +1,57 @@ +package Koha::OaiServers; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; + + +use Koha::Database; + +use Koha::OaiServer; + +use base qw(Koha::Objects); + +=head1 NAME + +Koha::OaiServers - Koha OaiServer Object set class + +=head1 API + +=head2 Class Methods + +=cut + +=head3 _type + +Return type of object, relating to Schema ResultSet + +=cut + +sub _type { + return 'Oaiserver'; +} + +=head3 object_class + +Return object class + +=cut + +sub object_class { + return 'Koha::OaiServer'; +} + +1; diff --git a/admin/oaiservers.pl b/admin/oaiservers.pl new file mode 100755 index 0000000000..cdbe26dcb3 --- /dev/null +++ b/admin/oaiservers.pl @@ -0,0 +1,122 @@ +#!/usr/bin/perl + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +# This script is used to maintain the OAI servers table. +# Parameter $op is operation: list, new, edit, add_validated, delete_confirmed. +# add_validated saves a validated record and goes to list view. +# delete_confirmed deletes a record and goes to list view. + +use Modern::Perl; +use CGI qw ( -utf8 ); +use C4::Context; +use C4::Auth qw( get_template_and_user ); +use C4::Output qw( output_html_with_http_headers ); +use Koha::Database; +use Koha::OaiServers; + +# Initialize CGI, template, database + +my $input = CGI->new; +my $op = $input->param('op') || 'list'; +my $id = $input->param('id') || 0; +my $type = $input->param('type') || ''; +my $searchfield = ''; + +my ( $template, $loggedinuser, $cookie ) = get_template_and_user( + { + template_name => "admin/oaiservers.tt", + query => $input, + type => "intranet", + flagsrequired => { parameters => 'manage_search_targets' }, + } +); +my $script_name = "/cgi-bin/koha/admin/oaiservers.pl"; +my $path = C4::Context->config('intrahtdocs') . "/prog/"; + +$template->param( script_name => $script_name, xslt_path => $path ); + +my $schema = Koha::Database->new()->schema(); + +# Main code +# First process a confirmed delete, or save a validated record + +if ( $op eq 'delete_confirmed' && $id ) { + my $server = Koha::OaiServers->find($id); + if ($server) { + $server->delete; + $template->param( msg_deleted => 1, msg_add => $server->servername ); + } else { + $template->param( msg_notfound => 1, msg_add => $id ); + } + $id = 0; +} elsif ( $op eq 'add_validated' ) { + my @fields = qw/endpoint oai_set dataformat + recordtype servername + add_xslt/; + my $formdata = _form_data_hashref( $input, \@fields ); + if ($id) { + my $server = Koha::OaiServers->find($id); + if ($server) { + $server->set($formdata)->store; + $template->param( msg_updated => 1, msg_add => $formdata->{servername} ); + } else { + $template->param( msg_notfound => 1, msg_add => $id ); + } + $id = 0; + } else { + Koha::OaiServer->new($formdata)->store; + $template->param( msg_added => 1, msg_add => $formdata->{servername} ); + } +} else { + + #use searchfield only in remaining operations + $searchfield = $input->param('searchfield') || ''; +} + +# Now list multiple records, or edit one record + +my $data = []; +if ( $op eq 'add' || $op eq 'edit' ) { + $data = ServerSearch( $schema, $id, $searchfield ) if $searchfield || $id; + delete $data->[0]->{id} if @$data && $op eq 'add'; #cloning record + $template->param( + add_form => 1, server => @$data ? $data->[0] : undef, + op => $op, type => $op eq 'add' ? lc $type : '' + ); +} else { + $data = ServerSearch( $schema, $id, $searchfield ); + $template->param( + loop => \@$data, searchfield => $searchfield, id => $id, + op => 'list' + ); +} +output_html_with_http_headers $input, $cookie, $template->output; + +# End of main code + +sub ServerSearch { #find server(s) by id or name + my ( $schema, $id, $searchstring ) = @_; + + return Koha::OaiServers->search( + $id ? { id => $id } : { servername => { like => $searchstring . '%' } }, + )->unblessed; +} + +sub _form_data_hashref { + my ( $input, $fieldref ) = @_; + return { map { ( $_ => scalar $input->param($_) // '' ) } @$fieldref }; +} diff --git a/installer/data/mysql/atomicupdate/Bug-35659.pl b/installer/data/mysql/atomicupdate/Bug-35659.pl new file mode 100755 index 0000000000..6627c812ce --- /dev/null +++ b/installer/data/mysql/atomicupdate/Bug-35659.pl @@ -0,0 +1,88 @@ +use Modern::Perl; + +return { + bug_number => "35659", + description => "OAI-PMH Harvester", + up => sub { + my ($args) = @_; + my ( $dbh, $out ) = @$args{qw(dbh out)}; + $dbh->do(q{DROP TABLE IF EXISTS `oaiservers`;}); + $dbh->do( + q{ + CREATE TABLE `oaiservers` ( + `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha', + `endpoint` varchar(255) NOT NULL COMMENT 'OAI endpoint (host + port + path)', + `oai_set` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest', + `servername` longtext NOT NULL COMMENT 'name given to the target by the library', + `dataformat` enum('oai_dc','marc-xml', 'marcxml') NOT NULL DEFAULT 'oai_dc' COMMENT 'data format', + `recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'server contains bibliographic or authority records', + `add_xslt` longtext DEFAULT NULL COMMENT 'zero or more paths to XSLT files to be processed on the search results', + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + } + ); + say $out "Added new table 'oaiservers'"; + + $dbh->do(q{DROP TABLE IF EXISTS `import_oaipmh_biblios`;}); + $dbh->do( + q{ + CREATE TABLE `import_oaipmh_biblios` ( + `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha', + `biblionumber` int(11) NOT NULL COMMENT 'unique identifier assigned to each koha record', + `identifier` varchar(255) NOT NULL COMMENT 'OAI record identifier', + `repository` varchar(255) NOT NULL COMMENT 'OAI repository', + `recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'is the record bibliographic or authority', + `datestamp` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest', + `last_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (id), + KEY biblionumber (biblionumber), + CONSTRAINT FK_import_oaipmh_biblios_1 FOREIGN KEY (biblionumber) REFERENCES biblio (biblionumber) ON DELETE CASCADE ON UPDATE NO ACTION + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + } + ); + + $dbh->do(q{DROP TABLE IF EXISTS `import_oaipmh_authorities`;}); + $dbh->do( + q{ + CREATE TABLE `import_oaipmh_authorities` ( + `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha', + `authid` bigint(20) unsigned NOT NULL COMMENT 'unique identifier assigned to each koha record', + `identifier` varchar(255) NOT NULL COMMENT 'OAI record identifier', + `repository` varchar(255) NOT NULL COMMENT 'OAI repository', + `recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'is the record bibliographic or authority', + `datestamp` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest', + `last_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (id), + KEY authid (authid), + CONSTRAINT FK_import_oaipmh_authorities_1 FOREIGN KEY (authid) REFERENCES auth_header (authid) ON DELETE CASCADE ON UPDATE NO ACTION + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + } + ); + say $out "Added new tables 'import_oaipmh_biblios' and import_oaipmh_authorities"; + + $dbh->do( + q{ + UPDATE `permissions` SET description='Manage Z39.50 and SRU servers, OAI repositories configuration' WHERE code='manage_search_targets'; + } + ); + say $out "Updated manage_search_targets permission description"; + + $dbh->do( + q{ + INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES + ('OAI-PMH:HarvestEmailReport','','','If set, a report email will be sent to this address after an OAI harvest','Free'); + } + ); + say $out "Added OAI-PMH:HarvestEmailReport system preference"; + + $dbh->do( + q{ + INSERT IGNORE INTO letter + (module, code, branchcode, name, is_html, title, content, message_transport_type, lang) + VALUES ('catalogue','OAI_HARVEST_REPORT','','OAI Harvest Report',0,'OAI Harvest report for [% servername %]','OAI Harvest report for [% servername %]:\n\nEndpoint: [% endpoint %]\nSet: [% set %]\nData format: [% dataformat %]\nRecord type: [% recordtype %]\n\n[% added %] records added\n[% updated %] records updated\n[% deleted %] records deleted\n[% skipped %] records skipped\n[% in_error %] records in error\n[% total %] total','email','default'); + } + ); + say $out "Added OAI_HARVEST_REPORT letter"; + + }, +}; diff --git a/installer/data/mysql/en/mandatory/sample_notices.yml b/installer/data/mysql/en/mandatory/sample_notices.yml index 9675b3eb4f..6451df8974 100644 --- a/installer/data/mysql/en/mandatory/sample_notices.yml +++ b/installer/data/mysql/en/mandatory/sample_notices.yml @@ -2467,3 +2467,26 @@ tables: - "Your hold on [% hold.biblio.title %] ([% hold.biblio.id %]) has been confirmed." - "You will be notified by the library when your item is available for pickup." - "Thank you!" + + - module: catalogue + code: OAI_HARVEST_REPORT + branchcode: "" + name: "OAI Harvest Report" + is_html: 0 + title: "OAI Harvest report for [% servername %]" + message_transport_type: email + lang: default + content: + - "OAI Harvest report for [% servername %]:" + - "" + - "Endpoint: [% endpoint %]" + - "Set: [% set %]" + - "Data format: [% dataformat %]" + - "Record type: [% recordtype %]" + - "" + - "[% added %] records added" + - "[% updated %] records updated" + - "[% deleted %] records deleted" + - "[% skipped %] records skipped" + - "[% in_error %] records in error" + - "[% total %] total" diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql index 16310f0f44..8750d9a536 100644 --- a/installer/data/mysql/kohastructure.sql +++ b/installer/data/mysql/kohastructure.sql @@ -4819,6 +4819,64 @@ CREATE TABLE `oai_sets_mappings` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; +-- +-- Table structure for table `oaiservers` +-- + +DROP TABLE IF EXISTS `oaiservers`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `oaiservers` ( + `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha', + `endpoint` varchar(255) NOT NULL COMMENT 'OAI endpoint (host + port + path)', + `oai_set` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest', + `servername` longtext NOT NULL COMMENT 'name given to the target by the library', + `dataformat` enum('oai_dc','marc-xml', 'marcxml') NOT NULL DEFAULT 'oai_dc' COMMENT 'data format', + `recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'server contains bibliographic or authority records', + `add_xslt` longtext DEFAULT NULL COMMENT 'zero or more paths to XSLT files to be processed on the search results', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- +-- Table structure for table `import_oaipmh_biblios` +-- + +DROP TABLE IF EXISTS `import_oaipmh_biblios`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `import_oaipmh_biblios` ( + `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha', + `biblionumber` int(11) NOT NULL COMMENT 'unique identifier assigned to each koha record', + `identifier` varchar(255) NOT NULL COMMENT 'OAI record identifier', + `repository` varchar(255) NOT NULL COMMENT 'OAI repository', + `recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'is the record bibliographic or authority', + `datestamp` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest', + `last_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (id), + KEY biblionumber (biblionumber), + CONSTRAINT FK_import_oaipmh_biblios_1 FOREIGN KEY (biblionumber) REFERENCES biblio (biblionumber) ON DELETE CASCADE ON UPDATE NO ACTION +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- +-- Table structure for table `import_oaipmh_authorities` +-- + +DROP TABLE IF EXISTS `import_oaipmh_authorities`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `import_oaipmh_authorities` ( + `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha', + `authid` bigint(20) unsigned NOT NULL COMMENT 'unique identifier assigned to each koha record', + `identifier` varchar(255) NOT NULL COMMENT 'OAI record identifier', + `repository` varchar(255) NOT NULL COMMENT 'OAI repository', + `recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'is the record bibliographic or authority', + `datestamp` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest', + `last_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (id), + KEY authid (authid), + CONSTRAINT FK_import_oaipmh_authorities_1 FOREIGN KEY (authid) REFERENCES auth_header (authid) ON DELETE CASCADE ON UPDATE NO ACTION +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + -- -- Table structure for table `oauth_access_tokens` -- diff --git a/installer/data/mysql/mandatory/sysprefs.sql b/installer/data/mysql/mandatory/sysprefs.sql index 669074aa50..75bd895e78 100644 --- a/installer/data/mysql/mandatory/sysprefs.sql +++ b/installer/data/mysql/mandatory/sysprefs.sql @@ -437,6 +437,7 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` ('OAI-PMH:AutoUpdateSetsEmbedItemData', '0', '', 'Embed item information when automatically updating OAI sets. Requires OAI-PMH:AutoUpdateSets syspref to be enabled', 'YesNo'), ('OAI-PMH:ConfFile','',NULL,'If empty, Koha OAI Server operates in normal mode, otherwise it operates in extended mode.','File'), ('OAI-PMH:DeletedRecord','persistent','Koha\'s deletedbiblio table will never be deleted (persistent), might be deleted (transient), or will never have any data in it (no)','transient|persistent|no','Choice'), +('OAI-PMH:HarvestEmailReport','','','If set, a report email will be sent to this address after an OAI harvest','Free'), ('OAI-PMH:MaxCount','50',NULL,'OAI-PMH maximum number of records by answer to ListRecords and ListIdentifiers queries','Integer'), ('OnSiteCheckoutAutoCheck','0','','Enable/Do not enable onsite checkout by default if last checkout was an onsite checkout','YesNo'), ('OnSiteCheckouts','0','','Enable/Disable the on-site checkouts feature','YesNo'), diff --git a/installer/data/mysql/mandatory/userpermissions.sql b/installer/data/mysql/mandatory/userpermissions.sql index 729c075548..a4dbed1247 100644 --- a/installer/data/mysql/mandatory/userpermissions.sql +++ b/installer/data/mysql/mandatory/userpermissions.sql @@ -29,7 +29,7 @@ INSERT INTO permissions (module_bit, code, description) VALUES ( 3, 'manage_item_search_fields', 'Manage item search fields'), ( 3, 'manage_search_engine_config', 'Manage search engine configuration'), ( 3, 'manage_marc_overlay_rules', 'Manage MARC overlay rules configuration'), - ( 3, 'manage_search_targets', 'Manage Z39.50 and SRU server configuration'), + ( 3, 'manage_search_targets', 'Manage Z39.50 and SRU servers, OAI repositories configuration'), ( 3, 'manage_didyoumean', 'Manage Did you mean? configuration'), ( 3, 'manage_column_config', 'Manage column configuration'), ( 3, 'manage_sms_providers', 'Manage SMS cellular providers'), diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc index c7caffc014..184d27cfa0 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc @@ -149,6 +149,7 @@ [% END %] [% IF ( CAN_user_parameters_manage_search_targets ) %]
  • Z39.50/SRU servers
    • +
  • OAI repositories
    • [% END %] [% IF ( CAN_user_parameters_manage_smtp_servers ) %]
  • SMTP servers
    • diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/oai-admin-search.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/oai-admin-search.inc new file mode 100644 index 0000000000..49feedd580 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/oai-admin-search.inc @@ -0,0 +1,43 @@ +[% USE Koha %] + + + diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/permissions.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/permissions.inc index 742e6e1c35..15ad115181 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/permissions.inc +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/permissions.inc @@ -244,7 +244,7 @@ ([% name | html %]) [%- CASE 'manage_search_targets' -%] - Manage Z39.50 and SRU server configuration + Manage Z39.50 and SRU servers, OAI repositories configuration ([% name | html %]) [%- CASE 'manage_didyoumean' -%] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt index d5582effc2..61fa6e5bd2 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt @@ -249,6 +249,8 @@ [% IF ( CAN_user_parameters_manage_search_targets ) %]
    Z39.50/SRU servers
    Define which external servers to query for MARC data
    +
    OAI repositories
    +
    Define which OAI repositories to harvest data from
    [% END %] [% IF ( CAN_user_parameters_manage_smtp_servers ) %]
    SMTP servers
    diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oaiservers.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oaiservers.tt new file mode 100644 index 0000000000..7f95726ab9 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oaiservers.tt @@ -0,0 +1,229 @@ +[% USE raw %] +[% USE Asset %] +[% SET footerjs = 1 %] +[% INCLUDE 'doc-head-open.inc' %] + + + [% IF op == 'edit' %] + Modify OAI repository [% server.servername | html %] › [% END %] + + [% IF op == 'add' %] + New OAI repository › [% END %] + + OAI repositories › Administration › Koha + +[% INCLUDE 'doc-head-close.inc' %] + + + +[% WRAPPER 'header.inc' %] + [% INCLUDE 'oai-admin-search.inc' %] +[% END %] + +[% WRAPPER 'sub-header.inc' %] + [% WRAPPER breadcrumbs %] + [% WRAPPER breadcrumb_item %] + Administration + [% END %] + + [% IF op == 'edit' || op == 'add' %] + [% WRAPPER breadcrumb_item %] + OAI repositories + [% END %] + [% END %] + + [% IF op == 'edit' %] + [% WRAPPER breadcrumb_item bc_active= 1 %] + Modify OAI repository [% server.servername | html %] + [% END %] + + [% ELSIF op == 'add' %] + [% WRAPPER breadcrumb_item bc_active= 1 %] + New OAI repository + [% END %] + + [% ELSE %] + [% WRAPPER breadcrumb_item bc_active= 1 %] + OAI repositories + [% END %] + [% END %] + [% END #/ WRAPPER breadcrumbs %] +[% END #/ WRAPPER sub-header.inc %] + +
    +
    +
    +
    + +[% IF msg_deleted %] +
    OAI repository deleted ([% msg_add | html %])
    +[% ELSIF msg_updated %] +
    OAI repository updated ([% msg_add | html %])
    +[% ELSIF msg_added %] +
    OAI repository added ([% msg_add | html %])
    +[% ELSIF msg_notfound %] +
    Error: Server with ID [% msg_add | html %] not found
    +[% END %] + +[% IF ( add_form ) %] +
    + + [% IF op == 'edit' %] +

    Modify OAI repository

    + + [% ELSE %] +

    New OAI repository

    + [% END %] +
    +
      +
    1. + [% IF server.servername %] + + [% ELSE %] + + [% END %] + Required +
      2. + +
    2. Required +
      3. +
    3. +
      4. +
    4. + + +
      OAI can send records in various formats. Choose one.
      +
      5. + +
    5. + +
      6. +
    6. + + +
      7. + +
    +
    + +
    Cancel
    +
    +[% END %] + +[% IF op == 'list' %] + +

    OAI repositories administration

    + [% IF id %] + You searched for record [% id | html %] + [% ELSIF searchfield %] + You searched for [% searchfield | html %] + [% END %] + +
    + + + + + + [% FOREACH loo IN loop %] + + + + + + + + [% END %] + +
    IdRepository nameEndpointSetData formatRecord typeActions
    [% loo.id | html %][% loo.servername | html %][% loo.endpoint | html %][% loo.oai_set | html %][% loo.dataformat | html %][% IF ( loo.recordtype == 'biblio' ) %] + Bibliographic + [% ELSIF ( loo.recordtype == 'authority' ) %] + Authority + [% END %] + +
    + + Actions + + +
    +
    +
    +[% END %] + +
    +
    + +
    + +
    +
    + +[% MACRO jsinclude BLOCK %] + [% Asset.js("js/admin-menu.js") | $raw %] + [% IF op == 'list' %] + [% INCLUDE 'datatables.inc' %] + [% END %] + + +[% END %] +[% INCLUDE 'intranet-bottom.inc' %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref index 1fb5c99916..ae5c298a05 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref @@ -80,6 +80,11 @@ Web services: transient: might be emptied or truncated at some point (transient) no: will never have any data in it (no) - "." + - + - If set, a report email will be sent to this address after an OAI harvest + - pref: "OAI-PMH:HarvestEmailReport" + class: Text + ILS-DI: - - pref: ILS-DI diff --git a/koha-tmpl/intranet-tmpl/prog/en/xslt/AuthOAIDCtoMARC21XML.xsl b/koha-tmpl/intranet-tmpl/prog/en/xslt/AuthOAIDCtoMARC21XML.xsl new file mode 100644 index 0000000000..e2dbe4d4d6 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/xslt/AuthOAIDCtoMARC21XML.xsl @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/koha-tmpl/intranet-tmpl/prog/en/xslt/AuthOAIDCtoUNIMARCXML.xsl b/koha-tmpl/intranet-tmpl/prog/en/xslt/AuthOAIDCtoUNIMARCXML.xsl new file mode 100644 index 0000000000..9dcd285c96 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/xslt/AuthOAIDCtoUNIMARCXML.xsl @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/koha-tmpl/intranet-tmpl/prog/en/xslt/OAIDCtoMARC21XML.xsl b/koha-tmpl/intranet-tmpl/prog/en/xslt/OAIDCtoMARC21XML.xsl new file mode 100644 index 0000000000..4e3d13a436 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/xslt/OAIDCtoMARC21XML.xsl @@ -0,0 +1,367 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + p + m + r + k + m + m + m + i + a + a + + + + + c + m + + + + + + + dc + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + . + + + + . + + + + + + + + + . + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + local + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Connect to this object online. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + () + + + + + + + + + + + + + + + + + . + + + + + + + + + + . + + + + + + + + + + diff --git a/koha-tmpl/intranet-tmpl/prog/en/xslt/OAIDCtoUNIMARCXML.xsl b/koha-tmpl/intranet-tmpl/prog/en/xslt/OAIDCtoUNIMARCXML.xsl new file mode 100644 index 0000000000..2abf4f7d53 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/xslt/OAIDCtoUNIMARCXML.xsl @@ -0,0 +1,341 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + . + + + + . + + + + + + + + + . + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + local + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Connect to this object online. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + () + + + + + + + + + + + + + + + + + . + + + + + + + + + + . + + + + + + + + + + diff --git a/koha-tmpl/intranet-tmpl/prog/en/xslt/OAIMarcxml2KohaMarcxml.xsl b/koha-tmpl/intranet-tmpl/prog/en/xslt/OAIMarcxml2KohaMarcxml.xsl new file mode 100644 index 0000000000..67669afdb0 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/xslt/OAIMarcxml2KohaMarcxml.xsl @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/misc/cronjobs/harvest_oai.pl b/misc/cronjobs/harvest_oai.pl new file mode 100755 index 0000000000..2cc212652d --- /dev/null +++ b/misc/cronjobs/harvest_oai.pl @@ -0,0 +1,110 @@ +#!/usr/bin/perl +# +# Copyright 2023 BibLibre +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use warnings; +use strict; +use utf8; + +use Getopt::Long qw( GetOptions ); +use Koha::Script -cron; +use Koha::OAI::Client::Harvester; +use Koha::OaiServers; +use C4::Log qw( cronlogaction ); +use Try::Tiny qw( catch try ); + +my $command_line_options = join( " ", @ARGV ); + +my ( $help, $verbose, $id, $days, $list, $force ); + +GetOptions( + 'h|help' => \$help, + 'v|verbose' => \$verbose, + 'r|repository:i' => \$id, + 'd|days:i' => \$days, + 'l|list' => \$list, + 'f|force' => \$force, +); +my $usage = << 'ENDUSAGE'; + +This script starts an OAI Harvest + +This script has the following parameters : + -h --help: this message + -v --verbose + -r --repository: id of the OAI repository + -d --days: number of days to harvest from (optional) + -l --list: list the OAI repositories + -f --force: force harvesting (ignore records datestamps) + +ENDUSAGE + +if ($help) { + print $usage; + exit; +} + +if ($list) { + my $servers = Koha::OaiServers->search( {}, { order_by => { -asc => 'id' } } )->unblessed; + print "The following repositories are available: \n\n"; + foreach my $server (@$servers) { + print $server->{'id'} . ": " + . $server->{'servername'} + . ", endpoint: " + . $server->{'endpoint'} + . ", set: " + . $server->{'oai_set'} + . ", recordtype: " + . $server->{'recordtype'} . "\n"; + } + print "\n"; + exit; +} + +if ( !$id ) { + print "The repository parameter is mandatory.\n"; + print $usage . "\n"; +} + +my $server = Koha::OaiServers->find($id); + +unless ($server) { + print "OAI Server $id unknown\n"; + exit; +} + +my $script_handler = Koha::Script->new( { script => $0 } ); + +try { + $script_handler->lock_exec; +} catch { + my $message = "Skipping execution of $0"; + print "$message\n" if $verbose; + cronlogaction( { info => $message } ); + exit; +}; + +cronlogaction( { action => 'Start', info => "Starting OAI Harvest" } ); +cronlogaction( { info => "Command line: $command_line_options" } ); + +my $harvester = + Koha::OAI::Client::Harvester->new( { server => $server, verbose => $verbose, days => $days, force => $force } ); +$harvester->init(); + +cronlogaction( { action => 'End', info => "Ending OAI Harvest" } ); +exit(0); diff --git a/t/db_dependent/Koha/OAIHarvester.t b/t/db_dependent/Koha/OAIHarvester.t new file mode 100755 index 0000000000..e42b9a08f4 --- /dev/null +++ b/t/db_dependent/Koha/OAIHarvester.t @@ -0,0 +1,78 @@ +#!/usr/bin/perl + +# Copyright 2024 BibLibre +# +# This file is part of Koha +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; +use Test::More tests => 5; +use Test::Exception; + +use t::lib::TestBuilder; +use HTTP::OAI; +use HTTP::OAI::Metadata::OAI_DC; +use HTTP::OAI::Record; +use HTTP::OAI::Encapsulation; +use Koha::Database; +use Koha::OaiServer; +use Koha::OaiServers; +use Koha::OAI::Client::Harvester; + +my $schema = Koha::Database->new->schema; +my $builder = t::lib::TestBuilder->new; + +$schema->storage->txn_begin; + +my $new_oai_1 = Koha::OaiServer->new( + { + endpoint => 'my_host1.org', + oai_set => 'set1', + servername => 'my_test_1', + dataformat => 'oai_dc', + recordtype => 'biblio', + add_xslt => '', + } +)->store; + +my $harvester = Koha::OAI::Client::Harvester->new( { server => $new_oai_1, verbose => 1, days => 1, force => 1 } ); + +my $record = + 'Pièces diverses ARCH/0320 [cote]FR-920509801 [RCR établissement]Central obrera boliviana [Fonds ou collection]1 carton1971/2000Archives et manuscrits'; + +my $r = HTTP::OAI::Record->new(); +$r->header->datestamp('2017-05-10T09:18:13Z'); +$r->metadata( HTTP::OAI::Metadata->new( dom => $record ) ); + +my $status = $harvester->processRecord($r); +is( $status, 'skipped', 'Record with no identifier is skipped' ); + +$r->header->identifier('oai:myarchive.org:oid-233'); + +$status = $harvester->processRecord($r); +is( $status, 'added', 'Record is added' ); + +$status = $harvester->processRecord($r); +is( $status, 'updated', 'When force is used, record is updated' ); + +$harvester = Koha::OAI::Client::Harvester->new( { server => $new_oai_1, verbose => 1, days => 1, force => 0 } ); +$status = $harvester->processRecord($r); +is( $status, 'skipped', 'When force is not used, record is skipped (already up to date)' ); + +$r->header->datestamp('2018-05-10T09:18:13Z'); +$status = $harvester->processRecord($r); +is( $status, 'updated', 'When force is not used, record is updated if newer' ); + +$schema->storage->txn_rollback; diff --git a/t/db_dependent/Koha/OAIServers.t b/t/db_dependent/Koha/OAIServers.t new file mode 100755 index 0000000000..9f135b77db --- /dev/null +++ b/t/db_dependent/Koha/OAIServers.t @@ -0,0 +1,95 @@ +#!/usr/bin/perl + +# Copyright 2015 Koha Development team +# +# This file is part of Koha +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; +use Test::More tests => 2; +use Test::Exception; + +use t::lib::TestBuilder; +use Koha::Database; +use Koha::OaiServer; +use Koha::OaiServers; + +my $schema = Koha::Database->new->schema; +my $builder = t::lib::TestBuilder->new; + +subtest 'new, find and delete tests' => sub { + plan tests => 4; + $schema->storage->txn_begin; + my $nb_of_oais = Koha::OaiServers->search->count; + my $new_oai_1 = Koha::OaiServer->new( + { + endpoint => 'my_host1.org', + oai_set => 'set1', + servername => 'my_test_1', + dataformat => 'oai_dc', + recordtype => 'biblio', + } + )->store; + my $new_oai_2 = Koha::OaiServer->new( + { + endpoint => 'my_host2.org', + oai_set => 'set2', + servername => 'my_test_2', + dataformat => 'marcxml', + recordtype => 'authority', + } + )->store; + + like( $new_oai_1->id, qr|^\d+$|, 'Adding a new oai server should have set the id' ); + is( Koha::OaiServers->search->count, $nb_of_oais + 2, 'The 2 oai servers should have been added' ); + + my $retrieved_oai_1 = Koha::OaiServers->find( $new_oai_1->id ); + is( + $retrieved_oai_1->servername, $new_oai_1->servername, + 'Find a oai server by id should return the correct oai server' + ); + + $retrieved_oai_1->delete; + is( Koha::OaiServers->search->count, $nb_of_oais + 1, 'Delete should have deleted the oai server' ); + + $schema->storage->txn_rollback; +}; + +subtest 'Check NOT NULL without default values' => sub { + plan tests => 5; + $schema->storage->txn_begin; + local $SIG{__WARN__} = sub { }; # TODO Needed it for suppressing DBIx warns + + my $server = Koha::OaiServer->new( + { + oai_set => 'set3', + dataformat => 'marcxml', + recordtype => 'biblio', + } + ); + + throws_ok { $server->store } 'DBIx::Class::Exception', 'Exception on empty endpoint'; + like( $@->{msg}, qr/'endpoint' doesn't have a default value/, 'Verified that DBIx blamed endpoint' ); + + $server->endpoint('endpoint_added'); + throws_ok { $server->store } 'DBIx::Class::Exception', 'Exception on empty servername'; + like( $@->{msg}, qr/'servername' doesn't have a default value/, 'Verified that DBIx blamed servername' ); + + $server->servername('servername_added'); + + lives_ok { $server->store } 'No exceptions anymore'; + + $schema->storage->txn_rollback; +}; -- 2.30.2