From b0237a0d193d08514ee2351e7e1c97dbe96ec56b Mon Sep 17 00:00:00 2001 From: David Cook Date: Tue, 8 Sep 2015 12:52:51 +1000 Subject: [PATCH] Bug 10662 - Build OAI-PMH Harvesting Client This patch set adds an OAI-PMH harvesting client to Koha. It provides a user interface (UI) for defining external servers from which to harvest records using the OAI-PMH protocol. After it downloads records, it checks the harvest database to see if it needs to add a new record, update an existing record, or delete a record in Koha. _TEST PLAN_ 1) Apply all patches 2) Run updatedatabase.pl (to apply the atomic update) 3) Go to Administration > OAI-PMH servers 4) Click "New OAI-PMH server target" 5) At a minimum, include a valid "Base URL" and a valid "Metadata prefix". 6) Click "Test HTTP and OAI-PMH parameters" 7) If successful, continue with this plan. If unsuccessful, address the warning messages displayed in red before testing the parameters again. 8) At this point, you might want to choose a preferred granularity. All OAI-PMH servers must support YYYY-MM-DD according to the spec, but in practice this isn't always the case, so you may need to choose a more particular granularity (note that this support isn't tested using the "Test" button). 9) You may also want to choose a "From" and "Until" range, at least for the purposes of testing, so that you don't accidentally try downloading thousands or millions of records. (You may also want to download by "Set".) 10) You must set the "Active" parameter to "Active" from "Inactive" for the harvester to work on this server target. 11) Optionally, you may provide a path to a XSLT to transform the incoming data. There is a parameter called "identifier" which is passed to the XSLT engine. This contains the unique OAI-PMH identifier for a record. You may wish to add this to the MARC, especially for the sake of provenance. (You may also want to strip 952, 942, and 999 fields, as well as $9 subfields from incoming records. You may also try the magic "default" keyword here which uses a XSLT I've already written and linked in the backend.) 12) Choose the MARC framework you would like to use (although Default is fine as well). 13) Optionally, you may wish to include a "Original system field". At this time, this has no real purpose. However, in the future, it may be used for linking downloaded holdings records with their original parent record. (e.g. the 004 of a holdings record would link to the 001 of the bibliographic record). This field uses the format of 001 or 999$c with the dollar sign as a subfield delimiter. 14) Click Save 15) You will now see a table containing your entry; click "View". 16) All the numbers on the following screen should be 0. -- 17) Set your environmental variables for KOHA_CONF and PERL5LIB 18) Run "perl /misc/cronjobs/oai/oai_harvester.pl -d -v" to download your records (NOTE: This downloader will run as long as it needs to, so try to only download a few records. Ctrl+C will stop the harvest if it gets out of control.) 19) Revisit the web app as per step #15 20) It should now say "Harvested records waiting to be imported: X" with X being higher than 0. 21) Run "perl /misc/cronjobs/oai/oai_harvester.pl -i -v" to import these records into Koha. 22) The terminal output should indicate the result of the import. This should also be reflected by the webapp as per step #15 (e.g. "Koha records created from harvested records: X"). -- Now, there are a few different scenarios to try: If you control the OAI-PMH repository, try editing a record you've downloaded, and try downloading records again (it might be necessary to change your "From" entry as this should be auto-updated after each harvest), and seeing if your Koha record is updated. If your repository also supports deleted records, try deleting a record that you've already imported into Koha. Koha should get a deletion notice and delete the record from Koha (unless it has items attached). If you delete a record from Koha, but that record still exists upstream, you'll still download updates for that record, but an error will be generated when trying to import into Koha. Each record in this "error" state will be recorded in the "View" section of the UI as "Harvested records in an error state: X". (In the future, I might make it so that the record gets re-added, or add more configuration options to control this behaviour.) If you want to reset the respository harvest (ie delete all your existing harvested records and re-harvest a repository), click "Reset repository harvest" in the "View" screen of the OAI-PMH server target. If errors are encountered while deleting an existing harvest, it should display hyperlinks to the problem records for manual intervention. --- Koha/OAI/Client/Importer.pm | 316 ++++++++++++++++ Koha/OAI/Client/Record.pm | 61 +++ Koha/OAI/Client/Records.pm | 58 +++ Koha/OAI/Client/Repositories.pm | 62 +++ Koha/OAI/Client/Repository.pm | 418 +++++++++++++++++++++ admin/oai_client.pl | 220 +++++++++++ ...0662-Build_OAI-PMH_Harvesting_Client_tables.sql | 51 +++ installer/data/mysql/kohastructure.sql | 49 +++ .../intranet-tmpl/prog/en/includes/admin-menu.inc | 1 + .../prog/en/modules/admin/admin-home.tt | 2 + .../prog/en/modules/admin/oai_client.tt | 326 ++++++++++++++++ .../prog/en/xslt/MARC21slimFromOAI.xsl | 82 ++++ .../intranet-tmpl/prog/en/xslt/StripOAIWrapper.xsl | 22 ++ misc/cronjobs/oai/oai_harvester.pl | 180 +++++++++ 14 files changed, 1848 insertions(+) create mode 100755 Koha/OAI/Client/Importer.pm create mode 100755 Koha/OAI/Client/Record.pm create mode 100755 Koha/OAI/Client/Records.pm create mode 100755 Koha/OAI/Client/Repositories.pm create mode 100755 Koha/OAI/Client/Repository.pm create mode 100755 admin/oai_client.pl create mode 100755 installer/data/mysql/atomicupdate/bug_10662-Build_OAI-PMH_Harvesting_Client_tables.sql create mode 100755 koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_client.tt create mode 100755 koha-tmpl/intranet-tmpl/prog/en/xslt/MARC21slimFromOAI.xsl create mode 100755 koha-tmpl/intranet-tmpl/prog/en/xslt/StripOAIWrapper.xsl create mode 100755 misc/cronjobs/oai/oai_harvester.pl diff --git a/Koha/OAI/Client/Importer.pm b/Koha/OAI/Client/Importer.pm new file mode 100755 index 0000000..cd38bbe --- /dev/null +++ b/Koha/OAI/Client/Importer.pm @@ -0,0 +1,316 @@ +package Koha::OAI::Client::Importer; + +# Copyright Prosentient Systems 2015 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, see . + +use Modern::Perl; + +use Koha::Database; +use Koha::XSLT_Handler; +use C4::Context; +use C4::Languages; +use C4::XSLT; +use C4::Biblio; + +use parent qw(Class::Accessor); +__PACKAGE__->mk_accessors( qw (xslt_handler verbosity) ); + +=head1 NAME + +Koha::OAI::Importer - Imports records, previously harvested via OAI-PMH, into Koha + +=head1 SYNOPSIS + + my $oai_importer = Koha::OAI::Importer->new({ + verbosity => $verbose, + }); + $oai_importer->import_records(); + +=head1 DESCRIPTION + +An OAI Importer is an object which retrieves records, previously harvested +via OAI-PMH, and determines whether they should be added as new records to the +catalogue or if they should update existing catalogue records. + +After this determination, the record's metadata is run through a XSLT, which +is configured on a per server basis, and then handed over to Koha's internal APIs. + +The record in the OAI harvesting queue is then updated to reflect the +action taken and the biblionumber to which it is linked. + +=head1 METHODS + +=head2 new + +The Importer constructor which sets the 'verbosity' and 'xslt_handler' +for the object. + +The 'verbosity' should be set by calling script. + +The 'xslt_handler' could also be set by a calling script, but defaults +to a new Koha::XSLT_Handler object with silenced warnings, so it is +not too noisy. + +=cut + +sub new { + my ($class, $args) = @_; + $args = {} unless defined $args; + $args->{xslt_handler} = Koha::XSLT_Handler->new({print_warns => 0,}) unless defined $args->{xslt_handler}; + $args->{verbosity} = 0 unless defined $args->{verbosity}; + return bless ($args, $class); +} + +sub delete_record { + my ( $self, $record ) = @_; + + my ($action, $system_number) = $self->determine_action_and_system_number({ + identifier => $record->identifier, + repository_id => $record->repository_id, + }); + my $error = C4::Biblio::DelBiblio($system_number); + if ($error){ + die $error; + } else { + $record->set({ + system_number => $system_number, + action => "deleted", + }); + $record->store(); + } +} + +=head2 import_record + +This is the method doing the heavy lifting. It runs the other +methods for determining what to do with the record and if it is +linked to an existing record, it transforms the record via XSLT, +and it ultimately adds or updates the record as need be. + +=cut + +sub import_record { + my ( $self, $record ) = @_; + + #Step one: determine action (i.e. determine if a record with this identifier from this repository has been successfully added before) + my ($action, $system_number) = $self->determine_action_and_system_number({ + identifier => $record->identifier, + repository_id => $record->repository_id, + }); + + #Step two: transform metadata (if necessary) && add/update in Koha + if ( ($action eq 'add') || ($action eq 'update') ){ + my $schema = Koha::Database->new()->schema(); + + #Lookup the repository so we can get the xslt_path and frameworkcode + my $repository = $schema->resultset('OaiHarvestRepository')->find($record->repository_id); + + my $metadata = $record->metadata; + if ($metadata && $repository->xslt_path){ + eval { + #If there is a XSLT path, try to transform metadata + my $transformed_metadata = $self->transform_metadata({ + xslt_path => $repository->xslt_path, + metadata => $metadata, + metadata_prefix => $record->metadata_prefix, + identifier => $record->identifier, + }); + if ($transformed_metadata){ + $metadata = $transformed_metadata; + } + }; + if ($@){ + warn "There was a non-fatal problem transforming the metadata during import: $@"; + } + } + if ($metadata){ + if ($record->record_type eq 'biblio'){ + my ( $biblionumber, $biblioitemnumber ); + #create MARC::Record object + my $marc_record = MARC::Record::new_from_xml( $metadata, "utf8", C4::Context->preference('marcflavour') ); + if ($marc_record){ + if ($action eq 'add'){ + #Add record to Koha + ($biblionumber,$biblioitemnumber) = C4::Biblio::AddBiblio($marc_record,$repository->frameworkcode); + } elsif ($action eq 'update'){ + #Update record in Koha + eval {C4::Biblio::ModBiblio( $marc_record,$system_number,$repository->frameworkcode)}; + if ($@){ + my $deleted_biblio = $schema->resultset('Deletedbiblio')->find($system_number); + my $existing_biblio = $schema->resultset('Biblio')->find($system_number); + if ( ( $deleted_biblio ) || (!$deleted_biblio && !$existing_biblio) ){ + die "It appears the original record referenced by biblionumber $system_number has been deleted."; + #FIXME: What should happen in this situation? You can't re-add the record without first removing + #the previous oai_harvest entries or there will be errors... + #($biblionumber,$biblioitemnumber) = C4::Biblio::AddBiblio($marc_record,$repository->frameworkcode); + #warn "Re-adding the incoming record as a new bibliographic record (biblionumber $system_number)."; + #$action = "re-add"; + } else { + die $@; + } + } + } + } + if ($biblionumber){ + $system_number = $biblionumber; + } + } + #TODO: Add handling for 'holdings' and 'auth' + } + if ($action && $system_number){ + #Update 'action' and 'system_number' columns for this record row + $record->set({ + action => $action, + system_number => $system_number, + }); + $record->store(); + } else { + die "Error importing record! action: ($action);"; + } + } +} + +=head 2 determine_action_and_system_number + +This method checks the database to see if a record from the same +OAI-PMH server with the same identifier has already been added +into Koha. If it hasn't, it should be added. If it has, it should +update the existing record. + +=cut + +sub determine_action_and_system_number { + my ( $self, $args ) = @_; + my ( $action, $system_number ); + if ( $args->{identifier} && $args->{repository_id} ){ + my $schema = Koha::Database->new()->schema(); + + #NOTE: $search_results is a DBIx::Class::ResultSet object + my $search_results = $schema->resultset('OaiHarvest')->search( + { + -and => [ + identifier => $args->{identifier}, + repository_id => $args->{repository_id}, + system_number => {'!=', undef}, + -or => [ + action => 'add', + action => 'update', + ], + ], + }, + { + columns => [ qw/system_number/ ], #Only retrieve system_number + group_by => [ qw/system_number/ ], #Group results by system_number so you only get 1 result if you have many rows sharing the same system_number + } + ); + + #NOTE: The DBIx::Class::ResultSet class has overloaded operators, + #which means a "count()" method is run whenever an object is called in a + #numerical context (like below). It will also always return 1 in a boolean context. + if ($search_results == 0){ + $action = 'add'; + } elsif ($search_results == 1){ + $action = 'update'; + while( my $result = $search_results->next ) { + $system_number = $result->system_number; + } + } else { + if ($search_results && $search_results > 1){ + die "More than one system_number matched the identifier " . $args->{identifier} . " with a repository_id of " . $args->{repository_id} . "."; + } else { + die "Unknown error scanning database for previous entries."; + } + } + } + return ($action,$system_number); +} + +=head2 transform_metadata + +This method takes a record's XML metadata and runs it through a XSLT +(using a Koha::XSLT_Handler object). The purpose of this is to remove +undesirable fields or to add desired ones. + +=cut + +sub transform_metadata { + my ( $self, $args ) = @_; + my $transformed_metadata; + + if ($self->xslt_handler){ + if ( ($args->{xslt_path}) && ($args->{metadata_prefix}) && ($args->{metadata}) && ($args->{identifier}) ){ + + my $xslt = _resolve_xslt_path($args->{xslt_path}, $args->{metadata_prefix}); + + #If there is a XSLT file, we try to transform the data + if ($xslt){ + require XML::LibXSLT; + #NOTE: You can use "register_function()" here to provide extension functions to the XSLT + #NOTE: For parameters, you must wrap XML::LibXSLT::xpath_to_string() around each key => value pair within a hashref + #{ XML::LibXSLT::xpath_to_string(identifier => $args->{identifier}), XML::LibXSLT::xpath_to_string(test => "tester"), } + $transformed_metadata = $self->xslt_handler->transform({ + xml => $args->{metadata}, + file => $xslt, + parameters => { XML::LibXSLT::xpath_to_string(identifier => $args->{identifier}), }, + }); + if ( $self->xslt_handler->err ){ + die $self->xslt_handler->errstr; + } + if (! $transformed_metadata){ + die "Unable to transform metadata"; + } + } + } + } + + return $transformed_metadata; +} + +=head2 _resolve_xslt_path + +An internal sub for turning magical values into actual filepaths to a XSLT. + +=cut + +sub _resolve_xslt_path { + my ( $xslfilename, $metadata_prefix ) = @_; + if ( $xslfilename ){ + if ( $xslfilename =~ /^\s*"?default"?\s*$/i ) { + if ($metadata_prefix =~ "marc"){ + #If the repository has a default XSLT and harvests MARCXML: + my $htdocs = C4::Context->config('intrahtdocs'); + my $theme = C4::Context->preference("template"); + my $lang = C4::Languages::getlanguage(); + my $xslfile = C4::Context->preference('marcflavour') . + "slimFromOAI.xsl"; + + $xslfilename = C4::XSLT::_get_best_default_xslt_filename($htdocs, $theme, $lang, $xslfile); + if ( ! -f $xslfilename ){ + #If the default XSLT doesn't exist, nullify this variable + $xslfilename = undef; + } + } else { + #If the repository has a default XSLT and doesn't harvest marc: + #We return a null because currently we only provide default XSLTs for MARC21 marcxml + $xslfilename = undef; + } + } + } + return $xslfilename; +} + + +1; diff --git a/Koha/OAI/Client/Record.pm b/Koha/OAI/Client/Record.pm new file mode 100755 index 0000000..82176d2 --- /dev/null +++ b/Koha/OAI/Client/Record.pm @@ -0,0 +1,61 @@ +package Koha::OAI::Client::Record; + +# Copyright Prosentient Systems 2015 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use Carp; + +use base qw(Koha::Object); +use C4::Biblio; + +=head1 NAME + +Koha::OAI::Client::Record - + +=head1 API + +=head2 Class Methods + +=cut + +sub delete { + my ( $self ) = @_; + if ($self->record_type eq 'biblio' && $self->system_number){ + my $error = C4::Biblio::DelBiblio($self->system_number); + if ($error){ + die $error; + } + } + $self->SUPER::delete(); +} + +=head3 type + +=cut + +sub type { + return 'OaiHarvest'; +} + +=head1 AUTHOR + +David Cook + +=cut + +1; diff --git a/Koha/OAI/Client/Records.pm b/Koha/OAI/Client/Records.pm new file mode 100755 index 0000000..40bd034 --- /dev/null +++ b/Koha/OAI/Client/Records.pm @@ -0,0 +1,58 @@ +package Koha::OAI::Client::Records; + +# Copyright Prosentient Systems 2015 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use Carp; + +use base qw(Koha::Objects); +use Koha::OAI::Client::Record; + +=head1 NAME + +Koha::OAI::Client::Records - + +=head1 API + +=head2 Class Methods + +=cut + +=head3 type + +=cut + +sub type { + return 'OaiHarvest'; +} + +=head3 object_class + +=cut + +sub object_class { + return 'Koha::OAI::Client::Record'; +} + +=head1 AUTHOR + +David Cook + +=cut + +1; diff --git a/Koha/OAI/Client/Repositories.pm b/Koha/OAI/Client/Repositories.pm new file mode 100755 index 0000000..00fe595 --- /dev/null +++ b/Koha/OAI/Client/Repositories.pm @@ -0,0 +1,62 @@ +package Koha::OAI::Client::Repositories; + +# Copyright Prosentient Systems 2015 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; + +use Carp; + +use Koha::Database; + +use Koha::OAI::Client::Repository; + +use base qw(Koha::Objects); + +=head1 NAME + +Koha::OAI::Client::Repositories - + +=head1 API + +=head2 Class Methods + +=cut + +=head3 type + +=cut + +sub type { + return 'OaiHarvestRepository'; +} + +=head3 object_class + +=cut + +sub object_class { + return 'Koha::OAI::Client::Repository'; +} + +=head1 AUTHOR + +David Cook + +=cut + +1; diff --git a/Koha/OAI/Client/Repository.pm b/Koha/OAI/Client/Repository.pm new file mode 100755 index 0000000..782f6d9 --- /dev/null +++ b/Koha/OAI/Client/Repository.pm @@ -0,0 +1,418 @@ +package Koha::OAI::Client::Repository; + +# Copyright Prosentient Systems 2015 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use Carp; +use POSIX qw(strftime); +use HTTP::OAI; +use URI; +use DateTime; +use DateTime::Format::Strptime; +use base qw(Koha::Object); + +use Koha::OAI::Client::Records; + +=head1 NAME + +Koha::OAI::Client::Repository - + +=head1 API + +=head2 Class Methods + +=cut + +=head3 test_settings + +=cut + +sub test_settings { + my ( $self ) = @_; + my $errors = {}; + my $harvester = $self->harvester(); + if ($harvester){ + my $identify = $harvester->Identify; + if ($identify->is_success){ + + #Test Granularity + my $granularity_setting = $self->datetime_granularity; + if ($granularity_setting eq "YYYY-MM-DDThh:mm:ssZ"){ + my $actual_granularity = $identify->granularity; + if ($granularity_setting ne $actual_granularity){ + $errors->{second_granularity_not_supported} = 1; + } + } + + #Test Set + my $set_setting = $self->opt_set; + if ($set_setting){ + my $sets = $harvester->ListSets(); + my $matched_set; + while ( my $set = $sets->next ){ + if ($set_setting eq $set->setSpec){ + $matched_set = 1; + last; + } + } + if ( ! $matched_set ){ + $errors->{set_does_not_exist} = 1; + } + } + + #Test Metadata prefix + my $metadata_prefix_setting = $self->metadata_prefix; + if ($metadata_prefix_setting){ + my $metadata_formats = $harvester->ListMetadataFormats(); + my $matched_format; + while ( my $metadata_format = $metadata_formats->next ){ + if ($metadata_prefix_setting eq $metadata_format->metadataPrefix){ + $matched_format = 1; + last; + } + } + if ( ! $matched_format ){ + $errors->{metadata_prefix_does_not_exist} = 1; + } + } + } else { + if ($identify->is_error()){ + foreach my $error ($identify->errors){ + if ($error->code =~ /^404$/){ + $errors->{url_not_found} = 1; + } elsif ($error->code =~ /^401$/){ + $errors->{failed_authentication} = 1; + } else { + $errors->{generic_identify_error} = 1; + } + } + } else { + #This should never happen, but you never know... + $errors->{generic_identify_error} = 1; + } + } + } else { + $errors->{no_harvester} = 1; + } + return $errors; +} + + +=head3 validate + +=cut + +sub validate { + my ( $self ) = @_; + my $errors = {}; + if ($self->base_url){ + my $uri = URI->new($self->base_url); + if ($uri){ + my $reference = ref $uri; + if ( ! grep { $reference eq $_ } ("URI::http","URI::https")){ + $errors->{base_url}->{invalid} = 1; + } + } + } elsif (! defined $self->base_url){ + #base_url is undefined + $errors->{base_url}->{required} = 1; + } + if ( ! defined $self->metadata_prefix ){ + $errors->{metadata_prefix}->{required} = 1; + } + return $errors; +} + +=head3 find_original_system_number + +=cut + +sub find_original_system_number { + my ( $self, $args ) = @_; + + my $original_system_number; + my $metadata = $args->{metadata}; + if ($metadata){ + my $original_system_field = $self->original_system_field; #e.g. 001 or 999$c + if ($original_system_field){ + #Select root element (e.g. ) + my $root = $metadata->documentElement(); + #Select the first child of root (ie the original metadata record) (eg ) + my $original_metadata = $root->firstChild; + + #Check to see if there is a MARC namespace + my $marcxml_prefix = $original_metadata->lookupNamespacePrefix("http://www.loc.gov/MARC21/slim"); + if (defined $marcxml_prefix){ + my ($tag,$code) = split(/\$/,$original_system_field); #e.g. split 999$c into 999 and c || split 001 into 001 + my $xpc = XML::LibXML::XPathContext->new; + $xpc->registerNs('marc', 'http://www.loc.gov/MARC21/slim'); + my $fields = $xpc->find(qq(//marc:record/node()[\@tag="$tag"]),$original_metadata); + if ($fields){ + #Use the first field found + my $field = $fields->get_node(1); + if ($field){ + my $node_name = $field->nodeName; + if ($node_name eq 'controlfield'){ + #If it's a controlfield, we can use the text without drilling down + $original_system_number = $field->textContent; + } elsif ($node_name eq 'datafield'){ + #If it's a datafield, we need to drill down into the subfields + if ($code){ + my $subfields = $field->find(qq(node()[\@code="$code"])); + if ($subfields){ + #Use the first subfield found + my $subfield = $subfields->get_node(1); + if ($subfield){ + $original_system_number = $subfield->textContent; + } + + } + } + } + } + } + } + } + } + return $original_system_number; +} + +=head3 queue_record + +=cut + +sub queue_record { + my ( $self, $args ) = @_; + my $record = $args->{record}; + my $stylesheet = $args->{stylesheet}; + + my $action = "to_import"; + if ( $record->is_deleted() ){ + my $already_deleted_rows = Koha::OAI::Client::Records->search({ identifier => $record->identifier, action => "deleted" }); + my $identified_rows = Koha::OAI::Client::Records->search({ identifier => $record->identifier }); + if ($identified_rows->count() > 0 && $already_deleted_rows->count() == 0){ + #Only queue records "to_delete", if the record has been previously harvested + $action = "to_delete"; + } else { + return; + } + } + + my $metadata = $record->metadata ? $record->metadata->dom : ''; + my $original_system_number; + if ($metadata){ + if ($stylesheet){ + #Use a stylesheet to strip the OAI-PMH metadata XML wrapper + eval { + my $result = $stylesheet->transform($metadata); + if ($result){ + $metadata = $result; + } + }; + if ($@){ + warn "Problem transforming harvested metadata with XSLT: $@"; + } + } + + $original_system_number = $self->find_original_system_number({metadata => $metadata}); + } + + my $entry = Koha::OAI::Client::Records->_resultset->find_or_new({ + repository_id => $self->repository_id, #Internal ID for the OAI repository + action => $action, + identifier => $record->identifier, #External ID for the OAI record + datestamp => $record->datestamp, #External datestamp for the OAI record + metadata_prefix => $self->metadata_prefix, #External metadataPrefix for the OAI record + metadata => $metadata ? $metadata->toString(1) : undef, + record_type => $self->record_type, + original_system_number => $original_system_number, + }, + { key => "harvest_identifier_datestamp" } + ); + if( !$entry->in_storage ) { + $entry->insert; + } +} + +sub _format_datetime { + my ( $self, $args ) = @_; + my $datetime_string = $args->{datetime_string}; + my $formatted_string; + if ($datetime_string && $datetime_string ne "0000-00-00 00:00:00"){ + my $strp = DateTime::Format::Strptime->new( + pattern => '%F %T', + ); + my $dt = $strp->parse_datetime($datetime_string); + if ($dt){ + if ($self->datetime_granularity && $self->datetime_granularity eq 'YYYY-MM-DDThh:mm:ssZ'){ + $formatted_string = $dt->strftime("%FT%TZ"); + } else { + $formatted_string = $dt->strftime("%F"); + } + } + } + return $formatted_string; +} + +sub harvester { + my ( $self ) = @_; + my $harvester; + if ($self->base_url){ + $harvester = new HTTP::OAI::Harvester( baseURL => $self->base_url ); + #Use basic http authentication credentials if they're set + # if ($self->basic_realm && $self->basic_username && $self->basic_password){ + #Decompose the URI in order to register the basic authentication credentials + my $uri = URI->new($self->base_url); + my $host = $uri->host; + my $port = $uri->port; + $harvester->credentials($host.":".$port, $self->basic_realm, $self->basic_username, $self->basic_password); + # } + } + return $harvester; +} + +=head3 query_repository + +=cut + +sub query_repository { + my ( $self ) = @_; + my $response; + my $h = $self->harvester(); + if ($h){ + my $opt_from = $self->opt_from; + my $opt_until = $self->opt_until; + foreach my $datetime ($opt_from,$opt_until){ + #Format the datetime strings from the database into the format and granularity + #expected by the target OAI-PMH server + $datetime = $self->_format_datetime({datetime_string => $datetime,}); + } + + #Issue the request to the OAI-PMH server + $response = $h->ListRecords( + metadataPrefix => $self->metadata_prefix, + from => $opt_from, + until => $opt_until, + set => $self->opt_set, + ); + } + return $response; +} + +=head3 harvest + +=cut + +sub harvest { + my ( $self, $args ) = @_; + + my $verbose = $args->{verbose} // ''; + my $stylesheet = $args->{stylesheet} // ''; + $verbose && print "Harvesting records from ".$self->base_url."\n"; + + #Record the exact time before sending the first OAI-PMH request, and set "opt_from" to this after the harvest is finished + my $new_from_date = strftime "%Y-%m-%d %H:%M:%S", localtime; + #Send a request to the OAI-PMH repository + my $oai_response = $self->query_repository(); + if ($oai_response){ + if( $oai_response->is_error ){ + warn "responseDate => " . $oai_response->responseDate . "\n"; + warn "requestURL => " . $oai_response->requestURL . "\n"; + warn "Error harvesting: " . $oai_response->message . "\n"; + } else { + while( my $rec = $oai_response->next ) { + $verbose && print $rec->identifier."\n"; + #Queue each record, so a different script can process them in Koha + $self->queue_record({record => $rec, stylesheet => $stylesheet, }); + } + #Update opt_from after a successful harvest + $self->opt_from($new_from_date); + $verbose && print "Updating `opt_from` to $new_from_date \n"; + $self->store(); + } + } +} + +#FIXME +=head3 reharvest + +=cut + +sub reharvest { + my ( $self, $args ) = @_; + my $verbose = $args->{verbose} // ''; + eval { + $self->delete_harvest({ verbose => $verbose, }); + }; + if ($@){ + warn $@; + } else { + $self->harvest({ verbose => $verbose, }); + } +} + +#FIXME +=head3 delete_harvest + +=cut + +sub delete_harvest { + my ( $self ) = @_; + + my $count_of_records = 0; + my $count_of_deleted_records = 0; + my $problem_records = []; + + if ($self->repository_id){ + my $records = Koha::OAI::Client::Records->search({ "repository_id" => $self->repository_id }); + while (my $record = $records->next){ + #Increment overall count + $count_of_records++; + eval { + my $is_deleted = $record->delete(); + if ($is_deleted){ + $count_of_deleted_records++; + } + }; + if ($@){ + push(@$problem_records,$record); + } + } + } + + if ($count_of_deleted_records < $count_of_records){ + #Not all of this repository's records could be deleted. Manual intervention may be required to remove items + die $problem_records; + } +} + +=head3 type + +=cut + +sub type { + return 'OaiHarvestRepository'; +} + +=head1 AUTHOR + +David Cook + +=cut + +1; diff --git a/admin/oai_client.pl b/admin/oai_client.pl new file mode 100755 index 0000000..cd8748d --- /dev/null +++ b/admin/oai_client.pl @@ -0,0 +1,220 @@ +#!/usr/bin/perl + +# Copyright Prosentient Systems 2015 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +=head1 NAME + +oai_client.pl + +=head1 DESCRIPTION + +Admin page to manage Koha's target OAI-PMH servers + +=cut + +use Modern::Perl; +use CGI qw ( -utf8 ); +use C4::Auth; +use C4::Output; +use C4::Koha qw/GetFrameworksLoop/; +use Koha::OAI::Client::Repository; +use Koha::OAI::Client::Repositories; +use Koha::OAI::Client::Records; + +my $input = new CGI; + +my ($template, $loggedinuser, $cookie, $flags) = get_template_and_user( { + template_name => 'admin/oai_client.tt', + query => $input, + type => 'intranet', + authnotrequired => 0, + flagsrequired => { 'parameters' => 'parameters_remaining_permissions' }, +} ); + +my $op = $input->param('op'); +my $test_settings = $input->param('test_settings'); +my $repository_id = $input->param('repository_id'); + +my $base_url = $input->param('base_url') || undef; +my $metadata_prefix = $input->param('metadata_prefix') || undef; +my $opt_from = $input->param('opt_from') || undef; +my $opt_until = $input->param('opt_until') || undef; +my $opt_set = $input->param('opt_set') || undef; +my $active = $input->param('active'); +my $xslt_path = $input->param('xslt_path') || undef; +my $frameworkcode = $input->param('frameworkcode'); #This can be empty as the default framework is an empty value +my $comments = $input->param('comments') || undef; +my $basic_username = $input->param('basic_username') || undef; +my $basic_password = $input->param('basic_password') || undef; +my $basic_realm = $input->param('basic_realm') || undef; +my $datetime_granularity = $input->param('datetime_granularity'); +my $record_type = $input->param('record_type'); +my $original_system_field = $input->param('original_system_field') || undef; + +if ( $op ){ + my $repository; + if ( $repository_id ){ + #Fetch repository from database... + $repository = Koha::OAI::Client::Repositories->find($repository_id); + } + + if ( $op eq "new" ){ + $template->param( + frameworkloop => C4::Koha::GetFrameworksLoop(), + ); + } elsif ( $op eq "create" ){ + my $new_repository = Koha::OAI::Client::Repository->new({ + base_url => $base_url, + metadata_prefix => $metadata_prefix, + opt_from => $opt_from, + opt_until => $opt_until, + opt_set => $opt_set, + active => $active, + xslt_path => $xslt_path, + frameworkcode => $frameworkcode, + comments => $comments, + basic_username => $basic_username, + basic_password => $basic_password, + basic_realm => $basic_realm, + datetime_granularity => $datetime_granularity, + record_type => $record_type, + original_system_field => $original_system_field, + }); + my $errors = $new_repository->validate(); + if (%$errors){ + $op = "new"; + $template->param( + repository => $new_repository, + frameworkloop => C4::Koha::GetFrameworksLoop($new_repository->frameworkcode), + errors => $errors, + ); + } else { + $new_repository->store(); + $op = "list"; + } + } elsif ( $op eq "edit" ){ + if ($repository){ + $template->param( + repository => $repository, + frameworkloop => C4::Koha::GetFrameworksLoop($repository->frameworkcode), + ); + } + } elsif ( ($op eq "view") || ($op eq "reset") ){ + if ($repository){ + + #If op eq reset, then delete the entire harvest for this repository + if ($op eq "reset"){ + eval { + $repository->delete_harvest(); + $template->param( reset_success => 1 ); + }; + if ($@){ + my $problem_records = $@; + $template->param( problem_records => $problem_records ); + } + } + + #"add", "update" + my $imported_records = Koha::OAI::Client::Records->search({ + repository_id => $repository->repository_id, + system_number => {'!=', undef}, + }, { + group_by => [ qw/system_number/ ], + #Group results by system_number so you only get 1 result if you have many rows sharing the same system_number + #Also, group by system number because you might have imported record 1, deleted it, and then it gets re-imported later as a different Koha record + }); + my $waiting_records = Koha::OAI::Client::Records->search({ repository_id => $repository->repository_id, action => 'to_import',}); + my $to_delete_records = Koha::OAI::Client::Records->search({ repository_id => $repository->repository_id, action => 'to_delete',}); + my $deleted_records = Koha::OAI::Client::Records->search({ repository_id => $repository->repository_id, action => 'deleted',}); + my $error_records = Koha::OAI::Client::Records->search({ repository_id => $repository->repository_id, action => 'error',}); + + $template->param( + repository => $repository, + imported_records_count => $imported_records->count(), + waiting_records_count => $waiting_records->count(), + to_delete_records_count => $to_delete_records->count(), + deleted_records_count => $deleted_records->count(), + error_records_count => $error_records->count(), + ); + } + if ($op eq "reset"){ + $op = "view"; + } + } elsif ( $op eq "update" ){ + if ($repository){ + $repository->set({ + base_url => $base_url, + metadata_prefix => $metadata_prefix, + opt_from => $opt_from, + opt_until => $opt_until, + opt_set => $opt_set, + active => $active, + xslt_path => $xslt_path, + frameworkcode => $frameworkcode, + comments => $comments, + basic_username => $basic_username, + basic_password => $basic_password, + basic_realm => $basic_realm, + datetime_granularity => $datetime_granularity, + record_type => $record_type, + original_system_field => $original_system_field, + }); + my $errors = $repository->validate(); + if (%$errors || $test_settings){ + if (! %$errors && $test_settings){ + my $test_errors = $repository->test_settings(); + if (%$test_errors){ + $template->param( test_errors => $test_errors ); + } else { + $template->param( test_success => 1 ); + } + } + $op = "edit"; + $template->param( + repository => $repository, + frameworkloop => C4::Koha::GetFrameworksLoop($repository->frameworkcode), + errors => $errors, + ); + } else { + $repository->store(); + $op = "list"; + } + } + } elsif ( $op eq "delete" ){ + if ($repository){ + $repository->delete; + } + $op = "list"; + } +} else { + $op = "list"; +} + +if ( $op eq "list" ){ + my @repositories = Koha::OAI::Client::Repositories->as_list(); + $template->param( + repositories => \@repositories, + frameworkloop => C4::Koha::GetFrameworksLoop(), + ); +} + +$template->param( + op => $op, +); + +output_html_with_http_headers $input, $cookie, $template->output; diff --git a/installer/data/mysql/atomicupdate/bug_10662-Build_OAI-PMH_Harvesting_Client_tables.sql b/installer/data/mysql/atomicupdate/bug_10662-Build_OAI-PMH_Harvesting_Client_tables.sql new file mode 100755 index 0000000..853ef46 --- /dev/null +++ b/installer/data/mysql/atomicupdate/bug_10662-Build_OAI-PMH_Harvesting_Client_tables.sql @@ -0,0 +1,51 @@ +DROP TABLE IF EXISTS oai_harvest; +DROP TABLE IF EXISTS oai_harvest_repositories; + +-- +-- Table structure for table `oai_harvest_repositories` +-- + +CREATE TABLE IF NOT EXISTS oai_harvest_repositories ( + repository_id int(11) NOT NULL AUTO_INCREMENT, -- primary key identifier + base_url text NOT NULL, -- baseURL of the remote OAI-PMH repository (mandatory) + basic_username text, -- username for basic http authentication + basic_password text, -- password for basic http authentication + basic_realm text, -- realm for basic http authentication + datetime_granularity enum('YYYY-MM-DDThh:mm:ssZ','YYYY-MM-DD') NOT NULL DEFAULT 'YYYY-MM-DD', -- granularity to use in harvest request + opt_from DATETIME DEFAULT NULL, -- "from" DATETIME for selective harvesting (optional - automatically updated by cronjob) + opt_until DATETIME DEFAULT NULL, -- "until" DATETIME for selective harvesting (optional) + opt_set varchar(45) DEFAULT NULL, -- the record set to harvest for selective harvesting (optional) + metadata_prefix varchar(45) NOT NULL, -- metadata format (e.g. oai_dc, dc, marcxml) + active int(1) NOT NULL DEFAULT '0', -- indicate whether this repo is actively harvested by Koha + timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- date last modified. Maybe `datelastmodified` would be a better column name. + xslt_path text, -- filepath to a local or external XSLT file to use for transforming incoming records into MARCXML + frameworkcode varchar(4) NOT NULL DEFAULT '', -- framework to use when ingesting records + comments text, -- limited number of characters (controlled by template) to describe the repository (optional - for librarian use rather than system use) + record_type enum('biblio','auth','holdings') NOT NULL DEFAULT 'biblio', + original_system_field varchar(45), -- Field where you can a find a record's original system number (e.g. 001 or 999$c) + -- NOTE: In future, may need to add fields for holdings-biblio relationship (e.g. field to designate 004 in holdings record as link to a biblio's original system number) + PRIMARY KEY (repository_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- +-- Table structure for table `oai_harvest` +-- + +CREATE TABLE IF NOT EXISTS oai_harvest ( + oai_harvest_id int(11) NOT NULL AUTO_INCREMENT, -- unique internal id for the OAI record + repository_id int(11) NOT NULL, -- OAI repository from which the record was harvested + system_number int(11) DEFAULT NULL, -- indicates the Koha record created from this record + action varchar(45) DEFAULT NULL, -- indicates whether this record has been processed and how it's been processed (e.g. add, update, ignore) + timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- last modified date in Koha. Maybe `datelastmodified` would be a better column name. + identifier varchar(255) NOT NULL, -- the record's OAI ID from the OAI repository + datestamp varchar(45) NOT NULL, -- the record's OAI datestamp from the OAI repository (e.g. YYYY-MM-DDTHH:MM:SSZ). + metadata_prefix varchar(45) NOT NULL, -- the type of metadata (e.g. marcxml, oai_dc, etc.) + metadata longtext, -- original metadata as transmitted via OAI-PMH + record_type enum('biblio','auth','holdings') NOT NULL DEFAULT 'biblio', + original_system_number varchar(45), -- original system number (e.g. the unique number in the 001 of the incoming record) + -- NOTE: In future, may need to add fields for holdings-biblio relationship + PRIMARY KEY (oai_harvest_id), + UNIQUE KEY harvest_identifier_datestamp (identifier,datestamp), + KEY FK_oai_harvest_1 (repository_id), + CONSTRAINT FK_oai_harvest_1 FOREIGN KEY (repository_id) REFERENCES oai_harvest_repositories (repository_id) ON UPDATE NO ACTION +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql index 146565c..cfac25f 100644 --- a/installer/data/mysql/kohastructure.sql +++ b/installer/data/mysql/kohastructure.sql @@ -3596,6 +3596,55 @@ CREATE TABLE audio_alerts ( KEY precedence (precedence) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; +-- +-- Table structure for table `oai_harvest_repositories` +-- + +CREATE TABLE IF NOT EXISTS oai_harvest_repositories ( + repository_id int(11) NOT NULL AUTO_INCREMENT, -- primary key identifier + base_url text NOT NULL, -- baseURL of the remote OAI-PMH repository (mandatory) + basic_username text, -- username for basic http authentication + basic_password text, -- password for basic http authentication + basic_realm text, -- realm for basic http authentication + datetime_granularity enum('YYYY-MM-DDThh:mm:ssZ','YYYY-MM-DD') NOT NULL DEFAULT 'YYYY-MM-DD', -- granularity to use in harvest request + opt_from DATETIME DEFAULT NULL, -- "from" DATETIME for selective harvesting (optional - automatically updated by cronjob) + opt_until DATETIME DEFAULT NULL, -- "until" DATETIME for selective harvesting (optional) + opt_set varchar(45) DEFAULT NULL, -- the record set to harvest for selective harvesting (optional) + metadata_prefix varchar(45) NOT NULL, -- metadata format (e.g. oai_dc, dc, marcxml) + active int(1) NOT NULL DEFAULT '0', -- indicate whether this repo is actively harvested by Koha + timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- date last modified. Maybe `datelastmodified` would be a better column name. + xslt_path text, -- filepath to a local or external XSLT file to use for transforming incoming records into MARCXML + frameworkcode varchar(4) NOT NULL DEFAULT '', -- framework to use when ingesting records + comments text, -- limited number of characters (controlled by template) to describe the repository (optional - for librarian use rather than system use) + record_type enum('biblio','auth','holdings') NOT NULL DEFAULT 'biblio', + original_system_field varchar(45), -- Field where you can a find a record's original system number (e.g. 001 or 999$c) + -- NOTE: In future, may need to add fields for holdings-biblio relationship (e.g. field to designate 004 in holdings record as link to a biblio's original system number) + PRIMARY KEY (repository_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- +-- Table structure for table `oai_harvest` +-- + +CREATE TABLE IF NOT EXISTS oai_harvest ( + oai_harvest_id int(11) NOT NULL AUTO_INCREMENT, -- unique internal id for the OAI record + repository_id int(11) NOT NULL, -- OAI repository from which the record was harvested + system_number int(11) DEFAULT NULL, -- indicates the Koha record created from this record + action varchar(45) DEFAULT NULL, -- indicates whether this record has been processed and how it's been processed (e.g. add, update, ignore) + timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- last modified date in Koha. Maybe `datelastmodified` would be a better column name. + identifier varchar(255) NOT NULL, -- the record's OAI ID from the OAI repository + datestamp varchar(45) NOT NULL, -- the record's OAI datestamp from the OAI repository (e.g. YYYY-MM-DDTHH:MM:SSZ). + metadata_prefix varchar(45) NOT NULL, -- the type of metadata (e.g. marcxml, oai_dc, etc.) + metadata longtext, -- original metadata as transmitted via OAI-PMH + record_type enum('biblio','auth','holdings') NOT NULL DEFAULT 'biblio', + original_system_number varchar(45), -- original system number (e.g. the unique number in the 001 of the incoming record) + -- NOTE: In future, may need to add fields for holdings-biblio relationship + PRIMARY KEY (oai_harvest_id), + UNIQUE KEY harvest_identifier_datestamp (identifier,datestamp), + KEY FK_oai_harvest_1 (repository_id), + CONSTRAINT FK_oai_harvest_1 FOREIGN KEY (repository_id) REFERENCES oai_harvest_repositories (repository_id) ON UPDATE NO ACTION +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc index e38b0b4..61a599f 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc @@ -62,6 +62,7 @@
  • Z39.50/SRU servers
    • +
  • OAI-PMH servers
  • Did you mean?
  • Columns settings
  • Audio alerts
    • diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt index 885654a..0ba39d0 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt @@ -107,6 +107,8 @@
    Printers (UNIX paths).
    -->
    Z39.50/SRU servers
    Define which external servers to query for MARC data.
    +
    OAI-PMH servers
    +
    Define external servers from which to harvest records via OAI-PMH.
    Did you mean?
    Choose which plugins to use to suggest searches to patrons and staff.
    Configure columns
    diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_client.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_client.tt new file mode 100755 index 0000000..d2453d6 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_client.tt @@ -0,0 +1,326 @@ +[% INCLUDE 'doc-head-open.inc' %] +Koha › Administration › OAI-PMH server targets +[% INCLUDE 'doc-head-close.inc' %] +[% INCLUDE 'calendar.inc' %] + +[% INCLUDE 'timepicker.inc' %] +[% IF ( op == "list" ) %] + + [% INCLUDE 'datatables.inc' %] + +[% ELSIF ( op == "edit" ) || ( op == "new" ) %] + + +[% END %] + + + +[% INCLUDE 'header.inc' %] +[% INCLUDE 'cat-search.inc' %] + + +
    + +
    +
    +
    + [% IF ( op ) %] + [% IF ( op == "list" ) %] + +

    OAI-PMH server targets

    + + + + + + + + + + + + + + + + + + + [% FOREACH repository IN repositories %] + + + + + + + + + + + + + + + [% END %] + +
    Base URLMetadata PrefixFromUntilSetActivePath to XSLTRecord typeMARC frameworkOriginal system fieldCommentsOptions
    [% repository.base_url %][% repository.metadata_prefix %][% IF ( repository.opt_from ) != "0000-00-00 00:00:00"; repository.opt_from; END; %][% IF ( repository.opt_until ) != "0000-00-00 00:00:00"; repository.opt_until; END; %][% repository.opt_set %] + [% IF ( repository.active == 1 ) %] + Active + [% ELSE %] + Inactive + [% END %] + [% repository.xslt_path %][% repository.record_type %] + [% + framework_name = ''; + FOREACH framework IN frameworkloop; + IF ( framework.value == repository.frameworkcode ); + framework_name = framework.description; + END; + END; + %] + [% IF ( framework_name ) %] + [% framework_name %] + [% ELSE %] + Default + [% END %] + [% repository.original_system_field %][% repository.comments %] + View + Edit + Delete +
    + [% ELSIF ( op == "view" ) %] +

    View OAI-PMH server target

    +

    [% repository.base_url %]

    +
    +
      +
    1. Koha records created from harvested records: [% imported_records_count %]
      2. +
    2. Koha records deleted by harvested records: [% deleted_records_count %]
      3. +
    3. Harvested records waiting to be imported: [% waiting_records_count %]
      4. +
    4. Harvested records waiting to be deleted from Koha: [% to_delete_records_count %]
      5. +
    5. Harvested records in an error state: [% error_records_count %]
      6. +
    +
    +
    +
    + + + +
    +
    + [% IF ( problem_records ) %] +

    Not all of this repository's records could be deleted. Manual intervention may be required to remove items. Review the following records:

    +
      + [% FOREACH problem_record IN problem_records %] + [% IF ( problem_record.record_type == "biblio" ) %] +
    1. [% problem_record.record_type %] - [% problem_record.system_number %]
      2. + [% ELSE %] +
    2. [% problem_record.record_type %] - [% problem_record.system_number %]
      3. + [% END %] + [% END %] +
    + [% END %] + [% IF ( reset_success ) %] +

    This repository no longer has any harvested records. Modify the "From" and "Until" dates to prepare the repository for the next harvest.

    + [% END %] + [% ELSIF ( op == "edit" ) || ( op == "new" ) %] + [% IF ( op == "new" ) %] +

    New OAI-PMH server target

    + [% ELSIF ( op == "edit" ) %] +

    Modify OAI-PMH server target

    + [% END %] +
    + [% IF ( op == "new" ) %] + + [% ELSIF ( op == "edit" ) %] + + + [% END %] +
    + HTTP parameters: +
      +
    1. + + + [% IF ( errors.base_url.required ) %]Required[% END %] + [% IF ( errors.base_url.invalid ) %](Invalid URL)[% END %] + [% IF ( test_errors.url_not_found ) %](404 - URL Not Found)[% END %] + +
      2. +
    2. + + + [% IF ( test_errors.failed_authentication ) %](Authentication failed)[% END %] +
      3. +
    3. + + + [% IF ( test_errors.failed_authentication ) %](Authentication failed)[% END %] +
      4. +
    4. + + + [% IF ( test_errors.failed_authentication ) %](Authentication failed)[% END %] +
      5. +
    +
    +
    + OAI-PMH parameters: +
      +
    1. + + + [% IF ( test_errors.set_does_not_exist ) %](Set unavailable)[% END %] +
      2. +
    2. + + + [% IF ( errors.metadata_prefix.required ) %]Required[% END %] + [% IF ( test_errors.metadata_prefix_does_not_exist ) %](Metadata prefix unavailable)[% END %] + +
      3. +
    3. + + + + [% IF ( test_errors.second_granularity_not_supported ) %](Second granularity is not supported)[% END %] +
      4. +
    4. + + +
      5. +
    5. + + +
      6. +
    +
    +
    + [% IF ( test_success ) %] +

    HTTP and OAI-PMH parameter tests completed successfully.

    + [% END %] + [% IF (test_errors.no_harvester) %] +

    Unable to create a OAI-PMH harvester with these settings

    + [% END %] + [% IF (test_errors.generic_identify_error) %] +

    An error was encountered using the "Identify" verb with the server target.

    + [% END %] + +
    +
    + Import parameters: +
      +
    1. + + [% IF ( repository.active == 1) %] + + [% ELSE %] + + [% END %] + Active + [% IF ( repository.active != 1) %] + + [% ELSE %] + + [% END %] + Inactive +
      2. +
    2. + + +
      3. +
    3. + + + +
      4. + [%# Only need framework for 'biblio' and 'holdings'. This will need to change for 'auth'. %] +
    4. + + + +
      5. +
    5. + + +
      6. +
    6. + + +
      7. +
    +
    +
    + + Cancel +
    +
    + [% END %] + [% END #/op %] +
    +
    +
    + [% INCLUDE 'admin-menu.inc' %] +
    +
    +[% INCLUDE 'intranet-bottom.inc' %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/xslt/MARC21slimFromOAI.xsl b/koha-tmpl/intranet-tmpl/prog/en/xslt/MARC21slimFromOAI.xsl new file mode 100755 index 0000000..e8b4050 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/xslt/MARC21slimFromOAI.xsl @@ -0,0 +1,82 @@ + + + + + + + + + + + + + + + + + + + + + + 8 + + 037 + + + a + + + + + b + OAI-PMH + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/koha-tmpl/intranet-tmpl/prog/en/xslt/StripOAIWrapper.xsl b/koha-tmpl/intranet-tmpl/prog/en/xslt/StripOAIWrapper.xsl new file mode 100755 index 0000000..db3749c --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/xslt/StripOAIWrapper.xsl @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/misc/cronjobs/oai/oai_harvester.pl b/misc/cronjobs/oai/oai_harvester.pl new file mode 100755 index 0000000..811f1ad --- /dev/null +++ b/misc/cronjobs/oai/oai_harvester.pl @@ -0,0 +1,180 @@ +#!/usr/bin/perl + +# Copyright Prosentient Systems 2015 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, see . + +use Modern::Perl; +use Getopt::Long; +use Pod::Usage; + + +=head1 NAME + +oai_harvester.pl - cron script to download records via OAI-PMH, and import +into Koha. + +=head1 SYNOPSIS + +perl oai_harvester.pl [ -r ][ -d ][ -i ] + +=head1 OPTIONS + +=over + +=item B<--help> + +Print a brief help message and exits. + +=item B<--man> + +Prints the manual page and exits. + +=item B<-v> + +Verbose. Without this flag set, only errors are reported. + +=item B<-r> + +Re-harvest the OAI server by resetting the "from" parameter to null. + +=item B<-d> + +Download records from the OAI-PMH server, and queue for importing. + +=item B<-i> + +Import queued records into Koha. + +=back + +=head1 DESCRIPTION + +With the "-d" option, this script iterates through a list of +OAI servers (ie repositories), issues a ListRecords OAI-PMH request, and +downloads the results into the Koha database to await further +processing (ie adding new records and updating existing ones). + +With the "-i" option, this script iterates through a list of +harvested OAI-PMH metadata records. Each record is checked +to see whether it needs to be added as a new record in Koha, +or if it needs to update a record that has previously been +harvested. After this checking, it is optionally transformed +using a user-defined XSLT. Finally, it is passed to +Koha's internal APIs to be added/updated in the database. + + +=cut + +my $help = 0; +my $man = 0; +my $verbose = 0; +my $reset = 0; +my $download = 0; +my $import = 0; + +GetOptions( + 'h|help|?' => \$help, + 'm|man' => \$man, + 'v' => \$verbose, + 'd' => \$download, + 'i' => \$import, + 'r' => \$reset, + )or pod2usage(2); +pod2usage(1) if $help; +pod2usage( -verbose => 2 ) if $man; + +if ($download){ + require C4::Context; + require XML::LibXSLT; + require Koha::OAI::Client::Repositories; + + my $stylesheet; + #Build path to XSLT filename + #FIXME: It would make more sense to have a utility XSLT directory outside of htdocs which is language independent... + my $htdocs = C4::Context->config('intrahtdocs'); + my $theme = C4::Context->preference("template"); + my $xslfilename = "$htdocs/$theme/en/xslt/StripOAIWrapper.xsl"; + if ( -f $xslfilename ){ + #FIXME: Ideally, it would be good to use Koha::XSLT_Handler here, but + #we want to output a XML::LibXML::Document instead of output_as_chars. + my $xslt = XML::LibXSLT->new(); + my $style_doc = XML::LibXML->load_xml(location => $xslfilename); + $stylesheet = $xslt->parse_stylesheet($style_doc); + } + + my $repositories = Koha::OAI::Client::Repositories->search({ + active => 1, + base_url => {'!=', undef}, + }); + while (my $repository = $repositories->next){ + if ($reset){ + $verbose && print "Resetting opt_from to undefined...\n"; + $repository->opt_from(undef); + } + $repository->harvest({ verbose => $verbose, stylesheet => $stylesheet, }); + } +} + +if ($import){ + require Koha::OAI::Client::Importer; + require Koha::OAI::Client::Records; + + #Import records, previously harvested via OAI-PMH, into Koha + my $oai_importer = Koha::OAI::Client::Importer->new(); + if ($oai_importer){ + my $records = Koha::OAI::Client::Records->search({ + system_number => undef, + action => ["to_import","to_delete"], + }); + + $verbose && print $records->count." records to process\n"; + if ($records){ + + my $number_imported = 0; + my $number_deleted = 0; + + #Iterate through the records and import (i.e. add/update/re-add) or delete them in Koha + while( my $record = $records->next ) { + if ($record->action eq "to_import"){ + eval { + $oai_importer->import_record($record); + }; + if ($@){ + warn "Error importing record (oai_harvest_id ".$record->oai_harvest_id.") into Koha:\n$@"; + $record->action("error"); + $record->store(); + } else { + $verbose && print $record->action . " identifier (".$record->identifier.") (".$record->record_type.") as system number (".$record->system_number.")\n"; + $number_imported++; + } + } elsif ($record->action eq "to_delete") { + eval { + $oai_importer->delete_record($record); + }; + if ($@){ + warn "Error deleting record (oai_harvest_id ".$record->oai_harvest_id.") from Koha:\n$@"; + } else { + $number_deleted++; + } + } + } + $verbose && print "$number_imported records imported (i.e. added/updated)\n"; + $verbose && print "$number_deleted records deleted \n"; + + + } + } +} -- 1.9.1