@@ -, +, @@ -- -- --- Koha/OAI/Client/Importer.pm | 316 ++++++++++++++++ Koha/OAI/Client/Record.pm | 61 +++ Koha/OAI/Client/Records.pm | 58 +++ Koha/OAI/Client/Repositories.pm | 62 +++ Koha/OAI/Client/Repository.pm | 418 +++++++++++++++++++++ admin/oai_client.pl | 220 +++++++++++ ...0662-Build_OAI-PMH_Harvesting_Client_tables.sql | 51 +++ installer/data/mysql/kohastructure.sql | 49 +++ .../intranet-tmpl/prog/en/includes/admin-menu.inc | 1 + .../prog/en/modules/admin/admin-home.tt | 2 + .../prog/en/modules/admin/oai_client.tt | 326 ++++++++++++++++ .../prog/en/xslt/MARC21slimFromOAI.xsl | 82 ++++ .../intranet-tmpl/prog/en/xslt/StripOAIWrapper.xsl | 22 ++ misc/cronjobs/oai/oai_harvester.pl | 180 +++++++++ 14 files changed, 1848 insertions(+) create mode 100755 Koha/OAI/Client/Importer.pm create mode 100755 Koha/OAI/Client/Record.pm create mode 100755 Koha/OAI/Client/Records.pm create mode 100755 Koha/OAI/Client/Repositories.pm create mode 100755 Koha/OAI/Client/Repository.pm create mode 100755 admin/oai_client.pl create mode 100755 installer/data/mysql/atomicupdate/bug_10662-Build_OAI-PMH_Harvesting_Client_tables.sql create mode 100755 koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_client.tt create mode 100755 koha-tmpl/intranet-tmpl/prog/en/xslt/MARC21slimFromOAI.xsl create mode 100755 koha-tmpl/intranet-tmpl/prog/en/xslt/StripOAIWrapper.xsl create mode 100755 misc/cronjobs/oai/oai_harvester.pl --- a/Koha/OAI/Client/Importer.pm +++ a/Koha/OAI/Client/Importer.pm @@ -0,0 +1,316 @@ +package Koha::OAI::Client::Importer; + +# Copyright Prosentient Systems 2015 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, see . + +use Modern::Perl; + +use Koha::Database; +use Koha::XSLT_Handler; +use C4::Context; +use C4::Languages; +use C4::XSLT; +use C4::Biblio; + +use parent qw(Class::Accessor); +__PACKAGE__->mk_accessors( qw (xslt_handler verbosity) ); + +=head1 NAME + +Koha::OAI::Importer - Imports records, previously harvested via OAI-PMH, into Koha + +=head1 SYNOPSIS + + my $oai_importer = Koha::OAI::Importer->new({ + verbosity => $verbose, + }); + $oai_importer->import_records(); + +=head1 DESCRIPTION + +An OAI Importer is an object which retrieves records, previously harvested +via OAI-PMH, and determines whether they should be added as new records to the +catalogue or if they should update existing catalogue records. + +After this determination, the record's metadata is run through a XSLT, which +is configured on a per server basis, and then handed over to Koha's internal APIs. + +The record in the OAI harvesting queue is then updated to reflect the +action taken and the biblionumber to which it is linked. + +=head1 METHODS + +=head2 new + +The Importer constructor which sets the 'verbosity' and 'xslt_handler' +for the object. + +The 'verbosity' should be set by calling script. + +The 'xslt_handler' could also be set by a calling script, but defaults +to a new Koha::XSLT_Handler object with silenced warnings, so it is +not too noisy. + +=cut + +sub new { + my ($class, $args) = @_; + $args = {} unless defined $args; + $args->{xslt_handler} = Koha::XSLT_Handler->new({print_warns => 0,}) unless defined $args->{xslt_handler}; + $args->{verbosity} = 0 unless defined $args->{verbosity}; + return bless ($args, $class); +} + +sub delete_record { + my ( $self, $record ) = @_; + + my ($action, $system_number) = $self->determine_action_and_system_number({ + identifier => $record->identifier, + repository_id => $record->repository_id, + }); + my $error = C4::Biblio::DelBiblio($system_number); + if ($error){ + die $error; + } else { + $record->set({ + system_number => $system_number, + action => "deleted", + }); + $record->store(); + } +} + +=head2 import_record + +This is the method doing the heavy lifting. It runs the other +methods for determining what to do with the record and if it is +linked to an existing record, it transforms the record via XSLT, +and it ultimately adds or updates the record as need be. + +=cut + +sub import_record { + my ( $self, $record ) = @_; + + #Step one: determine action (i.e. determine if a record with this identifier from this repository has been successfully added before) + my ($action, $system_number) = $self->determine_action_and_system_number({ + identifier => $record->identifier, + repository_id => $record->repository_id, + }); + + #Step two: transform metadata (if necessary) && add/update in Koha + if ( ($action eq 'add') || ($action eq 'update') ){ + my $schema = Koha::Database->new()->schema(); + + #Lookup the repository so we can get the xslt_path and frameworkcode + my $repository = $schema->resultset('OaiHarvestRepository')->find($record->repository_id); + + my $metadata = $record->metadata; + if ($metadata && $repository->xslt_path){ + eval { + #If there is a XSLT path, try to transform metadata + my $transformed_metadata = $self->transform_metadata({ + xslt_path => $repository->xslt_path, + metadata => $metadata, + metadata_prefix => $record->metadata_prefix, + identifier => $record->identifier, + }); + if ($transformed_metadata){ + $metadata = $transformed_metadata; + } + }; + if ($@){ + warn "There was a non-fatal problem transforming the metadata during import: $@"; + } + } + if ($metadata){ + if ($record->record_type eq 'biblio'){ + my ( $biblionumber, $biblioitemnumber ); + #create MARC::Record object + my $marc_record = MARC::Record::new_from_xml( $metadata, "utf8", C4::Context->preference('marcflavour') ); + if ($marc_record){ + if ($action eq 'add'){ + #Add record to Koha + ($biblionumber,$biblioitemnumber) = C4::Biblio::AddBiblio($marc_record,$repository->frameworkcode); + } elsif ($action eq 'update'){ + #Update record in Koha + eval {C4::Biblio::ModBiblio( $marc_record,$system_number,$repository->frameworkcode)}; + if ($@){ + my $deleted_biblio = $schema->resultset('Deletedbiblio')->find($system_number); + my $existing_biblio = $schema->resultset('Biblio')->find($system_number); + if ( ( $deleted_biblio ) || (!$deleted_biblio && !$existing_biblio) ){ + die "It appears the original record referenced by biblionumber $system_number has been deleted."; + #FIXME: What should happen in this situation? You can't re-add the record without first removing + #the previous oai_harvest entries or there will be errors... + #($biblionumber,$biblioitemnumber) = C4::Biblio::AddBiblio($marc_record,$repository->frameworkcode); + #warn "Re-adding the incoming record as a new bibliographic record (biblionumber $system_number)."; + #$action = "re-add"; + } else { + die $@; + } + } + } + } + if ($biblionumber){ + $system_number = $biblionumber; + } + } + #TODO: Add handling for 'holdings' and 'auth' + } + if ($action && $system_number){ + #Update 'action' and 'system_number' columns for this record row + $record->set({ + action => $action, + system_number => $system_number, + }); + $record->store(); + } else { + die "Error importing record! action: ($action);"; + } + } +} + +=head 2 determine_action_and_system_number + +This method checks the database to see if a record from the same +OAI-PMH server with the same identifier has already been added +into Koha. If it hasn't, it should be added. If it has, it should +update the existing record. + +=cut + +sub determine_action_and_system_number { + my ( $self, $args ) = @_; + my ( $action, $system_number ); + if ( $args->{identifier} && $args->{repository_id} ){ + my $schema = Koha::Database->new()->schema(); + + #NOTE: $search_results is a DBIx::Class::ResultSet object + my $search_results = $schema->resultset('OaiHarvest')->search( + { + -and => [ + identifier => $args->{identifier}, + repository_id => $args->{repository_id}, + system_number => {'!=', undef}, + -or => [ + action => 'add', + action => 'update', + ], + ], + }, + { + columns => [ qw/system_number/ ], #Only retrieve system_number + group_by => [ qw/system_number/ ], #Group results by system_number so you only get 1 result if you have many rows sharing the same system_number + } + ); + + #NOTE: The DBIx::Class::ResultSet class has overloaded operators, + #which means a "count()" method is run whenever an object is called in a + #numerical context (like below). It will also always return 1 in a boolean context. + if ($search_results == 0){ + $action = 'add'; + } elsif ($search_results == 1){ + $action = 'update'; + while( my $result = $search_results->next ) { + $system_number = $result->system_number; + } + } else { + if ($search_results && $search_results > 1){ + die "More than one system_number matched the identifier " . $args->{identifier} . " with a repository_id of " . $args->{repository_id} . "."; + } else { + die "Unknown error scanning database for previous entries."; + } + } + } + return ($action,$system_number); +} + +=head2 transform_metadata + +This method takes a record's XML metadata and runs it through a XSLT +(using a Koha::XSLT_Handler object). The purpose of this is to remove +undesirable fields or to add desired ones. + +=cut + +sub transform_metadata { + my ( $self, $args ) = @_; + my $transformed_metadata; + + if ($self->xslt_handler){ + if ( ($args->{xslt_path}) && ($args->{metadata_prefix}) && ($args->{metadata}) && ($args->{identifier}) ){ + + my $xslt = _resolve_xslt_path($args->{xslt_path}, $args->{metadata_prefix}); + + #If there is a XSLT file, we try to transform the data + if ($xslt){ + require XML::LibXSLT; + #NOTE: You can use "register_function()" here to provide extension functions to the XSLT + #NOTE: For parameters, you must wrap XML::LibXSLT::xpath_to_string() around each key => value pair within a hashref + #{ XML::LibXSLT::xpath_to_string(identifier => $args->{identifier}), XML::LibXSLT::xpath_to_string(test => "tester"), } + $transformed_metadata = $self->xslt_handler->transform({ + xml => $args->{metadata}, + file => $xslt, + parameters => { XML::LibXSLT::xpath_to_string(identifier => $args->{identifier}), }, + }); + if ( $self->xslt_handler->err ){ + die $self->xslt_handler->errstr; + } + if (! $transformed_metadata){ + die "Unable to transform metadata"; + } + } + } + } + + return $transformed_metadata; +} + +=head2 _resolve_xslt_path + +An internal sub for turning magical values into actual filepaths to a XSLT. + +=cut + +sub _resolve_xslt_path { + my ( $xslfilename, $metadata_prefix ) = @_; + if ( $xslfilename ){ + if ( $xslfilename =~ /^\s*"?default"?\s*$/i ) { + if ($metadata_prefix =~ "marc"){ + #If the repository has a default XSLT and harvests MARCXML: + my $htdocs = C4::Context->config('intrahtdocs'); + my $theme = C4::Context->preference("template"); + my $lang = C4::Languages::getlanguage(); + my $xslfile = C4::Context->preference('marcflavour') . + "slimFromOAI.xsl"; + + $xslfilename = C4::XSLT::_get_best_default_xslt_filename($htdocs, $theme, $lang, $xslfile); + if ( ! -f $xslfilename ){ + #If the default XSLT doesn't exist, nullify this variable + $xslfilename = undef; + } + } else { + #If the repository has a default XSLT and doesn't harvest marc: + #We return a null because currently we only provide default XSLTs for MARC21 marcxml + $xslfilename = undef; + } + } + } + return $xslfilename; +} + + +1; --- a/Koha/OAI/Client/Record.pm +++ a/Koha/OAI/Client/Record.pm @@ -0,0 +1,61 @@ +package Koha::OAI::Client::Record; + +# Copyright Prosentient Systems 2015 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use Carp; + +use base qw(Koha::Object); +use C4::Biblio; + +=head1 NAME + +Koha::OAI::Client::Record - + +=head1 API + +=head2 Class Methods + +=cut + +sub delete { + my ( $self ) = @_; + if ($self->record_type eq 'biblio' && $self->system_number){ + my $error = C4::Biblio::DelBiblio($self->system_number); + if ($error){ + die $error; + } + } + $self->SUPER::delete(); +} + +=head3 type + +=cut + +sub type { + return 'OaiHarvest'; +} + +=head1 AUTHOR + +David Cook + +=cut + +1; --- a/Koha/OAI/Client/Records.pm +++ a/Koha/OAI/Client/Records.pm @@ -0,0 +1,58 @@ +package Koha::OAI::Client::Records; + +# Copyright Prosentient Systems 2015 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use Carp; + +use base qw(Koha::Objects); +use Koha::OAI::Client::Record; + +=head1 NAME + +Koha::OAI::Client::Records - + +=head1 API + +=head2 Class Methods + +=cut + +=head3 type + +=cut + +sub type { + return 'OaiHarvest'; +} + +=head3 object_class + +=cut + +sub object_class { + return 'Koha::OAI::Client::Record'; +} + +=head1 AUTHOR + +David Cook + +=cut + +1; --- a/Koha/OAI/Client/Repositories.pm +++ a/Koha/OAI/Client/Repositories.pm @@ -0,0 +1,62 @@ +package Koha::OAI::Client::Repositories; + +# Copyright Prosentient Systems 2015 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; + +use Carp; + +use Koha::Database; + +use Koha::OAI::Client::Repository; + +use base qw(Koha::Objects); + +=head1 NAME + +Koha::OAI::Client::Repositories - + +=head1 API + +=head2 Class Methods + +=cut + +=head3 type + +=cut + +sub type { + return 'OaiHarvestRepository'; +} + +=head3 object_class + +=cut + +sub object_class { + return 'Koha::OAI::Client::Repository'; +} + +=head1 AUTHOR + +David Cook + +=cut + +1; --- a/Koha/OAI/Client/Repository.pm +++ a/Koha/OAI/Client/Repository.pm @@ -0,0 +1,418 @@ +package Koha::OAI::Client::Repository; + +# Copyright Prosentient Systems 2015 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use Carp; +use POSIX qw(strftime); +use HTTP::OAI; +use URI; +use DateTime; +use DateTime::Format::Strptime; +use base qw(Koha::Object); + +use Koha::OAI::Client::Records; + +=head1 NAME + +Koha::OAI::Client::Repository - + +=head1 API + +=head2 Class Methods + +=cut + +=head3 test_settings + +=cut + +sub test_settings { + my ( $self ) = @_; + my $errors = {}; + my $harvester = $self->harvester(); + if ($harvester){ + my $identify = $harvester->Identify; + if ($identify->is_success){ + + #Test Granularity + my $granularity_setting = $self->datetime_granularity; + if ($granularity_setting eq "YYYY-MM-DDThh:mm:ssZ"){ + my $actual_granularity = $identify->granularity; + if ($granularity_setting ne $actual_granularity){ + $errors->{second_granularity_not_supported} = 1; + } + } + + #Test Set + my $set_setting = $self->opt_set; + if ($set_setting){ + my $sets = $harvester->ListSets(); + my $matched_set; + while ( my $set = $sets->next ){ + if ($set_setting eq $set->setSpec){ + $matched_set = 1; + last; + } + } + if ( ! $matched_set ){ + $errors->{set_does_not_exist} = 1; + } + } + + #Test Metadata prefix + my $metadata_prefix_setting = $self->metadata_prefix; + if ($metadata_prefix_setting){ + my $metadata_formats = $harvester->ListMetadataFormats(); + my $matched_format; + while ( my $metadata_format = $metadata_formats->next ){ + if ($metadata_prefix_setting eq $metadata_format->metadataPrefix){ + $matched_format = 1; + last; + } + } + if ( ! $matched_format ){ + $errors->{metadata_prefix_does_not_exist} = 1; + } + } + } else { + if ($identify->is_error()){ + foreach my $error ($identify->errors){ + if ($error->code =~ /^404$/){ + $errors->{url_not_found} = 1; + } elsif ($error->code =~ /^401$/){ + $errors->{failed_authentication} = 1; + } else { + $errors->{generic_identify_error} = 1; + } + } + } else { + #This should never happen, but you never know... + $errors->{generic_identify_error} = 1; + } + } + } else { + $errors->{no_harvester} = 1; + } + return $errors; +} + + +=head3 validate + +=cut + +sub validate { + my ( $self ) = @_; + my $errors = {}; + if ($self->base_url){ + my $uri = URI->new($self->base_url); + if ($uri){ + my $reference = ref $uri; + if ( ! grep { $reference eq $_ } ("URI::http","URI::https")){ + $errors->{base_url}->{invalid} = 1; + } + } + } elsif (! defined $self->base_url){ + #base_url is undefined + $errors->{base_url}->{required} = 1; + } + if ( ! defined $self->metadata_prefix ){ + $errors->{metadata_prefix}->{required} = 1; + } + return $errors; +} + +=head3 find_original_system_number + +=cut + +sub find_original_system_number { + my ( $self, $args ) = @_; + + my $original_system_number; + my $metadata = $args->{metadata}; + if ($metadata){ + my $original_system_field = $self->original_system_field; #e.g. 001 or 999$c + if ($original_system_field){ + #Select root element (e.g. ) + my $root = $metadata->documentElement(); + #Select the first child of root (ie the original metadata record) (eg ) + my $original_metadata = $root->firstChild; + + #Check to see if there is a MARC namespace + my $marcxml_prefix = $original_metadata->lookupNamespacePrefix("http://www.loc.gov/MARC21/slim"); + if (defined $marcxml_prefix){ + my ($tag,$code) = split(/\$/,$original_system_field); #e.g. split 999$c into 999 and c || split 001 into 001 + my $xpc = XML::LibXML::XPathContext->new; + $xpc->registerNs('marc', 'http://www.loc.gov/MARC21/slim'); + my $fields = $xpc->find(qq(//marc:record/node()[\@tag="$tag"]),$original_metadata); + if ($fields){ + #Use the first field found + my $field = $fields->get_node(1); + if ($field){ + my $node_name = $field->nodeName; + if ($node_name eq 'controlfield'){ + #If it's a controlfield, we can use the text without drilling down + $original_system_number = $field->textContent; + } elsif ($node_name eq 'datafield'){ + #If it's a datafield, we need to drill down into the subfields + if ($code){ + my $subfields = $field->find(qq(node()[\@code="$code"])); + if ($subfields){ + #Use the first subfield found + my $subfield = $subfields->get_node(1); + if ($subfield){ + $original_system_number = $subfield->textContent; + } + + } + } + } + } + } + } + } + } + return $original_system_number; +} + +=head3 queue_record + +=cut + +sub queue_record { + my ( $self, $args ) = @_; + my $record = $args->{record}; + my $stylesheet = $args->{stylesheet}; + + my $action = "to_import"; + if ( $record->is_deleted() ){ + my $already_deleted_rows = Koha::OAI::Client::Records->search({ identifier => $record->identifier, action => "deleted" }); + my $identified_rows = Koha::OAI::Client::Records->search({ identifier => $record->identifier }); + if ($identified_rows->count() > 0 && $already_deleted_rows->count() == 0){ + #Only queue records "to_delete", if the record has been previously harvested + $action = "to_delete"; + } else { + return; + } + } + + my $metadata = $record->metadata ? $record->metadata->dom : ''; + my $original_system_number; + if ($metadata){ + if ($stylesheet){ + #Use a stylesheet to strip the OAI-PMH metadata XML wrapper + eval { + my $result = $stylesheet->transform($metadata); + if ($result){ + $metadata = $result; + } + }; + if ($@){ + warn "Problem transforming harvested metadata with XSLT: $@"; + } + } + + $original_system_number = $self->find_original_system_number({metadata => $metadata}); + } + + my $entry = Koha::OAI::Client::Records->_resultset->find_or_new({ + repository_id => $self->repository_id, #Internal ID for the OAI repository + action => $action, + identifier => $record->identifier, #External ID for the OAI record + datestamp => $record->datestamp, #External datestamp for the OAI record + metadata_prefix => $self->metadata_prefix, #External metadataPrefix for the OAI record + metadata => $metadata ? $metadata->toString(1) : undef, + record_type => $self->record_type, + original_system_number => $original_system_number, + }, + { key => "harvest_identifier_datestamp" } + ); + if( !$entry->in_storage ) { + $entry->insert; + } +} + +sub _format_datetime { + my ( $self, $args ) = @_; + my $datetime_string = $args->{datetime_string}; + my $formatted_string; + if ($datetime_string && $datetime_string ne "0000-00-00 00:00:00"){ + my $strp = DateTime::Format::Strptime->new( + pattern => '%F %T', + ); + my $dt = $strp->parse_datetime($datetime_string); + if ($dt){ + if ($self->datetime_granularity && $self->datetime_granularity eq 'YYYY-MM-DDThh:mm:ssZ'){ + $formatted_string = $dt->strftime("%FT%TZ"); + } else { + $formatted_string = $dt->strftime("%F"); + } + } + } + return $formatted_string; +} + +sub harvester { + my ( $self ) = @_; + my $harvester; + if ($self->base_url){ + $harvester = new HTTP::OAI::Harvester( baseURL => $self->base_url ); + #Use basic http authentication credentials if they're set + # if ($self->basic_realm && $self->basic_username && $self->basic_password){ + #Decompose the URI in order to register the basic authentication credentials + my $uri = URI->new($self->base_url); + my $host = $uri->host; + my $port = $uri->port; + $harvester->credentials($host.":".$port, $self->basic_realm, $self->basic_username, $self->basic_password); + # } + } + return $harvester; +} + +=head3 query_repository + +=cut + +sub query_repository { + my ( $self ) = @_; + my $response; + my $h = $self->harvester(); + if ($h){ + my $opt_from = $self->opt_from; + my $opt_until = $self->opt_until; + foreach my $datetime ($opt_from,$opt_until){ + #Format the datetime strings from the database into the format and granularity + #expected by the target OAI-PMH server + $datetime = $self->_format_datetime({datetime_string => $datetime,}); + } + + #Issue the request to the OAI-PMH server + $response = $h->ListRecords( + metadataPrefix => $self->metadata_prefix, + from => $opt_from, + until => $opt_until, + set => $self->opt_set, + ); + } + return $response; +} + +=head3 harvest + +=cut + +sub harvest { + my ( $self, $args ) = @_; + + my $verbose = $args->{verbose} // ''; + my $stylesheet = $args->{stylesheet} // ''; + $verbose && print "Harvesting records from ".$self->base_url."\n"; + + #Record the exact time before sending the first OAI-PMH request, and set "opt_from" to this after the harvest is finished + my $new_from_date = strftime "%Y-%m-%d %H:%M:%S", localtime; + #Send a request to the OAI-PMH repository + my $oai_response = $self->query_repository(); + if ($oai_response){ + if( $oai_response->is_error ){ + warn "responseDate => " . $oai_response->responseDate . "\n"; + warn "requestURL => " . $oai_response->requestURL . "\n"; + warn "Error harvesting: " . $oai_response->message . "\n"; + } else { + while( my $rec = $oai_response->next ) { + $verbose && print $rec->identifier."\n"; + #Queue each record, so a different script can process them in Koha + $self->queue_record({record => $rec, stylesheet => $stylesheet, }); + } + #Update opt_from after a successful harvest + $self->opt_from($new_from_date); + $verbose && print "Updating `opt_from` to $new_from_date \n"; + $self->store(); + } + } +} + +#FIXME +=head3 reharvest + +=cut + +sub reharvest { + my ( $self, $args ) = @_; + my $verbose = $args->{verbose} // ''; + eval { + $self->delete_harvest({ verbose => $verbose, }); + }; + if ($@){ + warn $@; + } else { + $self->harvest({ verbose => $verbose, }); + } +} + +#FIXME +=head3 delete_harvest + +=cut + +sub delete_harvest { + my ( $self ) = @_; + + my $count_of_records = 0; + my $count_of_deleted_records = 0; + my $problem_records = []; + + if ($self->repository_id){ + my $records = Koha::OAI::Client::Records->search({ "repository_id" => $self->repository_id }); + while (my $record = $records->next){ + #Increment overall count + $count_of_records++; + eval { + my $is_deleted = $record->delete(); + if ($is_deleted){ + $count_of_deleted_records++; + } + }; + if ($@){ + push(@$problem_records,$record); + } + } + } + + if ($count_of_deleted_records < $count_of_records){ + #Not all of this repository's records could be deleted. Manual intervention may be required to remove items + die $problem_records; + } +} + +=head3 type + +=cut + +sub type { + return 'OaiHarvestRepository'; +} + +=head1 AUTHOR + +David Cook + +=cut + +1; --- a/admin/oai_client.pl +++ a/admin/oai_client.pl @@ -0,0 +1,220 @@ +#!/usr/bin/perl + +# Copyright Prosentient Systems 2015 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +=head1 NAME + +oai_client.pl + +=head1 DESCRIPTION + +Admin page to manage Koha's target OAI-PMH servers + +=cut + +use Modern::Perl; +use CGI qw ( -utf8 ); +use C4::Auth; +use C4::Output; +use C4::Koha qw/GetFrameworksLoop/; +use Koha::OAI::Client::Repository; +use Koha::OAI::Client::Repositories; +use Koha::OAI::Client::Records; + +my $input = new CGI; + +my ($template, $loggedinuser, $cookie, $flags) = get_template_and_user( { + template_name => 'admin/oai_client.tt', + query => $input, + type => 'intranet', + authnotrequired => 0, + flagsrequired => { 'parameters' => 'parameters_remaining_permissions' }, +} ); + +my $op = $input->param('op'); +my $test_settings = $input->param('test_settings'); +my $repository_id = $input->param('repository_id'); + +my $base_url = $input->param('base_url') || undef; +my $metadata_prefix = $input->param('metadata_prefix') || undef; +my $opt_from = $input->param('opt_from') || undef; +my $opt_until = $input->param('opt_until') || undef; +my $opt_set = $input->param('opt_set') || undef; +my $active = $input->param('active'); +my $xslt_path = $input->param('xslt_path') || undef; +my $frameworkcode = $input->param('frameworkcode'); #This can be empty as the default framework is an empty value +my $comments = $input->param('comments') || undef; +my $basic_username = $input->param('basic_username') || undef; +my $basic_password = $input->param('basic_password') || undef; +my $basic_realm = $input->param('basic_realm') || undef; +my $datetime_granularity = $input->param('datetime_granularity'); +my $record_type = $input->param('record_type'); +my $original_system_field = $input->param('original_system_field') || undef; + +if ( $op ){ + my $repository; + if ( $repository_id ){ + #Fetch repository from database... + $repository = Koha::OAI::Client::Repositories->find($repository_id); + } + + if ( $op eq "new" ){ + $template->param( + frameworkloop => C4::Koha::GetFrameworksLoop(), + ); + } elsif ( $op eq "create" ){ + my $new_repository = Koha::OAI::Client::Repository->new({ + base_url => $base_url, + metadata_prefix => $metadata_prefix, + opt_from => $opt_from, + opt_until => $opt_until, + opt_set => $opt_set, + active => $active, + xslt_path => $xslt_path, + frameworkcode => $frameworkcode, + comments => $comments, + basic_username => $basic_username, + basic_password => $basic_password, + basic_realm => $basic_realm, + datetime_granularity => $datetime_granularity, + record_type => $record_type, + original_system_field => $original_system_field, + }); + my $errors = $new_repository->validate(); + if (%$errors){ + $op = "new"; + $template->param( + repository => $new_repository, + frameworkloop => C4::Koha::GetFrameworksLoop($new_repository->frameworkcode), + errors => $errors, + ); + } else { + $new_repository->store(); + $op = "list"; + } + } elsif ( $op eq "edit" ){ + if ($repository){ + $template->param( + repository => $repository, + frameworkloop => C4::Koha::GetFrameworksLoop($repository->frameworkcode), + ); + } + } elsif ( ($op eq "view") || ($op eq "reset") ){ + if ($repository){ + + #If op eq reset, then delete the entire harvest for this repository + if ($op eq "reset"){ + eval { + $repository->delete_harvest(); + $template->param( reset_success => 1 ); + }; + if ($@){ + my $problem_records = $@; + $template->param( problem_records => $problem_records ); + } + } + + #"add", "update" + my $imported_records = Koha::OAI::Client::Records->search({ + repository_id => $repository->repository_id, + system_number => {'!=', undef}, + }, { + group_by => [ qw/system_number/ ], + #Group results by system_number so you only get 1 result if you have many rows sharing the same system_number + #Also, group by system number because you might have imported record 1, deleted it, and then it gets re-imported later as a different Koha record + }); + my $waiting_records = Koha::OAI::Client::Records->search({ repository_id => $repository->repository_id, action => 'to_import',}); + my $to_delete_records = Koha::OAI::Client::Records->search({ repository_id => $repository->repository_id, action => 'to_delete',}); + my $deleted_records = Koha::OAI::Client::Records->search({ repository_id => $repository->repository_id, action => 'deleted',}); + my $error_records = Koha::OAI::Client::Records->search({ repository_id => $repository->repository_id, action => 'error',}); + + $template->param( + repository => $repository, + imported_records_count => $imported_records->count(), + waiting_records_count => $waiting_records->count(), + to_delete_records_count => $to_delete_records->count(), + deleted_records_count => $deleted_records->count(), + error_records_count => $error_records->count(), + ); + } + if ($op eq "reset"){ + $op = "view"; + } + } elsif ( $op eq "update" ){ + if ($repository){ + $repository->set({ + base_url => $base_url, + metadata_prefix => $metadata_prefix, + opt_from => $opt_from, + opt_until => $opt_until, + opt_set => $opt_set, + active => $active, + xslt_path => $xslt_path, + frameworkcode => $frameworkcode, + comments => $comments, + basic_username => $basic_username, + basic_password => $basic_password, + basic_realm => $basic_realm, + datetime_granularity => $datetime_granularity, + record_type => $record_type, + original_system_field => $original_system_field, + }); + my $errors = $repository->validate(); + if (%$errors || $test_settings){ + if (! %$errors && $test_settings){ + my $test_errors = $repository->test_settings(); + if (%$test_errors){ + $template->param( test_errors => $test_errors ); + } else { + $template->param( test_success => 1 ); + } + } + $op = "edit"; + $template->param( + repository => $repository, + frameworkloop => C4::Koha::GetFrameworksLoop($repository->frameworkcode), + errors => $errors, + ); + } else { + $repository->store(); + $op = "list"; + } + } + } elsif ( $op eq "delete" ){ + if ($repository){ + $repository->delete; + } + $op = "list"; + } +} else { + $op = "list"; +} + +if ( $op eq "list" ){ + my @repositories = Koha::OAI::Client::Repositories->as_list(); + $template->param( + repositories => \@repositories, + frameworkloop => C4::Koha::GetFrameworksLoop(), + ); +} + +$template->param( + op => $op, +); + +output_html_with_http_headers $input, $cookie, $template->output; --- a/installer/data/mysql/atomicupdate/bug_10662-Build_OAI-PMH_Harvesting_Client_tables.sql +++ a/installer/data/mysql/atomicupdate/bug_10662-Build_OAI-PMH_Harvesting_Client_tables.sql @@ -0,0 +1,51 @@ +DROP TABLE IF EXISTS oai_harvest; +DROP TABLE IF EXISTS oai_harvest_repositories; + +-- +-- Table structure for table `oai_harvest_repositories` +-- + +CREATE TABLE IF NOT EXISTS oai_harvest_repositories ( + repository_id int(11) NOT NULL AUTO_INCREMENT, -- primary key identifier + base_url text NOT NULL, -- baseURL of the remote OAI-PMH repository (mandatory) + basic_username text, -- username for basic http authentication + basic_password text, -- password for basic http authentication + basic_realm text, -- realm for basic http authentication + datetime_granularity enum('YYYY-MM-DDThh:mm:ssZ','YYYY-MM-DD') NOT NULL DEFAULT 'YYYY-MM-DD', -- granularity to use in harvest request + opt_from DATETIME DEFAULT NULL, -- "from" DATETIME for selective harvesting (optional - automatically updated by cronjob) + opt_until DATETIME DEFAULT NULL, -- "until" DATETIME for selective harvesting (optional) + opt_set varchar(45) DEFAULT NULL, -- the record set to harvest for selective harvesting (optional) + metadata_prefix varchar(45) NOT NULL, -- metadata format (e.g. oai_dc, dc, marcxml) + active int(1) NOT NULL DEFAULT '0', -- indicate whether this repo is actively harvested by Koha + timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- date last modified. Maybe `datelastmodified` would be a better column name. + xslt_path text, -- filepath to a local or external XSLT file to use for transforming incoming records into MARCXML + frameworkcode varchar(4) NOT NULL DEFAULT '', -- framework to use when ingesting records + comments text, -- limited number of characters (controlled by template) to describe the repository (optional - for librarian use rather than system use) + record_type enum('biblio','auth','holdings') NOT NULL DEFAULT 'biblio', + original_system_field varchar(45), -- Field where you can a find a record's original system number (e.g. 001 or 999$c) + -- NOTE: In future, may need to add fields for holdings-biblio relationship (e.g. field to designate 004 in holdings record as link to a biblio's original system number) + PRIMARY KEY (repository_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- +-- Table structure for table `oai_harvest` +-- + +CREATE TABLE IF NOT EXISTS oai_harvest ( + oai_harvest_id int(11) NOT NULL AUTO_INCREMENT, -- unique internal id for the OAI record + repository_id int(11) NOT NULL, -- OAI repository from which the record was harvested + system_number int(11) DEFAULT NULL, -- indicates the Koha record created from this record + action varchar(45) DEFAULT NULL, -- indicates whether this record has been processed and how it's been processed (e.g. add, update, ignore) + timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- last modified date in Koha. Maybe `datelastmodified` would be a better column name. + identifier varchar(255) NOT NULL, -- the record's OAI ID from the OAI repository + datestamp varchar(45) NOT NULL, -- the record's OAI datestamp from the OAI repository (e.g. YYYY-MM-DDTHH:MM:SSZ). + metadata_prefix varchar(45) NOT NULL, -- the type of metadata (e.g. marcxml, oai_dc, etc.) + metadata longtext, -- original metadata as transmitted via OAI-PMH + record_type enum('biblio','auth','holdings') NOT NULL DEFAULT 'biblio', + original_system_number varchar(45), -- original system number (e.g. the unique number in the 001 of the incoming record) + -- NOTE: In future, may need to add fields for holdings-biblio relationship + PRIMARY KEY (oai_harvest_id), + UNIQUE KEY harvest_identifier_datestamp (identifier,datestamp), + KEY FK_oai_harvest_1 (repository_id), + CONSTRAINT FK_oai_harvest_1 FOREIGN KEY (repository_id) REFERENCES oai_harvest_repositories (repository_id) ON UPDATE NO ACTION +) ENGINE=InnoDB DEFAULT CHARSET=utf8; --- a/installer/data/mysql/kohastructure.sql +++ a/installer/data/mysql/kohastructure.sql @@ -3549,6 +3549,55 @@ CREATE TABLE discharges ( CONSTRAINT borrower_discharges_ibfk1 FOREIGN KEY (borrower) REFERENCES borrowers (borrowernumber) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; +-- +-- Table structure for table `oai_harvest_repositories` +-- + +CREATE TABLE IF NOT EXISTS oai_harvest_repositories ( + repository_id int(11) NOT NULL AUTO_INCREMENT, -- primary key identifier + base_url text NOT NULL, -- baseURL of the remote OAI-PMH repository (mandatory) + basic_username text, -- username for basic http authentication + basic_password text, -- password for basic http authentication + basic_realm text, -- realm for basic http authentication + datetime_granularity enum('YYYY-MM-DDThh:mm:ssZ','YYYY-MM-DD') NOT NULL DEFAULT 'YYYY-MM-DD', -- granularity to use in harvest request + opt_from DATETIME DEFAULT NULL, -- "from" DATETIME for selective harvesting (optional - automatically updated by cronjob) + opt_until DATETIME DEFAULT NULL, -- "until" DATETIME for selective harvesting (optional) + opt_set varchar(45) DEFAULT NULL, -- the record set to harvest for selective harvesting (optional) + metadata_prefix varchar(45) NOT NULL, -- metadata format (e.g. oai_dc, dc, marcxml) + active int(1) NOT NULL DEFAULT '0', -- indicate whether this repo is actively harvested by Koha + timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- date last modified. Maybe `datelastmodified` would be a better column name. + xslt_path text, -- filepath to a local or external XSLT file to use for transforming incoming records into MARCXML + frameworkcode varchar(4) NOT NULL DEFAULT '', -- framework to use when ingesting records + comments text, -- limited number of characters (controlled by template) to describe the repository (optional - for librarian use rather than system use) + record_type enum('biblio','auth','holdings') NOT NULL DEFAULT 'biblio', + original_system_field varchar(45), -- Field where you can a find a record's original system number (e.g. 001 or 999$c) + -- NOTE: In future, may need to add fields for holdings-biblio relationship (e.g. field to designate 004 in holdings record as link to a biblio's original system number) + PRIMARY KEY (repository_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- +-- Table structure for table `oai_harvest` +-- + +CREATE TABLE IF NOT EXISTS oai_harvest ( + oai_harvest_id int(11) NOT NULL AUTO_INCREMENT, -- unique internal id for the OAI record + repository_id int(11) NOT NULL, -- OAI repository from which the record was harvested + system_number int(11) DEFAULT NULL, -- indicates the Koha record created from this record + action varchar(45) DEFAULT NULL, -- indicates whether this record has been processed and how it's been processed (e.g. add, update, ignore) + timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- last modified date in Koha. Maybe `datelastmodified` would be a better column name. + identifier varchar(255) NOT NULL, -- the record's OAI ID from the OAI repository + datestamp varchar(45) NOT NULL, -- the record's OAI datestamp from the OAI repository (e.g. YYYY-MM-DDTHH:MM:SSZ). + metadata_prefix varchar(45) NOT NULL, -- the type of metadata (e.g. marcxml, oai_dc, etc.) + metadata longtext, -- original metadata as transmitted via OAI-PMH + record_type enum('biblio','auth','holdings') NOT NULL DEFAULT 'biblio', + original_system_number varchar(45), -- original system number (e.g. the unique number in the 001 of the incoming record) + -- NOTE: In future, may need to add fields for holdings-biblio relationship + PRIMARY KEY (oai_harvest_id), + UNIQUE KEY harvest_identifier_datestamp (identifier,datestamp), + KEY FK_oai_harvest_1 (repository_id), + CONSTRAINT FK_oai_harvest_1 FOREIGN KEY (repository_id) REFERENCES oai_harvest_repositories (repository_id) ON UPDATE NO ACTION +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; --- a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc +++ a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc @@ -62,6 +62,7 @@ --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt +++ a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt @@ -107,6 +107,8 @@
Printers (UNIX paths).
-->
Z39.50/SRU servers
Define which external servers to query for MARC data.
+
OAI-PMH servers
+
Define external servers from which to harvest records via OAI-PMH.
Did you mean?
Choose which plugins to use to suggest searches to patrons and staff.
Configure columns
--- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_client.tt +++ a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_client.tt @@ -0,0 +1,326 @@ +[% INCLUDE 'doc-head-open.inc' %] +Koha › Administration › OAI-PMH server targets +[% INCLUDE 'doc-head-close.inc' %] +[% INCLUDE 'calendar.inc' %] + +[% INCLUDE 'timepicker.inc' %] +[% IF ( op == "list" ) %] + + [% INCLUDE 'datatables.inc' %] + +[% ELSIF ( op == "edit" ) || ( op == "new" ) %] + + +[% END %] + + + +[% INCLUDE 'header.inc' %] +[% INCLUDE 'cat-search.inc' %] + + +
+ +
+
+
+ [% IF ( op ) %] + [% IF ( op == "list" ) %] + +

OAI-PMH server targets

+ + + + + + + + + + + + + + + + + + + [% FOREACH repository IN repositories %] + + + + + + + + + + + + + + + [% END %] + +
Base URLMetadata PrefixFromUntilSetActivePath to XSLTRecord typeMARC frameworkOriginal system fieldCommentsOptions
[% repository.base_url %][% repository.metadata_prefix %][% IF ( repository.opt_from ) != "0000-00-00 00:00:00"; repository.opt_from; END; %][% IF ( repository.opt_until ) != "0000-00-00 00:00:00"; repository.opt_until; END; %][% repository.opt_set %] + [% IF ( repository.active == 1 ) %] + Active + [% ELSE %] + Inactive + [% END %] + [% repository.xslt_path %][% repository.record_type %] + [% + framework_name = ''; + FOREACH framework IN frameworkloop; + IF ( framework.value == repository.frameworkcode ); + framework_name = framework.description; + END; + END; + %] + [% IF ( framework_name ) %] + [% framework_name %] + [% ELSE %] + Default + [% END %] + [% repository.original_system_field %][% repository.comments %] + View + Edit + Delete +
+ [% ELSIF ( op == "view" ) %] +

View OAI-PMH server target

+

[% repository.base_url %]

+
+
    +
  1. Koha records created from harvested records: [% imported_records_count %]
  2. +
  3. Koha records deleted by harvested records: [% deleted_records_count %]
  4. +
  5. Harvested records waiting to be imported: [% waiting_records_count %]
  6. +
  7. Harvested records waiting to be deleted from Koha: [% to_delete_records_count %]
  8. +
  9. Harvested records in an error state: [% error_records_count %]
  10. +
+
+
+
+ + + +
+
+ [% IF ( problem_records ) %] +

Not all of this repository's records could be deleted. Manual intervention may be required to remove items. Review the following records:

+
    + [% FOREACH problem_record IN problem_records %] + [% IF ( problem_record.record_type == "biblio" ) %] +
  1. [% problem_record.record_type %] - [% problem_record.system_number %]
  2. + [% ELSE %] +
  3. [% problem_record.record_type %] - [% problem_record.system_number %]
  4. + [% END %] + [% END %] +
+ [% END %] + [% IF ( reset_success ) %] +

This repository no longer has any harvested records. Modify the "From" and "Until" dates to prepare the repository for the next harvest.

+ [% END %] + [% ELSIF ( op == "edit" ) || ( op == "new" ) %] + [% IF ( op == "new" ) %] +

New OAI-PMH server target

+ [% ELSIF ( op == "edit" ) %] +

Modify OAI-PMH server target

+ [% END %] +
+ [% IF ( op == "new" ) %] + + [% ELSIF ( op == "edit" ) %] + + + [% END %] +
+ HTTP parameters: +
    +
  1. + + + [% IF ( errors.base_url.required ) %]Required[% END %] + [% IF ( errors.base_url.invalid ) %](Invalid URL)[% END %] + [% IF ( test_errors.url_not_found ) %](404 - URL Not Found)[% END %] + +
  2. +
  3. + + + [% IF ( test_errors.failed_authentication ) %](Authentication failed)[% END %] +
  4. +
  5. + + + [% IF ( test_errors.failed_authentication ) %](Authentication failed)[% END %] +
  6. +
  7. + + + [% IF ( test_errors.failed_authentication ) %](Authentication failed)[% END %] +
  8. +
+
+
+ OAI-PMH parameters: +
    +
  1. + + + [% IF ( test_errors.set_does_not_exist ) %](Set unavailable)[% END %] +
  2. +
  3. + + + [% IF ( errors.metadata_prefix.required ) %]Required[% END %] + [% IF ( test_errors.metadata_prefix_does_not_exist ) %](Metadata prefix unavailable)[% END %] + +
  4. +
  5. + + + + [% IF ( test_errors.second_granularity_not_supported ) %](Second granularity is not supported)[% END %] +
  6. +
  7. + + +
  8. +
  9. + + +
  10. +
+
+
+ [% IF ( test_success ) %] +

HTTP and OAI-PMH parameter tests completed successfully.

+ [% END %] + [% IF (test_errors.no_harvester) %] +

Unable to create a OAI-PMH harvester with these settings

+ [% END %] + [% IF (test_errors.generic_identify_error) %] +

An error was encountered using the "Identify" verb with the server target.

+ [% END %] + +
+
+ Import parameters: +
    +
  1. + + [% IF ( repository.active == 1) %] + + [% ELSE %] + + [% END %] + Active + [% IF ( repository.active != 1) %] + + [% ELSE %] + + [% END %] + Inactive +
  2. +
  3. + + +
  4. +
  5. + + + +
  6. + [%# Only need framework for 'biblio' and 'holdings'. This will need to change for 'auth'. %] +
  7. + + + +
  8. +
  9. + + +
  10. +
  11. + + +
  12. +
+
+
+ + Cancel +
+
+ [% END %] + [% END #/op %] +
+
+
+ [% INCLUDE 'admin-menu.inc' %] +
+
+[% INCLUDE 'intranet-bottom.inc' %] --- a/koha-tmpl/intranet-tmpl/prog/en/xslt/MARC21slimFromOAI.xsl +++ a/koha-tmpl/intranet-tmpl/prog/en/xslt/MARC21slimFromOAI.xsl @@ -0,0 +1,82 @@ + + + + + + + + + + + + + + + + + + + + + + 8 + + 037 + + + a + + + + + b + OAI-PMH + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + --- a/koha-tmpl/intranet-tmpl/prog/en/xslt/StripOAIWrapper.xsl +++ a/koha-tmpl/intranet-tmpl/prog/en/xslt/StripOAIWrapper.xsl @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + --- a/misc/cronjobs/oai/oai_harvester.pl +++ a/misc/cronjobs/oai/oai_harvester.pl @@ -0,0 +1,180 @@ +#!/usr/bin/perl + +# Copyright Prosentient Systems 2015 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, see . + +use Modern::Perl; +use Getopt::Long; +use Pod::Usage; + + +=head1 NAME + +oai_harvester.pl - cron script to download records via OAI-PMH, and import +into Koha. + +=head1 SYNOPSIS + +perl oai_harvester.pl [ -r ][ -d ][ -i ] + +=head1 OPTIONS + +=over + +=item B<--help> + +Print a brief help message and exits. + +=item B<--man> + +Prints the manual page and exits. + +=item B<-v> + +Verbose. Without this flag set, only errors are reported. + +=item B<-r> + +Re-harvest the OAI server by resetting the "from" parameter to null. + +=item B<-d> + +Download records from the OAI-PMH server, and queue for importing. + +=item B<-i> + +Import queued records into Koha. + +=back + +=head1 DESCRIPTION + +With the "-d" option, this script iterates through a list of +OAI servers (ie repositories), issues a ListRecords OAI-PMH request, and +downloads the results into the Koha database to await further +processing (ie adding new records and updating existing ones). + +With the "-i" option, this script iterates through a list of +harvested OAI-PMH metadata records. Each record is checked +to see whether it needs to be added as a new record in Koha, +or if it needs to update a record that has previously been +harvested. After this checking, it is optionally transformed +using a user-defined XSLT. Finally, it is passed to +Koha's internal APIs to be added/updated in the database. + + +=cut + +my $help = 0; +my $man = 0; +my $verbose = 0; +my $reset = 0; +my $download = 0; +my $import = 0; + +GetOptions( + 'h|help|?' => \$help, + 'm|man' => \$man, + 'v' => \$verbose, + 'd' => \$download, + 'i' => \$import, + 'r' => \$reset, + )or pod2usage(2); +pod2usage(1) if $help; +pod2usage( -verbose => 2 ) if $man; + +if ($download){ + require C4::Context; + require XML::LibXSLT; + require Koha::OAI::Client::Repositories; + + my $stylesheet; + #Build path to XSLT filename + #FIXME: It would make more sense to have a utility XSLT directory outside of htdocs which is language independent... + my $htdocs = C4::Context->config('intrahtdocs'); + my $theme = C4::Context->preference("template"); + my $xslfilename = "$htdocs/$theme/en/xslt/StripOAIWrapper.xsl"; + if ( -f $xslfilename ){ + #FIXME: Ideally, it would be good to use Koha::XSLT_Handler here, but + #we want to output a XML::LibXML::Document instead of output_as_chars. + my $xslt = XML::LibXSLT->new(); + my $style_doc = XML::LibXML->load_xml(location => $xslfilename); + $stylesheet = $xslt->parse_stylesheet($style_doc); + } + + my $repositories = Koha::OAI::Client::Repositories->search({ + active => 1, + base_url => {'!=', undef}, + }); + while (my $repository = $repositories->next){ + if ($reset){ + $verbose && print "Resetting opt_from to undefined...\n"; + $repository->opt_from(undef); + } + $repository->harvest({ verbose => $verbose, stylesheet => $stylesheet, }); + } +} + +if ($import){ + require Koha::OAI::Client::Importer; + require Koha::OAI::Client::Records; + + #Import records, previously harvested via OAI-PMH, into Koha + my $oai_importer = Koha::OAI::Client::Importer->new(); + if ($oai_importer){ + my $records = Koha::OAI::Client::Records->search({ + system_number => undef, + action => ["to_import","to_delete"], + }); + + $verbose && print $records->count." records to process\n"; + if ($records){ + + my $number_imported = 0; + my $number_deleted = 0; + + #Iterate through the records and import (i.e. add/update/re-add) or delete them in Koha + while( my $record = $records->next ) { + if ($record->action eq "to_import"){ + eval { + $oai_importer->import_record($record); + }; + if ($@){ + warn "Error importing record (oai_harvest_id ".$record->oai_harvest_id.") into Koha:\n$@"; + $record->action("error"); + $record->store(); + } else { + $verbose && print $record->action . " identifier (".$record->identifier.") (".$record->record_type.") as system number (".$record->system_number.")\n"; + $number_imported++; + } + } elsif ($record->action eq "to_delete") { + eval { + $oai_importer->delete_record($record); + }; + if ($@){ + warn "Error deleting record (oai_harvest_id ".$record->oai_harvest_id.") from Koha:\n$@"; + } else { + $number_deleted++; + } + } + } + $verbose && print "$number_imported records imported (i.e. added/updated)\n"; + $verbose && print "$number_deleted records deleted \n"; + + + } + } +} --