From c7a5ba6129c1d3b2abd9559749e8e3dec7a79e26 Mon Sep 17 00:00:00 2001 From: David Cook Date: Thu, 6 Aug 2020 11:18:45 +0000 Subject: [PATCH] Bug 25905: Create /import/oaipmh/biblios API endpoint --- Koha/Import/Oaipmh.pm | 208 ++++++++++++++++++++++++ Koha/Import/Oaipmh/Biblio.pm | 40 +++++ Koha/Import/Oaipmh/Biblios.pm | 47 ++++++ Koha/REST/V1/Import/Oaipmh/Biblios.pm | 68 ++++++++ api/v1/swagger/paths.json | 3 + api/v1/swagger/paths/import_oaipmh_biblios.json | 63 +++++++ 6 files changed, 429 insertions(+) create mode 100644 Koha/Import/Oaipmh.pm create mode 100644 Koha/Import/Oaipmh/Biblio.pm create mode 100644 Koha/Import/Oaipmh/Biblios.pm create mode 100644 Koha/REST/V1/Import/Oaipmh/Biblios.pm create mode 100644 api/v1/swagger/paths/import_oaipmh_biblios.json diff --git a/Koha/Import/Oaipmh.pm b/Koha/Import/Oaipmh.pm new file mode 100644 index 0000000000..83ae6ca627 --- /dev/null +++ b/Koha/Import/Oaipmh.pm @@ -0,0 +1,208 @@ +package Koha::Import::Oaipmh; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License + +use Modern::Perl; +use XML::LibXML::Reader; +use MARC::Record; +use DateTime::Format::Strptime; + +use Koha::Import::Oaipmh::Biblios; +use C4::Context; +use C4::Biblio; + +my $xpc = XML::LibXML::XPathContext->new(); +$xpc->registerNs('oai','http://www.openarchives.org/OAI/2.0/'); +my $xpath_identifier = XML::LibXML::XPathExpression->new("oai:header/oai:identifier"); +my $xpath_datestamp = XML::LibXML::XPathExpression->new("oai:header/oai:datestamp"); +my $xpath_status = XML::LibXML::XPathExpression->new(q{oai:header/@status}); +my $xpath_metadata = XML::LibXML::XPathExpression->new(q{oai:metadata/*}); + +my $strp = DateTime::Format::Strptime->new( + pattern => '%F %T', + time_zone => 'floating', +); + +=head1 API Model + +=head2 Methods + +=cut + +sub new { + my ($class, $args) = @_; + $args = {} unless defined $args; + return bless ($args, $class); +} + +#NOTE: To improve performance, you could turn off autocommit at the start of this function, then commit at the end of the function +sub process_metadata { + my ($self, $args) = @_; + my $added = 0; + my $updated = 0; + my $deleted = 0; + my $skipped = 0; + + my $repository = $args->{repository}; + my $records = $args->{records}; + my $frameworkcode = $args->{frameworkcode} // ''; + if ($repository && $records) { + my $reader = XML::LibXML::Reader->new( string => $records ); + my $each_pattern = XML::LibXML::Pattern->new('//oai:record', { 'oai' => "http://www.openarchives.org/OAI/2.0/" }); + while (my $rv = $reader->nextPatternMatch($each_pattern)){ + #NOTE: We do this so we only get the opening tag of the element. + next unless $reader->nodeType == XML_READER_TYPE_ELEMENT; + if ($reader->localName eq "record"){ + my $record_node = $reader->copyCurrentNode(1); + if ($record_node){ + my $document = XML::LibXML::Document->new('1.0', 'UTF-8'); + $document->setDocumentElement($record_node); + my $root = $document->documentElement; + + my $identifier = _get_node_text({ + root => $root, + xpath_expression => $xpath_identifier, + }); + my $datestamp = _get_node_text({ + root => $root, + xpath_expression => $xpath_datestamp, + }); + #Remove UTC markers from timestamp + $datestamp =~ s/T/ /; + $datestamp =~ s/Z//; + + my $metadata; + my $metadata_node = _get_node({ + root => $root, + xpath_expression => $xpath_metadata, + }); + if ($metadata_node){ + $metadata = $metadata_node->toString(2); + } + + my $deleted = _get_node_text({ + root => $root, + xpath_expression => $xpath_status, + }); + + if ($self->{type} && $self->{type} eq 'biblio'){ + if ($identifier && $datestamp){ + my $record = Koha::Import::Oaipmh::Biblios->find({ + repository => $repository, + identifier => $identifier, + }); + if ($record){ + if ($metadata && ! $deleted){ + eval { + my $incoming_dt = $strp->parse_datetime($datestamp); + my $existing_dt = $strp->parse_datetime( $record->datestamp ); + warn $incoming_dt; + warn $existing_dt; + if ( $incoming_dt > $existing_dt ){ + MARC::Record::default_record_format("UNIMARC") + if ( C4::Context->preference("marcflavour") eq "UNIMARC" ); + my $marc_record = MARC::Record::new_from_xml( $metadata, 'UTF-8' ); + C4::Biblio::ModBiblio($marc_record, $record->biblionumber, $frameworkcode); + $record->update({ + datestamp => $datestamp, + metadata => $metadata, + }); + $updated++; + } else { + $skipped++; + } + }; + } elsif ($deleted){ + eval { + C4::Biblio::DelBiblio($record->biblionumber); + #$record->update({ + # datestamp => $datestamp, + # deleted => 1, + # metadata => '' + #}); + $deleted++; + }; + } + } else { + #NOTE: Having metadata and being deleted should be mutually exclusive, + #but it doesn't hurt to be thorough + if ($metadata && ! $deleted){ + eval { + MARC::Record::default_record_format("UNIMARC") + if ( C4::Context->preference("marcflavour") eq "UNIMARC" ); + my $marc_record = MARC::Record::new_from_xml( $metadata, 'UTF-8' ); + my ($biblionumber) = C4::Biblio::AddBiblio($marc_record,$frameworkcode); + Koha::Import::Oaipmh::Biblio->new({ + repository => $repository, + identifier => $identifier, + datestamp => $datestamp, + metadata => $metadata, + frameworkcode => $frameworkcode, + biblionumber => $biblionumber, + })->store; + $added++; + }; + } + } + } + } + } + } + } + } + my $results = { + added => $added, + updated => $updated, + deleted => $deleted, + skipped => $skipped, + }; + return $results; +} + +sub _get_node { + my ($args) = @_; + my $node; + my $root = $args->{root}; + my $xpath_expression = $args->{xpath_expression}; + if ($root && $xpath_expression){ + my $node_list = $xpc->findnodes($xpath_expression,$root); + if ($node_list){ + my $first_node = $node_list->shift; + if ($first_node){ + $node = $first_node; + } + } + } + return $node; +} + +sub _get_node_text { + my ($args) = @_; + my $text; + my $root = $args->{root}; + my $xpath_expression = $args->{xpath_expression}; + if ($root && $xpath_expression){ + my $node = _get_node({ + root => $root, + xpath_expression => $xpath_expression, + }); + if ($node){ + $text = $node->textContent; + } + } + return $text; +} + +1; diff --git a/Koha/Import/Oaipmh/Biblio.pm b/Koha/Import/Oaipmh/Biblio.pm new file mode 100644 index 0000000000..5f715e86bb --- /dev/null +++ b/Koha/Import/Oaipmh/Biblio.pm @@ -0,0 +1,40 @@ +package Koha::Import::Oaipmh::Biblio; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License + +use Modern::Perl; + +use base qw(Koha::Object); + +=head1 NAME + +Koha::Import::Oaipmh::Biblio + +This object represents an OAI-PMH record being imported as a biblio + +=head1 API + +=head2 Methods + + +=head3 type + +=cut + +sub _type { + return 'ImportOaipmhBiblio'; +} + +1; diff --git a/Koha/Import/Oaipmh/Biblios.pm b/Koha/Import/Oaipmh/Biblios.pm new file mode 100644 index 0000000000..6a62bd7dc2 --- /dev/null +++ b/Koha/Import/Oaipmh/Biblios.pm @@ -0,0 +1,47 @@ +package Koha::Import::Oaipmh::Biblios; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License + +use Modern::Perl; + +use Koha::Import::Oaipmh::Biblio; + +use base qw(Koha::Objects); + +=head1 NAME + +Koha::Import::Oaipmh::Biblios + +This object represents a collection of OAI-PMH records being imported as biblios + +=head1 API + +=head2 Methods + + +=head3 type + +=cut + +sub _type { + return 'ImportOaipmhBiblio'; +} + + +sub object_class { + return 'Koha::Import::Oaipmh::Biblio' +} + +1; diff --git a/Koha/REST/V1/Import/Oaipmh/Biblios.pm b/Koha/REST/V1/Import/Oaipmh/Biblios.pm new file mode 100644 index 0000000000..1e41f86715 --- /dev/null +++ b/Koha/REST/V1/Import/Oaipmh/Biblios.pm @@ -0,0 +1,68 @@ +package Koha::REST::V1::Import::Oaipmh::Biblios; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License + +use Modern::Perl; + +use Mojo::Base 'Mojolicious::Controller'; + +use Koha::Import::Oaipmh; + +#use Try::Tiny; + +=head1 API + +=head2 Class Methods + +=head3 Method that imports bibliographic records from OAI-PMH + +=cut + +sub import_biblios { + #NOTE: Validation will only work if body is marked as optional, as it's a non-JSON body + my $c = shift->openapi->valid_input or return; + + my $repository = $c->validation->param('x-koha-oaipmh-repository'); + #TODO: validate the repository (ie check that it is a URL) + + my $frameworkcode = $c->validation->param('x-koha-frameworkcode') // ''; + #TODO: validate the frameworkcode (ie check that it is a frameworkcode that exists) + + #NOTE: Mojolicious::Plugin::OpenAPI expects a JSON body, so we can't use the validation + #my $records = $c->validation->param('body'); + my $records = $c->req->body; + if ($records) { + my $importer = Koha::Import::Oaipmh->new({ + type => 'biblio', + }); + #NOTE: Don't use these variables as API output, since one day you want the API to just stage + #the records and not process them synchronously. Returning no "useful" output keeps the API + #externally consistent. + my $results = $importer->process_metadata({ + repository => $repository, + records => $records, + frameworkcode => $frameworkcode, + }); + use Data::Dumper; + warn Dumper($results); + } else { + return $c->render( status => 400, openapi => { error => "Records not found." } ); + } + return $c->render( status => 200, openapi => undef ); +} + +1; + + diff --git a/api/v1/swagger/paths.json b/api/v1/swagger/paths.json index 8f1af595ce..68bb924bb0 100644 --- a/api/v1/swagger/paths.json +++ b/api/v1/swagger/paths.json @@ -130,5 +130,8 @@ }, "/return_claims/{claim_id}": { "$ref": "paths/return_claims.json#/~1return_claims~1{claim_id}" + }, + "/import/oaipmh/biblios": { + "$ref": "paths/import_oaipmh_biblios.json#/~1import~1oaipmh~1biblios" } } diff --git a/api/v1/swagger/paths/import_oaipmh_biblios.json b/api/v1/swagger/paths/import_oaipmh_biblios.json new file mode 100644 index 0000000000..bf0239d47d --- /dev/null +++ b/api/v1/swagger/paths/import_oaipmh_biblios.json @@ -0,0 +1,63 @@ +{ + "/import/oaipmh/biblios": { + "post": { + "x-mojo-to": "Import::Oaipmh::Biblios#import_biblios", + "operationId": "import_biblios", + "tags": ["biblios"], + "parameters": [ + { + "name": "body", + "in": "body", + "description": "OAI-PMH XML bibliographic records", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "x-koha-oaipmh-repository", + "in": "header", + "required": true, + "description": "OAI-PMH repository sent as a request header", + "type": "string" + }, + { + "name": "x-koha-frameworkcode", + "in": "header", + "required": false, + "description": "MARC bibliographic framework code to use for importing bib records", + "type": "string" + } + ], + "consumes": [ + "text/xml" + ], + "produces": [ + "application/json" + ], + "responses": { + "200": { + "description": "Records accepted" + }, + "400": { + "description": "An error occured", + "schema": { + "type": "object", + "properties": { + "error": { + "description": "An explanation for the error", + "type": "string" + } + } + } + } + }, + "x-koha-authorization": { + "permissions": { + "editcatalogue": "edit_catalogue" + } + } + } + } +} + -- 2.11.0