Bugzilla – Attachment 107882 Details for
Bug 25905
REST API: create endpoint for importing OAI-PMH records from external OAI-PMH clients
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 25905: Create /import/oaipmh/biblios API endpoint
-Bug-25905-Create-importoaipmhbiblios-API-endpoint.patch (text/plain), 16.60 KB, created by
David Cook
on 2020-08-06 12:29:13 UTC
(
hide
)
Description:
Bug 25905: Create /import/oaipmh/biblios API endpoint
Filename:
MIME Type:
Creator:
David Cook
Created:
2020-08-06 12:29:13 UTC
Size:
16.60 KB
patch
obsolete
>From c7a5ba6129c1d3b2abd9559749e8e3dec7a79e26 Mon Sep 17 00:00:00 2001 >From: David Cook <dcook@prosentient.com.au> >Date: Thu, 6 Aug 2020 11:18:45 +0000 >Subject: [PATCH] Bug 25905: Create /import/oaipmh/biblios API endpoint > >--- > Koha/Import/Oaipmh.pm | 208 ++++++++++++++++++++++++ > Koha/Import/Oaipmh/Biblio.pm | 40 +++++ > Koha/Import/Oaipmh/Biblios.pm | 47 ++++++ > Koha/REST/V1/Import/Oaipmh/Biblios.pm | 68 ++++++++ > api/v1/swagger/paths.json | 3 + > api/v1/swagger/paths/import_oaipmh_biblios.json | 63 +++++++ > 6 files changed, 429 insertions(+) > create mode 100644 Koha/Import/Oaipmh.pm > create mode 100644 Koha/Import/Oaipmh/Biblio.pm > create mode 100644 Koha/Import/Oaipmh/Biblios.pm > create mode 100644 Koha/REST/V1/Import/Oaipmh/Biblios.pm > create mode 100644 api/v1/swagger/paths/import_oaipmh_biblios.json > >diff --git a/Koha/Import/Oaipmh.pm b/Koha/Import/Oaipmh.pm >new file mode 100644 >index 0000000000..83ae6ca627 >--- /dev/null >+++ b/Koha/Import/Oaipmh.pm >@@ -0,0 +1,208 @@ >+package Koha::Import::Oaipmh; >+ >+# This file is part of Koha. >+# >+# Koha is free software; you can redistribute it and/or modify it >+# under the terms of the GNU General Public License as published by >+# the Free Software Foundation; either version 3 of the License, or >+# (at your option) any later version. >+# >+# Koha is distributed in the hope that it will be useful, but >+# WITHOUT ANY WARRANTY; without even the implied warranty of >+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >+# GNU General Public License for more details. >+# >+# You should have received a copy of the GNU General Public License >+ >+use Modern::Perl; >+use XML::LibXML::Reader; >+use MARC::Record; >+use DateTime::Format::Strptime; >+ >+use Koha::Import::Oaipmh::Biblios; >+use C4::Context; >+use C4::Biblio; >+ >+my $xpc = XML::LibXML::XPathContext->new(); >+$xpc->registerNs('oai','http://www.openarchives.org/OAI/2.0/'); >+my $xpath_identifier = XML::LibXML::XPathExpression->new("oai:header/oai:identifier"); >+my $xpath_datestamp = XML::LibXML::XPathExpression->new("oai:header/oai:datestamp"); >+my $xpath_status = XML::LibXML::XPathExpression->new(q{oai:header/@status}); >+my $xpath_metadata = XML::LibXML::XPathExpression->new(q{oai:metadata/*}); >+ >+my $strp = DateTime::Format::Strptime->new( >+ pattern => '%F %T', >+ time_zone => 'floating', >+); >+ >+=head1 API Model >+ >+=head2 Methods >+ >+=cut >+ >+sub new { >+ my ($class, $args) = @_; >+ $args = {} unless defined $args; >+ return bless ($args, $class); >+} >+ >+#NOTE: To improve performance, you could turn off autocommit at the start of this function, then commit at the end of the function >+sub process_metadata { >+ my ($self, $args) = @_; >+ my $added = 0; >+ my $updated = 0; >+ my $deleted = 0; >+ my $skipped = 0; >+ >+ my $repository = $args->{repository}; >+ my $records = $args->{records}; >+ my $frameworkcode = $args->{frameworkcode} // ''; >+ if ($repository && $records) { >+ my $reader = XML::LibXML::Reader->new( string => $records ); >+ my $each_pattern = XML::LibXML::Pattern->new('//oai:record', { 'oai' => "http://www.openarchives.org/OAI/2.0/" }); >+ while (my $rv = $reader->nextPatternMatch($each_pattern)){ >+ #NOTE: We do this so we only get the opening tag of the element. >+ next unless $reader->nodeType == XML_READER_TYPE_ELEMENT; >+ if ($reader->localName eq "record"){ >+ my $record_node = $reader->copyCurrentNode(1); >+ if ($record_node){ >+ my $document = XML::LibXML::Document->new('1.0', 'UTF-8'); >+ $document->setDocumentElement($record_node); >+ my $root = $document->documentElement; >+ >+ my $identifier = _get_node_text({ >+ root => $root, >+ xpath_expression => $xpath_identifier, >+ }); >+ my $datestamp = _get_node_text({ >+ root => $root, >+ xpath_expression => $xpath_datestamp, >+ }); >+ #Remove UTC markers from timestamp >+ $datestamp =~ s/T/ /; >+ $datestamp =~ s/Z//; >+ >+ my $metadata; >+ my $metadata_node = _get_node({ >+ root => $root, >+ xpath_expression => $xpath_metadata, >+ }); >+ if ($metadata_node){ >+ $metadata = $metadata_node->toString(2); >+ } >+ >+ my $deleted = _get_node_text({ >+ root => $root, >+ xpath_expression => $xpath_status, >+ }); >+ >+ if ($self->{type} && $self->{type} eq 'biblio'){ >+ if ($identifier && $datestamp){ >+ my $record = Koha::Import::Oaipmh::Biblios->find({ >+ repository => $repository, >+ identifier => $identifier, >+ }); >+ if ($record){ >+ if ($metadata && ! $deleted){ >+ eval { >+ my $incoming_dt = $strp->parse_datetime($datestamp); >+ my $existing_dt = $strp->parse_datetime( $record->datestamp ); >+ warn $incoming_dt; >+ warn $existing_dt; >+ if ( $incoming_dt > $existing_dt ){ >+ MARC::Record::default_record_format("UNIMARC") >+ if ( C4::Context->preference("marcflavour") eq "UNIMARC" ); >+ my $marc_record = MARC::Record::new_from_xml( $metadata, 'UTF-8' ); >+ C4::Biblio::ModBiblio($marc_record, $record->biblionumber, $frameworkcode); >+ $record->update({ >+ datestamp => $datestamp, >+ metadata => $metadata, >+ }); >+ $updated++; >+ } else { >+ $skipped++; >+ } >+ }; >+ } elsif ($deleted){ >+ eval { >+ C4::Biblio::DelBiblio($record->biblionumber); >+ #$record->update({ >+ # datestamp => $datestamp, >+ # deleted => 1, >+ # metadata => '' >+ #}); >+ $deleted++; >+ }; >+ } >+ } else { >+ #NOTE: Having metadata and being deleted should be mutually exclusive, >+ #but it doesn't hurt to be thorough >+ if ($metadata && ! $deleted){ >+ eval { >+ MARC::Record::default_record_format("UNIMARC") >+ if ( C4::Context->preference("marcflavour") eq "UNIMARC" ); >+ my $marc_record = MARC::Record::new_from_xml( $metadata, 'UTF-8' ); >+ my ($biblionumber) = C4::Biblio::AddBiblio($marc_record,$frameworkcode); >+ Koha::Import::Oaipmh::Biblio->new({ >+ repository => $repository, >+ identifier => $identifier, >+ datestamp => $datestamp, >+ metadata => $metadata, >+ frameworkcode => $frameworkcode, >+ biblionumber => $biblionumber, >+ })->store; >+ $added++; >+ }; >+ } >+ } >+ } >+ } >+ } >+ } >+ } >+ } >+ my $results = { >+ added => $added, >+ updated => $updated, >+ deleted => $deleted, >+ skipped => $skipped, >+ }; >+ return $results; >+} >+ >+sub _get_node { >+ my ($args) = @_; >+ my $node; >+ my $root = $args->{root}; >+ my $xpath_expression = $args->{xpath_expression}; >+ if ($root && $xpath_expression){ >+ my $node_list = $xpc->findnodes($xpath_expression,$root); >+ if ($node_list){ >+ my $first_node = $node_list->shift; >+ if ($first_node){ >+ $node = $first_node; >+ } >+ } >+ } >+ return $node; >+} >+ >+sub _get_node_text { >+ my ($args) = @_; >+ my $text; >+ my $root = $args->{root}; >+ my $xpath_expression = $args->{xpath_expression}; >+ if ($root && $xpath_expression){ >+ my $node = _get_node({ >+ root => $root, >+ xpath_expression => $xpath_expression, >+ }); >+ if ($node){ >+ $text = $node->textContent; >+ } >+ } >+ return $text; >+} >+ >+1; >diff --git a/Koha/Import/Oaipmh/Biblio.pm b/Koha/Import/Oaipmh/Biblio.pm >new file mode 100644 >index 0000000000..5f715e86bb >--- /dev/null >+++ b/Koha/Import/Oaipmh/Biblio.pm >@@ -0,0 +1,40 @@ >+package Koha::Import::Oaipmh::Biblio; >+ >+# This file is part of Koha. >+# >+# Koha is free software; you can redistribute it and/or modify it >+# under the terms of the GNU General Public License as published by >+# the Free Software Foundation; either version 3 of the License, or >+# (at your option) any later version. >+# >+# Koha is distributed in the hope that it will be useful, but >+# WITHOUT ANY WARRANTY; without even the implied warranty of >+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >+# GNU General Public License for more details. >+# >+# You should have received a copy of the GNU General Public License >+ >+use Modern::Perl; >+ >+use base qw(Koha::Object); >+ >+=head1 NAME >+ >+Koha::Import::Oaipmh::Biblio >+ >+This object represents an OAI-PMH record being imported as a biblio >+ >+=head1 API >+ >+=head2 Methods >+ >+ >+=head3 type >+ >+=cut >+ >+sub _type { >+ return 'ImportOaipmhBiblio'; >+} >+ >+1; >diff --git a/Koha/Import/Oaipmh/Biblios.pm b/Koha/Import/Oaipmh/Biblios.pm >new file mode 100644 >index 0000000000..6a62bd7dc2 >--- /dev/null >+++ b/Koha/Import/Oaipmh/Biblios.pm >@@ -0,0 +1,47 @@ >+package Koha::Import::Oaipmh::Biblios; >+ >+# This file is part of Koha. >+# >+# Koha is free software; you can redistribute it and/or modify it >+# under the terms of the GNU General Public License as published by >+# the Free Software Foundation; either version 3 of the License, or >+# (at your option) any later version. >+# >+# Koha is distributed in the hope that it will be useful, but >+# WITHOUT ANY WARRANTY; without even the implied warranty of >+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >+# GNU General Public License for more details. >+# >+# You should have received a copy of the GNU General Public License >+ >+use Modern::Perl; >+ >+use Koha::Import::Oaipmh::Biblio; >+ >+use base qw(Koha::Objects); >+ >+=head1 NAME >+ >+Koha::Import::Oaipmh::Biblios >+ >+This object represents a collection of OAI-PMH records being imported as biblios >+ >+=head1 API >+ >+=head2 Methods >+ >+ >+=head3 type >+ >+=cut >+ >+sub _type { >+ return 'ImportOaipmhBiblio'; >+} >+ >+ >+sub object_class { >+ return 'Koha::Import::Oaipmh::Biblio' >+} >+ >+1; >diff --git a/Koha/REST/V1/Import/Oaipmh/Biblios.pm b/Koha/REST/V1/Import/Oaipmh/Biblios.pm >new file mode 100644 >index 0000000000..1e41f86715 >--- /dev/null >+++ b/Koha/REST/V1/Import/Oaipmh/Biblios.pm >@@ -0,0 +1,68 @@ >+package Koha::REST::V1::Import::Oaipmh::Biblios; >+ >+# This file is part of Koha. >+# >+# Koha is free software; you can redistribute it and/or modify it >+# under the terms of the GNU General Public License as published by >+# the Free Software Foundation; either version 3 of the License, or >+# (at your option) any later version. >+# >+# Koha is distributed in the hope that it will be useful, but >+# WITHOUT ANY WARRANTY; without even the implied warranty of >+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >+# GNU General Public License for more details. >+# >+# You should have received a copy of the GNU General Public License >+ >+use Modern::Perl; >+ >+use Mojo::Base 'Mojolicious::Controller'; >+ >+use Koha::Import::Oaipmh; >+ >+#use Try::Tiny; >+ >+=head1 API >+ >+=head2 Class Methods >+ >+=head3 Method that imports bibliographic records from OAI-PMH >+ >+=cut >+ >+sub import_biblios { >+ #NOTE: Validation will only work if body is marked as optional, as it's a non-JSON body >+ my $c = shift->openapi->valid_input or return; >+ >+ my $repository = $c->validation->param('x-koha-oaipmh-repository'); >+ #TODO: validate the repository (ie check that it is a URL) >+ >+ my $frameworkcode = $c->validation->param('x-koha-frameworkcode') // ''; >+ #TODO: validate the frameworkcode (ie check that it is a frameworkcode that exists) >+ >+ #NOTE: Mojolicious::Plugin::OpenAPI expects a JSON body, so we can't use the validation >+ #my $records = $c->validation->param('body'); >+ my $records = $c->req->body; >+ if ($records) { >+ my $importer = Koha::Import::Oaipmh->new({ >+ type => 'biblio', >+ }); >+ #NOTE: Don't use these variables as API output, since one day you want the API to just stage >+ #the records and not process them synchronously. Returning no "useful" output keeps the API >+ #externally consistent. >+ my $results = $importer->process_metadata({ >+ repository => $repository, >+ records => $records, >+ frameworkcode => $frameworkcode, >+ }); >+ use Data::Dumper; >+ warn Dumper($results); >+ } else { >+ return $c->render( status => 400, openapi => { error => "Records not found." } ); >+ } >+ return $c->render( status => 200, openapi => undef ); >+} >+ >+1; >+ >+ >diff --git a/api/v1/swagger/paths.json b/api/v1/swagger/paths.json >index 8f1af595ce..68bb924bb0 100644 >--- a/api/v1/swagger/paths.json >+++ b/api/v1/swagger/paths.json >@@ -130,5 +130,8 @@ > }, > "/return_claims/{claim_id}": { > "$ref": "paths/return_claims.json#/~1return_claims~1{claim_id}" >+ }, >+ "/import/oaipmh/biblios": { >+ "$ref": "paths/import_oaipmh_biblios.json#/~1import~1oaipmh~1biblios" > } > } >diff --git a/api/v1/swagger/paths/import_oaipmh_biblios.json b/api/v1/swagger/paths/import_oaipmh_biblios.json >new file mode 100644 >index 0000000000..bf0239d47d >--- /dev/null >+++ b/api/v1/swagger/paths/import_oaipmh_biblios.json >@@ -0,0 +1,63 @@ >+{ >+ "/import/oaipmh/biblios": { >+ "post": { >+ "x-mojo-to": "Import::Oaipmh::Biblios#import_biblios", >+ "operationId": "import_biblios", >+ "tags": ["biblios"], >+ "parameters": [ >+ { >+ "name": "body", >+ "in": "body", >+ "description": "OAI-PMH XML bibliographic records", >+ "required": false, >+ "schema": { >+ "type": "string" >+ } >+ }, >+ { >+ "name": "x-koha-oaipmh-repository", >+ "in": "header", >+ "required": true, >+ "description": "OAI-PMH repository sent as a request header", >+ "type": "string" >+ }, >+ { >+ "name": "x-koha-frameworkcode", >+ "in": "header", >+ "required": false, >+ "description": "MARC bibliographic framework code to use for importing bib records", >+ "type": "string" >+ } >+ ], >+ "consumes": [ >+ "text/xml" >+ ], >+ "produces": [ >+ "application/json" >+ ], >+ "responses": { >+ "200": { >+ "description": "Records accepted" >+ }, >+ "400": { >+ "description": "An error occured", >+ "schema": { >+ "type": "object", >+ "properties": { >+ "error": { >+ "description": "An explanation for the error", >+ "type": "string" >+ } >+ } >+ } >+ } >+ }, >+ "x-koha-authorization": { >+ "permissions": { >+ "editcatalogue": "edit_catalogue" >+ } >+ } >+ } >+ } >+} >+ >-- >2.11.0
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 25905
:
107880
|
107881
| 107882 |
107883