From 2aae98101bf3013f577bf50f5232fb88a84143fe Mon Sep 17 00:00:00 2001 From: David Cook Date: Mon, 16 May 2016 16:08:06 +1000 Subject: [PATCH] Bug 10662 - Create svc/import_oai API --- Koha/OAI/Client/Record.pm | 249 +++++++++++++++++++++ Koha/Schema/Result/ImportOai.pm | 152 +++++++++++++ docs/OAIPMH/README | 20 ++ .../bug_10662-Build_import_oai_table.sql | 15 ++ .../intranet-tmpl/prog/en/includes/tools-menu.inc | 1 + .../prog/en/modules/tools/manage-oai-import.tt | 154 +++++++++++++ .../prog/en/modules/tools/tools-home.tt | 3 + .../intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl | 74 ++++++ svc/import_oai | 160 +++++++++++++ t/Import/bib-deleted.xml | 7 + t/Import/bib-oaidc.xml | 19 ++ t/Import/bib.xml | 25 +++ tools/manage-oai-import.pl | 206 +++++++++++++++++ 13 files changed, 1085 insertions(+) create mode 100755 Koha/OAI/Client/Record.pm create mode 100755 Koha/Schema/Result/ImportOai.pm create mode 100755 docs/OAIPMH/README create mode 100644 installer/data/mysql/atomicupdate/bug_10662-Build_import_oai_table.sql create mode 100755 koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-oai-import.tt create mode 100755 koha-tmpl/intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl create mode 100755 svc/import_oai create mode 100755 t/Import/bib-deleted.xml create mode 100755 t/Import/bib-oaidc.xml create mode 100755 t/Import/bib.xml create mode 100755 tools/manage-oai-import.pl diff --git a/Koha/OAI/Client/Record.pm b/Koha/OAI/Client/Record.pm new file mode 100755 index 0000000..ca978b0 --- /dev/null +++ b/Koha/OAI/Client/Record.pm @@ -0,0 +1,249 @@ +package Koha::OAI::Client::Record; + +# Copyright 2016 Prosentient Systems +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . +# + +use Modern::Perl; +use XML::LibXML; +use XML::LibXSLT; +use MARC::Record; + +use C4::Context; +use C4::Biblio; +use C4::ImportBatch; +use C4::Matcher; + +use constant MAX_MATCHES => 99999; #NOTE: This is an arbitrary value. We want to get all matches. + +sub new { + my ($class, $args) = @_; + $args = {} unless defined $args; + + if (my $inxml = $args->{xml_string}){ + + #Parse the XML string into a XML::LibXML object + my $doc = XML::LibXML->load_xml(string => $inxml, { no_blanks => 1 }); + $args->{doc} = $doc; + #NOTE: Don't load blank nodes... + + #Get the root element + my $root = $doc->documentElement; + + #Register namespaces for searching purposes + my $xpc = XML::LibXML::XPathContext->new(); + $xpc->registerNs('oai','http://www.openarchives.org/OAI/2.0/'); + + my $xpath_identifier = XML::LibXML::XPathExpression->new("oai:header/oai:identifier"); + my $identifier = $xpc->findnodes($xpath_identifier,$root)->shift; + #my $identifier_string = $identifier->textContent; + $args->{header_identifier} = $identifier->textContent; + + my $xpath_datestamp = XML::LibXML::XPathExpression->new("oai:header/oai:datestamp"); + my $datestamp = $xpc->findnodes($xpath_datestamp,$root)->shift; + #my $datestamp_string = $datestamp->textContent; + $args->{header_datestamp} = $datestamp->textContent; + + my $xpath_status = XML::LibXML::XPathExpression->new(q{oai:header/@status}); + my $status_node = $xpc->findnodes($xpath_status,$root)->shift; + #my $status_string = $status_node ? $status_node->textContent : ""; + $args->{header_status} = $status_node ? $status_node->textContent : ""; + } + + return bless ($args, $class); +} + +sub is_deleted_upstream { + my ($self, $args) = @_; + if ($self->{header_status}){ + if ($self->{header_status} eq "deleted"){ + return 1; + } + } + return 0; +} + +sub filter { + my ($self, $args) = @_; + my $doc = $self->{doc}; + my $filter = $args->{filter}; + $self->{filter} = $filter; #FIXME + #FIXME: Check that it's an XSLT here... + if ( -f $filter ){ + #Filter is a valid filepath + + #FIXME: Ideally, it would be good to use Koha::XSLT_Handler here... (especially for persistent environments...) + my $xslt = XML::LibXSLT->new(); + my $style_doc = XML::LibXML->load_xml(location => $filter); + my $stylesheet = $xslt->parse_stylesheet($style_doc); + if ($stylesheet){ + my $results = $stylesheet->transform($doc); + my $metadata_xml = $stylesheet->output_as_bytes($results); + #If the XSLT outputs nothing, then we don't meet the following condition, and we'll return 0 instead. + if ($metadata_xml){ + $self->{filtered_record} = $metadata_xml; + return 1; + } + } + } + return 0; +} + + + + + + +sub import_record { + my ($self, $args) = @_; + my $koha_record_numbers = ""; + my $errors = []; + my $import_status = "error"; + my $match_status = "no_match"; + + my $batch_id = $args->{import_batch_id}; + $self->{import_batch_id} = $batch_id; #FIXME + my $matcher = $args->{matcher}; + my $framework = $args->{framework}; + my $import_mode = $args->{import_mode}; + + my $metadata_xml = $self->{filtered_record}; + + if ($metadata_xml){ + #Convert MARCXML into MARC::Record object + my $marcflavour = C4::Context->preference('marcflavour') || 'MARC21'; + my $marc_record = eval {MARC::Record::new_from_xml( $metadata_xml, "utf8", $marcflavour)}; + if ($@) { + warn "Error converting OAI-PMH filtered metadata into MARC::Record object: $@"; + #FIXME: Improve error handling + } + + if ($self->is_deleted_upstream){ + +=pod + my @matches = $matcher->get_matches($marc_record, MAX_MATCHES); + if (@matches){ + $match_status = "matched"; + } + my $delete_error; + foreach my $match (@matches){ + if (my $record_id = $match->{record_id}){ + #FIXME: This is biblio specific... what about authority records? + my $error = C4::Biblio::DelBiblio($record_id); + if ($error){ + $delete_error++; + $koha_record_numbers = []; + push(@$koha_record_numbers,$record_id); + + #FIXME: Find a better way of sending the errors in a predictable way... + push(@$errors,{"record_id" => $record_id, "error" => $error, }); + } + } + + } + + #If there are no delete errors, then the import was ok + if ( ! $delete_error){ + $import_status = "ok"; + } + #Deleted records will never actually have an records in them, so always mark them as cleaned so that other imports don't try to pick up the same batch. + C4::ImportBatch::SetImportBatchStatus($batch_id, 'cleaned'); +=cut + my $import_record_id = AddBiblioToBatch($batch_id, 0, $marc_record, "utf8", int(rand(99999))); + my $number_of_matches = BatchFindDuplicates($batch_id, $matcher, MAX_MATCHES); + if ($number_of_matches > 0){ + $match_status = "auto_match"; #See `import_records` table for other options... but this should be the right one. + } + my $results = GetImportRecordMatches($import_record_id); #Only works for biblio... + my $delete_error; + + my @result_record_numbers = (); + foreach my $result (@$results){ + if (my $record_id = $result->{biblionumber}){ + push(@result_record_numbers,$record_id); + + #FIXME: This is biblio specific... what about authority records? + my $error = C4::Biblio::DelBiblio($record_id); + if ($error){ + $delete_error++; + push(@$errors, { type => 'delete_failed', error_msg => $error, record_id => $record_id, }) ; + } + } + } + $koha_record_numbers = join(",",@result_record_numbers); #FIXME: check that this works... + + if ($delete_error){ + $import_status = "error"; + C4::ImportBatch::SetImportBatchStatus($batch_id, 'importing'); + } else { + $import_status = "ok"; + #Ideally, it would be nice to say what records were deleted, but Koha doesn't have that capacity at the moment, so just clean the batch. + CleanBatch($batch_id); + } + + + + + } else { + #Import the MARCXML record into Koha + my $import_record_id = AddBiblioToBatch($batch_id, 0, $marc_record, "utf8", int(rand(99999))); + #FIXME: Don't allow item imports do to the nature of OAI-PMH records updating over time... + #my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 'UPDATE COUNTS'); + my $number_of_matches = BatchFindDuplicates($batch_id, $matcher); + + # XXX we are ignoring the result of this; + BatchCommitRecords($batch_id, $framework) if lc($import_mode) eq 'direct'; + + my $dbh = C4::Context->dbh(); + my $sth = $dbh->prepare("SELECT matched_biblionumber FROM import_biblios WHERE import_record_id =?"); + $sth->execute($import_record_id); + $koha_record_numbers = $sth->fetchrow_arrayref->[0] || ''; + $sth = $dbh->prepare("SELECT overlay_status FROM import_records WHERE import_record_id =?"); + $sth->execute($import_record_id); + + $match_status = $sth->fetchrow_arrayref->[0] || 'no_match'; + $import_status = "ok"; + } + } else { + #There's no filtered metadata... + #Clean the batch, so future imports don't use the same batch. + CleanBatch($batch_id); + } + $self->{status} = $import_status; #FIXME + #$self->save_to_database(); + return ($import_status,$match_status,$koha_record_numbers, $errors); +} + +sub save_to_database { + my ($self,$args) = @_; + + my $header_identifier = $self->{header_identifier}; + my $header_datestamp = $self->{header_datestamp}; + my $header_status = $self->{header_status}; + my $metadata = $self->{doc}->toString(1); + my $import_batch_id = $self->{import_batch_id}; + my $filter = $self->{filter}; + my $status = $self->{status}; + + my $dbh = C4::Context->dbh; + my $sql = "INSERT INTO import_oai (header_identifier, header_datestamp, header_status, metadata, import_batch_id, filter, status) VALUES (?, ?, ?, ?, ?, ?, ?)"; + my $sth = $dbh->prepare($sql); + $sth->execute($header_identifier,$header_datestamp,$header_status,$metadata, $import_batch_id, $filter, $status); +} + + +1; \ No newline at end of file diff --git a/Koha/Schema/Result/ImportOai.pm b/Koha/Schema/Result/ImportOai.pm new file mode 100755 index 0000000..bfa1a25 --- /dev/null +++ b/Koha/Schema/Result/ImportOai.pm @@ -0,0 +1,152 @@ +use utf8; +package Koha::Schema::Result::ImportOai; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +=head1 NAME + +Koha::Schema::Result::ImportOai + +=cut + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +=head1 TABLE: C + +=cut + +__PACKAGE__->table("import_oai"); + +=head1 ACCESSORS + +=head2 import_oai_id + + data_type: 'integer' + extra: {unsigned => 1} + is_auto_increment: 1 + is_nullable: 0 + +=head2 header_identifier + + data_type: 'varchar' + is_nullable: 0 + size: 45 + +=head2 header_datestamp + + data_type: 'datetime' + datetime_undef_if_invalid: 1 + is_nullable: 0 + +=head2 header_status + + data_type: 'varchar' + is_nullable: 1 + size: 45 + +=head2 metadata + + data_type: 'longtext' + is_nullable: 0 + +=head2 last_modified + + data_type: 'timestamp' + datetime_undef_if_invalid: 1 + default_value: current_timestamp + is_nullable: 0 + +=head2 status + + data_type: 'varchar' + is_nullable: 0 + size: 45 + +=head2 import_batch_id + + data_type: 'integer' + is_foreign_key: 1 + is_nullable: 0 + +=head2 filter + + data_type: 'text' + is_nullable: 0 + +=cut + +__PACKAGE__->add_columns( + "import_oai_id", + { + data_type => "integer", + extra => { unsigned => 1 }, + is_auto_increment => 1, + is_nullable => 0, + }, + "header_identifier", + { data_type => "varchar", is_nullable => 0, size => 45 }, + "header_datestamp", + { + data_type => "datetime", + datetime_undef_if_invalid => 1, + is_nullable => 0, + }, + "header_status", + { data_type => "varchar", is_nullable => 1, size => 45 }, + "metadata", + { data_type => "longtext", is_nullable => 0 }, + "last_modified", + { + data_type => "timestamp", + datetime_undef_if_invalid => 1, + default_value => \"current_timestamp", + is_nullable => 0, + }, + "status", + { data_type => "varchar", is_nullable => 0, size => 45 }, + "import_batch_id", + { data_type => "integer", is_foreign_key => 1, is_nullable => 0 }, + "filter", + { data_type => "text", is_nullable => 0 }, +); + +=head1 PRIMARY KEY + +=over 4 + +=item * L + +=back + +=cut + +__PACKAGE__->set_primary_key("import_oai_id"); + +=head1 RELATIONS + +=head2 import_batch + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "import_batch", + "Koha::Schema::Result::ImportBatch", + { import_batch_id => "import_batch_id" }, + { is_deferrable => 1, on_delete => "RESTRICT", on_update => "RESTRICT" }, +); + + +# Created by DBIx::Class::Schema::Loader v0.07042 @ 2016-04-12 11:02:33 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:QmCetOjXql0gsAi+wZ74Ng + + +# You can replace this text with custom code or comments, and it will be preserved on regeneration +1; diff --git a/docs/OAIPMH/README b/docs/OAIPMH/README new file mode 100755 index 0000000..c217cb7 --- /dev/null +++ b/docs/OAIPMH/README @@ -0,0 +1,20 @@ +TODO: +- Change `import_oai` database table? + - import_oai's "metadata" should actually be "oai_record"... so it's not so confusing... it's NOT the metadata element... but rather the whole OAI record. + - Rename "last_modified" to "upload_timestamp" +- Fix https://bugs.koha-community.org/bugzilla3/show_bug.cgi?id=15541 + +- Add documentation to all code... +- Add unit tests +- Clean up the code +- Resolve all TODO/FIXME comments in the code + + +FUTURE: +- Add support for authority records and possibly holdings records? +- Add default OAI record matching rule? + - I thought about adding a SQL atomic update 'bug_10662-Add_oai_record_matching_rule.sql', but adding matching rules seems complex. This needs to be done in Perl. + - Should the rule include other fields like 022, 020, 245 rather than just 001 and 024a? +- Add entry to Cleanupdatabase.pl cronjob? + - You could remove all import_oai rows older than a certain age? +- Re-do the paging to use DataTables AJAX? Or, create a centralized/generalized server-side paging function for Koha... diff --git a/installer/data/mysql/atomicupdate/bug_10662-Build_import_oai_table.sql b/installer/data/mysql/atomicupdate/bug_10662-Build_import_oai_table.sql new file mode 100644 index 0000000..554d8ed --- /dev/null +++ b/installer/data/mysql/atomicupdate/bug_10662-Build_import_oai_table.sql @@ -0,0 +1,15 @@ +DROP TABLE IF EXISTS import_oai; +CREATE TABLE import_oai ( + import_oai_id int(10) unsigned NOT NULL AUTO_INCREMENT, + header_identifier varchar(45) CHARACTER SET utf8 NOT NULL, + header_datestamp datetime NOT NULL, + header_status varchar(45) CHARACTER SET utf8 DEFAULT NULL, + metadata longtext CHARACTER SET utf8 NOT NULL, + last_modified timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + status varchar(45) CHARACTER SET utf8 NOT NULL, + import_batch_id int(11) NOT NULL, + filter text COLLATE utf8_unicode_ci NOT NULL, + PRIMARY KEY (import_oai_id), + KEY FK_import_oai_1 (import_batch_id), + CONSTRAINT FK_import_oai_1 FOREIGN KEY (import_batch_id) REFERENCES import_batches (import_batch_id) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-menu.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-menu.inc index 9a2b1bf..124c6b3 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-menu.inc +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-menu.inc @@ -92,6 +92,7 @@ [% END %] [% IF ( CAN_user_tools_manage_staged_marc ) %]
  • Staged MARC management
    • +
  • OAI-PMH import management
    • [% END %] [% IF ( CAN_user_tools_upload_local_cover_images ) %]
  • Upload local cover image
    • diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-oai-import.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-oai-import.tt new file mode 100755 index 0000000..075f78a --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-oai-import.tt @@ -0,0 +1,154 @@ +[% INCLUDE 'doc-head-open.inc' %] +Koha › Tools › Manage OAI-PMH record imports +[% IF ( import_oai_id ) %] + › Record [% import_oai_id %] +[% END %] + +[% INCLUDE 'doc-head-close.inc' %] + +[% INCLUDE 'datatables.inc' %] + + + +[% INCLUDE 'header.inc' %] +[% INCLUDE 'cat-search.inc' %] + + + +
    +
    +
    +
    + [% IF ( import_oai_id ) %] + [% IF ( view_record ) %] +

    Record [% import_oai_id %]

    + [% IF ( oai_record.metadata ) %] +
    [% oai_record.metadata | xml %]
    + [% END %] + [% ELSIF ( retry ) %] +
    +
      +
    1. + Import status: + [% IF ( import_status ) %] + [% IF ( import_status == "ok" ) %] + OK + [% ELSIF ( import_status == "error" ) %] + ERROR + [% END %] + [% END %] +
      2. + [% IF ( errors ) %] + [% FOREACH error IN errors %] + [% IF ( error ) %] +
    2. + Error: + [% SWITCH error.type %] + [% CASE 'delete_failed' %] + [% error.error_msg # FIXME: These English messages come straight from C4::Biblio... %] + [% IF ( record_type ) && ( record_type == "biblio" ) %] + (View biblio record) + [% END %] + [% CASE 'filter_failed' %] + Filter failed to produce MARCXML. + Review the metadata at View record [% import_oai_id %]. + You should also review your filter at [% oai_record.filter %]. + [% CASE %] + [% error.error_msg %] + [% END %] +
      3. + [% END %] + [% END %] + [% END %] +
    +
    + [% END %] + [% ELSE %] +

    Manage OAI-PMH record imports

    + + + + + + + + + + [%# %] + + + + [% WHILE (oai_record = oai_records.next) %] + + + + + + + [%# oai_record.filter %] + + + [% END %] + + +
    Record identifierRecord datestampProvider statusImport statusImport batchOAI-PMH recordFilter
    [% oai_record.header_identifier %][% oai_record.header_datestamp %] + [% IF ( oai_record.header_status ) %] + [% IF ( oai_record.header_status == "deleted" ) %] + DELETED + [% END %] + [% END %] + + [% IF ( oai_record.status ) %] + [% IF ( oai_record.status == "ok" ) %] + OK + [% ELSIF ( oai_record.status == "error" ) %] + ERROR - Click to retry + [% END %] + + [% ELSE %] + Unknown + [% END %] + + [% IF ( oai_record.import_batch_id ) %] + View batch [% oai_record.import_batch_id %] + [% END %] + View record [% oai_record.import_oai_id %]
    +
    + [% IF ( page_first ) %] + First ([% page_first %]) + [% ELSE %] + First + [% END %] + [% IF ( page_previous ) %] + Previous ([% page_previous %]) + [% ELSE %] + Previous + [% END %] + [% IF ( page_next ) %] + Next ([% page_next %]) + [% ELSE %] + Next + [% END %] + [% IF ( page_last ) %] + Last ([% page_last %]) + [% ELSE %] + Last + [% END %] +
    + [% END %] +
    +
    +
    + [% INCLUDE 'tools-menu.inc' %] +
    +
    +
    +[% INCLUDE 'intranet-bottom.inc' %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/tools-home.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/tools-home.tt index a8f8ff4..505a08f 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/tools-home.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/tools-home.tt @@ -177,6 +177,9 @@ [% IF ( CAN_user_tools_manage_staged_marc ) %]
    Staged MARC record management
    Managed staged MARC records, including completing and reversing imports
    + +
    OAI-PMH import management
    +
    Manage import of OAI-PMH harvested records
    [% END %] [% IF ( CAN_user_tools_upload_local_cover_images ) %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl b/koha-tmpl/intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl new file mode 100755 index 0000000..0f1d6f0 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl @@ -0,0 +1,74 @@ + + + + + + + + + + + + + + + + + + + + + + + + + http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 7 + + 024 + + a + + + + 2 + uri + + + + + diff --git a/svc/import_oai b/svc/import_oai new file mode 100755 index 0000000..b005d01 --- /dev/null +++ b/svc/import_oai @@ -0,0 +1,160 @@ +#!/usr/bin/perl + +# Copyright 2016 Prosentient Systems +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . +# + +use Modern::Perl; +use XML::LibXML; +use URI; +use File::Basename; + +use CGI qw ( -utf8 ); +use C4::Auth qw/check_api_auth/; +use C4::Context; +use C4::ImportBatch; +use C4::Matcher; +use XML::Simple; +use C4::Biblio; + +use Koha::OAI::Client::Record; + +my $query = new CGI; +binmode STDOUT, ':encoding(UTF-8)'; + +my ($status, $cookie, $sessionID) = check_api_auth($query, { editcatalogue => 'edit_catalogue'} ); +unless ($status eq "ok") { + print $query->header(-type => 'text/xml', -status => '403 Forbidden'); + print XMLout({ auth_status => $status }, NoAttr => 1, RootName => 'response', XMLDecl => 1); + exit 0; +} + +my $xml; +if ($query->request_method eq "POST") { + $xml = $query->param('xml'); +} +if ($xml) { + #TODO: You could probably use $query->Vars here instead... + my %params = map { $_ => scalar $query->param($_) } $query->param; + my $result = import_oai($xml, \%params ); + print $query->header(-type => 'text/xml'); + print XMLout($result, NoAttr => 1, RootName => 'response', XMLDecl => 1); +} else { + print $query->header(-type => 'text/xml', -status => '400 Bad Request'); +} + +exit 0; + +sub import_oai { + my ($inxml, $params) = @_; + + #Create record object + my $oai_record = Koha::OAI::Client::Record->new({ + xml_string => $inxml, + }); + + my $result = {}; + my $result_errors = []; + + my $filter = delete $params->{filter} || ''; + my $import_mode = delete $params->{import_mode} || ''; + my $framework = delete $params->{framework} || ''; + + unless ($params->{comments}){ + $params->{comments} = "OAI-PMH import"; + if ($oai_record->{header_identifier}){ + $params->{comments} .= ": $oai_record->{header_identifier}"; + } + } + + if (my $matcher_code = delete $params->{match}) { + $params->{matcher_id} = C4::Matcher::GetMatcherId($matcher_code); + } + + my $batch_id = GetWebserviceBatchId($params); + #FIXME: Use the batch_id to create a more useful filename in the import_batches table... + unless ($batch_id) { + $result->{'status'} = "failed"; + $result->{'error'} = "Batch create error"; + return $result; + } + + #Source a default XSLT to use for filtering + my $htdocs = C4::Context->config('intrahtdocs'); + my $theme = C4::Context->preference("template"); + #FIXME: This doesn't work for UNIMARC! + my $xslfilename = "$htdocs/$theme/en/xslt/OAI2MARC21slim.xsl"; + + #FIXME: There's a better way to do these filters... + if ($filter){ + my $filter_uri = URI->new($filter); + if ($filter_uri){ + my $scheme = $filter_uri->scheme; + if ($scheme && $scheme eq "file"){ + my $path = $filter_uri->path; + #Filters may theoretically be .xsl or .pm files + my($filename, $dirs, $suffix) = fileparse($path,(".xsl",".pm")); + if ($suffix && $suffix eq ".xsl"){ + #If this new path exists, change the filter XSLT to it + if ( -f $path ){ + $xslfilename = $path; + } + } + } + } + } + + #Get matching rule matcher + my $matcher = C4::Matcher->new($params->{record_type} || 'biblio'); + $matcher = C4::Matcher->fetch($params->{matcher_id}); + + + + #Filter OAI-PMH into MARCXML + my $filtered = $oai_record->filter({ + filter => $xslfilename, + }); + + if (!$filtered){ + push(@$result_errors, { type => 'filter_failed', error_msg => '', record_id => '', }) ; + } + + my ($import_status, $match_status, $koha_record_numbers, $errors) = $oai_record->import_record({ + matcher => $matcher, + import_batch_id => $batch_id, + import_mode => $import_mode, + framework => $framework, + }); + if (@$errors){ + push(@$result_errors,@$errors); + } + + $oai_record->save_to_database(); + + $result->{'match_status'} = $match_status; + $result->{'import_batch_id'} = $batch_id; + $result->{'koha_record_numbers'} = $koha_record_numbers; + + if ($import_status && $import_status eq "ok"){ + $result->{'status'} = "ok"; + } else { + $result->{'status'} = "failed"; + $result->{'errors'} = {error => $result_errors}; + } + + return $result; +} diff --git a/t/Import/bib-deleted.xml b/t/Import/bib-deleted.xml new file mode 100755 index 0000000..5a5b4ab --- /dev/null +++ b/t/Import/bib-deleted.xml @@ -0,0 +1,7 @@ + + +
    + oai:koha-community.org:5000 + 2015-12-22T18:46:29Z +
    +     diff --git a/t/Import/bib-oaidc.xml b/t/Import/bib-oaidc.xml new file mode 100755 index 0000000..cf247cc --- /dev/null +++ b/t/Import/bib-oaidc.xml @@ -0,0 +1,19 @@ + + +
    + oai:koha-community.org:5000 + 2015-12-21T18:46:29Z +
    + + + + Everything you never wanted to know about OAI-PMH : a primer / + + Cook, David + + text + eng + + + +     diff --git a/t/Import/bib.xml b/t/Import/bib.xml new file mode 100755 index 0000000..ac1209a --- /dev/null +++ b/t/Import/bib.xml @@ -0,0 +1,25 @@ + + +
    + oai:koha-community.org:5000 + 2015-12-21T18:46:29Z +
    + + + + 01005cam a22003377a 4500 + 123456789 + 20151221185246.0 + 010203s2004 nyu 000 0 eng u + + Cook, David + + + Everything you never wanted to know about OAI-PMH : + a primer / + David Cook + + + + +     \ No newline at end of file diff --git a/tools/manage-oai-import.pl b/tools/manage-oai-import.pl new file mode 100755 index 0000000..f04da4b --- /dev/null +++ b/tools/manage-oai-import.pl @@ -0,0 +1,206 @@ +#!/usr/bin/perl + +# Copyright 2016 Prosentient Systems +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; +use POSIX qw//; + +use Koha::Database; + +use C4::Auth; +use C4::Output; +use C4::Koha; +use C4::Context; +use C4::Matcher; +use Koha::OAI::Client::Record; + +use C4::ImportBatch; + +my $script_name = "/cgi-bin/koha/tools/manage-oai-import.pl"; + +my $input = new CGI; +my $op = $input->param('op') || 'list'; +my $page_number = $input->param('page') && $input->param('page') =~ /^\d+$/ ? $input->param('page') : 1; #Only permit numeric parameter + +my $import_oai_id = $input->param('import_oai_id'); +#my $results_per_page = $input->param('results_per_page') || 25; + + + +my ($template, $loggedinuser, $cookie) = + get_template_and_user({template_name => "tools/manage-oai-import.tt", + query => $input, + type => "intranet", + authnotrequired => 0, + flagsrequired => {tools => 'manage_staged_marc'}, + debug => 1, + }); + +my $schema = Koha::Database->new()->schema(); +my $resultset = $schema->resultset('ImportOai'); + + +if ($import_oai_id){ + my $import_oai_record = $resultset->find($import_oai_id); + $template->param( + oai_record => $import_oai_record, + ); + + if ($op eq "view_record" && $import_oai_id){ + $template->param( + view_record => 1, + import_oai_id => $import_oai_id, + ); + } + + if ($op eq "retry" && $import_oai_record){ + my $result_errors = []; + + my $oai_record = Koha::OAI::Client::Record->new({ + xml_string => $import_oai_record->metadata, + }); + + my $filtered = $oai_record->filter({ + filter => $import_oai_record->filter, + }); + if (!$filtered){ + push(@$result_errors, { type => 'filter_failed', error_msg => '', record_id => '', }) ; + } + + my $import_batch_id = $import_oai_record->import_batch_id; + if ($import_batch_id){ + my $import_batch_rs = $schema->resultset('ImportBatch'); + my $import_batch = $import_batch_rs->find($import_batch_id); + my $matcher_id = $import_batch->matcher_id; + + my $record_type = $import_batch->record_type; + $template->param( + record_type => $record_type, + ); + + + #my $matcher = C4::Matcher->new($record_type || 'biblio'); + my $matcher = C4::Matcher->fetch($matcher_id); + + + #FIXME + my $import_mode = "direct"; + #FIXME + my $framework = ""; + + #Reset the batch before re-trying the import + C4::ImportBatch::CleanBatch($import_batch_id); + + my ($import_status, $match_status, $koha_record_numbers, $errors) = $oai_record->import_record({ + matcher => $matcher, + import_batch_id => $import_batch_id, + import_mode => $import_mode, + framework => $framework, + }); + if (@$errors){ + push(@$result_errors,@$errors); + } + + if ($import_status){ + if ($import_status eq 'ok'){ + $import_oai_record->status("ok"); + $import_oai_record->update(); + } else { + $template->param( + import_status => $import_status, + errors => $result_errors, + retry => 1, + import_oai_id => $import_oai_id, + ); + } + } + } + } +} + + + +$template->param( + script_name => $script_name, +); + +if ($op && $op eq "list"){ + #NOTE: It would be preferable if we centralized server-side paging code with generic functions... + + #Get grand total in the database + my $total_rows = $resultset->count; + + my $number_of_rows = 10; + my $number_of_pages = POSIX::ceil($total_rows / $number_of_rows); + + if ($page_number > 1){ + $template->{VARS}->{ page_first } = 1; + } + if ($number_of_pages > 1 && $page_number != $number_of_pages){ + $template->{VARS}->{ page_last } = $number_of_pages; + } + + #Do the search and define a limit + my $results = $resultset->search( + undef, + { + rows => $number_of_rows, + order_by => { -desc => 'last_modified' }, + }, + ); + + my $page = $results->page($page_number); + my $current_page_rows = $page->count(); + + if ($current_page_rows){ + if ($page_number == 1) { + #Can't page previous to the first page + if ( $current_page_rows == $total_rows ){ + #Can't page past the total count + } else { + #Signal that there's another page + $template->{VARS}->{ page_next } = $page_number + 1; + } + } else { + + #Signal that there's a previous page + $template->{VARS}->{ page_previous } = $page_number - 1; + + #Total rows of all previous pages + my $previous_pages_rows = ( $page_number - 1 ) * $number_of_rows; + + #If current and past rows are less than total rows, there must still be more to show + if ( ($current_page_rows + $previous_pages_rows) < $total_rows){ + $template->{VARS}->{ page_next } = $page_number + 1; + } + + } + } + + $template->param( + page_number => $page_number, + script_name => $script_name, + oai_records => $page, + ); +} + + + + + +output_html_with_http_headers $input, $cookie, $template->output; \ No newline at end of file -- 2.1.4