From 2aae98101bf3013f577bf50f5232fb88a84143fe Mon Sep 17 00:00:00 2001
From: David Cook <dcook@prosentient.com.au>
Date: Mon, 16 May 2016 16:08:06 +1000
Subject: [PATCH] Bug 10662 - Create svc/import_oai API

---
 Koha/OAI/Client/Record.pm                          | 249 +++++++++++++++++++++
 Koha/Schema/Result/ImportOai.pm                    | 152 +++++++++++++
 docs/OAIPMH/README                                 |  20 ++
 .../bug_10662-Build_import_oai_table.sql           |  15 ++
 .../intranet-tmpl/prog/en/includes/tools-menu.inc  |   1 +
 .../prog/en/modules/tools/manage-oai-import.tt     | 154 +++++++++++++
 .../prog/en/modules/tools/tools-home.tt            |   3 +
 .../intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl  |  74 ++++++
 svc/import_oai                                     | 160 +++++++++++++
 t/Import/bib-deleted.xml                           |   7 +
 t/Import/bib-oaidc.xml                             |  19 ++
 t/Import/bib.xml                                   |  25 +++
 tools/manage-oai-import.pl                         | 206 +++++++++++++++++
 13 files changed, 1085 insertions(+)
 create mode 100755 Koha/OAI/Client/Record.pm
 create mode 100755 Koha/Schema/Result/ImportOai.pm
 create mode 100755 docs/OAIPMH/README
 create mode 100644 installer/data/mysql/atomicupdate/bug_10662-Build_import_oai_table.sql
 create mode 100755 koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-oai-import.tt
 create mode 100755 koha-tmpl/intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl
 create mode 100755 svc/import_oai
 create mode 100755 t/Import/bib-deleted.xml
 create mode 100755 t/Import/bib-oaidc.xml
 create mode 100755 t/Import/bib.xml
 create mode 100755 tools/manage-oai-import.pl

diff --git a/Koha/OAI/Client/Record.pm b/Koha/OAI/Client/Record.pm
new file mode 100755
index 0000000..ca978b0
--- /dev/null
+++ b/Koha/OAI/Client/Record.pm
@@ -0,0 +1,249 @@
+package Koha::OAI::Client::Record;
+
+# Copyright 2016 Prosentient Systems
+#
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# Koha is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Koha; if not, see <http://www.gnu.org/licenses>.
+#
+
+use Modern::Perl;
+use XML::LibXML;
+use XML::LibXSLT;
+use MARC::Record;
+
+use C4::Context;
+use C4::Biblio;
+use C4::ImportBatch;
+use C4::Matcher;
+
+use constant MAX_MATCHES => 99999; #NOTE: This is an arbitrary value. We want to get all matches.
+
+sub new {
+    my ($class, $args) = @_;
+    $args = {} unless defined $args;
+
+    if (my $inxml = $args->{xml_string}){
+
+        #Parse the XML string into a XML::LibXML object
+        my $doc = XML::LibXML->load_xml(string => $inxml, { no_blanks => 1 });
+        $args->{doc} = $doc;
+        #NOTE: Don't load blank nodes...
+
+        #Get the root element
+        my $root = $doc->documentElement;
+
+        #Register namespaces for searching purposes
+        my $xpc = XML::LibXML::XPathContext->new();
+        $xpc->registerNs('oai','http://www.openarchives.org/OAI/2.0/');
+
+        my $xpath_identifier = XML::LibXML::XPathExpression->new("oai:header/oai:identifier");
+        my $identifier = $xpc->findnodes($xpath_identifier,$root)->shift;
+        #my $identifier_string = $identifier->textContent;
+        $args->{header_identifier} = $identifier->textContent;
+
+        my $xpath_datestamp = XML::LibXML::XPathExpression->new("oai:header/oai:datestamp");
+        my $datestamp = $xpc->findnodes($xpath_datestamp,$root)->shift;
+        #my $datestamp_string = $datestamp->textContent;
+        $args->{header_datestamp} = $datestamp->textContent;
+
+        my $xpath_status = XML::LibXML::XPathExpression->new(q{oai:header/@status});
+        my $status_node = $xpc->findnodes($xpath_status,$root)->shift;
+        #my $status_string = $status_node ? $status_node->textContent : "";
+        $args->{header_status} = $status_node ? $status_node->textContent : "";
+    }
+
+    return bless ($args, $class);
+}
+
+sub is_deleted_upstream {
+    my ($self, $args) = @_;
+    if ($self->{header_status}){
+        if ($self->{header_status} eq "deleted"){
+            return 1;
+        }
+    }
+    return 0;
+}
+
+sub filter {
+    my ($self, $args) = @_;
+    my $doc = $self->{doc};
+    my $filter = $args->{filter};
+    $self->{filter} = $filter; #FIXME
+    #FIXME: Check that it's an XSLT here...
+    if ( -f $filter ){
+        #Filter is a valid filepath
+
+        #FIXME: Ideally, it would be good to use Koha::XSLT_Handler here... (especially for persistent environments...)
+        my $xslt = XML::LibXSLT->new();
+        my $style_doc = XML::LibXML->load_xml(location => $filter);
+        my $stylesheet = $xslt->parse_stylesheet($style_doc);
+        if ($stylesheet){
+            my $results = $stylesheet->transform($doc);
+            my $metadata_xml = $stylesheet->output_as_bytes($results);
+            #If the XSLT outputs nothing, then we don't meet the following condition, and we'll return 0 instead.
+            if ($metadata_xml){
+                $self->{filtered_record} = $metadata_xml;
+                return 1;
+            }
+        }
+    }
+    return 0;
+}
+
+
+
+
+
+
+sub import_record {
+    my ($self, $args) = @_;
+    my $koha_record_numbers = "";
+    my $errors = [];
+    my $import_status = "error";
+    my $match_status = "no_match";
+
+    my $batch_id = $args->{import_batch_id};
+    $self->{import_batch_id} = $batch_id; #FIXME
+    my $matcher = $args->{matcher};
+    my $framework = $args->{framework};
+    my $import_mode = $args->{import_mode};
+
+    my $metadata_xml = $self->{filtered_record};
+
+    if ($metadata_xml){
+        #Convert MARCXML into MARC::Record object
+        my $marcflavour = C4::Context->preference('marcflavour') || 'MARC21';
+        my $marc_record = eval {MARC::Record::new_from_xml( $metadata_xml, "utf8", $marcflavour)};
+        if ($@) {
+            warn "Error converting OAI-PMH filtered metadata into MARC::Record object: $@";
+            #FIXME: Improve error handling
+        }
+
+        if ($self->is_deleted_upstream){
+
+=pod
+            my @matches = $matcher->get_matches($marc_record, MAX_MATCHES);
+            if (@matches){
+                $match_status = "matched";
+            }
+            my $delete_error;
+            foreach my $match (@matches){
+                    if (my $record_id = $match->{record_id}){
+                        #FIXME: This is biblio specific... what about authority records?
+                        my $error = C4::Biblio::DelBiblio($record_id);
+                        if ($error){
+                            $delete_error++;
+                            $koha_record_numbers = [];
+                            push(@$koha_record_numbers,$record_id);
+
+                            #FIXME: Find a better way of sending the errors in a predictable way...
+                            push(@$errors,{"record_id" => $record_id, "error" => $error, });
+                        }
+                    }
+
+            }
+
+            #If there are no delete errors, then the import was ok
+            if ( ! $delete_error){
+                $import_status = "ok";
+            }
+            #Deleted records will never actually have an records in them, so always mark them as cleaned so that other imports don't try to pick up the same batch.
+            C4::ImportBatch::SetImportBatchStatus($batch_id, 'cleaned');
+=cut
+            my $import_record_id = AddBiblioToBatch($batch_id, 0, $marc_record, "utf8", int(rand(99999)));
+            my $number_of_matches =  BatchFindDuplicates($batch_id, $matcher, MAX_MATCHES);
+            if ($number_of_matches > 0){
+                $match_status = "auto_match"; #See `import_records` table for other options... but this should be the right one.
+            }
+            my $results = GetImportRecordMatches($import_record_id); #Only works for biblio...
+            my $delete_error;
+
+            my @result_record_numbers = ();
+            foreach my $result (@$results){
+                if (my $record_id = $result->{biblionumber}){
+                    push(@result_record_numbers,$record_id);
+
+                    #FIXME: This is biblio specific... what about authority records?
+                    my $error = C4::Biblio::DelBiblio($record_id);
+                    if ($error){
+                        $delete_error++;
+                        push(@$errors, { type => 'delete_failed', error_msg => $error, record_id => $record_id, }) ;
+                    }
+                }
+            }
+            $koha_record_numbers = join(",",@result_record_numbers); #FIXME: check that this works...
+
+            if ($delete_error){
+                $import_status = "error";
+                C4::ImportBatch::SetImportBatchStatus($batch_id, 'importing');
+            } else {
+                $import_status = "ok";
+                #Ideally, it would be nice to say what records were deleted, but Koha doesn't have that capacity at the moment, so just clean the batch.
+                CleanBatch($batch_id);
+            }
+
+
+
+
+        } else {
+            #Import the MARCXML record into Koha
+            my $import_record_id = AddBiblioToBatch($batch_id, 0, $marc_record, "utf8", int(rand(99999)));
+            #FIXME: Don't allow item imports do to the nature of OAI-PMH records updating over time...
+            #my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 'UPDATE COUNTS');
+            my $number_of_matches =  BatchFindDuplicates($batch_id, $matcher);
+
+            # XXX we are ignoring the result of this;
+            BatchCommitRecords($batch_id, $framework) if lc($import_mode) eq 'direct';
+
+            my $dbh = C4::Context->dbh();
+            my $sth = $dbh->prepare("SELECT matched_biblionumber FROM import_biblios WHERE import_record_id =?");
+            $sth->execute($import_record_id);
+            $koha_record_numbers = $sth->fetchrow_arrayref->[0] || '';
+            $sth = $dbh->prepare("SELECT overlay_status FROM import_records WHERE import_record_id =?");
+            $sth->execute($import_record_id);
+
+            $match_status = $sth->fetchrow_arrayref->[0] || 'no_match';
+            $import_status = "ok";
+        }
+    } else {
+        #There's no filtered metadata...
+        #Clean the batch, so future imports don't use the same batch.
+        CleanBatch($batch_id);
+    }
+    $self->{status} = $import_status; #FIXME
+    #$self->save_to_database();
+    return ($import_status,$match_status,$koha_record_numbers, $errors);
+}
+
+sub save_to_database {
+    my ($self,$args) = @_;
+
+    my $header_identifier = $self->{header_identifier};
+    my $header_datestamp = $self->{header_datestamp};
+    my $header_status = $self->{header_status};
+    my $metadata = $self->{doc}->toString(1);
+    my $import_batch_id = $self->{import_batch_id};
+    my $filter = $self->{filter};
+    my $status = $self->{status};
+
+    my $dbh = C4::Context->dbh;
+    my $sql = "INSERT INTO import_oai (header_identifier, header_datestamp, header_status, metadata, import_batch_id, filter, status) VALUES (?, ?, ?, ?, ?, ?, ?)";
+    my $sth = $dbh->prepare($sql);
+    $sth->execute($header_identifier,$header_datestamp,$header_status,$metadata, $import_batch_id, $filter, $status);
+}
+
+
+1;
\ No newline at end of file
diff --git a/Koha/Schema/Result/ImportOai.pm b/Koha/Schema/Result/ImportOai.pm
new file mode 100755
index 0000000..bfa1a25
--- /dev/null
+++ b/Koha/Schema/Result/ImportOai.pm
@@ -0,0 +1,152 @@
+use utf8;
+package Koha::Schema::Result::ImportOai;
+
+# Created by DBIx::Class::Schema::Loader
+# DO NOT MODIFY THE FIRST PART OF THIS FILE
+
+=head1 NAME
+
+Koha::Schema::Result::ImportOai
+
+=cut
+
+use strict;
+use warnings;
+
+use base 'DBIx::Class::Core';
+
+=head1 TABLE: C<import_oai>
+
+=cut
+
+__PACKAGE__->table("import_oai");
+
+=head1 ACCESSORS
+
+=head2 import_oai_id
+
+  data_type: 'integer'
+  extra: {unsigned => 1}
+  is_auto_increment: 1
+  is_nullable: 0
+
+=head2 header_identifier
+
+  data_type: 'varchar'
+  is_nullable: 0
+  size: 45
+
+=head2 header_datestamp
+
+  data_type: 'datetime'
+  datetime_undef_if_invalid: 1
+  is_nullable: 0
+
+=head2 header_status
+
+  data_type: 'varchar'
+  is_nullable: 1
+  size: 45
+
+=head2 metadata
+
+  data_type: 'longtext'
+  is_nullable: 0
+
+=head2 last_modified
+
+  data_type: 'timestamp'
+  datetime_undef_if_invalid: 1
+  default_value: current_timestamp
+  is_nullable: 0
+
+=head2 status
+
+  data_type: 'varchar'
+  is_nullable: 0
+  size: 45
+
+=head2 import_batch_id
+
+  data_type: 'integer'
+  is_foreign_key: 1
+  is_nullable: 0
+
+=head2 filter
+
+  data_type: 'text'
+  is_nullable: 0
+
+=cut
+
+__PACKAGE__->add_columns(
+  "import_oai_id",
+  {
+    data_type => "integer",
+    extra => { unsigned => 1 },
+    is_auto_increment => 1,
+    is_nullable => 0,
+  },
+  "header_identifier",
+  { data_type => "varchar", is_nullable => 0, size => 45 },
+  "header_datestamp",
+  {
+    data_type => "datetime",
+    datetime_undef_if_invalid => 1,
+    is_nullable => 0,
+  },
+  "header_status",
+  { data_type => "varchar", is_nullable => 1, size => 45 },
+  "metadata",
+  { data_type => "longtext", is_nullable => 0 },
+  "last_modified",
+  {
+    data_type => "timestamp",
+    datetime_undef_if_invalid => 1,
+    default_value => \"current_timestamp",
+    is_nullable => 0,
+  },
+  "status",
+  { data_type => "varchar", is_nullable => 0, size => 45 },
+  "import_batch_id",
+  { data_type => "integer", is_foreign_key => 1, is_nullable => 0 },
+  "filter",
+  { data_type => "text", is_nullable => 0 },
+);
+
+=head1 PRIMARY KEY
+
+=over 4
+
+=item * L</import_oai_id>
+
+=back
+
+=cut
+
+__PACKAGE__->set_primary_key("import_oai_id");
+
+=head1 RELATIONS
+
+=head2 import_batch
+
+Type: belongs_to
+
+Related object: L<Koha::Schema::Result::ImportBatch>
+
+=cut
+
+__PACKAGE__->belongs_to(
+  "import_batch",
+  "Koha::Schema::Result::ImportBatch",
+  { import_batch_id => "import_batch_id" },
+  { is_deferrable => 1, on_delete => "RESTRICT", on_update => "RESTRICT" },
+);
+
+
+# Created by DBIx::Class::Schema::Loader v0.07042 @ 2016-04-12 11:02:33
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:QmCetOjXql0gsAi+wZ74Ng
+
+
+# You can replace this text with custom code or comments, and it will be preserved on regeneration
+1;
diff --git a/docs/OAIPMH/README b/docs/OAIPMH/README
new file mode 100755
index 0000000..c217cb7
--- /dev/null
+++ b/docs/OAIPMH/README
@@ -0,0 +1,20 @@
+TODO:
+- Change `import_oai` database table?
+    - import_oai's "metadata" should actually be "oai_record"... so it's not so confusing... it's NOT the metadata element... but rather the whole OAI record.
+    - Rename "last_modified" to "upload_timestamp"
+- Fix https://bugs.koha-community.org/bugzilla3/show_bug.cgi?id=15541
+
+- Add documentation to all code...
+- Add unit tests
+- Clean up the code
+- Resolve all TODO/FIXME comments in the code
+
+    
+FUTURE:
+- Add support for authority records and possibly holdings records?
+- Add default OAI record matching rule?
+    - I thought about adding a SQL atomic update 'bug_10662-Add_oai_record_matching_rule.sql', but adding matching rules seems complex. This needs to be done in Perl.
+    - Should the rule include other fields like 022, 020, 245 rather than just 001 and 024a?
+- Add entry to Cleanupdatabase.pl cronjob?
+    - You could remove all import_oai rows older than a certain age?
+- Re-do the paging to use DataTables AJAX? Or, create a centralized/generalized server-side paging function for Koha...
diff --git a/installer/data/mysql/atomicupdate/bug_10662-Build_import_oai_table.sql b/installer/data/mysql/atomicupdate/bug_10662-Build_import_oai_table.sql
new file mode 100644
index 0000000..554d8ed
--- /dev/null
+++ b/installer/data/mysql/atomicupdate/bug_10662-Build_import_oai_table.sql
@@ -0,0 +1,15 @@
+DROP TABLE IF EXISTS import_oai;
+CREATE TABLE  import_oai (
+  import_oai_id int(10) unsigned NOT NULL AUTO_INCREMENT,
+  header_identifier varchar(45) CHARACTER SET utf8 NOT NULL,
+  header_datestamp datetime NOT NULL,
+  header_status varchar(45) CHARACTER SET utf8 DEFAULT NULL,
+  metadata longtext CHARACTER SET utf8 NOT NULL,
+  last_modified timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  status varchar(45) CHARACTER SET utf8 NOT NULL,
+  import_batch_id int(11) NOT NULL,
+  filter text COLLATE utf8_unicode_ci NOT NULL,
+  PRIMARY KEY (import_oai_id),
+  KEY FK_import_oai_1 (import_batch_id),
+  CONSTRAINT FK_import_oai_1 FOREIGN KEY (import_batch_id) REFERENCES import_batches (import_batch_id)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-menu.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-menu.inc
index 9a2b1bf..124c6b3 100644
--- a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-menu.inc
+++ b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-menu.inc
@@ -92,6 +92,7 @@
     [% END %]
     [% IF ( CAN_user_tools_manage_staged_marc ) %]
 	<li><a href="/cgi-bin/koha/tools/manage-marc-import.pl">Staged MARC management</a></li>
+    <li><a href="/cgi-bin/koha/tools/manage-oai-import.pl">OAI-PMH import management</a></li>
     [% END %]
     [% IF ( CAN_user_tools_upload_local_cover_images ) %]
     <li><a href="/cgi-bin/koha/tools/upload-cover-image.pl">Upload local cover image</a></li>
diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-oai-import.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-oai-import.tt
new file mode 100755
index 0000000..075f78a
--- /dev/null
+++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-oai-import.tt
@@ -0,0 +1,154 @@
+[% INCLUDE 'doc-head-open.inc' %]
+<title>Koha &rsaquo; Tools &rsaquo; Manage OAI-PMH record imports
+[% IF ( import_oai_id ) %]
+ &rsaquo; Record [% import_oai_id %]
+[% END %]
+</title>
+[% INCLUDE 'doc-head-close.inc' %]
+<link rel="stylesheet" type="text/css" href="[% interface %]/[% theme %]/css/datatables.css" />
+[% INCLUDE 'datatables.inc' %]
+</head>
+
+<body id="tools_manage-oai-import" class="tools">
+[% INCLUDE 'header.inc' %]
+[% INCLUDE 'cat-search.inc' %]
+
+    <div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a>
+    [% IF ( import_oai_id ) %]
+     &rsaquo;
+     <a href="[% script_name %]">Manage OAI-PMH record imports</a>
+     &rsaquo; Record [% import_oai_id %]
+    [% ELSE %]
+     &rsaquo; Manage OAI-PMH record imports
+    [% END %]
+    </div>
+
+    <div id="doc3" class="yui-t2">
+        <div id="bd">
+            <div id="yui-main">
+                <div class="yui-b">
+                    [% IF ( import_oai_id ) %]
+                        [% IF ( view_record ) %]
+                            <h1>Record [% import_oai_id %]</h1>
+                            [% IF ( oai_record.metadata ) %]
+                                <div style="white-space:pre">[% oai_record.metadata | xml %]</div>
+                            [% END %]
+                        [% ELSIF ( retry ) %]
+                            <fieldset class="rows">
+                                <ol>
+                                    <li>
+                                        <span class="label">Import status:</span>
+                                        [% IF ( import_status ) %]
+                                            [% IF ( import_status == "ok" ) %]
+                                            OK
+                                            [% ELSIF ( import_status == "error" ) %]
+                                            ERROR
+                                            [% END %]
+                                        [% END %]
+                                    </li>
+                                    [% IF ( errors ) %]
+                                        [% FOREACH error IN errors %]
+                                            [% IF ( error ) %]
+                                                <li>
+                                                    <span class="label">Error:</span>
+                                                    [% SWITCH error.type %]
+                                                    [% CASE 'delete_failed' %]
+                                                        [% error.error_msg # FIXME: These English messages come straight from C4::Biblio... %]
+                                                        [% IF ( record_type ) && ( record_type == "biblio" ) %]
+                                                            <a title="View biblio record" href="/cgi-bin/koha/catalogue/detail.pl?biblionumber=[% error.record_id %]">(View biblio record)</a>
+                                                        [% END %]
+                                                    [% CASE 'filter_failed' %]
+                                                        Filter failed to produce MARCXML.
+                                                        Review the metadata at <a href="/cgi-bin/koha/tools/manage-oai-import.pl?op=view_record&import_oai_id=[% import_oai_id %]">View record [% import_oai_id %]</a>.
+                                                        You should also review your filter at [% oai_record.filter %].
+                                                    [% CASE %]
+                                                        [% error.error_msg %]
+                                                    [% END %]
+                                                </li>
+                                            [% END %]
+                                        [% END %]
+                                    [% END %]
+                                </ol>
+                            </fieldset>
+                        [% END %]
+                    [% ELSE %]
+                        <h1>Manage OAI-PMH record imports</h1>
+                        <table>
+                            <thead>
+                                <tr>
+                                    <th>Record identifier</th>
+                                    <th>Record datestamp</th>
+                                    <th>Provider status</th>
+                                    <th>Import status</th>
+                                    <th>Import batch</th>
+                                    <th>OAI-PMH record</th>
+                                    [%# <th>Filter</th> %]
+                                </tr>
+                            </thead>
+                            <tbody>
+                                [% WHILE (oai_record = oai_records.next) %]
+                                <tr>
+                                    <td>[% oai_record.header_identifier %]</td>
+                                    <td>[% oai_record.header_datestamp %]</td>
+                                    <td>
+                                        [% IF ( oai_record.header_status ) %]
+                                            [% IF ( oai_record.header_status == "deleted" ) %]
+                                                DELETED
+                                            [% END %]
+                                        [% END %]
+                                    </td>
+                                    <td>
+                                        [% IF ( oai_record.status ) %]
+                                            [% IF ( oai_record.status == "ok" ) %]
+                                                OK
+                                            [% ELSIF ( oai_record.status == "error" ) %]
+                                                <a title="Retry import" href="[% script_name %]?op=retry&import_oai_id=[% oai_record.import_oai_id %]">ERROR - Click to retry</a>
+                                            [% END %]
+
+                                        [% ELSE %]
+                                            Unknown
+                                        [% END %]
+                                    </td>
+                                    <td>
+                                        [% IF ( oai_record.import_batch_id ) %]
+                                            <a title="View import batch" href="/cgi-bin/koha/tools/manage-marc-import.pl?import_batch_id=[% oai_record.import_batch_id %]">View batch [% oai_record.import_batch_id %]</a>
+                                        [% END %]
+                                    </td>
+                                    [%# oai_record.filter %]
+                                    <td><a title="View OAI-PMH record" href="[% script_name %]?op=view_record&import_oai_id=[% oai_record.import_oai_id %]">View record [% oai_record.import_oai_id %]</a></td>
+                                </tr>
+                                [% END %]
+
+                            </tbody>
+                        </table>
+                        <div class="pager">
+                        [% IF ( page_first ) %]
+                            <a href="[% script_name %]?page=[% page_first %]">First ([% page_first %])</a>
+                        [% ELSE %]
+                            <a class="disabled">First</a>
+                        [% END %]
+                        [% IF ( page_previous ) %]
+                            <a href="[% script_name %]?page=[% page_previous %]">Previous ([% page_previous %])</a>
+                        [% ELSE %]
+                            <a class="disabled">Previous</a>
+                        [% END %]
+                        [% IF ( page_next ) %]
+                            <a href="[% script_name %]?page=[% page_next %]">Next ([% page_next %])</a>
+                        [% ELSE %]
+                            <a class="disabled">Next</a>
+                        [% END %]
+                        [% IF ( page_last ) %]
+                            <a href="[% script_name %]?page=[% page_last %]">Last ([% page_last %])</a>
+                        [% ELSE %]
+                            <a class="disabled">Last</a>
+                        [% END %]
+                        </div>
+                    [% END %]
+                </div>
+            </div>
+            <div class="yui-b">
+                [% INCLUDE 'tools-menu.inc' %]
+            </div>
+        </div>
+    </div>
+[% INCLUDE 'intranet-bottom.inc' %]
diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/tools-home.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/tools-home.tt
index a8f8ff4..505a08f 100644
--- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/tools-home.tt
+++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/tools-home.tt
@@ -177,6 +177,9 @@
     [% IF ( CAN_user_tools_manage_staged_marc ) %]
     <dt><a href="/cgi-bin/koha/tools/manage-marc-import.pl">Staged MARC record management</a></dt>
     <dd>Managed staged MARC records, including completing and reversing imports</dd>
+
+    <dt><a href="/cgi-bin/koha/tools/manage-oai-import.pl">OAI-PMH import management</a></dt>
+    <dd>Manage import of OAI-PMH harvested records</dd>
     [% END %]
 
     [% IF ( CAN_user_tools_upload_local_cover_images ) %]
diff --git a/koha-tmpl/intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl b/koha-tmpl/intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl
new file mode 100755
index 0000000..0f1d6f0
--- /dev/null
+++ b/koha-tmpl/intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl
@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet version="1.0"
+    xmlns:marc="http://www.loc.gov/MARC21/slim"
+    xmlns:oai="http://www.openarchives.org/OAI/2.0/"
+    xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>
+    <!-- NOTE: This XSLT strips the OAI-PMH wrapper from the metadata. -->
+    <!-- NOTE: This XSLT also adds the OAI-PMH identifier back in as a MARC field -->
+
+    <!-- Match the root oai:record element -->
+    <xsl:template match="oai:record">
+        <!-- Apply templates only when the oai record is for a deleted item -->
+        <xsl:apply-templates select="oai:header[@status='deleted']" />
+        <!-- Apply templates only to the child metadata element(s) -->
+        <xsl:apply-templates select="oai:metadata" />
+    </xsl:template>
+
+    <!-- Matches an oai:metadata element -->
+    <xsl:template match="oai:metadata">
+        <!-- Only apply further templates to marc:record elements -->
+        <!-- This prevents the identity transformation from outputting other non-MARC metadata formats -->
+        <xsl:apply-templates select="//marc:record"/>
+    </xsl:template>
+
+    <!-- We need to create a MARCXML record from OAI records marked "deleted" to handle OAI deletions correctly in Koha -->
+    <xsl:template match="oai:header[@status='deleted']">
+        <xsl:element name="record" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+            xmlns="http://www.loc.gov/MARC21/slim">
+            <xsl:attribute name="xsi:schemaLocation">http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd</xsl:attribute>
+            <xsl:call-template name="add_oai"/>
+        </xsl:element>
+    </xsl:template>
+
+    <!-- Identity transformation: this template copies attributes and nodes -->
+    <xsl:template match="@* | node()">
+        <!-- Create a copy of this attribute or node -->
+        <xsl:copy>
+            <!-- Recursively apply this template to the attributes and child nodes of this element -->
+            <xsl:apply-templates select="@* | node()" />
+        </xsl:copy>
+    </xsl:template>
+
+
+    <xsl:template match="marc:record">
+        <xsl:copy>
+            <!-- Apply all relevant templates for all attributes and elements -->
+            <xsl:apply-templates select="@* | node()"/>
+
+            <!-- Add new node (or whatever else you want to do after copying the existing record) -->
+            <xsl:call-template name="add_oai"/>
+
+            <!-- Newline -->
+            <xsl:text>&#xa;</xsl:text>
+        </xsl:copy>
+    </xsl:template>
+
+    <!-- Template for adding the OAI-PMH identifier as 024$a -->
+    <xsl:template name="add_oai">
+        <xsl:element name="datafield" xmlns="http://www.loc.gov/MARC21/slim">
+            <xsl:attribute name="ind1"><xsl:text>7</xsl:text></xsl:attribute>
+            <xsl:attribute name="ind2"><xsl:text> </xsl:text></xsl:attribute>
+            <xsl:attribute name="tag">024</xsl:attribute>
+            <xsl:element name="subfield">
+                <xsl:attribute name="code">a</xsl:attribute>
+                <xsl:value-of select="/oai:record/oai:header/oai:identifier"/>
+            </xsl:element>
+            <xsl:element name="subfield">
+                <xsl:attribute name="code">2</xsl:attribute>
+                <xsl:text>uri</xsl:text>
+            </xsl:element>
+         </xsl:element>
+    </xsl:template>
+
+</xsl:stylesheet>
diff --git a/svc/import_oai b/svc/import_oai
new file mode 100755
index 0000000..b005d01
--- /dev/null
+++ b/svc/import_oai
@@ -0,0 +1,160 @@
+#!/usr/bin/perl
+
+# Copyright 2016 Prosentient Systems
+#
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# Koha is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Koha; if not, see <http://www.gnu.org/licenses>.
+#
+
+use Modern::Perl;
+use XML::LibXML;
+use URI;
+use File::Basename;
+
+use CGI qw ( -utf8 );
+use C4::Auth qw/check_api_auth/;
+use C4::Context;
+use C4::ImportBatch;
+use C4::Matcher;
+use XML::Simple;
+use C4::Biblio;
+
+use Koha::OAI::Client::Record;
+
+my $query = new CGI;
+binmode STDOUT, ':encoding(UTF-8)';
+
+my ($status, $cookie, $sessionID) = check_api_auth($query, { editcatalogue => 'edit_catalogue'} );
+unless ($status eq "ok") {
+    print $query->header(-type => 'text/xml', -status => '403 Forbidden');
+    print XMLout({ auth_status => $status }, NoAttr => 1, RootName => 'response', XMLDecl => 1);
+    exit 0;
+}
+
+my $xml;
+if ($query->request_method eq "POST") {
+    $xml = $query->param('xml');
+}
+if ($xml) {
+    #TODO: You could probably use $query->Vars here instead...
+    my %params = map { $_ => scalar $query->param($_) } $query->param;
+    my $result = import_oai($xml, \%params );
+    print $query->header(-type => 'text/xml');
+    print XMLout($result, NoAttr => 1, RootName => 'response', XMLDecl => 1);
+} else {
+    print $query->header(-type => 'text/xml', -status => '400 Bad Request');
+}
+
+exit 0;
+
+sub import_oai {
+    my ($inxml, $params) = @_;
+
+    #Create record object
+    my $oai_record = Koha::OAI::Client::Record->new({
+        xml_string => $inxml,
+    });
+    
+    my $result = {};
+    my $result_errors = [];
+
+    my $filter      = delete $params->{filter}      || '';
+    my $import_mode = delete $params->{import_mode} || '';
+    my $framework   = delete $params->{framework}   || '';
+    
+    unless ($params->{comments}){
+        $params->{comments} = "OAI-PMH import";
+        if ($oai_record->{header_identifier}){
+            $params->{comments} .= ": $oai_record->{header_identifier}";
+        }
+    }
+
+    if (my $matcher_code = delete $params->{match}) {
+        $params->{matcher_id} = C4::Matcher::GetMatcherId($matcher_code);
+    }
+
+    my $batch_id = GetWebserviceBatchId($params);
+    #FIXME: Use the batch_id to create a more useful filename in the import_batches table...
+    unless ($batch_id) {
+        $result->{'status'} = "failed";
+        $result->{'error'} = "Batch create error";
+        return $result;
+    }
+
+    #Source a default XSLT to use for filtering
+    my $htdocs  = C4::Context->config('intrahtdocs');
+    my $theme   = C4::Context->preference("template");
+    #FIXME: This doesn't work for UNIMARC!
+    my $xslfilename = "$htdocs/$theme/en/xslt/OAI2MARC21slim.xsl";
+
+    #FIXME: There's a better way to do these filters...
+    if ($filter){
+        my $filter_uri = URI->new($filter);
+        if ($filter_uri){
+            my $scheme = $filter_uri->scheme;
+            if ($scheme && $scheme eq "file"){
+                my $path = $filter_uri->path;
+                #Filters may theoretically be .xsl or .pm files
+                my($filename, $dirs, $suffix) = fileparse($path,(".xsl",".pm"));
+                if ($suffix && $suffix eq ".xsl"){
+                    #If this new path exists, change the filter XSLT to it
+                    if ( -f $path ){
+                        $xslfilename = $path;
+                    }
+                }
+            }
+        }
+    }
+
+    #Get matching rule matcher
+    my $matcher = C4::Matcher->new($params->{record_type} || 'biblio');
+    $matcher = C4::Matcher->fetch($params->{matcher_id});
+
+    
+
+    #Filter OAI-PMH into MARCXML
+    my $filtered = $oai_record->filter({
+        filter => $xslfilename,
+    });
+
+    if (!$filtered){
+        push(@$result_errors, { type => 'filter_failed', error_msg => '', record_id => '', }) ;
+    }
+
+    my ($import_status, $match_status, $koha_record_numbers, $errors) = $oai_record->import_record({
+        matcher => $matcher,
+        import_batch_id => $batch_id,
+        import_mode => $import_mode,
+        framework => $framework,
+    });
+    if (@$errors){
+        push(@$result_errors,@$errors);
+    }
+
+    $oai_record->save_to_database();
+
+    $result->{'match_status'} = $match_status;
+    $result->{'import_batch_id'} = $batch_id;
+    $result->{'koha_record_numbers'} = $koha_record_numbers;
+
+    if ($import_status && $import_status eq "ok"){
+        $result->{'status'} = "ok";
+    } else {
+        $result->{'status'} = "failed";
+        $result->{'errors'} = {error => $result_errors};
+    }
+
+    return $result;
+}
diff --git a/t/Import/bib-deleted.xml b/t/Import/bib-deleted.xml
new file mode 100755
index 0000000..5a5b4ab
--- /dev/null
+++ b/t/Import/bib-deleted.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<record xmlns="http://www.openarchives.org/OAI/2.0/">
+  <header status="deleted">
+    <identifier>oai:koha-community.org:5000</identifier>
+    <datestamp>2015-12-22T18:46:29Z</datestamp>
+  </header>
+</record>
diff --git a/t/Import/bib-oaidc.xml b/t/Import/bib-oaidc.xml
new file mode 100755
index 0000000..cf247cc
--- /dev/null
+++ b/t/Import/bib-oaidc.xml
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<record xmlns="http://www.openarchives.org/OAI/2.0/">
+  <header>
+    <identifier>oai:koha-community.org:5000</identifier>
+    <datestamp>2015-12-21T18:46:29Z</datestamp>
+  </header>
+  <metadata>
+    <metadata xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+        <oai_dc:dc xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
+          <dc:title>Everything you never wanted to know about OAI-PMH : a primer /</dc:title>
+          <dc:creator>
+            Cook, David
+          </dc:creator>
+          <dc:type>text</dc:type>
+          <dc:language>eng</dc:language>
+        </oai_dc:dc>
+    </metadata>
+  </metadata>
+</record>
diff --git a/t/Import/bib.xml b/t/Import/bib.xml
new file mode 100755
index 0000000..ac1209a
--- /dev/null
+++ b/t/Import/bib.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<record xmlns="http://www.openarchives.org/OAI/2.0/">
+  <header>
+    <identifier>oai:koha-community.org:5000</identifier>
+    <datestamp>2015-12-21T18:46:29Z</datestamp>
+  </header>
+  <metadata>
+    <metadata xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+      <record xmlns="http://www.loc.gov/MARC21/slim" type="Bibliographic">
+        <leader>01005cam a22003377a 4500</leader>
+        <controlfield tag="001">123456789</controlfield>
+        <controlfield tag="005">20151221185246.0</controlfield>
+        <controlfield tag="008">010203s2004    nyu           000 0 eng u</controlfield>
+        <datafield ind1="1" tag="100" ind2=" ">
+          <subfield code="a">Cook, David</subfield>
+        </datafield>
+        <datafield ind1="1" ind2="0" tag="245">
+          <subfield code="a">Everything you never wanted to know about OAI-PMH :</subfield>
+          <subfield code="b">a primer /</subfield>
+          <subfield code="c">David Cook</subfield>
+        </datafield>
+      </record>
+    </metadata>
+  </metadata>
+</record>
\ No newline at end of file
diff --git a/tools/manage-oai-import.pl b/tools/manage-oai-import.pl
new file mode 100755
index 0000000..f04da4b
--- /dev/null
+++ b/tools/manage-oai-import.pl
@@ -0,0 +1,206 @@
+#!/usr/bin/perl
+
+# Copyright 2016 Prosentient Systems
+#
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# Koha is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Koha; if not, see <http://www.gnu.org/licenses>.
+
+use Modern::Perl;
+use POSIX qw//;
+
+use Koha::Database;
+
+use C4::Auth;
+use C4::Output;
+use C4::Koha;
+use C4::Context;
+use C4::Matcher;
+use Koha::OAI::Client::Record;
+
+use C4::ImportBatch;
+
+my $script_name = "/cgi-bin/koha/tools/manage-oai-import.pl";
+
+my $input = new CGI;
+my $op = $input->param('op') || 'list';
+my $page_number = $input->param('page') && $input->param('page') =~ /^\d+$/ ? $input->param('page') : 1; #Only permit numeric parameter
+
+my $import_oai_id = $input->param('import_oai_id');
+#my $results_per_page = $input->param('results_per_page') || 25;
+
+
+
+my ($template, $loggedinuser, $cookie) =
+    get_template_and_user({template_name => "tools/manage-oai-import.tt",
+        query => $input,
+        type => "intranet",
+        authnotrequired => 0,
+        flagsrequired => {tools => 'manage_staged_marc'},
+        debug => 1,
+    });
+
+my $schema = Koha::Database->new()->schema();
+my $resultset = $schema->resultset('ImportOai');
+
+
+if ($import_oai_id){
+    my $import_oai_record = $resultset->find($import_oai_id);
+    $template->param(
+        oai_record => $import_oai_record,
+    );
+
+    if ($op eq "view_record" && $import_oai_id){
+        $template->param(
+            view_record => 1,
+            import_oai_id => $import_oai_id,
+        );
+    }
+
+    if ($op eq "retry" && $import_oai_record){
+        my $result_errors = [];
+
+        my $oai_record = Koha::OAI::Client::Record->new({
+            xml_string => $import_oai_record->metadata,
+        });
+
+        my $filtered = $oai_record->filter({
+            filter => $import_oai_record->filter,
+        });
+        if (!$filtered){
+            push(@$result_errors, { type => 'filter_failed', error_msg => '', record_id => '', }) ;
+        }
+
+        my $import_batch_id = $import_oai_record->import_batch_id;
+        if ($import_batch_id){
+            my $import_batch_rs = $schema->resultset('ImportBatch');
+            my $import_batch = $import_batch_rs->find($import_batch_id);
+            my $matcher_id = $import_batch->matcher_id;
+
+            my $record_type = $import_batch->record_type;
+            $template->param(
+                record_type => $record_type,
+            );
+
+
+            #my $matcher = C4::Matcher->new($record_type || 'biblio');
+            my $matcher = C4::Matcher->fetch($matcher_id);
+
+
+            #FIXME
+            my $import_mode = "direct";
+            #FIXME
+            my $framework = "";
+
+            #Reset the batch before re-trying the import
+            C4::ImportBatch::CleanBatch($import_batch_id);
+
+            my ($import_status, $match_status, $koha_record_numbers, $errors) = $oai_record->import_record({
+                matcher => $matcher,
+                import_batch_id => $import_batch_id,
+                import_mode => $import_mode,
+                framework => $framework,
+            });
+            if (@$errors){
+                push(@$result_errors,@$errors);
+            }
+
+            if ($import_status){
+                if ($import_status eq 'ok'){
+                    $import_oai_record->status("ok");
+                    $import_oai_record->update();
+                } else {
+                    $template->param(
+                        import_status => $import_status,
+                        errors => $result_errors,
+                        retry => 1,
+                        import_oai_id => $import_oai_id,
+                    );
+                }
+            }
+        }
+    }
+}
+
+
+
+$template->param(
+    script_name => $script_name,
+);
+
+if ($op && $op eq "list"){
+    #NOTE: It would be preferable if we centralized server-side paging code with generic functions...
+
+    #Get grand total in the database
+    my $total_rows = $resultset->count;
+
+    my $number_of_rows = 10;
+    my $number_of_pages = POSIX::ceil($total_rows / $number_of_rows);
+
+    if ($page_number > 1){
+        $template->{VARS}->{ page_first } = 1;
+    }
+    if ($number_of_pages > 1 && $page_number != $number_of_pages){
+        $template->{VARS}->{ page_last } = $number_of_pages;
+    }
+
+    #Do the search and define a limit
+    my $results = $resultset->search(
+        undef,
+        {
+            rows => $number_of_rows,
+            order_by => { -desc => 'last_modified' },
+        },
+    );
+
+    my $page = $results->page($page_number);
+    my $current_page_rows = $page->count();
+
+    if ($current_page_rows){
+        if ($page_number == 1) {
+            #Can't page previous to the first page
+            if ( $current_page_rows == $total_rows ){
+                #Can't page past the total count
+            } else {
+                #Signal that there's another page
+                $template->{VARS}->{ page_next } = $page_number + 1;
+            }
+        } else {
+
+            #Signal that there's a previous page
+            $template->{VARS}->{ page_previous } = $page_number - 1;
+
+            #Total rows of all previous pages
+            my $previous_pages_rows = ( $page_number - 1 ) * $number_of_rows;
+
+            #If current and past rows are less than total rows, there must still be more to show
+            if ( ($current_page_rows + $previous_pages_rows) < $total_rows){
+                $template->{VARS}->{ page_next } = $page_number + 1;
+            }
+
+        }
+    }
+
+    $template->param(
+        page_number => $page_number,
+        script_name => $script_name,
+        oai_records => $page,
+    );
+}
+
+
+
+
+
+output_html_with_http_headers $input, $cookie, $template->output;
\ No newline at end of file
-- 
2.1.4