Attachment 51514 Details for Bug 10662 – Bug 10662 - Create svc/import_oai API

[patch] Bug 10662 - Create svc/import_oai API

Bug-10662---Create-svcimportoai-API.patch (text/plain), 45.01 KB, created by David Cook on 2016-05-16 07:12:40 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: David Cook

Created: 2016-05-16 07:12:40 UTC

Size: 45.01 KB

patch

obsolete

>From c1e213b43c3102eaff69da7802acf945b68daa70 Mon Sep 17 00:00:00 2001
>From: David Cook <dcook@prosentient.com.au>
>Date: Mon, 16 May 2016 16:08:06 +1000
>Subject: [PATCH] Bug 10662 - Create svc/import_oai API
>
>---
> Koha/OAI/Client/Record.pm                          | 249 +++++++++++++++++++++
> Koha/Schema/Result/ImportOai.pm                    | 152 +++++++++++++
> docs/OAIPMH/README                                 |  20 ++
> .../bug_10662-Build_import_oai_table.sql           |  15 ++
> .../intranet-tmpl/prog/en/includes/tools-menu.inc  |   1 +
> .../prog/en/modules/tools/manage-oai-import.tt     | 154 +++++++++++++
> .../prog/en/modules/tools/tools-home.tt            |   3 +
> .../intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl  |  74 ++++++
> svc/import_oai                                     | 160 +++++++++++++
> t/Import/bib-deleted.xml                           |   7 +
> t/Import/bib-oaidc.xml                             |  19 ++
> t/Import/bib.xml                                   |  25 +++
> tools/manage-oai-import.pl                         | 206 +++++++++++++++++
> 13 files changed, 1085 insertions(+)
> create mode 100755 Koha/OAI/Client/Record.pm
> create mode 100755 Koha/Schema/Result/ImportOai.pm
> create mode 100755 docs/OAIPMH/README
> create mode 100644 installer/data/mysql/atomicupdate/bug_10662-Build_import_oai_table.sql
> create mode 100755 koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-oai-import.tt
> create mode 100755 koha-tmpl/intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl
> create mode 100755 svc/import_oai
> create mode 100755 t/Import/bib-deleted.xml
> create mode 100755 t/Import/bib-oaidc.xml
> create mode 100755 t/Import/bib.xml
> create mode 100755 tools/manage-oai-import.pl
>
>diff --git a/Koha/OAI/Client/Record.pm b/Koha/OAI/Client/Record.pm
>new file mode 100755
>index 0000000..ca978b0
>--- /dev/null
>+++ b/Koha/OAI/Client/Record.pm
>@@ -0,0 +1,249 @@
>+package Koha::OAI::Client::Record;
>+
>+# Copyright 2016 Prosentient Systems
>+#
>+# This file is part of Koha.
>+#
>+# Koha is free software; you can redistribute it and/or modify it
>+# under the terms of the GNU General Public License as published by
>+# the Free Software Foundation; either version 3 of the License, or
>+# (at your option) any later version.
>+#
>+# Koha is distributed in the hope that it will be useful, but
>+# WITHOUT ANY WARRANTY; without even the implied warranty of
>+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
>+# GNU General Public License for more details.
>+#
>+# You should have received a copy of the GNU General Public License
>+# along with Koha; if not, see <http://www.gnu.org/licenses>.
>+#
>+
>+use Modern::Perl;
>+use XML::LibXML;
>+use XML::LibXSLT;
>+use MARC::Record;
>+
>+use C4::Context;
>+use C4::Biblio;
>+use C4::ImportBatch;
>+use C4::Matcher;
>+
>+use constant MAX_MATCHES => 99999; #NOTE: This is an arbitrary value. We want to get all matches.
>+
>+sub new {
>+    my ($class, $args) = @_;
>+    $args = {} unless defined $args;
>+
>+    if (my $inxml = $args->{xml_string}){
>+
>+        #Parse the XML string into a XML::LibXML object
>+        my $doc = XML::LibXML->load_xml(string => $inxml, { no_blanks => 1 });
>+        $args->{doc} = $doc;
>+        #NOTE: Don't load blank nodes...
>+
>+        #Get the root element
>+        my $root = $doc->documentElement;
>+
>+        #Register namespaces for searching purposes
>+        my $xpc = XML::LibXML::XPathContext->new();
>+        $xpc->registerNs('oai','http://www.openarchives.org/OAI/2.0/');
>+
>+        my $xpath_identifier = XML::LibXML::XPathExpression->new("oai:header/oai:identifier");
>+        my $identifier = $xpc->findnodes($xpath_identifier,$root)->shift;
>+        #my $identifier_string = $identifier->textContent;
>+        $args->{header_identifier} = $identifier->textContent;
>+
>+        my $xpath_datestamp = XML::LibXML::XPathExpression->new("oai:header/oai:datestamp");
>+        my $datestamp = $xpc->findnodes($xpath_datestamp,$root)->shift;
>+        #my $datestamp_string = $datestamp->textContent;
>+        $args->{header_datestamp} = $datestamp->textContent;
>+
>+        my $xpath_status = XML::LibXML::XPathExpression->new(q{oai:header/@status});
>+        my $status_node = $xpc->findnodes($xpath_status,$root)->shift;
>+        #my $status_string = $status_node ? $status_node->textContent : "";
>+        $args->{header_status} = $status_node ? $status_node->textContent : "";
>+    }
>+
>+    return bless ($args, $class);
>+}
>+
>+sub is_deleted_upstream {
>+    my ($self, $args) = @_;
>+    if ($self->{header_status}){
>+        if ($self->{header_status} eq "deleted"){
>+            return 1;
>+        }
>+    }
>+    return 0;
>+}
>+
>+sub filter {
>+    my ($self, $args) = @_;
>+    my $doc = $self->{doc};
>+    my $filter = $args->{filter};
>+    $self->{filter} = $filter; #FIXME
>+    #FIXME: Check that it's an XSLT here...
>+    if ( -f $filter ){
>+        #Filter is a valid filepath
>+
>+        #FIXME: Ideally, it would be good to use Koha::XSLT_Handler here... (especially for persistent environments...)
>+        my $xslt = XML::LibXSLT->new();
>+        my $style_doc = XML::LibXML->load_xml(location => $filter);
>+        my $stylesheet = $xslt->parse_stylesheet($style_doc);
>+        if ($stylesheet){
>+            my $results = $stylesheet->transform($doc);
>+            my $metadata_xml = $stylesheet->output_as_bytes($results);
>+            #If the XSLT outputs nothing, then we don't meet the following condition, and we'll return 0 instead.
>+            if ($metadata_xml){
>+                $self->{filtered_record} = $metadata_xml;
>+                return 1;
>+            }
>+        }
>+    }
>+    return 0;
>+}
>+
>+
>+
>+
>+
>+
>+sub import_record {
>+    my ($self, $args) = @_;
>+    my $koha_record_numbers = "";
>+    my $errors = [];
>+    my $import_status = "error";
>+    my $match_status = "no_match";
>+
>+    my $batch_id = $args->{import_batch_id};
>+    $self->{import_batch_id} = $batch_id; #FIXME
>+    my $matcher = $args->{matcher};
>+    my $framework = $args->{framework};
>+    my $import_mode = $args->{import_mode};
>+
>+    my $metadata_xml = $self->{filtered_record};
>+
>+    if ($metadata_xml){
>+        #Convert MARCXML into MARC::Record object
>+        my $marcflavour = C4::Context->preference('marcflavour') || 'MARC21';
>+        my $marc_record = eval {MARC::Record::new_from_xml( $metadata_xml, "utf8", $marcflavour)};
>+        if ($@) {
>+            warn "Error converting OAI-PMH filtered metadata into MARC::Record object: $@";
>+            #FIXME: Improve error handling
>+        }
>+
>+        if ($self->is_deleted_upstream){
>+
>+=pod
>+            my @matches = $matcher->get_matches($marc_record, MAX_MATCHES);
>+            if (@matches){
>+                $match_status = "matched";
>+            }
>+            my $delete_error;
>+            foreach my $match (@matches){
>+                    if (my $record_id = $match->{record_id}){
>+                        #FIXME: This is biblio specific... what about authority records?
>+                        my $error = C4::Biblio::DelBiblio($record_id);
>+                        if ($error){
>+                            $delete_error++;
>+                            $koha_record_numbers = [];
>+                            push(@$koha_record_numbers,$record_id);
>+
>+                            #FIXME: Find a better way of sending the errors in a predictable way...
>+                            push(@$errors,{"record_id" => $record_id, "error" => $error, });
>+                        }
>+                    }
>+
>+            }
>+
>+            #If there are no delete errors, then the import was ok
>+            if ( ! $delete_error){
>+                $import_status = "ok";
>+            }
>+            #Deleted records will never actually have an records in them, so always mark them as cleaned so that other imports don't try to pick up the same batch.
>+            C4::ImportBatch::SetImportBatchStatus($batch_id, 'cleaned');
>+=cut
>+            my $import_record_id = AddBiblioToBatch($batch_id, 0, $marc_record, "utf8", int(rand(99999)));
>+            my $number_of_matches =  BatchFindDuplicates($batch_id, $matcher, MAX_MATCHES);
>+            if ($number_of_matches > 0){
>+                $match_status = "auto_match"; #See `import_records` table for other options... but this should be the right one.
>+            }
>+            my $results = GetImportRecordMatches($import_record_id); #Only works for biblio...
>+            my $delete_error;
>+
>+            my @result_record_numbers = ();
>+            foreach my $result (@$results){
>+                if (my $record_id = $result->{biblionumber}){
>+                    push(@result_record_numbers,$record_id);
>+
>+                    #FIXME: This is biblio specific... what about authority records?
>+                    my $error = C4::Biblio::DelBiblio($record_id);
>+                    if ($error){
>+                        $delete_error++;
>+                        push(@$errors, { type => 'delete_failed', error_msg => $error, record_id => $record_id, }) ;
>+                    }
>+                }
>+            }
>+            $koha_record_numbers = join(",",@result_record_numbers); #FIXME: check that this works...
>+
>+            if ($delete_error){
>+                $import_status = "error";
>+                C4::ImportBatch::SetImportBatchStatus($batch_id, 'importing');
>+            } else {
>+                $import_status = "ok";
>+                #Ideally, it would be nice to say what records were deleted, but Koha doesn't have that capacity at the moment, so just clean the batch.
>+                CleanBatch($batch_id);
>+            }
>+
>+
>+
>+
>+        } else {
>+            #Import the MARCXML record into Koha
>+            my $import_record_id = AddBiblioToBatch($batch_id, 0, $marc_record, "utf8", int(rand(99999)));
>+            #FIXME: Don't allow item imports do to the nature of OAI-PMH records updating over time...
>+            #my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 'UPDATE COUNTS');
>+            my $number_of_matches =  BatchFindDuplicates($batch_id, $matcher);
>+
>+            # XXX we are ignoring the result of this;
>+            BatchCommitRecords($batch_id, $framework) if lc($import_mode) eq 'direct';
>+
>+            my $dbh = C4::Context->dbh();
>+            my $sth = $dbh->prepare("SELECT matched_biblionumber FROM import_biblios WHERE import_record_id =?");
>+            $sth->execute($import_record_id);
>+            $koha_record_numbers = $sth->fetchrow_arrayref->[0] || '';
>+            $sth = $dbh->prepare("SELECT overlay_status FROM import_records WHERE import_record_id =?");
>+            $sth->execute($import_record_id);
>+
>+            $match_status = $sth->fetchrow_arrayref->[0] || 'no_match';
>+            $import_status = "ok";
>+        }
>+    } else {
>+        #There's no filtered metadata...
>+        #Clean the batch, so future imports don't use the same batch.
>+        CleanBatch($batch_id);
>+    }
>+    $self->{status} = $import_status; #FIXME
>+    #$self->save_to_database();
>+    return ($import_status,$match_status,$koha_record_numbers, $errors);
>+}
>+
>+sub save_to_database {
>+    my ($self,$args) = @_;
>+
>+    my $header_identifier = $self->{header_identifier};
>+    my $header_datestamp = $self->{header_datestamp};
>+    my $header_status = $self->{header_status};
>+    my $metadata = $self->{doc}->toString(1);
>+    my $import_batch_id = $self->{import_batch_id};
>+    my $filter = $self->{filter};
>+    my $status = $self->{status};
>+
>+    my $dbh = C4::Context->dbh;
>+    my $sql = "INSERT INTO import_oai (header_identifier, header_datestamp, header_status, metadata, import_batch_id, filter, status) VALUES (?, ?, ?, ?, ?, ?, ?)";
>+    my $sth = $dbh->prepare($sql);
>+    $sth->execute($header_identifier,$header_datestamp,$header_status,$metadata, $import_batch_id, $filter, $status);
>+}
>+
>+
>+1;
>\ No newline at end of file
>diff --git a/Koha/Schema/Result/ImportOai.pm b/Koha/Schema/Result/ImportOai.pm
>new file mode 100755
>index 0000000..bfa1a25
>--- /dev/null
>+++ b/Koha/Schema/Result/ImportOai.pm
>@@ -0,0 +1,152 @@
>+use utf8;
>+package Koha::Schema::Result::ImportOai;
>+
>+# Created by DBIx::Class::Schema::Loader
>+# DO NOT MODIFY THE FIRST PART OF THIS FILE
>+
>+=head1 NAME
>+
>+Koha::Schema::Result::ImportOai
>+
>+=cut
>+
>+use strict;
>+use warnings;
>+
>+use base 'DBIx::Class::Core';
>+
>+=head1 TABLE: C<import_oai>
>+
>+=cut
>+
>+__PACKAGE__->table("import_oai");
>+
>+=head1 ACCESSORS
>+
>+=head2 import_oai_id
>+
>+  data_type: 'integer'
>+  extra: {unsigned => 1}
>+  is_auto_increment: 1
>+  is_nullable: 0
>+
>+=head2 header_identifier
>+
>+  data_type: 'varchar'
>+  is_nullable: 0
>+  size: 45
>+
>+=head2 header_datestamp
>+
>+  data_type: 'datetime'
>+  datetime_undef_if_invalid: 1
>+  is_nullable: 0
>+
>+=head2 header_status
>+
>+  data_type: 'varchar'
>+  is_nullable: 1
>+  size: 45
>+
>+=head2 metadata
>+
>+  data_type: 'longtext'
>+  is_nullable: 0
>+
>+=head2 last_modified
>+
>+  data_type: 'timestamp'
>+  datetime_undef_if_invalid: 1
>+  default_value: current_timestamp
>+  is_nullable: 0
>+
>+=head2 status
>+
>+  data_type: 'varchar'
>+  is_nullable: 0
>+  size: 45
>+
>+=head2 import_batch_id
>+
>+  data_type: 'integer'
>+  is_foreign_key: 1
>+  is_nullable: 0
>+
>+=head2 filter
>+
>+  data_type: 'text'
>+  is_nullable: 0
>+
>+=cut
>+
>+__PACKAGE__->add_columns(
>+  "import_oai_id",
>+  {
>+    data_type => "integer",
>+    extra => { unsigned => 1 },
>+    is_auto_increment => 1,
>+    is_nullable => 0,
>+  },
>+  "header_identifier",
>+  { data_type => "varchar", is_nullable => 0, size => 45 },
>+  "header_datestamp",
>+  {
>+    data_type => "datetime",
>+    datetime_undef_if_invalid => 1,
>+    is_nullable => 0,
>+  },
>+  "header_status",
>+  { data_type => "varchar", is_nullable => 1, size => 45 },
>+  "metadata",
>+  { data_type => "longtext", is_nullable => 0 },
>+  "last_modified",
>+  {
>+    data_type => "timestamp",
>+    datetime_undef_if_invalid => 1,
>+    default_value => \"current_timestamp",
>+    is_nullable => 0,
>+  },
>+  "status",
>+  { data_type => "varchar", is_nullable => 0, size => 45 },
>+  "import_batch_id",
>+  { data_type => "integer", is_foreign_key => 1, is_nullable => 0 },
>+  "filter",
>+  { data_type => "text", is_nullable => 0 },
>+);
>+
>+=head1 PRIMARY KEY
>+
>+=over 4
>+
>+=item * L</import_oai_id>
>+
>+=back
>+
>+=cut
>+
>+__PACKAGE__->set_primary_key("import_oai_id");
>+
>+=head1 RELATIONS
>+
>+=head2 import_batch
>+
>+Type: belongs_to
>+
>+Related object: L<Koha::Schema::Result::ImportBatch>
>+
>+=cut
>+
>+__PACKAGE__->belongs_to(
>+  "import_batch",
>+  "Koha::Schema::Result::ImportBatch",
>+  { import_batch_id => "import_batch_id" },
>+  { is_deferrable => 1, on_delete => "RESTRICT", on_update => "RESTRICT" },
>+);
>+
>+
>+# Created by DBIx::Class::Schema::Loader v0.07042 @ 2016-04-12 11:02:33
>+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:QmCetOjXql0gsAi+wZ74Ng
>+
>+
>+# You can replace this text with custom code or comments, and it will be preserved on regeneration
>+1;
>diff --git a/docs/OAIPMH/README b/docs/OAIPMH/README
>new file mode 100755
>index 0000000..c217cb7
>--- /dev/null
>+++ b/docs/OAIPMH/README
>@@ -0,0 +1,20 @@
>+TODO:
>+- Change `import_oai` database table?
>+    - import_oai's "metadata" should actually be "oai_record"... so it's not so confusing... it's NOT the metadata element... but rather the whole OAI record.
>+    - Rename "last_modified" to "upload_timestamp"
>+- Fix https://bugs.koha-community.org/bugzilla3/show_bug.cgi?id=15541
>+
>+- Add documentation to all code...
>+- Add unit tests
>+- Clean up the code
>+- Resolve all TODO/FIXME comments in the code
>+
>+    
>+FUTURE:
>+- Add support for authority records and possibly holdings records?
>+- Add default OAI record matching rule?
>+    - I thought about adding a SQL atomic update 'bug_10662-Add_oai_record_matching_rule.sql', but adding matching rules seems complex. This needs to be done in Perl.
>+    - Should the rule include other fields like 022, 020, 245 rather than just 001 and 024a?
>+- Add entry to Cleanupdatabase.pl cronjob?
>+    - You could remove all import_oai rows older than a certain age?
>+- Re-do the paging to use DataTables AJAX? Or, create a centralized/generalized server-side paging function for Koha...
>diff --git a/installer/data/mysql/atomicupdate/bug_10662-Build_import_oai_table.sql b/installer/data/mysql/atomicupdate/bug_10662-Build_import_oai_table.sql
>new file mode 100644
>index 0000000..554d8ed
>--- /dev/null
>+++ b/installer/data/mysql/atomicupdate/bug_10662-Build_import_oai_table.sql
>@@ -0,0 +1,15 @@
>+DROP TABLE IF EXISTS import_oai;
>+CREATE TABLE  import_oai (
>+  import_oai_id int(10) unsigned NOT NULL AUTO_INCREMENT,
>+  header_identifier varchar(45) CHARACTER SET utf8 NOT NULL,
>+  header_datestamp datetime NOT NULL,
>+  header_status varchar(45) CHARACTER SET utf8 DEFAULT NULL,
>+  metadata longtext CHARACTER SET utf8 NOT NULL,
>+  last_modified timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
>+  status varchar(45) CHARACTER SET utf8 NOT NULL,
>+  import_batch_id int(11) NOT NULL,
>+  filter text COLLATE utf8_unicode_ci NOT NULL,
>+  PRIMARY KEY (import_oai_id),
>+  KEY FK_import_oai_1 (import_batch_id),
>+  CONSTRAINT FK_import_oai_1 FOREIGN KEY (import_batch_id) REFERENCES import_batches (import_batch_id)
>+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
>diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-menu.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-menu.inc
>index 9a2b1bf..124c6b3 100644
>--- a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-menu.inc
>+++ b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-menu.inc
>@@ -92,6 +92,7 @@
>     [% END %]
>     [% IF ( CAN_user_tools_manage_staged_marc ) %]
> 	<li><a href="/cgi-bin/koha/tools/manage-marc-import.pl">Staged MARC management</a></li>
>+    <li><a href="/cgi-bin/koha/tools/manage-oai-import.pl">OAI-PMH import management</a></li>
>     [% END %]
>     [% IF ( CAN_user_tools_upload_local_cover_images ) %]
>     <li><a href="/cgi-bin/koha/tools/upload-cover-image.pl">Upload local cover image</a></li>
>diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-oai-import.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-oai-import.tt
>new file mode 100755
>index 0000000..0c28692
>--- /dev/null
>+++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-oai-import.tt
>@@ -0,0 +1,154 @@
>+[% INCLUDE 'doc-head-open.inc' %]
>+<title>Koha &rsaquo; Tools &rsaquo; Manage OAI-PMH record imports
>+[% IF ( import_oai_id ) %]
>+ &rsaquo; Record [% import_oai_id %]
>+[% END %]
>+</title>
>+[% INCLUDE 'doc-head-close.inc' %]
>+<link rel="stylesheet" type="text/css" href="[% themelang %]/css/datatables.css" />
>+[% INCLUDE 'datatables.inc' %]
>+</head>
>+
>+<body id="tools_manage-oai-import" class="tools">
>+[% INCLUDE 'header.inc' %]
>+[% INCLUDE 'cat-search.inc' %]
>+
>+    <div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a>
>+    [% IF ( import_oai_id ) %]
>+     &rsaquo;
>+     <a href="[% script_name %]">Manage OAI-PMH record imports</a>
>+     &rsaquo; Record [% import_oai_id %]
>+    [% ELSE %]
>+     &rsaquo; Manage OAI-PMH record imports
>+    [% END %]
>+    </div>
>+
>+    <div id="doc3" class="yui-t2">
>+        <div id="bd">
>+            <div id="yui-main">
>+                <div class="yui-b">
>+                    [% IF ( import_oai_id ) %]
>+                        [% IF ( view_record ) %]
>+                            <h1>Record [% import_oai_id %]</h1>
>+                            [% IF ( oai_record.metadata ) %]
>+                                <div style="white-space:pre">[% oai_record.metadata | xml %]</div>
>+                            [% END %]
>+                        [% ELSIF ( retry ) %]
>+                            <fieldset class="rows">
>+                                <ol>
>+                                    <li>
>+                                        <span class="label">Import status:</span>
>+                                        [% IF ( import_status ) %]
>+                                            [% IF ( import_status == "ok" ) %]
>+                                            OK
>+                                            [% ELSIF ( import_status == "error" ) %]
>+                                            ERROR
>+                                            [% END %]
>+                                        [% END %]
>+                                    </li>
>+                                    [% IF ( errors ) %]
>+                                        [% FOREACH error IN errors %]
>+                                            [% IF ( error ) %]
>+                                                <li>
>+                                                    <span class="label">Error:</span>
>+                                                    [% SWITCH error.type %]
>+                                                    [% CASE 'delete_failed' %]
>+                                                        [% error.error_msg # FIXME: These English messages come straight from C4::Biblio... %]
>+                                                        [% IF ( record_type ) && ( record_type == "biblio" ) %]
>+                                                            <a title="View biblio record" href="/cgi-bin/koha/catalogue/detail.pl?biblionumber=[% error.record_id %]">(View biblio record)</a>
>+                                                        [% END %]
>+                                                    [% CASE 'filter_failed' %]
>+                                                        Filter failed to produce MARCXML.
>+                                                        Review the metadata at <a href="/cgi-bin/koha/tools/manage-oai-import.pl?op=view_record&import_oai_id=[% import_oai_id %]">View record [% import_oai_id %]</a>.
>+                                                        You should also review your filter at [% oai_record.filter %].
>+                                                    [% CASE %]
>+                                                        [% error.error_msg %]
>+                                                    [% END %]
>+                                                </li>
>+                                            [% END %]
>+                                        [% END %]
>+                                    [% END %]
>+                                </ol>
>+                            </fieldset>
>+                        [% END %]
>+                    [% ELSE %]
>+                        <h1>Manage OAI-PMH record imports</h1>
>+                        <table>
>+                            <thead>
>+                                <tr>
>+                                    <th>Record identifier</th>
>+                                    <th>Record datestamp</th>
>+                                    <th>Provider status</th>
>+                                    <th>Import status</th>
>+                                    <th>Import batch</th>
>+                                    <th>OAI-PMH record</th>
>+                                    [%# <th>Filter</th> %]
>+                                </tr>
>+                            </thead>
>+                            <tbody>
>+                                [% WHILE (oai_record = oai_records.next) %]
>+                                <tr>
>+                                    <td>[% oai_record.header_identifier %]</td>
>+                                    <td>[% oai_record.header_datestamp %]</td>
>+                                    <td>
>+                                        [% IF ( oai_record.header_status ) %]
>+                                            [% IF ( oai_record.header_status == "deleted" ) %]
>+                                                DELETED
>+                                            [% END %]
>+                                        [% END %]
>+                                    </td>
>+                                    <td>
>+                                        [% IF ( oai_record.status ) %]
>+                                            [% IF ( oai_record.status == "ok" ) %]
>+                                                OK
>+                                            [% ELSIF ( oai_record.status == "error" ) %]
>+                                                <a title="Retry import" href="[% script_name %]?op=retry&import_oai_id=[% oai_record.import_oai_id %]">ERROR - Click to retry</a>
>+                                            [% END %]
>+
>+                                        [% ELSE %]
>+                                            Unknown
>+                                        [% END %]
>+                                    </td>
>+                                    <td>
>+                                        [% IF ( oai_record.import_batch_id ) %]
>+                                            <a title="View import batch" href="/cgi-bin/koha/tools/manage-marc-import.pl?import_batch_id=[% oai_record.import_batch_id %]">View batch [% oai_record.import_batch_id %]</a>
>+                                        [% END %]
>+                                    </td>
>+                                    [%# oai_record.filter %]
>+                                    <td><a title="View OAI-PMH record" href="[% script_name %]?op=view_record&import_oai_id=[% oai_record.import_oai_id %]">View record [% oai_record.import_oai_id %]</a></td>
>+                                </tr>
>+                                [% END %]
>+
>+                            </tbody>
>+                        </table>
>+                        <div class="pager">
>+                        [% IF ( page_first ) %]
>+                            <a href="[% script_name %]?page=[% page_first %]">First ([% page_first %])</a>
>+                        [% ELSE %]
>+                            <a class="disabled">First</a>
>+                        [% END %]
>+                        [% IF ( page_previous ) %]
>+                            <a href="[% script_name %]?page=[% page_previous %]">Previous ([% page_previous %])</a>
>+                        [% ELSE %]
>+                            <a class="disabled">Previous</a>
>+                        [% END %]
>+                        [% IF ( page_next ) %]
>+                            <a href="[% script_name %]?page=[% page_next %]">Next ([% page_next %])</a>
>+                        [% ELSE %]
>+                            <a class="disabled">Next</a>
>+                        [% END %]
>+                        [% IF ( page_last ) %]
>+                            <a href="[% script_name %]?page=[% page_last %]">Last ([% page_last %])</a>
>+                        [% ELSE %]
>+                            <a class="disabled">Last</a>
>+                        [% END %]
>+                        </div>
>+                    [% END %]
>+                </div>
>+            </div>
>+            <div class="yui-b">
>+                [% INCLUDE 'tools-menu.inc' %]
>+            </div>
>+        </div>
>+    </div>
>+[% INCLUDE 'intranet-bottom.inc' %]
>diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/tools-home.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/tools-home.tt
>index a8f8ff4..505a08f 100644
>--- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/tools-home.tt
>+++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/tools-home.tt
>@@ -177,6 +177,9 @@
>     [% IF ( CAN_user_tools_manage_staged_marc ) %]
>     <dt><a href="/cgi-bin/koha/tools/manage-marc-import.pl">Staged MARC record management</a></dt>
>     <dd>Managed staged MARC records, including completing and reversing imports</dd>
>+
>+    <dt><a href="/cgi-bin/koha/tools/manage-oai-import.pl">OAI-PMH import management</a></dt>
>+    <dd>Manage import of OAI-PMH harvested records</dd>
>     [% END %]
> 
>     [% IF ( CAN_user_tools_upload_local_cover_images ) %]
>diff --git a/koha-tmpl/intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl b/koha-tmpl/intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl
>new file mode 100755
>index 0000000..0f1d6f0
>--- /dev/null
>+++ b/koha-tmpl/intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl
>@@ -0,0 +1,74 @@
>+<?xml version="1.0" encoding="UTF-8"?>
>+<xsl:stylesheet version="1.0"
>+    xmlns:marc="http://www.loc.gov/MARC21/slim"
>+    xmlns:oai="http://www.openarchives.org/OAI/2.0/"
>+    xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
>+    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>
>+    
>+    
>+
>+    
>+    <xsl:template match="oai:record">
>+        
>+        <xsl:apply-templates select="oai:header[@status='deleted']" />
>+        
>+        <xsl:apply-templates select="oai:metadata" />
>+    </xsl:template>
>+
>+    
>+    <xsl:template match="oai:metadata">
>+        
>+        
>+        <xsl:apply-templates select="//marc:record"/>
>+    </xsl:template>
>+
>+    
>+    <xsl:template match="oai:header[@status='deleted']">
>+        <xsl:element name="record" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
>+            xmlns="http://www.loc.gov/MARC21/slim">
>+            <xsl:attribute name="xsi:schemaLocation">http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd</xsl:attribute>
>+            <xsl:call-template name="add_oai"/>
>+        </xsl:element>
>+    </xsl:template>
>+
>+    
>+    <xsl:template match="@* | node()">
>+        
>+        <xsl:copy>
>+            
>+            <xsl:apply-templates select="@* | node()" />
>+        </xsl:copy>
>+    </xsl:template>
>+
>+
>+    <xsl:template match="marc:record">
>+        <xsl:copy>
>+            
>+            <xsl:apply-templates select="@* | node()"/>
>+
>+            
>+            <xsl:call-template name="add_oai"/>
>+
>+            
>+            <xsl:text>&#xa;</xsl:text>
>+        </xsl:copy>
>+    </xsl:template>
>+
>+    
>+    <xsl:template name="add_oai">
>+        <xsl:element name="datafield" xmlns="http://www.loc.gov/MARC21/slim">
>+            <xsl:attribute name="ind1"><xsl:text>7</xsl:text></xsl:attribute>
>+            <xsl:attribute name="ind2"><xsl:text> </xsl:text></xsl:attribute>
>+            <xsl:attribute name="tag">024</xsl:attribute>
>+            <xsl:element name="subfield">
>+                <xsl:attribute name="code">a</xsl:attribute>
>+                <xsl:value-of select="/oai:record/oai:header/oai:identifier"/>
>+            </xsl:element>
>+            <xsl:element name="subfield">
>+                <xsl:attribute name="code">2</xsl:attribute>
>+                <xsl:text>uri</xsl:text>
>+            </xsl:element>
>+         </xsl:element>
>+    </xsl:template>
>+
>+</xsl:stylesheet>
>diff --git a/svc/import_oai b/svc/import_oai
>new file mode 100755
>index 0000000..b005d01
>--- /dev/null
>+++ b/svc/import_oai
>@@ -0,0 +1,160 @@
>+#!/usr/bin/perl
>+
>+# Copyright 2016 Prosentient Systems
>+#
>+# This file is part of Koha.
>+#
>+# Koha is free software; you can redistribute it and/or modify it
>+# under the terms of the GNU General Public License as published by
>+# the Free Software Foundation; either version 3 of the License, or
>+# (at your option) any later version.
>+#
>+# Koha is distributed in the hope that it will be useful, but
>+# WITHOUT ANY WARRANTY; without even the implied warranty of
>+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
>+# GNU General Public License for more details.
>+#
>+# You should have received a copy of the GNU General Public License
>+# along with Koha; if not, see <http://www.gnu.org/licenses>.
>+#
>+
>+use Modern::Perl;
>+use XML::LibXML;
>+use URI;
>+use File::Basename;
>+
>+use CGI qw ( -utf8 );
>+use C4::Auth qw/check_api_auth/;
>+use C4::Context;
>+use C4::ImportBatch;
>+use C4::Matcher;
>+use XML::Simple;
>+use C4::Biblio;
>+
>+use Koha::OAI::Client::Record;
>+
>+my $query = new CGI;
>+binmode STDOUT, ':encoding(UTF-8)';
>+
>+my ($status, $cookie, $sessionID) = check_api_auth($query, { editcatalogue => 'edit_catalogue'} );
>+unless ($status eq "ok") {
>+    print $query->header(-type => 'text/xml', -status => '403 Forbidden');
>+    print XMLout({ auth_status => $status }, NoAttr => 1, RootName => 'response', XMLDecl => 1);
>+    exit 0;
>+}
>+
>+my $xml;
>+if ($query->request_method eq "POST") {
>+    $xml = $query->param('xml');
>+}
>+if ($xml) {
>+    #TODO: You could probably use $query->Vars here instead...
>+    my %params = map { $_ => scalar $query->param($_) } $query->param;
>+    my $result = import_oai($xml, \%params );
>+    print $query->header(-type => 'text/xml');
>+    print XMLout($result, NoAttr => 1, RootName => 'response', XMLDecl => 1);
>+} else {
>+    print $query->header(-type => 'text/xml', -status => '400 Bad Request');
>+}
>+
>+exit 0;
>+
>+sub import_oai {
>+    my ($inxml, $params) = @_;
>+
>+    #Create record object
>+    my $oai_record = Koha::OAI::Client::Record->new({
>+        xml_string => $inxml,
>+    });
>+    
>+    my $result = {};
>+    my $result_errors = [];
>+
>+    my $filter      = delete $params->{filter}      || '';
>+    my $import_mode = delete $params->{import_mode} || '';
>+    my $framework   = delete $params->{framework}   || '';
>+    
>+    unless ($params->{comments}){
>+        $params->{comments} = "OAI-PMH import";
>+        if ($oai_record->{header_identifier}){
>+            $params->{comments} .= ": $oai_record->{header_identifier}";
>+        }
>+    }
>+
>+    if (my $matcher_code = delete $params->{match}) {
>+        $params->{matcher_id} = C4::Matcher::GetMatcherId($matcher_code);
>+    }
>+
>+    my $batch_id = GetWebserviceBatchId($params);
>+    #FIXME: Use the batch_id to create a more useful filename in the import_batches table...
>+    unless ($batch_id) {
>+        $result->{'status'} = "failed";
>+        $result->{'error'} = "Batch create error";
>+        return $result;
>+    }
>+
>+    #Source a default XSLT to use for filtering
>+    my $htdocs  = C4::Context->config('intrahtdocs');
>+    my $theme   = C4::Context->preference("template");
>+    #FIXME: This doesn't work for UNIMARC!
>+    my $xslfilename = "$htdocs/$theme/en/xslt/OAI2MARC21slim.xsl";
>+
>+    #FIXME: There's a better way to do these filters...
>+    if ($filter){
>+        my $filter_uri = URI->new($filter);
>+        if ($filter_uri){
>+            my $scheme = $filter_uri->scheme;
>+            if ($scheme && $scheme eq "file"){
>+                my $path = $filter_uri->path;
>+                #Filters may theoretically be .xsl or .pm files
>+                my($filename, $dirs, $suffix) = fileparse($path,(".xsl",".pm"));
>+                if ($suffix && $suffix eq ".xsl"){
>+                    #If this new path exists, change the filter XSLT to it
>+                    if ( -f $path ){
>+                        $xslfilename = $path;
>+                    }
>+                }
>+            }
>+        }
>+    }
>+
>+    #Get matching rule matcher
>+    my $matcher = C4::Matcher->new($params->{record_type} || 'biblio');
>+    $matcher = C4::Matcher->fetch($params->{matcher_id});
>+
>+    
>+
>+    #Filter OAI-PMH into MARCXML
>+    my $filtered = $oai_record->filter({
>+        filter => $xslfilename,
>+    });
>+
>+    if (!$filtered){
>+        push(@$result_errors, { type => 'filter_failed', error_msg => '', record_id => '', }) ;
>+    }
>+
>+    my ($import_status, $match_status, $koha_record_numbers, $errors) = $oai_record->import_record({
>+        matcher => $matcher,
>+        import_batch_id => $batch_id,
>+        import_mode => $import_mode,
>+        framework => $framework,
>+    });
>+    if (@$errors){
>+        push(@$result_errors,@$errors);
>+    }
>+
>+    $oai_record->save_to_database();
>+
>+    $result->{'match_status'} = $match_status;
>+    $result->{'import_batch_id'} = $batch_id;
>+    $result->{'koha_record_numbers'} = $koha_record_numbers;
>+
>+    if ($import_status && $import_status eq "ok"){
>+        $result->{'status'} = "ok";
>+    } else {
>+        $result->{'status'} = "failed";
>+        $result->{'errors'} = {error => $result_errors};
>+    }
>+
>+    return $result;
>+}
>diff --git a/t/Import/bib-deleted.xml b/t/Import/bib-deleted.xml
>new file mode 100755
>index 0000000..5a5b4ab
>--- /dev/null
>+++ b/t/Import/bib-deleted.xml
>@@ -0,0 +1,7 @@
>+<?xml version="1.0" encoding="UTF-8"?>
>+<record xmlns="http://www.openarchives.org/OAI/2.0/">
>+  <header status="deleted">
>+    <identifier>oai:koha-community.org:5000</identifier>
>+    <datestamp>2015-12-22T18:46:29Z</datestamp>
>+  </header>
>+</record>
>diff --git a/t/Import/bib-oaidc.xml b/t/Import/bib-oaidc.xml
>new file mode 100755
>index 0000000..cf247cc
>--- /dev/null
>+++ b/t/Import/bib-oaidc.xml
>@@ -0,0 +1,19 @@
>+<?xml version="1.0" encoding="UTF-8"?>
>+<record xmlns="http://www.openarchives.org/OAI/2.0/">
>+  <header>
>+    <identifier>oai:koha-community.org:5000</identifier>
>+    <datestamp>2015-12-21T18:46:29Z</datestamp>
>+  </header>
>+  <metadata>
>+    <metadata xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
>+        <oai_dc:dc xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
>+          <dc:title>Everything you never wanted to know about OAI-PMH : a primer /</dc:title>
>+          <dc:creator>
>+            Cook, David
>+          </dc:creator>
>+          <dc:type>text</dc:type>
>+          <dc:language>eng</dc:language>
>+        </oai_dc:dc>
>+    </metadata>
>+  </metadata>
>+</record>
>diff --git a/t/Import/bib.xml b/t/Import/bib.xml
>new file mode 100755
>index 0000000..ac1209a
>--- /dev/null
>+++ b/t/Import/bib.xml
>@@ -0,0 +1,25 @@
>+<?xml version="1.0" encoding="UTF-8"?>
>+<record xmlns="http://www.openarchives.org/OAI/2.0/">
>+  <header>
>+    <identifier>oai:koha-community.org:5000</identifier>
>+    <datestamp>2015-12-21T18:46:29Z</datestamp>
>+  </header>
>+  <metadata>
>+    <metadata xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
>+      <record xmlns="http://www.loc.gov/MARC21/slim" type="Bibliographic">
>+        <leader>01005cam a22003377a 4500</leader>
>+        <controlfield tag="001">123456789</controlfield>
>+        <controlfield tag="005">20151221185246.0</controlfield>
>+        <controlfield tag="008">010203s2004    nyu           000 0 eng u</controlfield>
>+        <datafield ind1="1" tag="100" ind2=" ">
>+          <subfield code="a">Cook, David</subfield>
>+        </datafield>
>+        <datafield ind1="1" ind2="0" tag="245">
>+          <subfield code="a">Everything you never wanted to know about OAI-PMH :</subfield>
>+          <subfield code="b">a primer /</subfield>
>+          <subfield code="c">David Cook</subfield>
>+        </datafield>
>+      </record>
>+    </metadata>
>+  </metadata>
>+</record>
>\ No newline at end of file
>diff --git a/tools/manage-oai-import.pl b/tools/manage-oai-import.pl
>new file mode 100755
>index 0000000..f04da4b
>--- /dev/null
>+++ b/tools/manage-oai-import.pl
>@@ -0,0 +1,206 @@
>+#!/usr/bin/perl
>+
>+# Copyright 2016 Prosentient Systems
>+#
>+# This file is part of Koha.
>+#
>+# Koha is free software; you can redistribute it and/or modify it
>+# under the terms of the GNU General Public License as published by
>+# the Free Software Foundation; either version 3 of the License, or
>+# (at your option) any later version.
>+#
>+# Koha is distributed in the hope that it will be useful, but
>+# WITHOUT ANY WARRANTY; without even the implied warranty of
>+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
>+# GNU General Public License for more details.
>+#
>+# You should have received a copy of the GNU General Public License
>+# along with Koha; if not, see <http://www.gnu.org/licenses>.
>+
>+use Modern::Perl;
>+use POSIX qw//;
>+
>+use Koha::Database;
>+
>+use C4::Auth;
>+use C4::Output;
>+use C4::Koha;
>+use C4::Context;
>+use C4::Matcher;
>+use Koha::OAI::Client::Record;
>+
>+use C4::ImportBatch;
>+
>+my $script_name = "/cgi-bin/koha/tools/manage-oai-import.pl";
>+
>+my $input = new CGI;
>+my $op = $input->param('op') || 'list';
>+my $page_number = $input->param('page') && $input->param('page') =~ /^\d+$/ ? $input->param('page') : 1; #Only permit numeric parameter
>+
>+my $import_oai_id = $input->param('import_oai_id');
>+#my $results_per_page = $input->param('results_per_page') || 25;
>+
>+
>+
>+my ($template, $loggedinuser, $cookie) =
>+    get_template_and_user({template_name => "tools/manage-oai-import.tt",
>+        query => $input,
>+        type => "intranet",
>+        authnotrequired => 0,
>+        flagsrequired => {tools => 'manage_staged_marc'},
>+        debug => 1,
>+    });
>+
>+my $schema = Koha::Database->new()->schema();
>+my $resultset = $schema->resultset('ImportOai');
>+
>+
>+if ($import_oai_id){
>+    my $import_oai_record = $resultset->find($import_oai_id);
>+    $template->param(
>+        oai_record => $import_oai_record,
>+    );
>+
>+    if ($op eq "view_record" && $import_oai_id){
>+        $template->param(
>+            view_record => 1,
>+            import_oai_id => $import_oai_id,
>+        );
>+    }
>+
>+    if ($op eq "retry" && $import_oai_record){
>+        my $result_errors = [];
>+
>+        my $oai_record = Koha::OAI::Client::Record->new({
>+            xml_string => $import_oai_record->metadata,
>+        });
>+
>+        my $filtered = $oai_record->filter({
>+            filter => $import_oai_record->filter,
>+        });
>+        if (!$filtered){
>+            push(@$result_errors, { type => 'filter_failed', error_msg => '', record_id => '', }) ;
>+        }
>+
>+        my $import_batch_id = $import_oai_record->import_batch_id;
>+        if ($import_batch_id){
>+            my $import_batch_rs = $schema->resultset('ImportBatch');
>+            my $import_batch = $import_batch_rs->find($import_batch_id);
>+            my $matcher_id = $import_batch->matcher_id;
>+
>+            my $record_type = $import_batch->record_type;
>+            $template->param(
>+                record_type => $record_type,
>+            );
>+
>+
>+            #my $matcher = C4::Matcher->new($record_type || 'biblio');
>+            my $matcher = C4::Matcher->fetch($matcher_id);
>+
>+
>+            #FIXME
>+            my $import_mode = "direct";
>+            #FIXME
>+            my $framework = "";
>+
>+            #Reset the batch before re-trying the import
>+            C4::ImportBatch::CleanBatch($import_batch_id);
>+
>+            my ($import_status, $match_status, $koha_record_numbers, $errors) = $oai_record->import_record({
>+                matcher => $matcher,
>+                import_batch_id => $import_batch_id,
>+                import_mode => $import_mode,
>+                framework => $framework,
>+            });
>+            if (@$errors){
>+                push(@$result_errors,@$errors);
>+            }
>+
>+            if ($import_status){
>+                if ($import_status eq 'ok'){
>+                    $import_oai_record->status("ok");
>+                    $import_oai_record->update();
>+                } else {
>+                    $template->param(
>+                        import_status => $import_status,
>+                        errors => $result_errors,
>+                        retry => 1,
>+                        import_oai_id => $import_oai_id,
>+                    );
>+                }
>+            }
>+        }
>+    }
>+}
>+
>+
>+
>+$template->param(
>+    script_name => $script_name,
>+);
>+
>+if ($op && $op eq "list"){
>+    #NOTE: It would be preferable if we centralized server-side paging code with generic functions...
>+
>+    #Get grand total in the database
>+    my $total_rows = $resultset->count;
>+
>+    my $number_of_rows = 10;
>+    my $number_of_pages = POSIX::ceil($total_rows / $number_of_rows);
>+
>+    if ($page_number > 1){
>+        $template->{VARS}->{ page_first } = 1;
>+    }
>+    if ($number_of_pages > 1 && $page_number != $number_of_pages){
>+        $template->{VARS}->{ page_last } = $number_of_pages;
>+    }
>+
>+    #Do the search and define a limit
>+    my $results = $resultset->search(
>+        undef,
>+        {
>+            rows => $number_of_rows,
>+            order_by => { -desc => 'last_modified' },
>+        },
>+    );
>+
>+    my $page = $results->page($page_number);
>+    my $current_page_rows = $page->count();
>+
>+    if ($current_page_rows){
>+        if ($page_number == 1) {
>+            #Can't page previous to the first page
>+            if ( $current_page_rows == $total_rows ){
>+                #Can't page past the total count
>+            } else {
>+                #Signal that there's another page
>+                $template->{VARS}->{ page_next } = $page_number + 1;
>+            }
>+        } else {
>+
>+            #Signal that there's a previous page
>+            $template->{VARS}->{ page_previous } = $page_number - 1;
>+
>+            #Total rows of all previous pages
>+            my $previous_pages_rows = ( $page_number - 1 ) * $number_of_rows;
>+
>+            #If current and past rows are less than total rows, there must still be more to show
>+            if ( ($current_page_rows + $previous_pages_rows) < $total_rows){
>+                $template->{VARS}->{ page_next } = $page_number + 1;
>+            }
>+
>+        }
>+    }
>+
>+    $template->param(
>+        page_number => $page_number,
>+        script_name => $script_name,
>+        oai_records => $page,
>+    );
>+}
>+
>+
>+
>+
>+
>+output_html_with_http_headers $input, $cookie, $template->output;
>\ No newline at end of file
>-- 
>2.1.4

Actions: View | Diff | Splinter Review

Attachments on bug 10662: 20757 | 20758 | 21954 | 42446 | 42447 | 42448 | 44792 | 44793 | 47675 | 48096 | 49832 | 50254 | 50980 | 51510 | 51511 | 51512 | 51514 | 51562 | 51563 | 51564 | 51565 | 51705 | 53258 | 53259 | 53260 | 53361 | 53362 | 64444 | 64445 | 64446 | 64479 | 64480 | 64481 | 65016 | 65017 | 65018 | 65019 | 68508 | 71008 | 71009 | 71010 | 71011 | 71012 | 71013 | 71014 | 71015 | 78569 | 78570 | 78571 | 78572 | 78573 | 78574 | 78575 | 78576 | 78577 | 78583 | 78601 | 78617 | 78754 | 78756 | 78758 | 78760 | 78762 | 78764 | 78767 | 78769 | 78771 | 78859 | 78860 | 78914 | 78956 | 79039 | 79045 | 81783 | 81784 | 81786 | 81789 | 81790 | 81855 | 81856 | 81861 | 81862 | 81863 | 81864 | 81902 | 82219 | 84318 | 84319 | 84320 | 84321 | 84322 | 85152 | 85224 | 85225 | 85226 | 85227 | 85228 | 85229 | 99739 | 99740 | 99741 | 99742 | 99743 | 99744 | 99745