Attachment 52423 Details for Bug 15108 – Bug 15108 - OAI-PMH provider improvements:

[patch] Bug 15108 - OAI-PMH provider improvements:

Bug-15108---OAI-PMH-provider-improvements.patch (text/plain), 33.14 KB, created by Ere Maijala on 2016-06-15 12:08:13 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Ere Maijala

Created: 2016-06-15 12:08:13 UTC

Size: 33.14 KB

patch

obsolete

>From fc328d69270cfe3d9b76ace7269fc84a08efe158 Mon Sep 17 00:00:00 2001
>From: Ere Maijala <ere.maijala@helsinki.fi>
>Date: Wed, 15 Jun 2016 13:42:55 +0300
>Subject: [PATCH] Bug 15108 - OAI-PMH provider improvements:
>
>- Fixed date handling to use UTC as specs require.
>- Added support for second precision in time stamps.
>- Added support for marc21 metadata prefix as recommended in the guidelines (synonym for marcxml).
>- ListIdentifiers and ListRecords always return records in timestamp order (faster than biblionumber order with the new index).
>- Improved performance of database queries especially for large collections.
>- Unified functionality of ListRecords and ListIdentifiers to a common base class.
>- If items are included in the records, their timestamps are taken into account everywhere so that whichever is the most recent (timestamp of biblioitem or any of its items) is considered the record's timestamp.
>
>https://bugs.koha-community.org/show_bug.cgi?id=15108
>---
> Koha/OAI/Server/GetRecord.pm                       |  88 +++++++---
> Koha/OAI/Server/Identify.pm                        |   5 +-
> Koha/OAI/Server/ListBase.pm                        | 179 +++++++++++++++++++++
> Koha/OAI/Server/ListIdentifiers.pm                 |  59 +------
> Koha/OAI/Server/ListMetadataFormats.pm             |   7 +-
> Koha/OAI/Server/ListRecords.pm                     |  76 +--------
> Koha/OAI/Server/Record.pm                          |   2 +-
> Koha/OAI/Server/Repository.pm                      |  27 +++-
> Koha/OAI/Server/ResumptionToken.pm                 |  13 +-
> .../Bug15108-OAI-PMH_provider_improvements.pl      |  34 ++++
> installer/data/mysql/kohastructure.sql             |  24 +--
> 11 files changed, 342 insertions(+), 172 deletions(-)
> create mode 100644 Koha/OAI/Server/ListBase.pm
> create mode 100644 installer/data/mysql/atomicupdate/Bug15108-OAI-PMH_provider_improvements.pl
>
>diff --git a/Koha/OAI/Server/GetRecord.pm b/Koha/OAI/Server/GetRecord.pm
>index bf0c3ae..d4870e3 100644
>--- a/Koha/OAI/Server/GetRecord.pm
>+++ b/Koha/OAI/Server/GetRecord.pm
>@@ -32,20 +32,62 @@ sub new {
> 
>     my $self = HTTP::OAI::GetRecord->new(%args);
> 
>+    my $prefix = $repository->{koha_identifier} . ':';
>+    my ($biblionumber) = $args{identifier} =~ /^$prefix(.*)/;
>+    my $items_included = $repository->items_included( $args{metadataPrefix} );
>     my $dbh = C4::Context->dbh;
>-    my $sth = $dbh->prepare("
>+    my $sql = "
>         SELECT timestamp
>         FROM   biblioitems
>-        WHERE  biblionumber=? " );
>-    my $prefix = $repository->{koha_identifier} . ':';
>-    my ($biblionumber) = $args{identifier} =~ /^$prefix(.*)/;
>-    $sth->execute( $biblionumber );
>+        WHERE  biblionumber=?
>+    ";
>+    my @bind_params = ($biblionumber);
>+    if ( $items_included ) {
>+        # Take latest timestamp of biblio and any items
>+        $sql .= "
>+            UNION
>+            SELECT timestamp from deleteditems
>+            WHERE biblionumber=?
>+            UNION
>+            SELECT timestamp from items
>+            WHERE biblionumber=?
>+        ";
>+        push @bind_params, $biblionumber;
>+        push @bind_params, $biblionumber;
>+        $sql = "
>+            SELECT max(timestamp)
>+            FROM ($sql) bib
>+        ";
>+    }
>+
>+    my $sth = $dbh->prepare( $sql ) || die( 'Could not prepare statement: ' . $dbh->errstr );
>+    $sth->execute( @bind_params ) || die( 'Could not execute statement: ' . $sth->errstr );
>     my ($timestamp, $deleted);
>-    unless ( ($timestamp) = $sth->fetchrow ) {
>-        unless ( ($timestamp) = $dbh->selectrow_array(q/
>+    unless ( ($timestamp = $sth->fetchrow) ) {
>+        $sql = "
>             SELECT timestamp
>             FROM deletedbiblio
>-            WHERE biblionumber=? /, undef, $biblionumber ))
>+            WHERE biblionumber=?
>+        ";
>+        @bind_params = ($biblionumber);
>+
>+        if ( $items_included ) {
>+            # Take latest timestamp among biblio and items
>+            $sql .= "
>+                UNION
>+                SELECT timestamp from deleteditems
>+                WHERE biblionumber=?
>+            ";
>+            push @bind_params, $biblionumber;
>+            $sql = "
>+                SELECT max(timestamp)
>+                FROM ($sql) bib
>+            ";
>+        }
>+
>+        $sth = $dbh->prepare($sql) || die('Could not prepare statement: ' . $dbh->errstr);
>+        $sth->execute( @bind_params ) || die('Could not execute statement: ' . $sth->errstr);
>+        unless ( ($timestamp = $sth->fetchrow) )
>         {
>             return HTTP::OAI::Response->new(
>              requestURL  => $repository->self_url(),
>@@ -55,28 +97,30 @@ sub new {
>                 ) ],
>             );
>         }
>-        else {
>-            $deleted = 1;
>-        }
>+        $deleted = 1;
>     }
> 
>-    # We fetch it using this method, rather than the database directly,
>-    # so it'll include the item data
>-    my $marcxml;
>-    $marcxml = $repository->get_biblio_marcxml($biblionumber, $args{metadataPrefix})
>-        unless $deleted;
>     my $oai_sets = GetOAISetsBiblio($biblionumber);
>     my @setSpecs;
>     foreach (@$oai_sets) {
>         push @setSpecs, $_->{spec};
>     }
> 
>-    #$self->header( HTTP::OAI::Header->new( identifier  => $args{identifier} ) );
>-    $self->record(
>-        $deleted
>-        ? Koha::OAI::Server::DeletedRecord->new($timestamp, \@setSpecs, %args)
>-        : Koha::OAI::Server::Record->new($repository, $marcxml, $timestamp, \@setSpecs, %args)
>-    );
>+    if ($deleted) {
>+        $self->record(
>+            Koha::OAI::Server::DeletedRecord->new($timestamp, \@setSpecs, %args)
>+        );
>+    } else {
>+        # We fetch it using this method, rather than the database directly,
>+        # so it'll include the item data
>+        my $marcxml;
>+        $marcxml = $repository->get_biblio_marcxml($biblionumber, $args{metadataPrefix})
>+            unless $deleted;
>+
>+        $self->record(
>+            Koha::OAI::Server::Record->new($repository, $marcxml, $timestamp, \@setSpecs, %args)
>+        );
>+    }
>     return $self;
> }
> 
>diff --git a/Koha/OAI/Server/Identify.pm b/Koha/OAI/Server/Identify.pm
>index 3ab2188..569472a 100644
>--- a/Koha/OAI/Server/Identify.pm
>+++ b/Koha/OAI/Server/Identify.pm
>@@ -1,5 +1,6 @@
> # Copyright Tamil s.a.r.l. 2008-2015
> # Copyright Biblibre 2008-2015
>+# Copyright The National Library of Finland, University of Helsinki 2016
> #
> # This file is part of Koha.
> #
>@@ -33,8 +34,8 @@ sub new {
>         repositoryName      => C4::Context->preference("LibraryName"),
>         adminEmail          => C4::Context->preference("KohaAdminEmailAddress"),
>         MaxCount            => C4::Context->preference("OAI-PMH:MaxCount"),
>-        granularity         => 'YYYY-MM-DD',
>-        earliestDatestamp   => '0001-01-01',
>+        granularity         => 'YYYY-MM-DDThh:mm:ssZ',
>+        earliestDatestamp   => '0001-01-01T00:00:00Z',
>         deletedRecord       => C4::Context->preference("OAI-PMH:DeletedRecord") || 'no',
>     );
> 
>diff --git a/Koha/OAI/Server/ListBase.pm b/Koha/OAI/Server/ListBase.pm
>new file mode 100644
>index 0000000..e225371
>--- /dev/null
>+++ b/Koha/OAI/Server/ListBase.pm
>@@ -0,0 +1,179 @@
>+package Koha::OAI::Server::ListBase;
>+
>+# Copyright The National Library of Finland, University of Helsinki 2016
>+#
>+# This file is part of Koha.
>+#
>+# Koha is free software; you can redistribute it and/or modify it
>+# under the terms of the GNU General Public License as published by
>+# the Free Software Foundation; either version 3 of the License, or
>+# (at your option) any later version.
>+#
>+# Koha is distributed in the hope that it will be useful, but
>+# WITHOUT ANY WARRANTY; without even the implied warranty of
>+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
>+# GNU General Public License for more details.
>+#
>+# You should have received a copy of the GNU General Public License
>+# along with Koha; if not, see <http://www.gnu.org/licenses>.
>+
>+=head1 NAME
>+
>+Koha::OAI::Server::ListBase - OAI ListIdentifiers/ListRecords shared functionality
>+
>+=head1 DESCRIPTION
>+
>+Koha::OAI::Server::ListBase contains OAI-PMH functions shared by ListIdentifiers and ListRecords.
>+
>+=cut
>+
>+use Modern::Perl;
>+use C4::Biblio;
>+use HTTP::OAI;
>+use Koha::OAI::Server::ResumptionToken;
>+use Koha::OAI::Server::Record;
>+use Koha::OAI::Server::DeletedRecord;
>+use C4::OAI::Sets;
>+use MARC::File::XML;
>+
>+sub GetRecords {
>+    my ($class, $self, $repository, $metadata, %args) = @_;
>+
>+    my $token = new Koha::OAI::Server::ResumptionToken( %args );
>+    my $dbh = C4::Context->dbh;
>+    my $set;
>+    if ( defined $token->{'set'} ) {
>+        $set = GetOAISetBySpec($token->{'set'});
>+    }
>+    my $offset = $token->{offset};
>+    my $deleted = defined $token->{deleted} ? $token->{deleted} : 1;
>+    my $max = $repository->{koha_max_count};
>+    my $count = 0;
>+    my $format = $args{metadataPrefix} || $token->{metadata_prefix};
>+    my $include_items = $repository->items_included( $format );
>+
>+    # Since creating a union of normal and deleted record tables would be a heavy
>+    # operation in a large database, build results in two stages:
>+    # first deleted records ($deleted == 1), then normal records ($deleted == 0)
>+    STAGELOOP:
>+    for ( ; $deleted >= 0; $deleted-- ) {
>+        my $table = $deleted ? 'deletedbiblioitems' : 'biblioitems';
>+        my $sql = "
>+            SELECT biblionumber
>+            FROM $table
>+            WHERE (timestamp >= ? AND timestamp <= ?)
>+        ";
>+        my @bind_params = ($token->{'from_arg'}, $token->{'until_arg'});
>+
>+        if ($include_items) {
>+            $sql .= "
>+                OR biblionumber IN (SELECT biblionumber from deleteditems WHERE timestamp >= ? AND timestamp <= ?)
>+            ";
>+            push @bind_params, ($token->{'from_arg'}, $token->{'until_arg'});
>+            if (!$deleted) {
>+                $sql .= "
>+                    OR biblionumber IN (SELECT biblionumber from items WHERE timestamp >= ? AND timestamp <= ?)
>+                ";
>+                push @bind_params, ($token->{'from_arg'}, $token->{'until_arg'});
>+            }
>+        }
>+
>+        $sql .= "
>+            ORDER BY timestamp
>+        ";
>+
>+        # Use a subquery for sets since it allows us to use an index in
>+        # biblioitems table and is quite a bit faster than a join.
>+        if (defined $set) {
>+            $sql = "
>+                SELECT bi.* FROM ($sql) bi
>+                  WHERE bi.biblionumber in (SELECT osb.biblionumber FROM oai_sets_biblios osb WHERE osb.set_id = ?)
>+            ";
>+            push @bind_params, $set->{'id'};
>+        }
>+
>+        $sql .= "
>+            LIMIT " . ($max + 1) . "
>+            OFFSET $offset
>+        ";
>+
>+        my $sth = $dbh->prepare( $sql ) || die( 'Could not prepare statement: ' . $dbh->errstr );
>+
>+        if ( $deleted ) {
>+            $sql = "
>+                SELECT MAX(timestamp)
>+                FROM (
>+                    SELECT timestamp FROM deletedbiblioitems WHERE biblionumber = ?
>+                    UNION
>+                    SELECT timestamp FROM deleteditems WHERE biblionumber = ?
>+                ) bis
>+            ";
>+        } else {
>+            $sql = "
>+                SELECT MAX(timestamp)
>+                FROM (
>+                    SELECT timestamp FROM biblioitems WHERE biblionumber = ?
>+                    UNION
>+                    SELECT timestamp FROM deleteditems WHERE biblionumber = ?
>+                    UNION
>+                    SELECT timestamp FROM items WHERE biblionumber = ?
>+                ) bi
>+            ";
>+        }
>+        my $record_sth = $dbh->prepare( $sql ) || die( 'Could not prepare statement: ' . $dbh->errstr );
>+
>+        $sth->execute( @bind_params ) || die( 'Could not execute statement: ' . $sth->errstr );
>+        while ( my ($biblionumber) = $sth->fetchrow ) {
>+            $count++;
>+            if ( $count > $max ) {
>+                $self->resumptionToken(
>+                    new Koha::OAI::Server::ResumptionToken(
>+                        metadataPrefix  => $token->{metadata_prefix},
>+                        from            => $token->{from},
>+                        until           => $token->{until},
>+                        offset          => $offset + $max,
>+                        set             => $token->{set},
>+                        deleted         => $deleted
>+                    )
>+                );
>+                last STAGELOOP;
>+            }
>+            my @params = $deleted ? ( $biblionumber, $biblionumber ) : ( $biblionumber, $biblionumber, $biblionumber );
>+            $record_sth->execute( @params ) || die( 'Could not execute statement: ' . $sth->errstr );
>+
>+            my ($timestamp) = $record_sth->fetchrow;
>+
>+            my $oai_sets = GetOAISetsBiblio($biblionumber);
>+            my @setSpecs;
>+            foreach ( @$oai_sets ) {
>+                push @setSpecs, $_->{spec};
>+            }
>+            if ( $metadata ) {
>+                my $marcxml = !$deleted ? $repository->get_biblio_marcxml($biblionumber, $format) : undef;
>+                if ( $marcxml ) {
>+                  $self->record( Koha::OAI::Server::Record->new(
>+                      $repository, $marcxml, $timestamp, \@setSpecs,
>+                      identifier      => $repository->{ koha_identifier } . ':' . $biblionumber,
>+                      metadataPrefix  => $token->{metadata_prefix}
>+                  ) );
>+                } else {
>+                  $self->record( Koha::OAI::Server::DeletedRecord->new(
>+                      $timestamp, \@setSpecs, identifier => $repository->{ koha_identifier } . ':' . $biblionumber
>+                  ) );
>+                }
>+            } else {
>+                $timestamp =~ s/ /T/, $timestamp .= 'Z';
>+                $self->identifier( new HTTP::OAI::Header(
>+                    identifier => $repository->{ koha_identifier} . ':' . $biblionumber,
>+                    datestamp  => $timestamp,
>+                    status     => $deleted ? 'deleted' : undef
>+                ) );
>+            }
>+        }
>+        # Reset offset for next stage
>+        $offset = 0;
>+    }
>+    return $count;
>+}
>+
>+1;
>diff --git a/Koha/OAI/Server/ListIdentifiers.pm b/Koha/OAI/Server/ListIdentifiers.pm
>index 8b6e830..428f6ed 100644
>--- a/Koha/OAI/Server/ListIdentifiers.pm
>+++ b/Koha/OAI/Server/ListIdentifiers.pm
>@@ -1,5 +1,6 @@
> # Copyright Tamil s.a.r.l. 2008-2015
> # Copyright Biblibre 2008-2015
>+# Copyright The National Library of Finland, University of Helsinki 2016
> #
> # This file is part of Koha.
> #
>@@ -19,69 +20,15 @@
> package Koha::OAI::Server::ListIdentifiers;
> 
> use Modern::Perl;
>-use HTTP::OAI;
>-use C4::OAI::Sets;
>-
>-use base ("HTTP::OAI::ListIdentifiers");
> 
>+use base qw(HTTP::OAI::ListIdentifiers Koha::OAI::Server::ListBase);
> 
> sub new {
>     my ($class, $repository, %args) = @_;
> 
>     my $self = HTTP::OAI::ListIdentifiers->new(%args);
> 
>-    my $token = new Koha::OAI::Server::ResumptionToken( %args );
>-    my $dbh = C4::Context->dbh;
>-    my $set;
>-    if(defined $token->{'set'}) {
>-        $set = GetOAISetBySpec($token->{'set'});
>-    }
>-    my $max = $repository->{koha_max_count};
>-    my $sql = "
>-        (SELECT biblioitems.biblionumber, biblioitems.timestamp
>-        FROM biblioitems
>-    ";
>-    $sql .= " JOIN oai_sets_biblios ON biblioitems.biblionumber = oai_sets_biblios.biblionumber " if defined $set;
>-    $sql .= " WHERE timestamp >= ? AND timestamp <= ? ";
>-    $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set;
>-    $sql .= ") UNION
>-        (SELECT deletedbiblio.biblionumber, timestamp FROM deletedbiblio";
>-    $sql .= " JOIN oai_sets_biblios ON deletedbiblio.biblionumber = oai_sets_biblios.biblionumber " if defined $set;
>-    $sql .= " WHERE DATE(timestamp) >= ? AND DATE(timestamp) <= ? ";
>-    $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set;
>-
>-    $sql .= ") ORDER BY biblionumber
>-        LIMIT " . ($max+1) . "
>-        OFFSET $token->{offset}
>-    ";
>-    my $sth = $dbh->prepare( $sql );
>-    my @bind_params = ($token->{'from_arg'}, $token->{'until_arg'});
>-    push @bind_params, $set->{'id'} if defined $set;
>-    push @bind_params, ($token->{'from'}, $token->{'until'});
>-    push @bind_params, $set->{'id'} if defined $set;
>-    $sth->execute( @bind_params );
>-
>-    my $count = 0;
>-    while ( my ($biblionumber, $timestamp) = $sth->fetchrow ) {
>-        $count++;
>-        if ( $count > $max ) {
>-            $self->resumptionToken(
>-                new Koha::OAI::Server::ResumptionToken(
>-                    metadataPrefix  => $token->{metadata_prefix},
>-                    from            => $token->{from},
>-                    until           => $token->{until},
>-                    offset          => $token->{offset} + $max,
>-                    set             => $token->{set}
>-                )
>-            );
>-            last;
>-        }
>-        $timestamp =~ s/ /T/, $timestamp .= 'Z';
>-        $self->identifier( new HTTP::OAI::Header(
>-            identifier => $repository->{ koha_identifier} . ':' . $biblionumber,
>-            datestamp  => $timestamp,
>-        ) );
>-    }
>+    my $count = $class->GetRecords($self, $repository, 0, %args);
> 
>     # Return error if no results
>     unless ($count) {
>diff --git a/Koha/OAI/Server/ListMetadataFormats.pm b/Koha/OAI/Server/ListMetadataFormats.pm
>index 47463a7..f6849cd 100644
>--- a/Koha/OAI/Server/ListMetadataFormats.pm
>+++ b/Koha/OAI/Server/ListMetadataFormats.pm
>@@ -1,5 +1,6 @@
> # Copyright Tamil s.a.r.l. 2008-2015
> # Copyright Biblibre 2008-2015
>+# Copyright The National Library of Finland, University of Helsinki 2016
> #
> # This file is part of Koha.
> #
>@@ -23,7 +24,6 @@ use HTTP::OAI;
> 
> use base ("HTTP::OAI::ListMetadataFormats");
> 
>-
> sub new {
>     my ($class, $repository) = @_;
> 
>@@ -45,6 +45,11 @@ sub new {
>             metadataNamespace => 'http://www.openarchives.org/OAI/2.0/oai_dc/'
>         ) );
>         $self->metadataFormat( HTTP::OAI::MetadataFormat->new(
>+            metadataPrefix    => 'marc21',
>+            schema            => 'http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd',
>+            metadataNamespace => 'http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim'
>+        ) );
>+        $self->metadataFormat( HTTP::OAI::MetadataFormat->new(
>             metadataPrefix    => 'marcxml',
>             schema            => 'http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd',
>             metadataNamespace => 'http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim'
>diff --git a/Koha/OAI/Server/ListRecords.pm b/Koha/OAI/Server/ListRecords.pm
>index 4411d66..08a54a1 100644
>--- a/Koha/OAI/Server/ListRecords.pm
>+++ b/Koha/OAI/Server/ListRecords.pm
>@@ -1,5 +1,6 @@
> # Copyright Tamil s.a.r.l. 2008-2015
> # Copyright Biblibre 2008-2015
>+# Copyright The National Library of Finland, University of Helsinki 2016
> #
> # This file is part of Koha.
> #
>@@ -19,86 +20,15 @@
> package Koha::OAI::Server::ListRecords;
> 
> use Modern::Perl;
>-use C4::Biblio;
>-use HTTP::OAI;
>-use Koha::OAI::Server::ResumptionToken;
>-use Koha::OAI::Server::Record;
>-use Koha::OAI::Server::DeletedRecord;
>-use C4::OAI::Sets;
>-use MARC::File::XML;
>-
>-use base ("HTTP::OAI::ListRecords");
> 
>+use base qw(HTTP::OAI::ListRecords Koha::OAI::Server::ListBase);
> 
> sub new {
>     my ($class, $repository, %args) = @_;
> 
>     my $self = HTTP::OAI::ListRecords->new(%args);
> 
>-    my $token = new Koha::OAI::Server::ResumptionToken( %args );
>-    my $dbh = C4::Context->dbh;
>-    my $set;
>-    if(defined $token->{'set'}) {
>-        $set = GetOAISetBySpec($token->{'set'});
>-    }
>-    my $max = $repository->{koha_max_count};
>-    my $sql = "
>-        (SELECT biblioitems.biblionumber, biblioitems.timestamp, marcxml
>-        FROM biblioitems
>-    ";
>-    $sql .= " JOIN oai_sets_biblios ON biblioitems.biblionumber = oai_sets_biblios.biblionumber " if defined $set;
>-    $sql .= " WHERE timestamp >= ? AND timestamp <= ? ";
>-    $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set;
>-    $sql .= ") UNION
>-        (SELECT deletedbiblio.biblionumber, null as marcxml, timestamp FROM deletedbiblio";
>-    $sql .= " JOIN oai_sets_biblios ON deletedbiblio.biblionumber = oai_sets_biblios.biblionumber " if defined $set;
>-    $sql .= " WHERE DATE(timestamp) >= ? AND DATE(timestamp) <= ? ";
>-    $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set;
>-
>-    $sql .= ") ORDER BY biblionumber
>-        LIMIT " . ($max + 1) . "
>-        OFFSET $token->{offset}
>-    ";
>-    my $sth = $dbh->prepare( $sql );
>-    my @bind_params = ($token->{'from_arg'}, $token->{'until_arg'});
>-    push @bind_params, $set->{'id'} if defined $set;
>-    push @bind_params, ($token->{'from'}, $token->{'until'});
>-    push @bind_params, $set->{'id'} if defined $set;
>-    $sth->execute( @bind_params );
>-
>-    my $count = 0;
>-    my $format = $args{metadataPrefix} || $token->{metadata_prefix};
>-    while ( my ($biblionumber, $timestamp) = $sth->fetchrow ) {
>-        $count++;
>-        if ( $count > $max ) {
>-            $self->resumptionToken(
>-                new Koha::OAI::Server::ResumptionToken(
>-                    metadataPrefix  => $token->{metadata_prefix},
>-                    from            => $token->{from},
>-                    until           => $token->{until},
>-                    offset          => $token->{offset} + $max,
>-                    set             => $token->{set}
>-                )
>-            );
>-            last;
>-        }
>-        my $marcxml = $repository->get_biblio_marcxml($biblionumber, $format);
>-        my $oai_sets = GetOAISetsBiblio($biblionumber);
>-        my @setSpecs;
>-        foreach (@$oai_sets) {
>-            push @setSpecs, $_->{spec};
>-        }
>-        if ($marcxml) {
>-          $self->record( Koha::OAI::Server::Record->new(
>-              $repository, $marcxml, $timestamp, \@setSpecs,
>-              identifier      => $repository->{ koha_identifier } . ':' . $biblionumber,
>-              metadataPrefix  => $token->{metadata_prefix}
>-          ) );
>-        } else {
>-          $self->record( Koha::OAI::Server::DeletedRecord->new(
>-          $timestamp, \@setSpecs, identifier => $repository->{ koha_identifier } . ':' . $biblionumber ) );
>-        }
>-    }
>+    my $count = $class->GetRecords($self, $repository, 1, %args);
> 
>     # Return error if no results
>     unless ($count) {
>diff --git a/Koha/OAI/Server/Record.pm b/Koha/OAI/Server/Record.pm
>index 542dfc4..cf86495 100644
>--- a/Koha/OAI/Server/Record.pm
>+++ b/Koha/OAI/Server/Record.pm
>@@ -43,7 +43,7 @@ sub new {
>     my $parser = XML::LibXML->new();
>     my $record_dom = $parser->parse_string( $marcxml );
>     my $format =  $args{metadataPrefix};
>-    if ( $format ne 'marcxml' ) {
>+    if ( $format ne 'marc21' && $format ne 'marcxml' ) {
>         my %args = (
>             OPACBaseURL => "'" . C4::Context->preference('OPACBaseURL') . "'"
>         );
>diff --git a/Koha/OAI/Server/Repository.pm b/Koha/OAI/Server/Repository.pm
>index 374ce29..efad788 100644
>--- a/Koha/OAI/Server/Repository.pm
>+++ b/Koha/OAI/Server/Repository.pm
>@@ -1,5 +1,6 @@
> # Copyright Tamil s.a.r.l. 2008-2015
> # Copyright Biblibre 2008-2015
>+# Copyright The National Library of Finland, University of Helsinki 2016
> #
> # This file is part of Koha.
> #
>@@ -77,6 +78,11 @@ mode. A configuration file koha-oai.conf can look like that:
>       metadataNamespace: http://veryspecial.tamil.fr/vs/format-pivot/1.1/vs
>       schema: http://veryspecial.tamil.fr/vs/format-pivot/1.1/vs.xsd
>       xsl_file: /usr/local/koha/xslt/vs.xsl
>+    marc21:
>+      metadataPrefix: marc21
>+      metadataNamespace: http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim
>+      schema: http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd
>+      include_items: 1
>     marcxml:
>       metadataPrefix: marxml
>       metadataNamespace: http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim
>@@ -88,7 +94,7 @@ mode. A configuration file koha-oai.conf can look like that:
>       schema: http://www.openarchives.org/OAI/2.0/oai_dc.xsd
>       xsl_file: /usr/local/koha/koha-tmpl/intranet-tmpl/xslt/UNIMARCslim2OAIDC.xsl
> 
>-Note de 'include_items' parameter which is the only mean to return item-level info.
>+Note the 'include_items' parameter which is the only mean to return item-level info.
> 
> =cut
> 
>@@ -99,7 +105,7 @@ sub new {
> 
>     $self->{ koha_identifier      } = C4::Context->preference("OAI-PMH:archiveID");
>     $self->{ koha_max_count       } = C4::Context->preference("OAI-PMH:MaxCount");
>-    $self->{ koha_metadata_format } = ['oai_dc', 'marcxml'];
>+    $self->{ koha_metadata_format } = ['oai_dc', 'marc21', 'marcxml'];
>     $self->{ koha_stylesheet      } = { }; # Build when needed
> 
>     # Load configuration file if defined in OAI-PMH:ConfFile syspref
>@@ -109,10 +115,14 @@ sub new {
>         $self->{ koha_metadata_format } =  \@formats;
>     }
> 
>+    # OAI-PMH handles dates in UTC, so do that on the database level to avoid need for
>+    # any conversions
>+    C4::Context->dbh->prepare("SET time_zone='+00:00'")->execute();
>+
>     # Check for grammatical errors in the request
>     my @errs = validate_request( CGI::Vars() );
> 
>-    # Is metadataPrefix supported by the respository?
>+    # Is metadataPrefix supported by the repository?
>     my $mdp = param('metadataPrefix') || '';
>     if ( $mdp && !grep { $_ eq $mdp } @{$self->{ koha_metadata_format }} ) {
>         push @errs, new HTTP::OAI::Error(
>@@ -166,6 +176,7 @@ sub stylesheet {
>                          '/prog/en/xslt/' .
>                          C4::Context->preference('marcflavour') .
>                          'slim2OAIDC.xsl' );
>+        $xsl_file || die( "No stylesheet found for $format" );
>         my $parser = XML::LibXML->new();
>         my $xslt = XML::LibXSLT->new();
>         my $style_doc = $parser->parse_file( $xsl_file );
>@@ -176,4 +187,14 @@ sub stylesheet {
>     return $stylesheet;
> }
> 
>+
>+sub items_included {
>+    my ( $self, $format ) = @_;
>+
>+    if ( my $conf = $self->{ conf } ) {
>+        return $conf->{ format }->{ $format }->{ include_items };
>+    }
>+    return 0;
>+}
>+
> 1;
>diff --git a/Koha/OAI/Server/ResumptionToken.pm b/Koha/OAI/Server/ResumptionToken.pm
>index 9798059..5fbf9cb 100644
>--- a/Koha/OAI/Server/ResumptionToken.pm
>+++ b/Koha/OAI/Server/ResumptionToken.pm
>@@ -1,5 +1,6 @@
> # Copyright Tamil s.a.r.l. 2008-2015
> # Copyright Biblibre 2008-2015
>+# Copyright The National Library of Finland, University of Helsinki 2016
> #
> # This file is part of Koha.
> #
>@@ -31,16 +32,18 @@ use base ("HTTP::OAI::ResumptionToken");
> # - from
> # - until
> # - offset
>+# - deleted
> 
>+sub deleted { shift->_elem('deleted',@_) }
> 
> sub new {
>     my ($class, %args) = @_;
> 
>     my $self = $class->SUPER::new(%args);
> 
>-    my ($metadata_prefix, $offset, $from, $until, $set);
>+    my ($metadata_prefix, $offset, $from, $until, $set, $deleted);
>     if ( $args{ resumptionToken } ) {
>-        ($metadata_prefix, $offset, $from, $until, $set)
>+        ($metadata_prefix, $offset, $from, $until, $set, $deleted)
>             = split( '/', $args{resumptionToken} );
>     }
>     else {
>@@ -55,7 +58,8 @@ sub new {
>         $from .= 'T00:00:00Z' if length($from) == 10;
>         $until .= 'T23:59:59Z' if length($until) == 10;
>         $offset = $args{ offset } || 0;
>-        $set = $args{set} || '';
>+        $set = $args{ set } || '';
>+        $deleted = defined $args{ deleted } ? $args{ deleted } : 1;
>     }
> 
>     $self->{ metadata_prefix } = $metadata_prefix;
>@@ -65,9 +69,10 @@ sub new {
>     $self->{ set             } = $set;
>     $self->{ from_arg        } = _strip_UTC_designators($from);
>     $self->{ until_arg       } = _strip_UTC_designators($until);
>+    $self->{ deleted         } = $deleted;
> 
>     $self->resumptionToken(
>-        join( '/', $metadata_prefix, $offset, $from, $until, $set ) );
>+        join( '/', $metadata_prefix, $offset, $from, $until, $set, $deleted ) );
>     $self->cursor( $offset );
> 
>     return $self;
>diff --git a/installer/data/mysql/atomicupdate/Bug15108-OAI-PMH_provider_improvements.pl b/installer/data/mysql/atomicupdate/Bug15108-OAI-PMH_provider_improvements.pl
>new file mode 100644
>index 0000000..46cf4d1
>--- /dev/null
>+++ b/installer/data/mysql/atomicupdate/Bug15108-OAI-PMH_provider_improvements.pl
>@@ -0,0 +1,34 @@
>+#!/usr/bin/perl
>+
>+# Copyright Open Source Freedom Fighters
>+#
>+# This file is part of Koha.
>+#
>+# Koha is free software; you can redistribute it and/or modify it under the
>+# terms of the GNU General Public License as published by the Free Software
>+# Foundation; either version 3 of the License, or (at your option) any later
>+# version.
>+#
>+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
>+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
>+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
>+#
>+# You should have received a copy of the GNU General Public License along
>+# with Koha; if not, write to the Free Software Foundation, Inc.,
>+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
>+
>+use C4::Context;
>+use Koha::AtomicUpdater;
>+
>+my $dbh = C4::Context->dbh();
>+my $atomicUpdater = Koha::AtomicUpdater->new();
>+
>+unless($atomicUpdater->find('Bug15108')) {
>+
>+    $dbh->do("ALTER TABLE biblioitems ADD KEY `timestamp` (`timestamp`);");
>+    $dbh->do("ALTER TABLE deletedbiblioitems ADD KEY `timestamp` (`timestamp`);");
>+    $dbh->do("ALTER TABLE items ADD KEY `timestamp` (`timestamp`);");
>+    $dbh->do("ALTER TABLE deleteditems ADD KEY `timestamp` (`timestamp`);");
>+
>+    print "Upgrade done (Bug 15108 - OAI-PMH provider improvements)\n";
>+}
>diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql
>index 42bff46..090f0eb 100644
>--- a/installer/data/mysql/kohastructure.sql
>+++ b/installer/data/mysql/kohastructure.sql
>@@ -190,6 +190,7 @@ CREATE TABLE `biblioitems` ( -- information related to bibliographic records in
>   KEY `isbn` (`isbn`(255)),
>   KEY `issn` (`issn`(255)),
>   KEY `publishercode` (`publishercode`),
>+  KEY `timestamp` (`timestamp`),
>   CONSTRAINT `biblioitems_ibfk_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE CASCADE
> ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
> 
>@@ -542,7 +543,8 @@ CREATE TABLE `deletedbiblioitems` ( -- information about bibliographic records t
>   KEY `bibnoidx` (`biblionumber`),
>   KEY `itemtype_idx` (`itemtype`),
>   KEY `isbn` (`isbn`(255)),
>-  KEY `publishercode` (`publishercode`)
>+  KEY `publishercode` (`publishercode`),
>+  KEY `timestamp` (`timestamp`)
> ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
> 
> --
>@@ -680,7 +682,8 @@ CREATE TABLE `deleteditems` (
>   KEY `delitembibnoidx` (`biblionumber`),
>   KEY `delhomebranch` (`homebranch`),
>   KEY `delholdingbranch` (`holdingbranch`),
>-  KEY `itype_idx` (`itype`)
>+  KEY `itype_idx` (`itype`),
>+  KEY `timestamp` (`timestamp`)
> ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
> 
> --
>@@ -871,7 +874,7 @@ CREATE TABLE `issuingrules` ( -- circulation and fine rules
> --
> 
> DROP TABLE IF EXISTS `items`;
>-CREATE TABLE `items` ( -- holdings/item information 
>+CREATE TABLE `items` ( -- holdings/item information
>   `itemnumber` int(11) NOT NULL auto_increment, -- primary key and unique identifier added by Koha
>   `biblionumber` int(11) NOT NULL default 0, -- foreign key from biblio table used to link this item to the right bib record
>   `biblioitemnumber` int(11) NOT NULL default 0, -- foreign key from the biblioitems table to link to item to additional information
>@@ -927,6 +930,7 @@ CREATE TABLE `items` ( -- holdings/item information
>   KEY `items_location` (`location`),
>   KEY `items_ccode` (`ccode`),
>   KEY `itype_idx` (`itype`),
>+  KEY `timestamp` (`timestamp`),
>   CONSTRAINT `items_ibfk_1` FOREIGN KEY (`biblioitemnumber`) REFERENCES `biblioitems` (`biblioitemnumber`) ON DELETE CASCADE ON UPDATE CASCADE,
>   CONSTRAINT `items_ibfk_2` FOREIGN KEY (`homebranch`) REFERENCES `branches` (`branchcode`) ON UPDATE CASCADE,
>   CONSTRAINT `items_ibfk_3` FOREIGN KEY (`holdingbranch`) REFERENCES `branches` (`branchcode`) ON UPDATE CASCADE,
>@@ -2202,7 +2206,7 @@ CREATE TABLE `userflags` (
> --
> 
> DROP TABLE IF EXISTS `virtualshelves`;
>-CREATE TABLE `virtualshelves` ( -- information about lists (or virtual shelves) 
>+CREATE TABLE `virtualshelves` ( -- information about lists (or virtual shelves)
>   `shelfnumber` int(11) NOT NULL auto_increment, -- unique identifier assigned by Koha
>   `shelfname` varchar(255) default NULL, -- name of the list
>   `owner` int default NULL, -- foreign key linking to the borrowers table (using borrowernumber) for the creator of this list (changed from varchar(80) to int)
>@@ -2397,12 +2401,12 @@ CREATE TABLE `permissions` (
> 
> DROP TABLE IF EXISTS `serialitems`;
> CREATE TABLE `serialitems` (
>-	`itemnumber` int(11) NOT NULL,
>-	`serialid` int(11) NOT NULL,
>-	UNIQUE KEY `serialitemsidx` (`itemnumber`),
>-	KEY `serialitems_sfk_1` (`serialid`),
>-	CONSTRAINT `serialitems_sfk_1` FOREIGN KEY (`serialid`) REFERENCES `serial` (`serialid`) ON DELETE CASCADE ON UPDATE CASCADE,
>-	CONSTRAINT `serialitems_sfk_2` FOREIGN KEY (`itemnumber`) REFERENCES `items` (`itemnumber`) ON DELETE CASCADE ON UPDATE CASCADE
>+  `itemnumber` int(11) NOT NULL,
>+  `serialid` int(11) NOT NULL,
>+  UNIQUE KEY `serialitemsidx` (`itemnumber`),
>+  KEY `serialitems_sfk_1` (`serialid`),
>+  CONSTRAINT `serialitems_sfk_1` FOREIGN KEY (`serialid`) REFERENCES `serial` (`serialid`) ON DELETE CASCADE ON UPDATE CASCADE,
>+  CONSTRAINT `serialitems_sfk_2` FOREIGN KEY (`itemnumber`) REFERENCES `items` (`itemnumber`) ON DELETE CASCADE ON UPDATE CASCADE
> ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
> 
> --
>-- 
>1.9.1

Actions: View | Diff | Splinter Review

Attachments on bug 15108: 44317 | 44318 | 51962 | 52416 | 52422 | 52423 | 53853 | 53854 | 56813 | 56844 | 58058 | 58059 | 58060 | 58061 | 58062 | 58063 | 58064 | 58065 | 58072 | 58080 | 58245 | 58246 | 58247 | 58248 | 58249 | 58250 | 58251 | 60260 | 60331 | 60608 | 60835 | 60836 | 63172 | 63173 | 63174