Attachment 58058 Details for Bug 15108 – Bug 15108 - OAI-PMH provider improvements

[patch] Bug 15108 - OAI-PMH provider improvements

Bug-15108---OAI-PMH-provider-improvements.patch (text/plain), 33.22 KB, created by Marcel de Rooy on 2016-12-09 10:15:16 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Marcel de Rooy

Created: 2016-12-09 10:15:16 UTC

Size: 33.22 KB

patch

obsolete

>From 4d40e4a47fc778dada50dd9e86b4d2da9311fc03 Mon Sep 17 00:00:00 2001
>From: Ere Maijala <ere.maijala@helsinki.fi>
>Date: Wed, 15 Jun 2016 13:42:55 +0300
>Subject: [PATCH] Bug 15108 - OAI-PMH provider improvements
>Content-Type: text/plain; charset=utf-8
>
>- Fixed date handling to use UTC as specs require.
>
>- Added support for second precision in time stamps.
>
>- Added support for marc21 metadata prefix as recommended in the
>  guidelines (synonym for marcxml).
>
>- ListIdentifiers and ListRecords always return records in timestamp
>  order (faster than biblionumber order with the new index).
>
>- Improved performance of database queries especially for large
>  collections.
>
>- Unified functionality of ListRecords and ListIdentifiers to a common
>  base class.
>
>- If items are included in the records, their timestamps are taken into
>  account everywhere so that whichever is the most recent (timestamp of
>  biblioitem or any of its items) is considered the record's timestamp.
>
>Signed-off-by: Frederic Demians <f.demians@tamil.fr>
>---
> Koha/OAI/Server/GetRecord.pm                       |  88 +++++++---
> Koha/OAI/Server/Identify.pm                        |   5 +-
> Koha/OAI/Server/ListBase.pm                        | 179 +++++++++++++++++++++
> Koha/OAI/Server/ListIdentifiers.pm                 |  59 +------
> Koha/OAI/Server/ListMetadataFormats.pm             |   7 +-
> Koha/OAI/Server/ListRecords.pm                     |  76 +--------
> Koha/OAI/Server/Record.pm                          |   2 +-
> Koha/OAI/Server/Repository.pm                      |  27 +++-
> Koha/OAI/Server/ResumptionToken.pm                 |  13 +-
> .../Bug15108-OAI-PMH_provider_improvements.pl      |  34 ++++
> installer/data/mysql/kohastructure.sql             |  24 +--
> 11 files changed, 342 insertions(+), 172 deletions(-)
> create mode 100644 Koha/OAI/Server/ListBase.pm
> create mode 100644 installer/data/mysql/atomicupdate/Bug15108-OAI-PMH_provider_improvements.pl
>
>diff --git a/Koha/OAI/Server/GetRecord.pm b/Koha/OAI/Server/GetRecord.pm
>index bf0c3ae..d4870e3 100644
>--- a/Koha/OAI/Server/GetRecord.pm
>+++ b/Koha/OAI/Server/GetRecord.pm
>@@ -32,20 +32,62 @@ sub new {
> 
>     my $self = HTTP::OAI::GetRecord->new(%args);
> 
>+    my $prefix = $repository->{koha_identifier} . ':';
>+    my ($biblionumber) = $args{identifier} =~ /^$prefix(.*)/;
>+    my $items_included = $repository->items_included( $args{metadataPrefix} );
>     my $dbh = C4::Context->dbh;
>-    my $sth = $dbh->prepare("
>+    my $sql = "
>         SELECT timestamp
>         FROM   biblioitems
>-        WHERE  biblionumber=? " );
>-    my $prefix = $repository->{koha_identifier} . ':';
>-    my ($biblionumber) = $args{identifier} =~ /^$prefix(.*)/;
>-    $sth->execute( $biblionumber );
>+        WHERE  biblionumber=?
>+    ";
>+    my @bind_params = ($biblionumber);
>+    if ( $items_included ) {
>+        # Take latest timestamp of biblio and any items
>+        $sql .= "
>+            UNION
>+            SELECT timestamp from deleteditems
>+            WHERE biblionumber=?
>+            UNION
>+            SELECT timestamp from items
>+            WHERE biblionumber=?
>+        ";
>+        push @bind_params, $biblionumber;
>+        push @bind_params, $biblionumber;
>+        $sql = "
>+            SELECT max(timestamp)
>+            FROM ($sql) bib
>+        ";
>+    }
>+
>+    my $sth = $dbh->prepare( $sql ) || die( 'Could not prepare statement: ' . $dbh->errstr );
>+    $sth->execute( @bind_params ) || die( 'Could not execute statement: ' . $sth->errstr );
>     my ($timestamp, $deleted);
>-    unless ( ($timestamp) = $sth->fetchrow ) {
>-        unless ( ($timestamp) = $dbh->selectrow_array(q/
>+    unless ( ($timestamp = $sth->fetchrow) ) {
>+        $sql = "
>             SELECT timestamp
>             FROM deletedbiblio
>-            WHERE biblionumber=? /, undef, $biblionumber ))
>+            WHERE biblionumber=?
>+        ";
>+        @bind_params = ($biblionumber);
>+
>+        if ( $items_included ) {
>+            # Take latest timestamp among biblio and items
>+            $sql .= "
>+                UNION
>+                SELECT timestamp from deleteditems
>+                WHERE biblionumber=?
>+            ";
>+            push @bind_params, $biblionumber;
>+            $sql = "
>+                SELECT max(timestamp)
>+                FROM ($sql) bib
>+            ";
>+        }
>+
>+        $sth = $dbh->prepare($sql) || die('Could not prepare statement: ' . $dbh->errstr);
>+        $sth->execute( @bind_params ) || die('Could not execute statement: ' . $sth->errstr);
>+        unless ( ($timestamp = $sth->fetchrow) )
>         {
>             return HTTP::OAI::Response->new(
>              requestURL  => $repository->self_url(),
>@@ -55,28 +97,30 @@ sub new {
>                 ) ],
>             );
>         }
>-        else {
>-            $deleted = 1;
>-        }
>+        $deleted = 1;
>     }
> 
>-    # We fetch it using this method, rather than the database directly,
>-    # so it'll include the item data
>-    my $marcxml;
>-    $marcxml = $repository->get_biblio_marcxml($biblionumber, $args{metadataPrefix})
>-        unless $deleted;
>     my $oai_sets = GetOAISetsBiblio($biblionumber);
>     my @setSpecs;
>     foreach (@$oai_sets) {
>         push @setSpecs, $_->{spec};
>     }
> 
>-    #$self->header( HTTP::OAI::Header->new( identifier  => $args{identifier} ) );
>-    $self->record(
>-        $deleted
>-        ? Koha::OAI::Server::DeletedRecord->new($timestamp, \@setSpecs, %args)
>-        : Koha::OAI::Server::Record->new($repository, $marcxml, $timestamp, \@setSpecs, %args)
>-    );
>+    if ($deleted) {
>+        $self->record(
>+            Koha::OAI::Server::DeletedRecord->new($timestamp, \@setSpecs, %args)
>+        );
>+    } else {
>+        # We fetch it using this method, rather than the database directly,
>+        # so it'll include the item data
>+        my $marcxml;
>+        $marcxml = $repository->get_biblio_marcxml($biblionumber, $args{metadataPrefix})
>+            unless $deleted;
>+
>+        $self->record(
>+            Koha::OAI::Server::Record->new($repository, $marcxml, $timestamp, \@setSpecs, %args)
>+        );
>+    }
>     return $self;
> }
> 
>diff --git a/Koha/OAI/Server/Identify.pm b/Koha/OAI/Server/Identify.pm
>index 3ab2188..569472a 100644
>--- a/Koha/OAI/Server/Identify.pm
>+++ b/Koha/OAI/Server/Identify.pm
>@@ -1,5 +1,6 @@
> # Copyright Tamil s.a.r.l. 2008-2015
> # Copyright Biblibre 2008-2015
>+# Copyright The National Library of Finland, University of Helsinki 2016
> #
> # This file is part of Koha.
> #
>@@ -33,8 +34,8 @@ sub new {
>         repositoryName      => C4::Context->preference("LibraryName"),
>         adminEmail          => C4::Context->preference("KohaAdminEmailAddress"),
>         MaxCount            => C4::Context->preference("OAI-PMH:MaxCount"),
>-        granularity         => 'YYYY-MM-DD',
>-        earliestDatestamp   => '0001-01-01',
>+        granularity         => 'YYYY-MM-DDThh:mm:ssZ',
>+        earliestDatestamp   => '0001-01-01T00:00:00Z',
>         deletedRecord       => C4::Context->preference("OAI-PMH:DeletedRecord") || 'no',
>     );
> 
>diff --git a/Koha/OAI/Server/ListBase.pm b/Koha/OAI/Server/ListBase.pm
>new file mode 100644
>index 0000000..e225371
>--- /dev/null
>+++ b/Koha/OAI/Server/ListBase.pm
>@@ -0,0 +1,179 @@
>+package Koha::OAI::Server::ListBase;
>+
>+# Copyright The National Library of Finland, University of Helsinki 2016
>+#
>+# This file is part of Koha.
>+#
>+# Koha is free software; you can redistribute it and/or modify it
>+# under the terms of the GNU General Public License as published by
>+# the Free Software Foundation; either version 3 of the License, or
>+# (at your option) any later version.
>+#
>+# Koha is distributed in the hope that it will be useful, but
>+# WITHOUT ANY WARRANTY; without even the implied warranty of
>+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
>+# GNU General Public License for more details.
>+#
>+# You should have received a copy of the GNU General Public License
>+# along with Koha; if not, see <http://www.gnu.org/licenses>.
>+
>+=head1 NAME
>+
>+Koha::OAI::Server::ListBase - OAI ListIdentifiers/ListRecords shared functionality
>+
>+=head1 DESCRIPTION
>+
>+Koha::OAI::Server::ListBase contains OAI-PMH functions shared by ListIdentifiers and ListRecords.
>+
>+=cut
>+
>+use Modern::Perl;
>+use C4::Biblio;
>+use HTTP::OAI;
>+use Koha::OAI::Server::ResumptionToken;
>+use Koha::OAI::Server::Record;
>+use Koha::OAI::Server::DeletedRecord;
>+use C4::OAI::Sets;
>+use MARC::File::XML;
>+
>+sub GetRecords {
>+    my ($class, $self, $repository, $metadata, %args) = @_;
>+
>+    my $token = new Koha::OAI::Server::ResumptionToken( %args );
>+    my $dbh = C4::Context->dbh;
>+    my $set;
>+    if ( defined $token->{'set'} ) {
>+        $set = GetOAISetBySpec($token->{'set'});
>+    }
>+    my $offset = $token->{offset};
>+    my $deleted = defined $token->{deleted} ? $token->{deleted} : 1;
>+    my $max = $repository->{koha_max_count};
>+    my $count = 0;
>+    my $format = $args{metadataPrefix} || $token->{metadata_prefix};
>+    my $include_items = $repository->items_included( $format );
>+
>+    # Since creating a union of normal and deleted record tables would be a heavy
>+    # operation in a large database, build results in two stages:
>+    # first deleted records ($deleted == 1), then normal records ($deleted == 0)
>+    STAGELOOP:
>+    for ( ; $deleted >= 0; $deleted-- ) {
>+        my $table = $deleted ? 'deletedbiblioitems' : 'biblioitems';
>+        my $sql = "
>+            SELECT biblionumber
>+            FROM $table
>+            WHERE (timestamp >= ? AND timestamp <= ?)
>+        ";
>+        my @bind_params = ($token->{'from_arg'}, $token->{'until_arg'});
>+
>+        if ($include_items) {
>+            $sql .= "
>+                OR biblionumber IN (SELECT biblionumber from deleteditems WHERE timestamp >= ? AND timestamp <= ?)
>+            ";
>+            push @bind_params, ($token->{'from_arg'}, $token->{'until_arg'});
>+            if (!$deleted) {
>+                $sql .= "
>+                    OR biblionumber IN (SELECT biblionumber from items WHERE timestamp >= ? AND timestamp <= ?)
>+                ";
>+                push @bind_params, ($token->{'from_arg'}, $token->{'until_arg'});
>+            }
>+        }
>+
>+        $sql .= "
>+            ORDER BY timestamp
>+        ";
>+
>+        # Use a subquery for sets since it allows us to use an index in
>+        # biblioitems table and is quite a bit faster than a join.
>+        if (defined $set) {
>+            $sql = "
>+                SELECT bi.* FROM ($sql) bi
>+                  WHERE bi.biblionumber in (SELECT osb.biblionumber FROM oai_sets_biblios osb WHERE osb.set_id = ?)
>+            ";
>+            push @bind_params, $set->{'id'};
>+        }
>+
>+        $sql .= "
>+            LIMIT " . ($max + 1) . "
>+            OFFSET $offset
>+        ";
>+
>+        my $sth = $dbh->prepare( $sql ) || die( 'Could not prepare statement: ' . $dbh->errstr );
>+
>+        if ( $deleted ) {
>+            $sql = "
>+                SELECT MAX(timestamp)
>+                FROM (
>+                    SELECT timestamp FROM deletedbiblioitems WHERE biblionumber = ?
>+                    UNION
>+                    SELECT timestamp FROM deleteditems WHERE biblionumber = ?
>+                ) bis
>+            ";
>+        } else {
>+            $sql = "
>+                SELECT MAX(timestamp)
>+                FROM (
>+                    SELECT timestamp FROM biblioitems WHERE biblionumber = ?
>+                    UNION
>+                    SELECT timestamp FROM deleteditems WHERE biblionumber = ?
>+                    UNION
>+                    SELECT timestamp FROM items WHERE biblionumber = ?
>+                ) bi
>+            ";
>+        }
>+        my $record_sth = $dbh->prepare( $sql ) || die( 'Could not prepare statement: ' . $dbh->errstr );
>+
>+        $sth->execute( @bind_params ) || die( 'Could not execute statement: ' . $sth->errstr );
>+        while ( my ($biblionumber) = $sth->fetchrow ) {
>+            $count++;
>+            if ( $count > $max ) {
>+                $self->resumptionToken(
>+                    new Koha::OAI::Server::ResumptionToken(
>+                        metadataPrefix  => $token->{metadata_prefix},
>+                        from            => $token->{from},
>+                        until           => $token->{until},
>+                        offset          => $offset + $max,
>+                        set             => $token->{set},
>+                        deleted         => $deleted
>+                    )
>+                );
>+                last STAGELOOP;
>+            }
>+            my @params = $deleted ? ( $biblionumber, $biblionumber ) : ( $biblionumber, $biblionumber, $biblionumber );
>+            $record_sth->execute( @params ) || die( 'Could not execute statement: ' . $sth->errstr );
>+
>+            my ($timestamp) = $record_sth->fetchrow;
>+
>+            my $oai_sets = GetOAISetsBiblio($biblionumber);
>+            my @setSpecs;
>+            foreach ( @$oai_sets ) {
>+                push @setSpecs, $_->{spec};
>+            }
>+            if ( $metadata ) {
>+                my $marcxml = !$deleted ? $repository->get_biblio_marcxml($biblionumber, $format) : undef;
>+                if ( $marcxml ) {
>+                  $self->record( Koha::OAI::Server::Record->new(
>+                      $repository, $marcxml, $timestamp, \@setSpecs,
>+                      identifier      => $repository->{ koha_identifier } . ':' . $biblionumber,
>+                      metadataPrefix  => $token->{metadata_prefix}
>+                  ) );
>+                } else {
>+                  $self->record( Koha::OAI::Server::DeletedRecord->new(
>+                      $timestamp, \@setSpecs, identifier => $repository->{ koha_identifier } . ':' . $biblionumber
>+                  ) );
>+                }
>+            } else {
>+                $timestamp =~ s/ /T/, $timestamp .= 'Z';
>+                $self->identifier( new HTTP::OAI::Header(
>+                    identifier => $repository->{ koha_identifier} . ':' . $biblionumber,
>+                    datestamp  => $timestamp,
>+                    status     => $deleted ? 'deleted' : undef
>+                ) );
>+            }
>+        }
>+        # Reset offset for next stage
>+        $offset = 0;
>+    }
>+    return $count;
>+}
>+
>+1;
>diff --git a/Koha/OAI/Server/ListIdentifiers.pm b/Koha/OAI/Server/ListIdentifiers.pm
>index 8b6e830..428f6ed 100644
>--- a/Koha/OAI/Server/ListIdentifiers.pm
>+++ b/Koha/OAI/Server/ListIdentifiers.pm
>@@ -1,5 +1,6 @@
> # Copyright Tamil s.a.r.l. 2008-2015
> # Copyright Biblibre 2008-2015
>+# Copyright The National Library of Finland, University of Helsinki 2016
> #
> # This file is part of Koha.
> #
>@@ -19,69 +20,15 @@
> package Koha::OAI::Server::ListIdentifiers;
> 
> use Modern::Perl;
>-use HTTP::OAI;
>-use C4::OAI::Sets;
>-
>-use base ("HTTP::OAI::ListIdentifiers");
> 
>+use base qw(HTTP::OAI::ListIdentifiers Koha::OAI::Server::ListBase);
> 
> sub new {
>     my ($class, $repository, %args) = @_;
> 
>     my $self = HTTP::OAI::ListIdentifiers->new(%args);
> 
>-    my $token = new Koha::OAI::Server::ResumptionToken( %args );
>-    my $dbh = C4::Context->dbh;
>-    my $set;
>-    if(defined $token->{'set'}) {
>-        $set = GetOAISetBySpec($token->{'set'});
>-    }
>-    my $max = $repository->{koha_max_count};
>-    my $sql = "
>-        (SELECT biblioitems.biblionumber, biblioitems.timestamp
>-        FROM biblioitems
>-    ";
>-    $sql .= " JOIN oai_sets_biblios ON biblioitems.biblionumber = oai_sets_biblios.biblionumber " if defined $set;
>-    $sql .= " WHERE timestamp >= ? AND timestamp <= ? ";
>-    $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set;
>-    $sql .= ") UNION
>-        (SELECT deletedbiblio.biblionumber, timestamp FROM deletedbiblio";
>-    $sql .= " JOIN oai_sets_biblios ON deletedbiblio.biblionumber = oai_sets_biblios.biblionumber " if defined $set;
>-    $sql .= " WHERE DATE(timestamp) >= ? AND DATE(timestamp) <= ? ";
>-    $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set;
>-
>-    $sql .= ") ORDER BY biblionumber
>-        LIMIT " . ($max+1) . "
>-        OFFSET $token->{offset}
>-    ";
>-    my $sth = $dbh->prepare( $sql );
>-    my @bind_params = ($token->{'from_arg'}, $token->{'until_arg'});
>-    push @bind_params, $set->{'id'} if defined $set;
>-    push @bind_params, ($token->{'from'}, $token->{'until'});
>-    push @bind_params, $set->{'id'} if defined $set;
>-    $sth->execute( @bind_params );
>-
>-    my $count = 0;
>-    while ( my ($biblionumber, $timestamp) = $sth->fetchrow ) {
>-        $count++;
>-        if ( $count > $max ) {
>-            $self->resumptionToken(
>-                new Koha::OAI::Server::ResumptionToken(
>-                    metadataPrefix  => $token->{metadata_prefix},
>-                    from            => $token->{from},
>-                    until           => $token->{until},
>-                    offset          => $token->{offset} + $max,
>-                    set             => $token->{set}
>-                )
>-            );
>-            last;
>-        }
>-        $timestamp =~ s/ /T/, $timestamp .= 'Z';
>-        $self->identifier( new HTTP::OAI::Header(
>-            identifier => $repository->{ koha_identifier} . ':' . $biblionumber,
>-            datestamp  => $timestamp,
>-        ) );
>-    }
>+    my $count = $class->GetRecords($self, $repository, 0, %args);
> 
>     # Return error if no results
>     unless ($count) {
>diff --git a/Koha/OAI/Server/ListMetadataFormats.pm b/Koha/OAI/Server/ListMetadataFormats.pm
>index 47463a7..f6849cd 100644
>--- a/Koha/OAI/Server/ListMetadataFormats.pm
>+++ b/Koha/OAI/Server/ListMetadataFormats.pm
>@@ -1,5 +1,6 @@
> # Copyright Tamil s.a.r.l. 2008-2015
> # Copyright Biblibre 2008-2015
>+# Copyright The National Library of Finland, University of Helsinki 2016
> #
> # This file is part of Koha.
> #
>@@ -23,7 +24,6 @@ use HTTP::OAI;
> 
> use base ("HTTP::OAI::ListMetadataFormats");
> 
>-
> sub new {
>     my ($class, $repository) = @_;
> 
>@@ -45,6 +45,11 @@ sub new {
>             metadataNamespace => 'http://www.openarchives.org/OAI/2.0/oai_dc/'
>         ) );
>         $self->metadataFormat( HTTP::OAI::MetadataFormat->new(
>+            metadataPrefix    => 'marc21',
>+            schema            => 'http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd',
>+            metadataNamespace => 'http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim'
>+        ) );
>+        $self->metadataFormat( HTTP::OAI::MetadataFormat->new(
>             metadataPrefix    => 'marcxml',
>             schema            => 'http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd',
>             metadataNamespace => 'http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim'
>diff --git a/Koha/OAI/Server/ListRecords.pm b/Koha/OAI/Server/ListRecords.pm
>index 4411d66..08a54a1 100644
>--- a/Koha/OAI/Server/ListRecords.pm
>+++ b/Koha/OAI/Server/ListRecords.pm
>@@ -1,5 +1,6 @@
> # Copyright Tamil s.a.r.l. 2008-2015
> # Copyright Biblibre 2008-2015
>+# Copyright The National Library of Finland, University of Helsinki 2016
> #
> # This file is part of Koha.
> #
>@@ -19,86 +20,15 @@
> package Koha::OAI::Server::ListRecords;
> 
> use Modern::Perl;
>-use C4::Biblio;
>-use HTTP::OAI;
>-use Koha::OAI::Server::ResumptionToken;
>-use Koha::OAI::Server::Record;
>-use Koha::OAI::Server::DeletedRecord;
>-use C4::OAI::Sets;
>-use MARC::File::XML;
>-
>-use base ("HTTP::OAI::ListRecords");
> 
>+use base qw(HTTP::OAI::ListRecords Koha::OAI::Server::ListBase);
> 
> sub new {
>     my ($class, $repository, %args) = @_;
> 
>     my $self = HTTP::OAI::ListRecords->new(%args);
> 
>-    my $token = new Koha::OAI::Server::ResumptionToken( %args );
>-    my $dbh = C4::Context->dbh;
>-    my $set;
>-    if(defined $token->{'set'}) {
>-        $set = GetOAISetBySpec($token->{'set'});
>-    }
>-    my $max = $repository->{koha_max_count};
>-    my $sql = "
>-        (SELECT biblioitems.biblionumber, biblioitems.timestamp, marcxml
>-        FROM biblioitems
>-    ";
>-    $sql .= " JOIN oai_sets_biblios ON biblioitems.biblionumber = oai_sets_biblios.biblionumber " if defined $set;
>-    $sql .= " WHERE timestamp >= ? AND timestamp <= ? ";
>-    $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set;
>-    $sql .= ") UNION
>-        (SELECT deletedbiblio.biblionumber, null as marcxml, timestamp FROM deletedbiblio";
>-    $sql .= " JOIN oai_sets_biblios ON deletedbiblio.biblionumber = oai_sets_biblios.biblionumber " if defined $set;
>-    $sql .= " WHERE DATE(timestamp) >= ? AND DATE(timestamp) <= ? ";
>-    $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set;
>-
>-    $sql .= ") ORDER BY biblionumber
>-        LIMIT " . ($max + 1) . "
>-        OFFSET $token->{offset}
>-    ";
>-    my $sth = $dbh->prepare( $sql );
>-    my @bind_params = ($token->{'from_arg'}, $token->{'until_arg'});
>-    push @bind_params, $set->{'id'} if defined $set;
>-    push @bind_params, ($token->{'from'}, $token->{'until'});
>-    push @bind_params, $set->{'id'} if defined $set;
>-    $sth->execute( @bind_params );
>-
>-    my $count = 0;
>-    my $format = $args{metadataPrefix} || $token->{metadata_prefix};
>-    while ( my ($biblionumber, $timestamp) = $sth->fetchrow ) {
>-        $count++;
>-        if ( $count > $max ) {
>-            $self->resumptionToken(
>-                new Koha::OAI::Server::ResumptionToken(
>-                    metadataPrefix  => $token->{metadata_prefix},
>-                    from            => $token->{from},
>-                    until           => $token->{until},
>-                    offset          => $token->{offset} + $max,
>-                    set             => $token->{set}
>-                )
>-            );
>-            last;
>-        }
>-        my $marcxml = $repository->get_biblio_marcxml($biblionumber, $format);
>-        my $oai_sets = GetOAISetsBiblio($biblionumber);
>-        my @setSpecs;
>-        foreach (@$oai_sets) {
>-            push @setSpecs, $_->{spec};
>-        }
>-        if ($marcxml) {
>-          $self->record( Koha::OAI::Server::Record->new(
>-              $repository, $marcxml, $timestamp, \@setSpecs,
>-              identifier      => $repository->{ koha_identifier } . ':' . $biblionumber,
>-              metadataPrefix  => $token->{metadata_prefix}
>-          ) );
>-        } else {
>-          $self->record( Koha::OAI::Server::DeletedRecord->new(
>-          $timestamp, \@setSpecs, identifier => $repository->{ koha_identifier } . ':' . $biblionumber ) );
>-        }
>-    }
>+    my $count = $class->GetRecords($self, $repository, 1, %args);
> 
>     # Return error if no results
>     unless ($count) {
>diff --git a/Koha/OAI/Server/Record.pm b/Koha/OAI/Server/Record.pm
>index 542dfc4..cf86495 100644
>--- a/Koha/OAI/Server/Record.pm
>+++ b/Koha/OAI/Server/Record.pm
>@@ -43,7 +43,7 @@ sub new {
>     my $parser = XML::LibXML->new();
>     my $record_dom = $parser->parse_string( $marcxml );
>     my $format =  $args{metadataPrefix};
>-    if ( $format ne 'marcxml' ) {
>+    if ( $format ne 'marc21' && $format ne 'marcxml' ) {
>         my %args = (
>             OPACBaseURL => "'" . C4::Context->preference('OPACBaseURL') . "'"
>         );
>diff --git a/Koha/OAI/Server/Repository.pm b/Koha/OAI/Server/Repository.pm
>index 374ce29..efad788 100644
>--- a/Koha/OAI/Server/Repository.pm
>+++ b/Koha/OAI/Server/Repository.pm
>@@ -1,5 +1,6 @@
> # Copyright Tamil s.a.r.l. 2008-2015
> # Copyright Biblibre 2008-2015
>+# Copyright The National Library of Finland, University of Helsinki 2016
> #
> # This file is part of Koha.
> #
>@@ -77,6 +78,11 @@ mode. A configuration file koha-oai.conf can look like that:
>       metadataNamespace: http://veryspecial.tamil.fr/vs/format-pivot/1.1/vs
>       schema: http://veryspecial.tamil.fr/vs/format-pivot/1.1/vs.xsd
>       xsl_file: /usr/local/koha/xslt/vs.xsl
>+    marc21:
>+      metadataPrefix: marc21
>+      metadataNamespace: http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim
>+      schema: http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd
>+      include_items: 1
>     marcxml:
>       metadataPrefix: marxml
>       metadataNamespace: http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim
>@@ -88,7 +94,7 @@ mode. A configuration file koha-oai.conf can look like that:
>       schema: http://www.openarchives.org/OAI/2.0/oai_dc.xsd
>       xsl_file: /usr/local/koha/koha-tmpl/intranet-tmpl/xslt/UNIMARCslim2OAIDC.xsl
> 
>-Note de 'include_items' parameter which is the only mean to return item-level info.
>+Note the 'include_items' parameter which is the only mean to return item-level info.
> 
> =cut
> 
>@@ -99,7 +105,7 @@ sub new {
> 
>     $self->{ koha_identifier      } = C4::Context->preference("OAI-PMH:archiveID");
>     $self->{ koha_max_count       } = C4::Context->preference("OAI-PMH:MaxCount");
>-    $self->{ koha_metadata_format } = ['oai_dc', 'marcxml'];
>+    $self->{ koha_metadata_format } = ['oai_dc', 'marc21', 'marcxml'];
>     $self->{ koha_stylesheet      } = { }; # Build when needed
> 
>     # Load configuration file if defined in OAI-PMH:ConfFile syspref
>@@ -109,10 +115,14 @@ sub new {
>         $self->{ koha_metadata_format } =  \@formats;
>     }
> 
>+    # OAI-PMH handles dates in UTC, so do that on the database level to avoid need for
>+    # any conversions
>+    C4::Context->dbh->prepare("SET time_zone='+00:00'")->execute();
>+
>     # Check for grammatical errors in the request
>     my @errs = validate_request( CGI::Vars() );
> 
>-    # Is metadataPrefix supported by the respository?
>+    # Is metadataPrefix supported by the repository?
>     my $mdp = param('metadataPrefix') || '';
>     if ( $mdp && !grep { $_ eq $mdp } @{$self->{ koha_metadata_format }} ) {
>         push @errs, new HTTP::OAI::Error(
>@@ -166,6 +176,7 @@ sub stylesheet {
>                          '/prog/en/xslt/' .
>                          C4::Context->preference('marcflavour') .
>                          'slim2OAIDC.xsl' );
>+        $xsl_file || die( "No stylesheet found for $format" );
>         my $parser = XML::LibXML->new();
>         my $xslt = XML::LibXSLT->new();
>         my $style_doc = $parser->parse_file( $xsl_file );
>@@ -176,4 +187,14 @@ sub stylesheet {
>     return $stylesheet;
> }
> 
>+
>+sub items_included {
>+    my ( $self, $format ) = @_;
>+
>+    if ( my $conf = $self->{ conf } ) {
>+        return $conf->{ format }->{ $format }->{ include_items };
>+    }
>+    return 0;
>+}
>+
> 1;
>diff --git a/Koha/OAI/Server/ResumptionToken.pm b/Koha/OAI/Server/ResumptionToken.pm
>index 9798059..5fbf9cb 100644
>--- a/Koha/OAI/Server/ResumptionToken.pm
>+++ b/Koha/OAI/Server/ResumptionToken.pm
>@@ -1,5 +1,6 @@
> # Copyright Tamil s.a.r.l. 2008-2015
> # Copyright Biblibre 2008-2015
>+# Copyright The National Library of Finland, University of Helsinki 2016
> #
> # This file is part of Koha.
> #
>@@ -31,16 +32,18 @@ use base ("HTTP::OAI::ResumptionToken");
> # - from
> # - until
> # - offset
>+# - deleted
> 
>+sub deleted { shift->_elem('deleted',@_) }
> 
> sub new {
>     my ($class, %args) = @_;
> 
>     my $self = $class->SUPER::new(%args);
> 
>-    my ($metadata_prefix, $offset, $from, $until, $set);
>+    my ($metadata_prefix, $offset, $from, $until, $set, $deleted);
>     if ( $args{ resumptionToken } ) {
>-        ($metadata_prefix, $offset, $from, $until, $set)
>+        ($metadata_prefix, $offset, $from, $until, $set, $deleted)
>             = split( '/', $args{resumptionToken} );
>     }
>     else {
>@@ -55,7 +58,8 @@ sub new {
>         $from .= 'T00:00:00Z' if length($from) == 10;
>         $until .= 'T23:59:59Z' if length($until) == 10;
>         $offset = $args{ offset } || 0;
>-        $set = $args{set} || '';
>+        $set = $args{ set } || '';
>+        $deleted = defined $args{ deleted } ? $args{ deleted } : 1;
>     }
> 
>     $self->{ metadata_prefix } = $metadata_prefix;
>@@ -65,9 +69,10 @@ sub new {
>     $self->{ set             } = $set;
>     $self->{ from_arg        } = _strip_UTC_designators($from);
>     $self->{ until_arg       } = _strip_UTC_designators($until);
>+    $self->{ deleted         } = $deleted;
> 
>     $self->resumptionToken(
>-        join( '/', $metadata_prefix, $offset, $from, $until, $set ) );
>+        join( '/', $metadata_prefix, $offset, $from, $until, $set, $deleted ) );
>     $self->cursor( $offset );
> 
>     return $self;
>diff --git a/installer/data/mysql/atomicupdate/Bug15108-OAI-PMH_provider_improvements.pl b/installer/data/mysql/atomicupdate/Bug15108-OAI-PMH_provider_improvements.pl
>new file mode 100644
>index 0000000..46cf4d1
>--- /dev/null
>+++ b/installer/data/mysql/atomicupdate/Bug15108-OAI-PMH_provider_improvements.pl
>@@ -0,0 +1,34 @@
>+#!/usr/bin/perl
>+
>+# Copyright Open Source Freedom Fighters
>+#
>+# This file is part of Koha.
>+#
>+# Koha is free software; you can redistribute it and/or modify it under the
>+# terms of the GNU General Public License as published by the Free Software
>+# Foundation; either version 3 of the License, or (at your option) any later
>+# version.
>+#
>+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
>+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
>+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
>+#
>+# You should have received a copy of the GNU General Public License along
>+# with Koha; if not, write to the Free Software Foundation, Inc.,
>+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
>+
>+use C4::Context;
>+use Koha::AtomicUpdater;
>+
>+my $dbh = C4::Context->dbh();
>+my $atomicUpdater = Koha::AtomicUpdater->new();
>+
>+unless($atomicUpdater->find('Bug15108')) {
>+
>+    $dbh->do("ALTER TABLE biblioitems ADD KEY `timestamp` (`timestamp`);");
>+    $dbh->do("ALTER TABLE deletedbiblioitems ADD KEY `timestamp` (`timestamp`);");
>+    $dbh->do("ALTER TABLE items ADD KEY `timestamp` (`timestamp`);");
>+    $dbh->do("ALTER TABLE deleteditems ADD KEY `timestamp` (`timestamp`);");
>+
>+    print "Upgrade done (Bug 15108 - OAI-PMH provider improvements)\n";
>+}
>diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql
>index 514c13d..25d6760 100644
>--- a/installer/data/mysql/kohastructure.sql
>+++ b/installer/data/mysql/kohastructure.sql
>@@ -201,6 +201,7 @@ CREATE TABLE `biblioitems` ( -- information related to bibliographic records in
>   KEY `isbn` (`isbn`(255)),
>   KEY `issn` (`issn`(255)),
>   KEY `publishercode` (`publishercode`),
>+  KEY `timestamp` (`timestamp`),
>   CONSTRAINT `biblioitems_ibfk_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE CASCADE
> ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
> 
>@@ -553,7 +554,8 @@ CREATE TABLE `deletedbiblioitems` ( -- information about bibliographic records t
>   KEY `bibnoidx` (`biblionumber`),
>   KEY `itemtype_idx` (`itemtype`),
>   KEY `isbn` (`isbn`(255)),
>-  KEY `publishercode` (`publishercode`)
>+  KEY `publishercode` (`publishercode`),
>+  KEY `timestamp` (`timestamp`)
> ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
> 
> --
>@@ -694,7 +696,8 @@ CREATE TABLE `deleteditems` (
>   KEY `delitembibnoidx` (`biblionumber`),
>   KEY `delhomebranch` (`homebranch`),
>   KEY `delholdingbranch` (`holdingbranch`),
>-  KEY `itype_idx` (`itype`)
>+  KEY `itype_idx` (`itype`),
>+  KEY `timestamp` (`timestamp`)
> ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
> 
> --
>@@ -899,7 +902,7 @@ CREATE TABLE `refund_lost_item_fee_rules` ( -- refund lost item fee rules tbale
> --
> 
> DROP TABLE IF EXISTS `items`;
>-CREATE TABLE `items` ( -- holdings/item information 
>+CREATE TABLE `items` ( -- holdings/item information
>   `itemnumber` int(11) NOT NULL auto_increment, -- primary key and unique identifier added by Koha
>   `biblionumber` int(11) NOT NULL default 0, -- foreign key from biblio table used to link this item to the right bib record
>   `biblioitemnumber` int(11) NOT NULL default 0, -- foreign key from the biblioitems table to link to item to additional information
>@@ -955,6 +958,7 @@ CREATE TABLE `items` ( -- holdings/item information
>   KEY `items_location` (`location`),
>   KEY `items_ccode` (`ccode`),
>   KEY `itype_idx` (`itype`),
>+  KEY `timestamp` (`timestamp`),
>   CONSTRAINT `items_ibfk_1` FOREIGN KEY (`biblioitemnumber`) REFERENCES `biblioitems` (`biblioitemnumber`) ON DELETE CASCADE ON UPDATE CASCADE,
>   CONSTRAINT `items_ibfk_2` FOREIGN KEY (`homebranch`) REFERENCES `branches` (`branchcode`) ON UPDATE CASCADE,
>   CONSTRAINT `items_ibfk_3` FOREIGN KEY (`holdingbranch`) REFERENCES `branches` (`branchcode`) ON UPDATE CASCADE,
>@@ -2236,7 +2240,7 @@ CREATE TABLE `userflags` (
> --
> 
> DROP TABLE IF EXISTS `virtualshelves`;
>-CREATE TABLE `virtualshelves` ( -- information about lists (or virtual shelves) 
>+CREATE TABLE `virtualshelves` ( -- information about lists (or virtual shelves)
>   `shelfnumber` int(11) NOT NULL auto_increment, -- unique identifier assigned by Koha
>   `shelfname` varchar(255) default NULL, -- name of the list
>   `owner` int default NULL, -- foreign key linking to the borrowers table (using borrowernumber) for the creator of this list (changed from varchar(80) to int)
>@@ -2431,12 +2435,12 @@ CREATE TABLE `permissions` (
> 
> DROP TABLE IF EXISTS `serialitems`;
> CREATE TABLE `serialitems` (
>-	`itemnumber` int(11) NOT NULL,
>-	`serialid` int(11) NOT NULL,
>-	UNIQUE KEY `serialitemsidx` (`itemnumber`),
>-	KEY `serialitems_sfk_1` (`serialid`),
>-	CONSTRAINT `serialitems_sfk_1` FOREIGN KEY (`serialid`) REFERENCES `serial` (`serialid`) ON DELETE CASCADE ON UPDATE CASCADE,
>-	CONSTRAINT `serialitems_sfk_2` FOREIGN KEY (`itemnumber`) REFERENCES `items` (`itemnumber`) ON DELETE CASCADE ON UPDATE CASCADE
>+  `itemnumber` int(11) NOT NULL,
>+  `serialid` int(11) NOT NULL,
>+  UNIQUE KEY `serialitemsidx` (`itemnumber`),
>+  KEY `serialitems_sfk_1` (`serialid`),
>+  CONSTRAINT `serialitems_sfk_1` FOREIGN KEY (`serialid`) REFERENCES `serial` (`serialid`) ON DELETE CASCADE ON UPDATE CASCADE,
>+  CONSTRAINT `serialitems_sfk_2` FOREIGN KEY (`itemnumber`) REFERENCES `items` (`itemnumber`) ON DELETE CASCADE ON UPDATE CASCADE
> ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
> 
> --
>-- 
>2.1.4

Actions: View | Diff | Splinter Review

Attachments on bug 15108: 44317 | 44318 | 51962 | 52416 | 52422 | 52423 | 53853 | 53854 | 56813 | 56844 | 58058 | 58059 | 58060 | 58061 | 58062 | 58063 | 58064 | 58065 | 58072 | 58080 | 58245 | 58246 | 58247 | 58248 | 58249 | 58250 | 58251 | 60260 | 60331 | 60608 | 60835 | 60836 | 63172 | 63173 | 63174