Bugzilla – Attachment 44317 Details for
Bug 15108
OAI-PMH provider improvements
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 15108 - OAI-PMH provider improvements
Bug-15108---OAI-PMH-provider-improvements.patch (text/plain), 18.32 KB, created by
Olli-Antti Kivilahti
on 2015-11-03 09:04:50 UTC
(
hide
)
Description:
Bug 15108 - OAI-PMH provider improvements
Filename:
MIME Type:
Creator:
Olli-Antti Kivilahti
Created:
2015-11-03 09:04:50 UTC
Size:
18.32 KB
patch
obsolete
>From 6558ba57a4f17aa45540c2887f7450a291ef3c8b Mon Sep 17 00:00:00 2001 >From: Ere Maijala <ere.maijala@helsinki.fi> >Date: Tue, 27 Oct 2015 10:10:11 +0200 >Subject: [PATCH] Bug 15108 - OAI-PMH provider improvements > >- Fixed date handling to use UTC as specs require. >- Added support for deleted records. >- Added support for second precision in time stamps. >- Added support for marc21 metadata prefix as recommended in the guidelines (synonym for marcxml). >- ListIdentifiers and ListRecords always return records in biblionumber order. >- Improved performance of database queries especially for large collections. >--- > installer/data/mysql/kohastructure.sql | 4 +- > opac/oai.pl | 294 +++++++++++++++++++++------------ > 2 files changed, 194 insertions(+), 104 deletions(-) > >diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql >index e9d836c..c2eec84 100644 >--- a/installer/data/mysql/kohastructure.sql >+++ b/installer/data/mysql/kohastructure.sql >@@ -209,6 +209,7 @@ CREATE TABLE `biblioitems` ( -- information related to bibliographic records in > KEY `isbn` (`isbn`(255)), > KEY `issn` (`issn`(255)), > KEY `publishercode` (`publishercode`), >+ KEY `timestamp_bibno` (`timestamp`, `biblionumber`), > CONSTRAINT `biblioitems_ibfk_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE CASCADE > ) ENGINE=InnoDB DEFAULT CHARSET=utf8; > >@@ -866,7 +867,8 @@ CREATE TABLE `deletedbiblioitems` ( -- information about bibliographic records t > KEY `bibnoidx` (`biblionumber`), > KEY `itemtype_idx` (`itemtype`), > KEY `isbn` (`isbn`(255)), >- KEY `publishercode` (`publishercode`) >+ KEY `publishercode` (`publishercode`), >+ KEY `timestamp_bibno` (`timestamp`, `biblionumber`) > ) ENGINE=InnoDB DEFAULT CHARSET=utf8; > > -- >diff --git a/opac/oai.pl b/opac/oai.pl >index b453408..2491a5a 100755 >--- a/opac/oai.pl >+++ b/opac/oai.pl >@@ -1,27 +1,28 @@ > #!/usr/bin/perl > > # Copyright Biblibre 2008 >+# Copyright The National Library of Finland 2015 > # > # This file is part of Koha. > # >-# Koha is free software; you can redistribute it and/or modify it under the >-# terms of the GNU General Public License as published by the Free Software >-# Foundation; either version 2 of the License, or (at your option) any later >-# version. >+# Koha is free software; you can redistribute it and/or modify it >+# under the terms of the GNU General Public License as published by >+# the Free Software Foundation; either version 3 of the License, or >+# (at your option) any later version. > # >-# Koha is distributed in the hope that it will be useful, but WITHOUT ANY >-# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR >-# A PARTICULAR PURPOSE. See the GNU General Public License for more details. >+# Koha is distributed in the hope that it will be useful, but >+# WITHOUT ANY WARRANTY; without even the implied warranty of >+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >+# GNU General Public License for more details. > # >-# You should have received a copy of the GNU General Public License along >-# with Koha; if not, write to the Free Software Foundation, Inc., >-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. >+# You should have received a copy of the GNU General Public License >+# along with Koha; if not, see <http://www.gnu.org/licenses>. > > > use strict; > use warnings; > >-use CGI qw/:standard -oldstyle_urls/; >+use CGI qw( :standard -oldstyle_urls -utf8 ); > use vars qw( $GZIP ); > use C4::Context; > >@@ -59,6 +60,9 @@ else { > } > > binmode STDOUT, ':encoding(UTF-8)'; >+# OAI-PMH handles dates in UTC, so do that on the database level to avoid the >+# need for any conversions >+C4::Context->dbh->prepare("SET time_zone='+00:00'")->execute(); > my $repository = C4::OAI::Repository->new(); > > # __END__ Main Prog >@@ -71,7 +75,7 @@ my $repository = C4::OAI::Repository->new(); > # - from > # - until > # - offset >-# >+# > package C4::OAI::ResumptionToken; > > use strict; >@@ -86,9 +90,9 @@ sub new { > > my $self = $class->SUPER::new(%args); > >- my ($metadata_prefix, $offset, $from, $until, $set); >+ my ($metadata_prefix, $offset, $from, $until, $set, $stage); > if ( $args{ resumptionToken } ) { >- ($metadata_prefix, $offset, $from, $until, $set) >+ ($metadata_prefix, $offset, $from, $until, $set, $stage) > = split( '/', $args{resumptionToken} ); > } > else { >@@ -101,6 +105,7 @@ sub new { > } > $offset = $args{ offset } || 0; > $set = $args{set}; >+ $stage = (defined $args{stage}) ? $args{stage} : 0; > } > > $self->{ metadata_prefix } = $metadata_prefix; >@@ -108,9 +113,10 @@ sub new { > $self->{ from } = $from; > $self->{ until } = $until; > $self->{ set } = $set; >+ $self->{ stage } = $stage; > > $self->resumptionToken( >- join( '/', $metadata_prefix, $offset, $from, $until, $set ) ); >+ join( '/', $metadata_prefix, $offset, $from, $until, $set, $stage ) ); > $self->cursor( $offset ); > > return $self; >@@ -138,9 +144,9 @@ sub new { > repositoryName => C4::Context->preference("LibraryName"), > adminEmail => C4::Context->preference("KohaAdminEmailAddress"), > MaxCount => C4::Context->preference("OAI-PMH:MaxCount"), >- granularity => 'YYYY-MM-DD', >- earliestDatestamp => '0001-01-01', >- deletedRecord => 'no', >+ granularity => 'YYYY-MM-DDThh:mm:ssZ', >+ earliestDatestamp => '0001-01-01T00:00:00Z', >+ deletedRecord => 'persistent', > ); > > # FIXME - alas, the description element is not so simple; to validate >@@ -187,6 +193,11 @@ sub new { > metadataNamespace => 'http://www.openarchives.org/OAI/2.0/oai_dc/' > ) ); > $self->metadataFormat( HTTP::OAI::MetadataFormat->new( >+ metadataPrefix => 'marc21', >+ schema => 'http://www.loc.gov/MARC21/slim http://www.loc.gov/ standards/marcxml/schema/MARC21slim.xsd', >+ metadataNamespace => 'http://www.loc.gov/MARC21/slim http://www.loc.gov/ standards/marcxml/schema/MARC21slim' >+ ) ); >+ $self->metadataFormat( HTTP::OAI::MetadataFormat->new( > metadataPrefix => 'marcxml', > schema => 'http://www.loc.gov/MARC21/slim http://www.loc.gov/ standards/marcxml/schema/MARC21slim.xsd', > metadataNamespace => 'http://www.loc.gov/MARC21/slim http://www.loc.gov/ standards/marcxml/schema/MARC21slim' >@@ -210,7 +221,7 @@ use HTTP::OAI::Metadata::OAI_DC; > use base ("HTTP::OAI::Record"); > > sub new { >- my ($class, $repository, $marcxml, $timestamp, $setSpecs, %args) = @_; >+ my ($class, $repository, $marcxml, $timestamp, $setSpecs, $deleted, %args) = @_; > > my $self = $class->SUPER::new(%args); > >@@ -218,22 +229,25 @@ sub new { > $self->header( new HTTP::OAI::Header( > identifier => $args{identifier}, > datestamp => $timestamp, >+ status => $deleted ? 'deleted' : undef > ) ); > > foreach my $setSpec (@$setSpecs) { > $self->header->setSpec($setSpec); > } > >- my $parser = XML::LibXML->new(); >- my $record_dom = $parser->parse_string( $marcxml ); >- my $format = $args{metadataPrefix}; >- if ( $format ne 'marcxml' ) { >- my %args = ( >- OPACBaseURL => "'" . C4::Context->preference('OPACBaseURL') . "'" >- ); >- $record_dom = $repository->stylesheet($format)->transform($record_dom, %args); >+ if (!$deleted) { >+ my $parser = XML::LibXML->new(); >+ my $record_dom = $parser->parse_string( $marcxml ); >+ my $format = $args{metadataPrefix}; >+ if ( $format ne 'marcxml' && $format ne 'marc21' ) { >+ my %args = ( >+ OPACBaseURL => "'" . C4::Context->preference('OPACBaseURL') . "'" >+ ); >+ $record_dom = $repository->stylesheet($format)->transform($record_dom, %args); >+ } >+ $self->metadata( HTTP::OAI::Metadata->new( dom => $record_dom ) ); > } >- $self->metadata( HTTP::OAI::Metadata->new( dom => $record_dom ) ); > > return $self; > } >@@ -314,40 +328,48 @@ sub new { > if(defined $token->{'set'}) { > $set = GetOAISetBySpec($token->{'set'}); > } >- my $sql = " >- SELECT biblioitems.biblionumber, biblioitems.timestamp >- FROM biblioitems >- "; >- $sql .= " JOIN oai_sets_biblios ON biblioitems.biblionumber = oai_sets_biblios.biblionumber " if defined $set; >- $sql .= " WHERE DATE(timestamp) >= ? AND DATE(timestamp) <= ? "; >- $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set; >- $sql .= " >- LIMIT $repository->{'koha_max_count'} >- OFFSET $token->{'offset'} >- "; >- my $sth = $dbh->prepare( $sql ); >- my @bind_params = ($token->{'from'}, $token->{'until'}); >- push @bind_params, $set->{'id'} if defined $set; >- $sth->execute( @bind_params ); >- >- my $pos = $token->{offset}; >- while ( my ($biblionumber, $timestamp) = $sth->fetchrow ) { >- $timestamp =~ s/ /T/, $timestamp .= 'Z'; >- $self->identifier( new HTTP::OAI::Header( >- identifier => $repository->{ koha_identifier} . ':' . $biblionumber, >- datestamp => $timestamp, >- ) ); >- $pos++; >+ my $max = $repository->{'koha_max_count'}; >+ my $offset = $token->{'offset'}; >+ >+ my $count = 0; >+ # Since creating a union of normal and deleted record tables would be a heavy >+ # operation, build results in two stages: first deleted records, then normal records >+ MAINLOOP: >+ for ( my $stage = $token->{'stage'}; $stage < 2; $stage++ ) { >+ my $sql = C4::OAI::Utils->build_list_sql($stage == 0, $set, $max, $offset); >+ my $sth = $dbh->prepare( $sql ) || die('Query preparation failed: ' . $dbh->errstr); >+ my @bind_params = (C4::OAI::Utils->fix_date($token->{'from'}, 0), C4::OAI::Utils->fix_date($token->{'until'}, 1)); >+ push @bind_params, $set->{'id'} if defined $set; >+ $sth->execute( @bind_params ) || die('Query failed: ' . $dbh->errstr); >+ >+ while ( my ($biblionumber, $timestamp) = $sth->fetchrow ) { >+ $count++; >+ if ( $count > $max ) { >+ $self->resumptionToken( >+ new C4::OAI::ResumptionToken( >+ metadataPrefix => $token->{metadata_prefix}, >+ from => $token->{from}, >+ until => $token->{until}, >+ offset => $offset + $max, >+ set => $token->{set}, >+ stage => $stage >+ ) >+ ); >+ last MAINLOOP; >+ } >+ $timestamp =~ s/ /T/, $timestamp .= 'Z'; >+ my $header = new HTTP::OAI::Header( >+ identifier => $repository->{ koha_identifier} . ':' . $biblionumber, >+ datestamp => $timestamp, >+ ); >+ if ($stage == 0) { >+ $header->status( 'deleted' ); >+ } >+ $self->identifier( $header ); >+ } >+ # Reset offset for next stage >+ $offset = 0; > } >- $self->resumptionToken( >- new C4::OAI::ResumptionToken( >- metadataPrefix => $token->{metadata_prefix}, >- from => $token->{from}, >- until => $token->{until}, >- offset => $pos, >- set => $token->{set} >- ) >- ) if ($pos > $token->{offset}); > > return $self; > } >@@ -467,52 +489,118 @@ sub new { > if(defined $token->{'set'}) { > $set = GetOAISetBySpec($token->{'set'}); > } >- my $sql = " >- SELECT biblioitems.biblionumber, biblioitems.marcxml, biblioitems.timestamp >- FROM biblioitems >- "; >- $sql .= " JOIN oai_sets_biblios ON biblioitems.biblionumber = oai_sets_biblios.biblionumber " if defined $set; >- $sql .= " WHERE DATE(timestamp) >= ? AND DATE(timestamp) <= ? "; >- $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set; >- $sql .= " >- LIMIT $repository->{'koha_max_count'} >- OFFSET $token->{'offset'} >- "; >- >- my $sth = $dbh->prepare( $sql ); >- my @bind_params = ($token->{'from'}, $token->{'until'}); >- push @bind_params, $set->{'id'} if defined $set; >- $sth->execute( @bind_params ); >- >- my $pos = $token->{offset}; >- while ( my ($biblionumber, $marcxml, $timestamp) = $sth->fetchrow ) { >- my $oai_sets = GetOAISetsBiblio($biblionumber); >- my @setSpecs; >- foreach (@$oai_sets) { >- push @setSpecs, $_->{spec}; >+ my $offset = $token->{'offset'}; >+ my $max = $repository->{'koha_max_count'}; >+ my $query = ' >+SELECT biblioitems.marcxml >+FROM biblioitems >+WHERE biblionumber = ? >+'; >+ my $marc_sth = $dbh->prepare($query) || die('Query preparation failed: ' . $dbh->errstr); >+ >+ my $count = 0; >+ # Since creating a union of normal and deleted record tables would be a heavy >+ # operation, build results in two stages: first deleted records, then normal records >+ MAINLOOP: >+ for ( my $stage = $token->{'stage'}; $stage < 2; $stage++ ) { >+ my $sql = C4::OAI::Utils->build_list_sql($stage == 0, $set, $max, $offset); >+ my $sth = $dbh->prepare( $sql ) || die('Query preparation failed: ' . $dbh->errstr); >+ my @bind_params = (C4::OAI::Utils->fix_date($token->{'from'}, 0), C4::OAI::Utils->fix_date($token->{'until'}, 1)); >+ push @bind_params, $set->{'id'} if defined $set; >+ $sth->execute( @bind_params ) || die('Query failed: ' . $dbh->errstr); >+ >+ while ( my ($biblionumber, $timestamp) = $sth->fetchrow ) { >+ $count++; >+ if ( $count > $max ) { >+ $self->resumptionToken( >+ new C4::OAI::ResumptionToken( >+ metadataPrefix => $token->{metadata_prefix}, >+ from => $token->{from}, >+ until => $token->{until}, >+ offset => $offset + $max, >+ set => $token->{set}, >+ stage => $stage >+ ) >+ ); >+ last MAINLOOP; >+ } >+ my $marcxml = ''; >+ if ($stage == 1) { >+ $marc_sth->execute(($biblionumber)) || die('Query failed: ' . $dbh->errstr); >+ my @marc_results = $marc_sth->fetchrow_array(); >+ if (@marc_results) { >+ $marcxml = $marc_results[0]; >+ } >+ } >+ my $oai_sets = GetOAISetsBiblio($biblionumber); >+ my @setSpecs; >+ foreach (@$oai_sets) { >+ push @setSpecs, $_->{spec}; >+ } >+ $self->record( C4::OAI::Record->new( >+ $repository, $marcxml, $timestamp, \@setSpecs, $stage == 0 || !$marcxml, >+ identifier => $repository->{ koha_identifier } . ':' . $biblionumber, >+ metadataPrefix => $token->{metadata_prefix} >+ ) ); > } >- $self->record( C4::OAI::Record->new( >- $repository, $marcxml, $timestamp, \@setSpecs, >- identifier => $repository->{ koha_identifier } . ':' . $biblionumber, >- metadataPrefix => $token->{metadata_prefix} >- ) ); >- $pos++; >+ # Reset offset for next stage >+ $offset = 0; > } >- $self->resumptionToken( >- new C4::OAI::ResumptionToken( >- metadataPrefix => $token->{metadata_prefix}, >- from => $token->{from}, >- until => $token->{until}, >- offset => $pos, >- set => $token->{set} >- ) >- ) if ($pos > $token->{offset}); > > return $self; > } > > # __END__ C4::OAI::ListRecords > >+package C4::OAI::Utils; >+ >+use strict; >+ >+sub fix_date { >+ my ($class, $date, $end) = @_; >+ >+ if (length($date) == 10) { >+ if ($end) { >+ $date .= ' 23:59:59'; >+ } else { >+ $date .= ' 00:00:00'; >+ } >+ } else { >+ $date = substr($date, 0, 10) . ' ' . substr($date, 11, 8); >+ } >+ return $date; >+} >+ >+sub build_list_sql { >+ my ($class, $deleted, $set, $max, $offset) = @_; >+ >+ my $table = $deleted ? 'deletedbiblioitems' : 'biblioitems'; >+ >+ my $sql = " >+SELECT bi.biblionumber, bi.timestamp >+ FROM $table bi >+ WHERE bi.timestamp >= ? AND bi.timestamp <= ? >+ ORDER BY bi.biblionumber >+"; >+ >+ # Use a subquery for sets since it allows us to use an index for the subquery in >+ # biblioitems table and is quite a bit faster than a join. >+ if (defined $set) { >+ $sql = " >+SELECT bi.* FROM ($sql) bi >+ WHERE bi.biblionumber in (SELECT osb.biblionumber FROM oai_sets_biblios osb WHERE osb.set_id = ?) >+"; >+ } >+ $sql .= ' >+ LIMIT ' . ($max + 1) . " >+ OFFSET $offset >+"; >+ >+ return $sql; >+} >+ >+# __END__ C4::OAI::Utils; >+ > > > package C4::OAI::Repository; >@@ -541,7 +629,7 @@ sub new { > > $self->{ koha_identifier } = C4::Context->preference("OAI-PMH:archiveID"); > $self->{ koha_max_count } = C4::Context->preference("OAI-PMH:MaxCount"); >- $self->{ koha_metadata_format } = ['oai_dc', 'marcxml']; >+ $self->{ koha_metadata_format } = ['oai_dc', 'marc21', 'marcxml']; > $self->{ koha_stylesheet } = { }; # Build when needed > > # Load configuration file if defined in OAI-PMH:ConfFile syspref >@@ -639,7 +727,7 @@ C4::OAI::Repository - Handles OAI-PMH requests for a Koha database. > This object extend HTTP::OAI::Repository object. > It accepts OAI-PMH HTTP requests and returns result. > >-This OAI-PMH server can operate in a simple mode and extended one. >+This OAI-PMH server can operate in a simple mode and extended one. > > In simple mode, repository configuration comes entirely from Koha system > preferences (OAI-PMH:archiveID and OAI-PMH:MaxCount) and the server returns >@@ -662,8 +750,8 @@ configuration file koha-oai.conf can look like that: > metadataNamespace: http://veryspecial.tamil.fr/vs/format-pivot/1.1/vs > schema: http://veryspecial.tamil.fr/vs/format-pivot/1.1/vs.xsd > xsl_file: /usr/local/koha/xslt/vs.xsl >- marcxml: >- metadataPrefix: marxml >+ marc21 or marcxml: >+ metadataPrefix: marc21 or marxml > metadataNamespace: http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim > schema: http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd > oai_dc: >-- >1.9.1
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 15108
:
44317
|
44318
|
51962
|
52416
|
52422
|
52423
|
53853
|
53854
|
56813
|
56844
|
58058
|
58059
|
58060
|
58061
|
58062
|
58063
|
58064
|
58065
|
58072
|
58080
|
58245
|
58246
|
58247
|
58248
|
58249
|
58250
|
58251
|
60260
|
60331
|
60608
|
60835
|
60836
|
63172
|
63173
|
63174