Bugzilla – Attachment 73765 Details for
Bug 20271
Merge deleted biblio, biblioitems, biblio_metadata, and items tables
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 20271 - Use a single pass in the OAI-PMH provider
Bug-20271---Use-a-single-pass-in-the-OAI-PMH-provi.patch (text/plain), 17.99 KB, created by
Benjamin Rokseth
on 2018-04-06 12:25:38 UTC
(
hide
)
Description:
Bug 20271 - Use a single pass in the OAI-PMH provider
Filename:
MIME Type:
Creator:
Benjamin Rokseth
Created:
2018-04-06 12:25:38 UTC
Size:
17.99 KB
patch
obsolete
>From a6eb1ad8247b09b665a799086035b09d3f57f15d Mon Sep 17 00:00:00 2001 >From: Ere Maijala <ere.maijala@helsinki.fi> >Date: Wed, 4 Apr 2018 10:49:13 +0300 >Subject: [PATCH] Bug 20271 - Use a single pass in the OAI-PMH provider > >--- > Koha/OAI/Server/ListBase.pm | 194 ++++++++++++++++--------------------- > Koha/OAI/Server/ResumptionToken.pm | 15 +-- > t/db_dependent/OAI/Server.t | 34 +++---- > 3 files changed, 102 insertions(+), 141 deletions(-) > >diff --git a/Koha/OAI/Server/ListBase.pm b/Koha/OAI/Server/ListBase.pm >index c2a835d..da03014 100644 >--- a/Koha/OAI/Server/ListBase.pm >+++ b/Koha/OAI/Server/ListBase.pm >@@ -1,6 +1,6 @@ > package Koha::OAI::Server::ListBase; > >-# Copyright The National Library of Finland, University of Helsinki 2016-2017 >+# Copyright The National Library of Finland, University of Helsinki 2016-2018 > # > # This file is part of Koha. > # >@@ -46,136 +46,104 @@ sub GetRecords { > $set = GetOAISetBySpec($token->{'set'}); > } > my $offset = $token->{offset}; >- my $deleted = defined $token->{deleted} ? $token->{deleted} : 0; >- my $deleted_count = defined $token->{deleted_count} ? $token->{deleted_count} : 0; > my $max = $repository->{koha_max_count}; > my $count = 0; > my $format = $args{metadataPrefix} || $token->{metadata_prefix}; > my $include_items = $repository->items_included( $format ); > >- # Since creating a union of normal and deleted record tables would be a heavy >- # operation in a large database, build results in two stages: >- # first deleted records ($deleted == 1), then normal records ($deleted == 0) >- STAGELOOP: >- for ( ; $deleted >= 0; $deleted-- ) { >- my $table = $deleted ? 'deletedbiblio_metadata' : 'biblio_metadata'; >- my $sql = " >- SELECT biblionumber >- FROM $table >- WHERE (timestamp >= ? AND timestamp <= ?) >- "; >- my @bind_params = ($token->{'from_arg'}, $token->{'until_arg'}); >- >- if ($include_items) { >- $sql .= " >- OR biblionumber IN (SELECT biblionumber from items WHERE timestamp >= ? AND timestamp <= ? AND deleted_at IS NOT NULL) >- "; >- push @bind_params, ($token->{'from_arg'}, $token->{'until_arg'}); >- if (!$deleted) { >- $sql .= " >- OR biblionumber IN (SELECT biblionumber from items WHERE timestamp >= ? AND timestamp <= ? AND deleted_at IS NULL) >- "; >- push @bind_params, ($token->{'from_arg'}, $token->{'until_arg'}); >- } >- } >+ my $sql = " >+ SELECT biblionumber, deleted_at >+ FROM biblio_metadata >+ WHERE (timestamp >= ? AND timestamp <= ?) >+ "; >+ my @bind_params = ($token->{'from_arg'}, $token->{'until_arg'}); > >+ if ($include_items) { > $sql .= " >- ORDER BY biblionumber >+ OR biblionumber IN (SELECT biblionumber from items WHERE timestamp >= ? AND timestamp <= ?) > "; >+ push @bind_params, ($token->{'from_arg'}, $token->{'until_arg'}); >+ } > >- # Use a subquery for sets since it allows us to use an index in >- # biblioitems table and is quite a bit faster than a join. >- if (defined $set) { >- $sql = " >- SELECT bi.* FROM ($sql) bi >- WHERE bi.biblionumber in (SELECT osb.biblionumber FROM oai_sets_biblios osb WHERE osb.set_id = ?) >- "; >- push @bind_params, $set->{'id'}; >- } >+ $sql .= " >+ ORDER BY biblionumber >+ "; > >- $sql .= " >- LIMIT " . ($max + 1) . " >- OFFSET " . ($offset - $deleted_count); >- >- my $sth = $dbh->prepare( $sql ) || die( 'Could not prepare statement: ' . $dbh->errstr ); >- >- if ( $deleted ) { >- $sql = " >- SELECT MAX(timestamp) >- FROM ( >- SELECT timestamp FROM deletedbiblio_metadata WHERE biblionumber = ? >- UNION >- SELECT timestamp FROM items WHERE biblionumber = ? AND deleted_at IS NOT NULL >- ) bis >- "; >- } else { >- $sql = " >- SELECT MAX(timestamp) >- FROM ( >- SELECT timestamp FROM biblio_metadata WHERE biblionumber = ? >- UNION >- SELECT timestamp FROM items WHERE biblionumber = ? AND deleted_at IS NULL >- UNION >- SELECT timestamp FROM items WHERE biblionumber = ? AND deleted_at IS NOT NULL >- ) bi >- "; >+ # Use a subquery for sets since it allows us to use an index in >+ # biblioitems table and is quite a bit faster than a join. >+ if (defined $set) { >+ $sql = " >+ SELECT bi.* FROM ($sql) bi >+ WHERE bi.biblionumber in (SELECT osb.biblionumber FROM oai_sets_biblios osb WHERE osb.set_id = ?) >+ "; >+ push @bind_params, $set->{'id'}; >+ } >+ >+ $sql .= " >+ LIMIT " . ($max + 1) . " >+ OFFSET $offset"; >+ >+ my $sth = $dbh->prepare( $sql ) || die( 'Could not prepare statement: ' . $dbh->errstr ); >+ >+ $sql = " >+ SELECT MAX(timestamp) >+ FROM ( >+ SELECT timestamp FROM biblio_metadata WHERE biblionumber = ? >+ UNION >+ SELECT timestamp FROM items WHERE biblionumber = ? >+ ) bi >+ "; >+ my $record_sth = $dbh->prepare( $sql ) || die( 'Could not prepare statement: ' . $dbh->errstr ); >+ >+ $sth->execute( @bind_params ) || die( 'Could not execute statement: ' . $sth->errstr ); >+ while ( my ($biblionumber, $deleted_at) = $sth->fetchrow ) { >+ $count++; >+ if ( $count > $max ) { >+ $self->resumptionToken( >+ new Koha::OAI::Server::ResumptionToken( >+ metadataPrefix => $token->{metadata_prefix}, >+ from => $token->{from}, >+ until => $token->{until}, >+ offset => $token->{offset} + $max, >+ set => $token->{set}, >+ ) >+ ); >+ last; > } >- my $record_sth = $dbh->prepare( $sql ) || die( 'Could not prepare statement: ' . $dbh->errstr ); >- >- $sth->execute( @bind_params ) || die( 'Could not execute statement: ' . $sth->errstr ); >- while ( my ($biblionumber) = $sth->fetchrow ) { >- $count++; >- if ( $count > $max ) { >- $self->resumptionToken( >- new Koha::OAI::Server::ResumptionToken( >- metadataPrefix => $token->{metadata_prefix}, >- from => $token->{from}, >- until => $token->{until}, >- offset => $token->{offset} + $max, >- set => $token->{set}, >- deleted => $deleted, >- deleted_count => $deleted_count >- ) >- ); >- last STAGELOOP; >- } >- my @params = $deleted ? ( $biblionumber, $biblionumber ) : ( $biblionumber, $biblionumber, $biblionumber ); >- $record_sth->execute( @params ) || die( 'Could not execute statement: ' . $sth->errstr ); >+ $record_sth->execute( ( $biblionumber, $biblionumber ) ) || die( 'Could not execute statement: ' . $sth->errstr ); > >- my ($timestamp) = $record_sth->fetchrow; >+ my ($timestamp) = $record_sth->fetchrow; > >- my $oai_sets = GetOAISetsBiblio($biblionumber); >- my @setSpecs; >- foreach ( @$oai_sets ) { >- push @setSpecs, $_->{spec}; >- } >- if ( $metadata ) { >- my $marcxml = !$deleted ? $repository->get_biblio_marcxml($biblionumber, $format) : undef; >- if ( $marcxml ) { >- $self->record( Koha::OAI::Server::Record->new( >- $repository, $marcxml, $timestamp, \@setSpecs, >- identifier => $repository->{ koha_identifier } . ':' . $biblionumber, >- metadataPrefix => $token->{metadata_prefix} >- ) ); >- } else { >- $self->record( Koha::OAI::Server::DeletedRecord->new( >- $timestamp, \@setSpecs, identifier => $repository->{ koha_identifier } . ':' . $biblionumber >- ) ); >- } >+ my $oai_sets = GetOAISetsBiblio($biblionumber); >+ my @setSpecs; >+ foreach ( @$oai_sets ) { >+ push @setSpecs, $_->{spec}; >+ } >+ if ( $metadata ) { >+ my $marcxml = !$deleted_at ? $repository->get_biblio_marcxml($biblionumber, $format) : undef; >+ if ( $marcxml ) { >+ $self->record( Koha::OAI::Server::Record->new( >+ $repository, $marcxml, $timestamp, \@setSpecs, >+ identifier => $repository->{ koha_identifier } . ':' . $biblionumber, >+ metadataPrefix => $token->{metadata_prefix} >+ ) ); > } else { >- $timestamp =~ s/ /T/; >- $timestamp .= 'Z'; >- $self->identifier( new HTTP::OAI::Header( >- identifier => $repository->{ koha_identifier} . ':' . $biblionumber, >- datestamp => $timestamp, >- status => $deleted ? 'deleted' : undef >+ $self->record( Koha::OAI::Server::DeletedRecord->new( >+ $timestamp, \@setSpecs, identifier => $repository->{ koha_identifier } . ':' . $biblionumber > ) ); > } >+ } else { >+ $timestamp =~ s/ /T/; >+ $timestamp .= 'Z'; >+ $self->identifier( new HTTP::OAI::Header( >+ identifier => $repository->{ koha_identifier} . ':' . $biblionumber, >+ datestamp => $timestamp, >+ status => $deleted_at ? 'deleted' : undef >+ ) ); > } >- # Store offset and deleted record count >- $offset += $count; >- $deleted_count = $offset if ($deleted); > } >+ # Store offset >+ $offset += $count; > return $count; > } > >diff --git a/Koha/OAI/Server/ResumptionToken.pm b/Koha/OAI/Server/ResumptionToken.pm >index ee97771..c308042 100644 >--- a/Koha/OAI/Server/ResumptionToken.pm >+++ b/Koha/OAI/Server/ResumptionToken.pm >@@ -1,6 +1,6 @@ > # Copyright Tamil s.a.r.l. 2008-2015 > # Copyright Biblibre 2008-2015 >-# Copyright The National Library of Finland, University of Helsinki 2016-2017 >+# Copyright The National Library of Finland, University of Helsinki 2016-2018 > # > # This file is part of Koha. > # >@@ -32,17 +32,15 @@ use base ("HTTP::OAI::ResumptionToken"); > # - from > # - until > # - offset >-# - deleted > > sub new { > my ($class, %args) = @_; > > my $self = $class->SUPER::new(%args); > >- my ($metadata_prefix, $offset, $from, $until, $set, $deleted, $deleted_count); >+ my ($metadata_prefix, $offset, $from, $until, $set); > if ( $args{ resumptionToken } ) { >- ($metadata_prefix, $offset, $from, $until, $set, $deleted, $deleted_count) >- = split( '/', $args{resumptionToken} ); >+ ($metadata_prefix, $offset, $from, $until, $set) = split( '/', $args{resumptionToken} ); > } > else { > $metadata_prefix = $args{ metadataPrefix }; >@@ -57,8 +55,6 @@ sub new { > $until .= 'T23:59:59Z' if length($until) == 10; > $offset = $args{ offset } || 0; > $set = $args{ set } || ''; >- $deleted = defined $args{ deleted } ? $args{ deleted } : 1; >- $deleted_count = defined $args{ deleted_count } ? $args{ deleted_count } : 0; > } > > $self->{ metadata_prefix } = $metadata_prefix; >@@ -68,11 +64,8 @@ sub new { > $self->{ set } = $set; > $self->{ from_arg } = _strip_UTC_designators($from); > $self->{ until_arg } = _strip_UTC_designators($until); >- $self->{ deleted } = $deleted; >- $self->{ deleted_count } = $deleted_count; > >- $self->resumptionToken( >- join( '/', $metadata_prefix, $offset, $from, $until, $set, $deleted, $deleted_count ) ); >+ $self->resumptionToken( join( '/', $metadata_prefix, $offset, $from, $until, $set ) ); > $self->cursor( $offset ); > > return $self; >diff --git a/t/db_dependent/OAI/Server.t b/t/db_dependent/OAI/Server.t >index 87d5c8b..b2da96a 100644 >--- a/t/db_dependent/OAI/Server.t >+++ b/t/db_dependent/OAI/Server.t >@@ -194,7 +194,7 @@ test_query('ListIdentifiers', {verb => 'ListIdentifiers', metadataPrefix => 'mar > ListIdentifiers => { > header => [ @header[0..2] ], > resumptionToken => { >- content => "marcxml/3/1970-01-01T00:00:00Z/$date_to//0/0", >+ content => "marcxml/3/1970-01-01T00:00:00Z/$date_to/", > cursor => 3, > }, > }, >@@ -204,7 +204,7 @@ test_query('ListIdentifiers', {verb => 'ListIdentifiers', metadataPrefix => 'mar > ListIdentifiers => { > header => [ @header[0..2] ], > resumptionToken => { >- content => "marcxml/3/1970-01-01T00:00:00Z/$date_to//0/0", >+ content => "marcxml/3/1970-01-01T00:00:00Z/$date_to/", > cursor => 3, > }, > }, >@@ -217,7 +217,7 @@ test_query( > ListIdentifiers => { > header => [ @header[3..5] ], > resumptionToken => { >- content => "marcxml/6/1970-01-01T00:00:00Z/$date_to//0/0", >+ content => "marcxml/6/1970-01-01T00:00:00Z/$date_to/", > cursor => 6, > }, > }, >@@ -231,7 +231,7 @@ test_query( > ListIdentifiers => { > header => [ @header[6..8] ], > resumptionToken => { >- content => "marcxml/9/1970-01-01T00:00:00Z/$date_to//0/0", >+ content => "marcxml/9/1970-01-01T00:00:00Z/$date_to/", > cursor => 9, > }, > }, >@@ -240,7 +240,7 @@ test_query( > > test_query( > 'ListIdentifiers with resumptionToken 3, response without resumption', >- { verb => 'ListIdentifiers', resumptionToken => "marcxml/9/1970-01-01T00:00:00Z/$date_to//0/0" }, >+ { verb => 'ListIdentifiers', resumptionToken => "marcxml/9/1970-01-01T00:00:00Z/$date_to/" }, > { > ListIdentifiers => { > header => $header[9], >@@ -259,7 +259,7 @@ test_query('ListRecords marcxml', {verb => 'ListRecords', metadataPrefix => 'mar > ListRecords => { > record => [ @marcxml[0..2] ], > resumptionToken => { >- content => "marcxml/3/1970-01-01T00:00:00Z/$date_to//0/0", >+ content => "marcxml/3/1970-01-01T00:00:00Z/$date_to/", > cursor => 3, > }, > }, >@@ -267,11 +267,11 @@ test_query('ListRecords marcxml', {verb => 'ListRecords', metadataPrefix => 'mar > > test_query( > 'ListRecords marcxml with resumptionToken 1', >- { verb => 'ListRecords', resumptionToken => "marcxml/3/1970-01-01T00:00:00Z/$date_to//0/0" }, >+ { verb => 'ListRecords', resumptionToken => "marcxml/3/1970-01-01T00:00:00Z/$date_to/" }, > { ListRecords => { > record => [ @marcxml[3..5] ], > resumptionToken => { >- content => "marcxml/6/1970-01-01T00:00:00Z/$date_to//0/0", >+ content => "marcxml/6/1970-01-01T00:00:00Z/$date_to/", > cursor => 6, > }, > }, >@@ -279,11 +279,11 @@ test_query( > > test_query( > 'ListRecords marcxml with resumptionToken 2', >- { verb => 'ListRecords', resumptionToken => "marcxml/6/1970-01-01T00:00:00Z/$date_to//0/0" }, >+ { verb => 'ListRecords', resumptionToken => "marcxml/6/1970-01-01T00:00:00Z/$date_to/" }, > { ListRecords => { > record => [ @marcxml[6..8] ], > resumptionToken => { >- content => "marcxml/9/1970-01-01T00:00:00Z/$date_to//0/0", >+ content => "marcxml/9/1970-01-01T00:00:00Z/$date_to/", > cursor => 9, > }, > }, >@@ -292,7 +292,7 @@ test_query( > # Last record, so no resumption token > test_query( > 'ListRecords marcxml with resumptionToken 3, response without resumption', >- { verb => 'ListRecords', resumptionToken => "marcxml/9/1970-01-01T00:00:00Z/$date_to//0/0" }, >+ { verb => 'ListRecords', resumptionToken => "marcxml/9/1970-01-01T00:00:00Z/$date_to/" }, > { ListRecords => { > record => $marcxml[9], > }, >@@ -302,7 +302,7 @@ test_query('ListRecords oai_dc', {verb => 'ListRecords', metadataPrefix => 'oai_ > ListRecords => { > record => [ @oaidc[0..2] ], > resumptionToken => { >- content => "oai_dc/3/1970-01-01T00:00:00Z/$date_to//0/0", >+ content => "oai_dc/3/1970-01-01T00:00:00Z/$date_to/", > cursor => 3, > }, > }, >@@ -310,11 +310,11 @@ test_query('ListRecords oai_dc', {verb => 'ListRecords', metadataPrefix => 'oai_ > > test_query( > 'ListRecords oai_dc with resumptionToken 1', >- { verb => 'ListRecords', resumptionToken => "oai_dc/3/1970-01-01T00:00:00Z/$date_to//0/0" }, >+ { verb => 'ListRecords', resumptionToken => "oai_dc/3/1970-01-01T00:00:00Z/$date_to/" }, > { ListRecords => { > record => [ @oaidc[3..5] ], > resumptionToken => { >- content => "oai_dc/6/1970-01-01T00:00:00Z/$date_to//0/0", >+ content => "oai_dc/6/1970-01-01T00:00:00Z/$date_to/", > cursor => 6, > }, > }, >@@ -322,11 +322,11 @@ test_query( > > test_query( > 'ListRecords oai_dc with resumptionToken 2', >- { verb => 'ListRecords', resumptionToken => "oai_dc/6/1970-01-01T00:00:00Z/$date_to//0/0" }, >+ { verb => 'ListRecords', resumptionToken => "oai_dc/6/1970-01-01T00:00:00Z/$date_to/" }, > { ListRecords => { > record => [ @oaidc[6..8] ], > resumptionToken => { >- content => "oai_dc/9/1970-01-01T00:00:00Z/$date_to//0/0", >+ content => "oai_dc/9/1970-01-01T00:00:00Z/$date_to/", > cursor => 9, > }, > }, >@@ -335,7 +335,7 @@ test_query( > # Last record, so no resumption token > test_query( > 'ListRecords oai_dc with resumptionToken 3, response without resumption', >- { verb => 'ListRecords', resumptionToken => "oai_dc/9/1970-01-01T00:00:00Z/$date_to//0/0" }, >+ { verb => 'ListRecords', resumptionToken => "oai_dc/9/1970-01-01T00:00:00Z/$date_to/" }, > { ListRecords => { > record => $oaidc[9], > }, >-- >2.1.4
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 20271
:
72300
|
72799
|
72800
|
73091
|
73092
|
73093
|
73535
|
73536
|
73537
|
73538
|
73539
|
73540
|
73597
|
73759
|
73760
|
73761
|
73762
|
73763
|
73764
|
73765
|
74127
|
74128
|
74129
|
74130
|
74131
|
74132
|
74133
|
74134
|
74135
|
74136
|
74137
|
74138
|
74139
|
74140
|
74141
|
74142
|
74182
|
74183
|
74184
|
74185
|
74186
|
74187
|
74188
|
74189
|
74190
|
74223
|
74407
|
74408
|
74409
|
74410
|
74411
|
74412
|
74413
|
74414
|
74415
|
74416
|
75265
|
75347
|
75894
|
75897
|
75898
|
75899
|
75910
|
75977
|
75978
|
75979
|
75980
|
75981
|
75982
|
75983
|
75984
|
75985
|
75986
|
75987
|
75988
|
75989
|
75990
|
75991
|
75992
|
75993
|
76047
|
76048
|
76255
|
76256
|
76257
|
76258
|
76259
|
76260
|
76261
|
76262
|
76263
|
76264
|
76265
|
76266
|
76267
|
76268
|
76269
|
76270
|
76271
|
76272
|
76273
|
76274
|
76275
|
76276
|
76277
|
76493
|
76494
|
76495
|
76496
|
76497
|
76498
|
76499
|
76500
|
76501
|
76502
|
76503
|
76504
|
76505
|
76506
|
76507
|
76508
|
76509
|
76510
|
76511
|
76512
|
76513
|
76514