View | Details | Raw Unified | Return to bug 20271
Collapse All | Expand All

(-)a/Koha/OAI/Server/ListBase.pm (-113 / +81 lines)
Lines 1-6 Link Here
1
package Koha::OAI::Server::ListBase;
1
package Koha::OAI::Server::ListBase;
2
2
3
# Copyright The National Library of Finland, University of Helsinki 2016-2017
3
# Copyright The National Library of Finland, University of Helsinki 2016-2018
4
#
4
#
5
# This file is part of Koha.
5
# This file is part of Koha.
6
#
6
#
Lines 46-181 sub GetRecords { Link Here
46
        $set = GetOAISetBySpec($token->{'set'});
46
        $set = GetOAISetBySpec($token->{'set'});
47
    }
47
    }
48
    my $offset = $token->{offset};
48
    my $offset = $token->{offset};
49
    my $deleted = defined $token->{deleted} ? $token->{deleted} : 0;
50
    my $deleted_count = defined $token->{deleted_count} ? $token->{deleted_count} : 0;
51
    my $max = $repository->{koha_max_count};
49
    my $max = $repository->{koha_max_count};
52
    my $count = 0;
50
    my $count = 0;
53
    my $format = $args{metadataPrefix} || $token->{metadata_prefix};
51
    my $format = $args{metadataPrefix} || $token->{metadata_prefix};
54
    my $include_items = $repository->items_included( $format );
52
    my $include_items = $repository->items_included( $format );
55
53
56
    # Since creating a union of normal and deleted record tables would be a heavy
54
    my $sql = "
57
    # operation in a large database, build results in two stages:
55
        SELECT biblionumber, deleted_at
58
    # first deleted records ($deleted == 1), then normal records ($deleted == 0)
56
        FROM biblio_metadata
59
    STAGELOOP:
57
        WHERE (timestamp >= ? AND timestamp <= ?)
60
    for ( ; $deleted >= 0; $deleted-- ) {
58
    ";
61
        my $table = $deleted ? 'deletedbiblio_metadata' : 'biblio_metadata';
59
    my @bind_params = ($token->{'from_arg'}, $token->{'until_arg'});
62
        my $sql = "
63
            SELECT biblionumber
64
            FROM $table
65
            WHERE (timestamp >= ? AND timestamp <= ?)
66
        ";
67
        my @bind_params = ($token->{'from_arg'}, $token->{'until_arg'});
68
69
        if ($include_items) {
70
            $sql .= "
71
                OR biblionumber IN (SELECT biblionumber from items WHERE timestamp >= ? AND timestamp <= ? AND deleted_at IS NOT NULL)
72
            ";
73
            push @bind_params, ($token->{'from_arg'}, $token->{'until_arg'});
74
            if (!$deleted) {
75
                $sql .= "
76
                    OR biblionumber IN (SELECT biblionumber from items WHERE timestamp >= ? AND timestamp <= ? AND deleted_at IS NULL)
77
                ";
78
                push @bind_params, ($token->{'from_arg'}, $token->{'until_arg'});
79
            }
80
        }
81
60
61
    if ($include_items) {
82
        $sql .= "
62
        $sql .= "
83
            ORDER BY biblionumber
63
            OR biblionumber IN (SELECT biblionumber from items WHERE timestamp >= ? AND timestamp <= ?)
84
        ";
64
        ";
65
        push @bind_params, ($token->{'from_arg'}, $token->{'until_arg'});
66
    }
85
67
86
        # Use a subquery for sets since it allows us to use an index in
68
    $sql .= "
87
        # biblioitems table and is quite a bit faster than a join.
69
        ORDER BY biblionumber
88
        if (defined $set) {
70
    ";
89
            $sql = "
90
                SELECT bi.* FROM ($sql) bi
91
                  WHERE bi.biblionumber in (SELECT osb.biblionumber FROM oai_sets_biblios osb WHERE osb.set_id = ?)
92
            ";
93
            push @bind_params, $set->{'id'};
94
        }
95
71
96
        $sql .= "
72
    # Use a subquery for sets since it allows us to use an index in
97
            LIMIT " . ($max + 1) . "
73
    # biblioitems table and is quite a bit faster than a join.
98
            OFFSET " . ($offset - $deleted_count);
74
    if (defined $set) {
99
75
        $sql = "
100
        my $sth = $dbh->prepare( $sql ) || die( 'Could not prepare statement: ' . $dbh->errstr );
76
            SELECT bi.* FROM ($sql) bi
101
77
                WHERE bi.biblionumber in (SELECT osb.biblionumber FROM oai_sets_biblios osb WHERE osb.set_id = ?)
102
        if ( $deleted ) {
78
        ";
103
            $sql = "
79
        push @bind_params, $set->{'id'};
104
                SELECT MAX(timestamp)
80
    }
105
                FROM (
81
106
                    SELECT timestamp FROM deletedbiblio_metadata WHERE biblionumber = ?
82
    $sql .= "
107
                    UNION
83
        LIMIT " . ($max + 1) . "
108
                    SELECT timestamp FROM items WHERE biblionumber = ? AND deleted_at IS NOT NULL
84
        OFFSET $offset";
109
                ) bis
85
110
            ";
86
    my $sth = $dbh->prepare( $sql ) || die( 'Could not prepare statement: ' . $dbh->errstr );
111
        } else {
87
112
            $sql = "
88
    $sql = "
113
                SELECT MAX(timestamp)
89
        SELECT MAX(timestamp)
114
                FROM (
90
        FROM (
115
                    SELECT timestamp FROM biblio_metadata WHERE biblionumber = ?
91
            SELECT timestamp FROM biblio_metadata WHERE biblionumber = ?
116
                    UNION
92
            UNION
117
                    SELECT timestamp FROM items WHERE biblionumber = ? AND deleted_at IS NULL
93
            SELECT timestamp FROM items WHERE biblionumber = ?
118
                    UNION
94
        ) bi
119
                    SELECT timestamp FROM items WHERE biblionumber = ? AND deleted_at IS NOT NULL
95
    ";
120
                ) bi
96
    my $record_sth = $dbh->prepare( $sql ) || die( 'Could not prepare statement: ' . $dbh->errstr );
121
            ";
97
98
    $sth->execute( @bind_params ) || die( 'Could not execute statement: ' . $sth->errstr );
99
    while ( my ($biblionumber, $deleted_at) = $sth->fetchrow ) {
100
        $count++;
101
        if ( $count > $max ) {
102
            $self->resumptionToken(
103
                new Koha::OAI::Server::ResumptionToken(
104
                    metadataPrefix  => $token->{metadata_prefix},
105
                    from            => $token->{from},
106
                    until           => $token->{until},
107
                    offset          => $token->{offset} + $max,
108
                    set             => $token->{set},
109
                )
110
            );
111
            last;
122
        }
112
        }
123
        my $record_sth = $dbh->prepare( $sql ) || die( 'Could not prepare statement: ' . $dbh->errstr );
113
        $record_sth->execute( ( $biblionumber, $biblionumber ) ) || die( 'Could not execute statement: ' . $sth->errstr );
124
125
        $sth->execute( @bind_params ) || die( 'Could not execute statement: ' . $sth->errstr );
126
        while ( my ($biblionumber) = $sth->fetchrow ) {
127
            $count++;
128
            if ( $count > $max ) {
129
                $self->resumptionToken(
130
                    new Koha::OAI::Server::ResumptionToken(
131
                        metadataPrefix  => $token->{metadata_prefix},
132
                        from            => $token->{from},
133
                        until           => $token->{until},
134
                        offset          => $token->{offset} + $max,
135
                        set             => $token->{set},
136
                        deleted         => $deleted,
137
                        deleted_count   => $deleted_count
138
                    )
139
                );
140
                last STAGELOOP;
141
            }
142
            my @params = $deleted ? ( $biblionumber, $biblionumber ) : ( $biblionumber, $biblionumber, $biblionumber );
143
            $record_sth->execute( @params ) || die( 'Could not execute statement: ' . $sth->errstr );
144
114
145
            my ($timestamp) = $record_sth->fetchrow;
115
        my ($timestamp) = $record_sth->fetchrow;
146
116
147
            my $oai_sets = GetOAISetsBiblio($biblionumber);
117
        my $oai_sets = GetOAISetsBiblio($biblionumber);
148
            my @setSpecs;
118
        my @setSpecs;
149
            foreach ( @$oai_sets ) {
119
        foreach ( @$oai_sets ) {
150
                push @setSpecs, $_->{spec};
120
            push @setSpecs, $_->{spec};
151
            }
121
        }
152
            if ( $metadata ) {
122
        if ( $metadata ) {
153
                my $marcxml = !$deleted ? $repository->get_biblio_marcxml($biblionumber, $format) : undef;
123
            my $marcxml = !$deleted_at ? $repository->get_biblio_marcxml($biblionumber, $format) : undef;
154
                if ( $marcxml ) {
124
            if ( $marcxml ) {
155
                  $self->record( Koha::OAI::Server::Record->new(
125
                $self->record( Koha::OAI::Server::Record->new(
156
                      $repository, $marcxml, $timestamp, \@setSpecs,
126
                    $repository, $marcxml, $timestamp, \@setSpecs,
157
                      identifier      => $repository->{ koha_identifier } . ':' . $biblionumber,
127
                    identifier      => $repository->{ koha_identifier } . ':' . $biblionumber,
158
                      metadataPrefix  => $token->{metadata_prefix}
128
                    metadataPrefix  => $token->{metadata_prefix}
159
                  ) );
129
                ) );
160
                } else {
161
                  $self->record( Koha::OAI::Server::DeletedRecord->new(
162
                      $timestamp, \@setSpecs, identifier => $repository->{ koha_identifier } . ':' . $biblionumber
163
                  ) );
164
                }
165
            } else {
130
            } else {
166
                $timestamp =~ s/ /T/;
131
                $self->record( Koha::OAI::Server::DeletedRecord->new(
167
                $timestamp .= 'Z';
132
                    $timestamp, \@setSpecs, identifier => $repository->{ koha_identifier } . ':' . $biblionumber
168
                $self->identifier( new HTTP::OAI::Header(
169
                    identifier => $repository->{ koha_identifier} . ':' . $biblionumber,
170
                    datestamp  => $timestamp,
171
                    status     => $deleted ? 'deleted' : undef
172
                ) );
133
                ) );
173
            }
134
            }
135
        } else {
136
            $timestamp =~ s/ /T/;
137
            $timestamp .= 'Z';
138
            $self->identifier( new HTTP::OAI::Header(
139
                identifier => $repository->{ koha_identifier} . ':' . $biblionumber,
140
                datestamp  => $timestamp,
141
                status     => $deleted_at ? 'deleted' : undef
142
            ) );
174
        }
143
        }
175
        # Store offset and deleted record count
176
        $offset += $count;
177
        $deleted_count = $offset if ($deleted);
178
    }
144
    }
145
    # Store offset
146
    $offset += $count;
179
    return $count;
147
    return $count;
180
}
148
}
181
149
(-)a/Koha/OAI/Server/ResumptionToken.pm (-11 / +4 lines)
Lines 1-6 Link Here
1
# Copyright Tamil s.a.r.l. 2008-2015
1
# Copyright Tamil s.a.r.l. 2008-2015
2
# Copyright Biblibre 2008-2015
2
# Copyright Biblibre 2008-2015
3
# Copyright The National Library of Finland, University of Helsinki 2016-2017
3
# Copyright The National Library of Finland, University of Helsinki 2016-2018
4
#
4
#
5
# This file is part of Koha.
5
# This file is part of Koha.
6
#
6
#
Lines 32-48 use base ("HTTP::OAI::ResumptionToken"); Link Here
32
# - from
32
# - from
33
# - until
33
# - until
34
# - offset
34
# - offset
35
# - deleted
36
35
37
sub new {
36
sub new {
38
    my ($class, %args) = @_;
37
    my ($class, %args) = @_;
39
38
40
    my $self = $class->SUPER::new(%args);
39
    my $self = $class->SUPER::new(%args);
41
40
42
    my ($metadata_prefix, $offset, $from, $until, $set, $deleted, $deleted_count);
41
    my ($metadata_prefix, $offset, $from, $until, $set);
43
    if ( $args{ resumptionToken } ) {
42
    if ( $args{ resumptionToken } ) {
44
        ($metadata_prefix, $offset, $from, $until, $set, $deleted, $deleted_count)
43
        ($metadata_prefix, $offset, $from, $until, $set) = split( '/', $args{resumptionToken} );
45
            = split( '/', $args{resumptionToken} );
46
    }
44
    }
47
    else {
45
    else {
48
        $metadata_prefix = $args{ metadataPrefix };
46
        $metadata_prefix = $args{ metadataPrefix };
Lines 57-64 sub new { Link Here
57
        $until .= 'T23:59:59Z' if length($until) == 10;
55
        $until .= 'T23:59:59Z' if length($until) == 10;
58
        $offset = $args{ offset } || 0;
56
        $offset = $args{ offset } || 0;
59
        $set = $args{ set } || '';
57
        $set = $args{ set } || '';
60
        $deleted = defined $args{ deleted } ? $args{ deleted } : 1;
61
        $deleted_count = defined $args{ deleted_count } ? $args{ deleted_count } : 0;
62
    }
58
    }
63
59
64
    $self->{ metadata_prefix } = $metadata_prefix;
60
    $self->{ metadata_prefix } = $metadata_prefix;
Lines 68-78 sub new { Link Here
68
    $self->{ set             } = $set;
64
    $self->{ set             } = $set;
69
    $self->{ from_arg        } = _strip_UTC_designators($from);
65
    $self->{ from_arg        } = _strip_UTC_designators($from);
70
    $self->{ until_arg       } = _strip_UTC_designators($until);
66
    $self->{ until_arg       } = _strip_UTC_designators($until);
71
    $self->{ deleted         } = $deleted;
72
    $self->{ deleted_count   } = $deleted_count;
73
67
74
    $self->resumptionToken(
68
    $self->resumptionToken( join( '/', $metadata_prefix, $offset, $from, $until, $set ) );
75
        join( '/', $metadata_prefix, $offset, $from, $until, $set, $deleted, $deleted_count ) );
76
    $self->cursor( $offset );
69
    $self->cursor( $offset );
77
70
78
    return $self;
71
    return $self;
(-)a/t/db_dependent/OAI/Server.t (-18 / +17 lines)
Lines 194-200 test_query('ListIdentifiers', {verb => 'ListIdentifiers', metadataPrefix => 'mar Link Here
194
    ListIdentifiers => {
194
    ListIdentifiers => {
195
        header => [ @header[0..2] ],
195
        header => [ @header[0..2] ],
196
        resumptionToken => {
196
        resumptionToken => {
197
            content => "marcxml/3/1970-01-01T00:00:00Z/$date_to//0/0",
197
            content => "marcxml/3/1970-01-01T00:00:00Z/$date_to/",
198
            cursor  => 3,
198
            cursor  => 3,
199
        },
199
        },
200
    },
200
    },
Lines 204-210 test_query('ListIdentifiers', {verb => 'ListIdentifiers', metadataPrefix => 'mar Link Here
204
    ListIdentifiers => {
204
    ListIdentifiers => {
205
        header => [ @header[0..2] ],
205
        header => [ @header[0..2] ],
206
        resumptionToken => {
206
        resumptionToken => {
207
            content => "marcxml/3/1970-01-01T00:00:00Z/$date_to//0/0",
207
            content => "marcxml/3/1970-01-01T00:00:00Z/$date_to/",
208
            cursor  => 3,
208
            cursor  => 3,
209
        },
209
        },
210
    },
210
    },
Lines 217-223 test_query( Link Here
217
        ListIdentifiers => {
217
        ListIdentifiers => {
218
            header => [ @header[3..5] ],
218
            header => [ @header[3..5] ],
219
            resumptionToken => {
219
            resumptionToken => {
220
              content => "marcxml/6/1970-01-01T00:00:00Z/$date_to//0/0",
220
              content => "marcxml/6/1970-01-01T00:00:00Z/$date_to/",
221
              cursor  => 6,
221
              cursor  => 6,
222
            },
222
            },
223
          },
223
          },
Lines 231-237 test_query( Link Here
231
        ListIdentifiers => {
231
        ListIdentifiers => {
232
            header => [ @header[6..8] ],
232
            header => [ @header[6..8] ],
233
            resumptionToken => {
233
            resumptionToken => {
234
              content => "marcxml/9/1970-01-01T00:00:00Z/$date_to//0/0",
234
              content => "marcxml/9/1970-01-01T00:00:00Z/$date_to/",
235
              cursor  => 9,
235
              cursor  => 9,
236
            },
236
            },
237
          },
237
          },
Lines 240-246 test_query( Link Here
240
240
241
test_query(
241
test_query(
242
    'ListIdentifiers with resumptionToken 3, response without resumption',
242
    'ListIdentifiers with resumptionToken 3, response without resumption',
243
    { verb => 'ListIdentifiers', resumptionToken => "marcxml/9/1970-01-01T00:00:00Z/$date_to//0/0" },
243
    { verb => 'ListIdentifiers', resumptionToken => "marcxml/9/1970-01-01T00:00:00Z/$date_to/" },
244
    {
244
    {
245
        ListIdentifiers => {
245
        ListIdentifiers => {
246
            header => $header[9],
246
            header => $header[9],
Lines 259-265 test_query('ListRecords marcxml', {verb => 'ListRecords', metadataPrefix => 'mar Link Here
259
    ListRecords => {
259
    ListRecords => {
260
        record => [ @marcxml[0..2] ],
260
        record => [ @marcxml[0..2] ],
261
        resumptionToken => {
261
        resumptionToken => {
262
          content => "marcxml/3/1970-01-01T00:00:00Z/$date_to//0/0",
262
          content => "marcxml/3/1970-01-01T00:00:00Z/$date_to/",
263
          cursor  => 3,
263
          cursor  => 3,
264
        },
264
        },
265
    },
265
    },
Lines 267-277 test_query('ListRecords marcxml', {verb => 'ListRecords', metadataPrefix => 'mar Link Here
267
267
268
test_query(
268
test_query(
269
    'ListRecords marcxml with resumptionToken 1',
269
    'ListRecords marcxml with resumptionToken 1',
270
    { verb => 'ListRecords', resumptionToken => "marcxml/3/1970-01-01T00:00:00Z/$date_to//0/0" },
270
    { verb => 'ListRecords', resumptionToken => "marcxml/3/1970-01-01T00:00:00Z/$date_to/" },
271
    { ListRecords => {
271
    { ListRecords => {
272
        record => [ @marcxml[3..5] ],
272
        record => [ @marcxml[3..5] ],
273
        resumptionToken => {
273
        resumptionToken => {
274
          content => "marcxml/6/1970-01-01T00:00:00Z/$date_to//0/0",
274
          content => "marcxml/6/1970-01-01T00:00:00Z/$date_to/",
275
          cursor  => 6,
275
          cursor  => 6,
276
        },
276
        },
277
    },
277
    },
Lines 279-289 test_query( Link Here
279
279
280
test_query(
280
test_query(
281
    'ListRecords marcxml with resumptionToken 2',
281
    'ListRecords marcxml with resumptionToken 2',
282
    { verb => 'ListRecords', resumptionToken => "marcxml/6/1970-01-01T00:00:00Z/$date_to//0/0" },
282
    { verb => 'ListRecords', resumptionToken => "marcxml/6/1970-01-01T00:00:00Z/$date_to/" },
283
    { ListRecords => {
283
    { ListRecords => {
284
        record => [ @marcxml[6..8] ],
284
        record => [ @marcxml[6..8] ],
285
        resumptionToken => {
285
        resumptionToken => {
286
          content => "marcxml/9/1970-01-01T00:00:00Z/$date_to//0/0",
286
          content => "marcxml/9/1970-01-01T00:00:00Z/$date_to/",
287
          cursor  => 9,
287
          cursor  => 9,
288
        },
288
        },
289
    },
289
    },
Lines 292-298 test_query( Link Here
292
# Last record, so no resumption token
292
# Last record, so no resumption token
293
test_query(
293
test_query(
294
    'ListRecords marcxml with resumptionToken 3, response without resumption',
294
    'ListRecords marcxml with resumptionToken 3, response without resumption',
295
    { verb => 'ListRecords', resumptionToken => "marcxml/9/1970-01-01T00:00:00Z/$date_to//0/0" },
295
    { verb => 'ListRecords', resumptionToken => "marcxml/9/1970-01-01T00:00:00Z/$date_to/" },
296
    { ListRecords => {
296
    { ListRecords => {
297
        record => $marcxml[9],
297
        record => $marcxml[9],
298
    },
298
    },
Lines 302-308 test_query('ListRecords oai_dc', {verb => 'ListRecords', metadataPrefix => 'oai_ Link Here
302
    ListRecords => {
302
    ListRecords => {
303
        record => [ @oaidc[0..2] ],
303
        record => [ @oaidc[0..2] ],
304
        resumptionToken => {
304
        resumptionToken => {
305
          content => "oai_dc/3/1970-01-01T00:00:00Z/$date_to//0/0",
305
          content => "oai_dc/3/1970-01-01T00:00:00Z/$date_to/",
306
          cursor  => 3,
306
          cursor  => 3,
307
        },
307
        },
308
    },
308
    },
Lines 310-320 test_query('ListRecords oai_dc', {verb => 'ListRecords', metadataPrefix => 'oai_ Link Here
310
310
311
test_query(
311
test_query(
312
    'ListRecords oai_dc with resumptionToken 1',
312
    'ListRecords oai_dc with resumptionToken 1',
313
    { verb => 'ListRecords', resumptionToken => "oai_dc/3/1970-01-01T00:00:00Z/$date_to//0/0" },
313
    { verb => 'ListRecords', resumptionToken => "oai_dc/3/1970-01-01T00:00:00Z/$date_to/" },
314
    { ListRecords => {
314
    { ListRecords => {
315
        record => [ @oaidc[3..5] ],
315
        record => [ @oaidc[3..5] ],
316
        resumptionToken => {
316
        resumptionToken => {
317
          content => "oai_dc/6/1970-01-01T00:00:00Z/$date_to//0/0",
317
          content => "oai_dc/6/1970-01-01T00:00:00Z/$date_to/",
318
          cursor  => 6,
318
          cursor  => 6,
319
        },
319
        },
320
    },
320
    },
Lines 322-332 test_query( Link Here
322
322
323
test_query(
323
test_query(
324
    'ListRecords oai_dc with resumptionToken 2',
324
    'ListRecords oai_dc with resumptionToken 2',
325
    { verb => 'ListRecords', resumptionToken => "oai_dc/6/1970-01-01T00:00:00Z/$date_to//0/0" },
325
    { verb => 'ListRecords', resumptionToken => "oai_dc/6/1970-01-01T00:00:00Z/$date_to/" },
326
    { ListRecords => {
326
    { ListRecords => {
327
        record => [ @oaidc[6..8] ],
327
        record => [ @oaidc[6..8] ],
328
        resumptionToken => {
328
        resumptionToken => {
329
          content => "oai_dc/9/1970-01-01T00:00:00Z/$date_to//0/0",
329
          content => "oai_dc/9/1970-01-01T00:00:00Z/$date_to/",
330
          cursor  => 9,
330
          cursor  => 9,
331
        },
331
        },
332
    },
332
    },
Lines 335-341 test_query( Link Here
335
# Last record, so no resumption token
335
# Last record, so no resumption token
336
test_query(
336
test_query(
337
    'ListRecords oai_dc with resumptionToken 3, response without resumption',
337
    'ListRecords oai_dc with resumptionToken 3, response without resumption',
338
    { verb => 'ListRecords', resumptionToken => "oai_dc/9/1970-01-01T00:00:00Z/$date_to//0/0" },
338
    { verb => 'ListRecords', resumptionToken => "oai_dc/9/1970-01-01T00:00:00Z/$date_to/" },
339
    { ListRecords => {
339
    { ListRecords => {
340
        record => $oaidc[9],
340
        record => $oaidc[9],
341
    },
341
    },
342
- 

Return to bug 20271