View | Details | Raw Unified | Return to bug 20271
Collapse All | Expand All

(-)a/Koha/OAI/Server/ListBase.pm (-113 / +81 lines)
Lines 1-6 Link Here
1
package Koha::OAI::Server::ListBase;
1
package Koha::OAI::Server::ListBase;
2
2
3
# Copyright The National Library of Finland, University of Helsinki 2016-2017
3
# Copyright The National Library of Finland, University of Helsinki 2016-2018
4
#
4
#
5
# This file is part of Koha.
5
# This file is part of Koha.
6
#
6
#
Lines 46-181 sub GetRecords { Link Here
46
        $set = GetOAISetBySpec($token->{'set'});
46
        $set = GetOAISetBySpec($token->{'set'});
47
    }
47
    }
48
    my $offset = $token->{offset};
48
    my $offset = $token->{offset};
49
    my $deleted = defined $token->{deleted} ? $token->{deleted} : 0;
50
    my $deleted_count = defined $token->{deleted_count} ? $token->{deleted_count} : 0;
51
    my $max = $repository->{koha_max_count};
49
    my $max = $repository->{koha_max_count};
52
    my $count = 0;
50
    my $count = 0;
53
    my $format = $args{metadataPrefix} || $token->{metadata_prefix};
51
    my $format = $args{metadataPrefix} || $token->{metadata_prefix};
54
    my $include_items = $repository->items_included( $format );
52
    my $include_items = $repository->items_included( $format );
55
53
56
    # Since creating a union of normal and deleted record tables would be a heavy
54
    my $sql = "
57
    # operation in a large database, build results in two stages:
55
        SELECT biblionumber, deleted_at
58
    # first deleted records ($deleted == 1), then normal records ($deleted == 0)
56
        FROM biblio_metadata
59
    STAGELOOP:
57
        WHERE (timestamp >= ? AND timestamp <= ?)
60
    for ( ; $deleted >= 0; $deleted-- ) {
58
    ";
61
        my $table = $deleted ? 'deletedbiblio_metadata' : 'biblio_metadata';
59
    my @bind_params = ($token->{'from_arg'}, $token->{'until_arg'});
62
        my $sql = "
63
            SELECT biblionumber
64
            FROM $table
65
            WHERE (timestamp >= ? AND timestamp <= ?)
66
        ";
67
        my @bind_params = ($token->{'from_arg'}, $token->{'until_arg'});
68
69
        if ($include_items) {
70
            $sql .= "
71
                OR biblionumber IN (SELECT biblionumber from items WHERE timestamp >= ? AND timestamp <= ? AND deleted_at IS NOT NULL)
72
            ";
73
            push @bind_params, ($token->{'from_arg'}, $token->{'until_arg'});
74
            if (!$deleted) {
75
                $sql .= "
76
                    OR biblionumber IN (SELECT biblionumber from items WHERE timestamp >= ? AND timestamp <= ? AND deleted_at IS NULL)
77
                ";
78
                push @bind_params, ($token->{'from_arg'}, $token->{'until_arg'});
79
            }
80
        }
81
60
61
    if ($include_items) {
82
        $sql .= "
62
        $sql .= "
83
            ORDER BY biblionumber
63
            OR biblionumber IN (SELECT biblionumber from items WHERE timestamp >= ? AND timestamp <= ?)
84
        ";
64
        ";
65
        push @bind_params, ($token->{'from_arg'}, $token->{'until_arg'});
66
    }
85
67
86
        # Use a subquery for sets since it allows us to use an index in
68
    $sql .= "
87
        # biblioitems table and is quite a bit faster than a join.
69
        ORDER BY biblionumber
88
        if (defined $set) {
70
    ";
89
            $sql = "
90
                SELECT bi.* FROM ($sql) bi
91
                  WHERE bi.biblionumber in (SELECT osb.biblionumber FROM oai_sets_biblios osb WHERE osb.set_id = ?)
92
            ";
93
            push @bind_params, $set->{'id'};
94
        }
95
71
96
        $sql .= "
72
    # Use a subquery for sets since it allows us to use an index in
97
            LIMIT " . ($max + 1) . "
73
    # biblioitems table and is quite a bit faster than a join.
98
            OFFSET " . ($offset - $deleted_count);
74
    if (defined $set) {
99
75
        $sql = "
100
        my $sth = $dbh->prepare( $sql ) || die( 'Could not prepare statement: ' . $dbh->errstr );
76
            SELECT bi.* FROM ($sql) bi
101
77
                WHERE bi.biblionumber in (SELECT osb.biblionumber FROM oai_sets_biblios osb WHERE osb.set_id = ?)
102
        if ( $deleted ) {
78
        ";
103
            $sql = "
79
        push @bind_params, $set->{'id'};
104
                SELECT MAX(timestamp)
80
    }
105
                FROM (
81
106
                    SELECT timestamp FROM deletedbiblio_metadata WHERE biblionumber = ?
82
    $sql .= "
107
                    UNION
83
        LIMIT " . ($max + 1) . "
108
                    SELECT timestamp FROM items WHERE biblionumber = ? AND deleted_at IS NOT NULL
84
        OFFSET $offset";
109
                ) bis
85
110
            ";
86
    my $sth = $dbh->prepare( $sql ) || die( 'Could not prepare statement: ' . $dbh->errstr );
111
        } else {
87
112
            $sql = "
88
    $sql = "
113
                SELECT MAX(timestamp)
89
        SELECT MAX(timestamp)
114
                FROM (
90
        FROM (
115
                    SELECT timestamp FROM biblio_metadata WHERE biblionumber = ?
91
            SELECT timestamp FROM biblio_metadata WHERE biblionumber = ?
116
                    UNION
92
            UNION
117
                    SELECT timestamp FROM items WHERE biblionumber = ? AND deleted_at IS NULL
93
            SELECT timestamp FROM items WHERE biblionumber = ?
118
                    UNION
94
        ) bi
119
                    SELECT timestamp FROM items WHERE biblionumber = ? AND deleted_at IS NOT NULL
95
    ";
120
                ) bi
96
    my $record_sth = $dbh->prepare( $sql ) || die( 'Could not prepare statement: ' . $dbh->errstr );
121
            ";
97
98
    $sth->execute( @bind_params ) || die( 'Could not execute statement: ' . $sth->errstr );
99
    while ( my ($biblionumber, $deleted_at) = $sth->fetchrow ) {
100
        $count++;
101
        if ( $count > $max ) {
102
            $self->resumptionToken(
103
                new Koha::OAI::Server::ResumptionToken(
104
                    metadataPrefix  => $token->{metadata_prefix},
105
                    from            => $token->{from},
106
                    until           => $token->{until},
107
                    offset          => $token->{offset} + $max,
108
                    set             => $token->{set},
109
                )
110
            );
111
            last;
122
        }
112
        }
123
        my $record_sth = $dbh->prepare( $sql ) || die( 'Could not prepare statement: ' . $dbh->errstr );
113
        $record_sth->execute( ( $biblionumber, $biblionumber ) ) || die( 'Could not execute statement: ' . $sth->errstr );
124
125
        $sth->execute( @bind_params ) || die( 'Could not execute statement: ' . $sth->errstr );
126
        while ( my ($biblionumber) = $sth->fetchrow ) {
127
            $count++;
128
            if ( $count > $max ) {
129
                $self->resumptionToken(
130
                    new Koha::OAI::Server::ResumptionToken(
131
                        metadataPrefix  => $token->{metadata_prefix},
132
                        from            => $token->{from},
133
                        until           => $token->{until},
134
                        offset          => $token->{offset} + $max,
135
                        set             => $token->{set},
136
                        deleted         => $deleted,
137
                        deleted_count   => $deleted_count
138
                    )
139
                );
140
                last STAGELOOP;
141
            }
142
            my @params = $deleted ? ( $biblionumber, $biblionumber ) : ( $biblionumber, $biblionumber, $biblionumber );
143
            $record_sth->execute( @params ) || die( 'Could not execute statement: ' . $sth->errstr );
144
114
145
            my ($timestamp) = $record_sth->fetchrow;
115
        my ($timestamp) = $record_sth->fetchrow;
146
116
147
            my $oai_sets = GetOAISetsBiblio($biblionumber);
117
        my $oai_sets = GetOAISetsBiblio($biblionumber);
148
            my @setSpecs;
118
        my @setSpecs;
149
            foreach ( @$oai_sets ) {
119
        foreach ( @$oai_sets ) {
150
                push @setSpecs, $_->{spec};
120
            push @setSpecs, $_->{spec};
151
            }
121
        }
152
            if ( $metadata ) {
122
        if ( $metadata ) {
153
                my $marcxml = !$deleted ? $repository->get_biblio_marcxml($biblionumber, $format) : undef;
123
            my $marcxml = !$deleted_at ? $repository->get_biblio_marcxml($biblionumber, $format) : undef;
154
                if ( $marcxml ) {
124
            if ( $marcxml ) {
155
                  $self->record( Koha::OAI::Server::Record->new(
125
                $self->record( Koha::OAI::Server::Record->new(
156
                      $repository, $marcxml, $timestamp, \@setSpecs,
126
                    $repository, $marcxml, $timestamp, \@setSpecs,
157
                      identifier      => $repository->{ koha_identifier } . ':' . $biblionumber,
127
                    identifier      => $repository->{ koha_identifier } . ':' . $biblionumber,
158
                      metadataPrefix  => $token->{metadata_prefix}
128
                    metadataPrefix  => $token->{metadata_prefix}
159
                  ) );
129
                ) );
160
                } else {
161
                  $self->record( Koha::OAI::Server::DeletedRecord->new(
162
                      $timestamp, \@setSpecs, identifier => $repository->{ koha_identifier } . ':' . $biblionumber
163
                  ) );
164
                }
165
            } else {
130
            } else {
166
                $timestamp =~ s/ /T/;
131
                $self->record( Koha::OAI::Server::DeletedRecord->new(
167
                $timestamp .= 'Z';
132
                    $timestamp, \@setSpecs, identifier => $repository->{ koha_identifier } . ':' . $biblionumber
168
                $self->identifier( new HTTP::OAI::Header(
169
                    identifier => $repository->{ koha_identifier} . ':' . $biblionumber,
170
                    datestamp  => $timestamp,
171
                    status     => $deleted ? 'deleted' : undef
172
                ) );
133
                ) );
173
            }
134
            }
135
        } else {
136
            $timestamp =~ s/ /T/;
137
            $timestamp .= 'Z';
138
            $self->identifier( new HTTP::OAI::Header(
139
                identifier => $repository->{ koha_identifier} . ':' . $biblionumber,
140
                datestamp  => $timestamp,
141
                status     => $deleted_at ? 'deleted' : undef
142
            ) );
174
        }
143
        }
175
        # Store offset and deleted record count
176
        $offset += $count;
177
        $deleted_count = $offset if ($deleted);
178
    }
144
    }
145
    # Store offset
146
    $offset += $count;
179
    return $count;
147
    return $count;
180
}
148
}
181
149
(-)a/Koha/OAI/Server/ResumptionToken.pm (-11 / +4 lines)
Lines 1-6 Link Here
1
# Copyright Tamil s.a.r.l. 2008-2015
1
# Copyright Tamil s.a.r.l. 2008-2015
2
# Copyright Biblibre 2008-2015
2
# Copyright Biblibre 2008-2015
3
# Copyright The National Library of Finland, University of Helsinki 2016-2017
3
# Copyright The National Library of Finland, University of Helsinki 2016-2018
4
#
4
#
5
# This file is part of Koha.
5
# This file is part of Koha.
6
#
6
#
Lines 32-48 use base ("HTTP::OAI::ResumptionToken"); Link Here
32
# - from
32
# - from
33
# - until
33
# - until
34
# - offset
34
# - offset
35
# - deleted
36
35
37
sub new {
36
sub new {
38
    my ($class, %args) = @_;
37
    my ($class, %args) = @_;
39
38
40
    my $self = $class->SUPER::new(%args);
39
    my $self = $class->SUPER::new(%args);
41
40
42
    my ($metadata_prefix, $offset, $from, $until, $set, $deleted, $deleted_count);
41
    my ($metadata_prefix, $offset, $from, $until, $set);
43
    if ( $args{ resumptionToken } ) {
42
    if ( $args{ resumptionToken } ) {
44
        ($metadata_prefix, $offset, $from, $until, $set, $deleted, $deleted_count)
43
        ($metadata_prefix, $offset, $from, $until, $set) = split( '/', $args{resumptionToken} );
45
            = split( '/', $args{resumptionToken} );
46
    }
44
    }
47
    else {
45
    else {
48
        $metadata_prefix = $args{ metadataPrefix };
46
        $metadata_prefix = $args{ metadataPrefix };
Lines 57-64 sub new { Link Here
57
        $until .= 'T23:59:59Z' if length($until) == 10;
55
        $until .= 'T23:59:59Z' if length($until) == 10;
58
        $offset = $args{ offset } || 0;
56
        $offset = $args{ offset } || 0;
59
        $set = $args{ set } || '';
57
        $set = $args{ set } || '';
60
        $deleted = defined $args{ deleted } ? $args{ deleted } : 1;
61
        $deleted_count = defined $args{ deleted_count } ? $args{ deleted_count } : 0;
62
    }
58
    }
63
59
64
    $self->{ metadata_prefix } = $metadata_prefix;
60
    $self->{ metadata_prefix } = $metadata_prefix;
Lines 68-78 sub new { Link Here
68
    $self->{ set             } = $set;
64
    $self->{ set             } = $set;
69
    $self->{ from_arg        } = _strip_UTC_designators($from);
65
    $self->{ from_arg        } = _strip_UTC_designators($from);
70
    $self->{ until_arg       } = _strip_UTC_designators($until);
66
    $self->{ until_arg       } = _strip_UTC_designators($until);
71
    $self->{ deleted         } = $deleted;
72
    $self->{ deleted_count   } = $deleted_count;
73
67
74
    $self->resumptionToken(
68
    $self->resumptionToken( join( '/', $metadata_prefix, $offset, $from, $until, $set ) );
75
        join( '/', $metadata_prefix, $offset, $from, $until, $set, $deleted, $deleted_count ) );
76
    $self->cursor( $offset );
69
    $self->cursor( $offset );
77
70
78
    return $self;
71
    return $self;
(-)a/t/db_dependent/OAI/Server.t (-18 / +17 lines)
Lines 201-207 test_query('ListIdentifiers', {verb => 'ListIdentifiers', metadataPrefix => 'mar Link Here
201
    ListIdentifiers => {
201
    ListIdentifiers => {
202
        header => [ @header[0..2] ],
202
        header => [ @header[0..2] ],
203
        resumptionToken => {
203
        resumptionToken => {
204
            content => "marcxml/3/1970-01-01T00:00:00Z/$date_to//0/0",
204
            content => "marcxml/3/1970-01-01T00:00:00Z/$date_to/",
205
            cursor  => 3,
205
            cursor  => 3,
206
        },
206
        },
207
    },
207
    },
Lines 211-217 test_query('ListIdentifiers', {verb => 'ListIdentifiers', metadataPrefix => 'mar Link Here
211
    ListIdentifiers => {
211
    ListIdentifiers => {
212
        header => [ @header[0..2] ],
212
        header => [ @header[0..2] ],
213
        resumptionToken => {
213
        resumptionToken => {
214
            content => "marcxml/3/1970-01-01T00:00:00Z/$date_to//0/0",
214
            content => "marcxml/3/1970-01-01T00:00:00Z/$date_to/",
215
            cursor  => 3,
215
            cursor  => 3,
216
        },
216
        },
217
    },
217
    },
Lines 224-230 test_query( Link Here
224
        ListIdentifiers => {
224
        ListIdentifiers => {
225
            header => [ @header[3..5] ],
225
            header => [ @header[3..5] ],
226
            resumptionToken => {
226
            resumptionToken => {
227
              content => "marcxml/6/1970-01-01T00:00:00Z/$date_to//0/0",
227
              content => "marcxml/6/1970-01-01T00:00:00Z/$date_to/",
228
              cursor  => 6,
228
              cursor  => 6,
229
            },
229
            },
230
          },
230
          },
Lines 238-244 test_query( Link Here
238
        ListIdentifiers => {
238
        ListIdentifiers => {
239
            header => [ @header[6..8] ],
239
            header => [ @header[6..8] ],
240
            resumptionToken => {
240
            resumptionToken => {
241
              content => "marcxml/9/1970-01-01T00:00:00Z/$date_to//0/0",
241
              content => "marcxml/9/1970-01-01T00:00:00Z/$date_to/",
242
              cursor  => 9,
242
              cursor  => 9,
243
            },
243
            },
244
          },
244
          },
Lines 247-253 test_query( Link Here
247
247
248
test_query(
248
test_query(
249
    'ListIdentifiers with resumptionToken 3, response without resumption',
249
    'ListIdentifiers with resumptionToken 3, response without resumption',
250
    { verb => 'ListIdentifiers', resumptionToken => "marcxml/9/1970-01-01T00:00:00Z/$date_to//0/0" },
250
    { verb => 'ListIdentifiers', resumptionToken => "marcxml/9/1970-01-01T00:00:00Z/$date_to/" },
251
    {
251
    {
252
        ListIdentifiers => {
252
        ListIdentifiers => {
253
            header => $header[9],
253
            header => $header[9],
Lines 266-272 test_query('ListRecords marcxml', {verb => 'ListRecords', metadataPrefix => 'mar Link Here
266
    ListRecords => {
266
    ListRecords => {
267
        record => [ @marcxml[0..2] ],
267
        record => [ @marcxml[0..2] ],
268
        resumptionToken => {
268
        resumptionToken => {
269
          content => "marcxml/3/1970-01-01T00:00:00Z/$date_to//0/0",
269
          content => "marcxml/3/1970-01-01T00:00:00Z/$date_to/",
270
          cursor  => 3,
270
          cursor  => 3,
271
        },
271
        },
272
    },
272
    },
Lines 274-284 test_query('ListRecords marcxml', {verb => 'ListRecords', metadataPrefix => 'mar Link Here
274
274
275
test_query(
275
test_query(
276
    'ListRecords marcxml with resumptionToken 1',
276
    'ListRecords marcxml with resumptionToken 1',
277
    { verb => 'ListRecords', resumptionToken => "marcxml/3/1970-01-01T00:00:00Z/$date_to//0/0" },
277
    { verb => 'ListRecords', resumptionToken => "marcxml/3/1970-01-01T00:00:00Z/$date_to/" },
278
    { ListRecords => {
278
    { ListRecords => {
279
        record => [ @marcxml[3..5] ],
279
        record => [ @marcxml[3..5] ],
280
        resumptionToken => {
280
        resumptionToken => {
281
          content => "marcxml/6/1970-01-01T00:00:00Z/$date_to//0/0",
281
          content => "marcxml/6/1970-01-01T00:00:00Z/$date_to/",
282
          cursor  => 6,
282
          cursor  => 6,
283
        },
283
        },
284
    },
284
    },
Lines 286-296 test_query( Link Here
286
286
287
test_query(
287
test_query(
288
    'ListRecords marcxml with resumptionToken 2',
288
    'ListRecords marcxml with resumptionToken 2',
289
    { verb => 'ListRecords', resumptionToken => "marcxml/6/1970-01-01T00:00:00Z/$date_to//0/0" },
289
    { verb => 'ListRecords', resumptionToken => "marcxml/6/1970-01-01T00:00:00Z/$date_to/" },
290
    { ListRecords => {
290
    { ListRecords => {
291
        record => [ @marcxml[6..8] ],
291
        record => [ @marcxml[6..8] ],
292
        resumptionToken => {
292
        resumptionToken => {
293
          content => "marcxml/9/1970-01-01T00:00:00Z/$date_to//0/0",
293
          content => "marcxml/9/1970-01-01T00:00:00Z/$date_to/",
294
          cursor  => 9,
294
          cursor  => 9,
295
        },
295
        },
296
    },
296
    },
Lines 299-305 test_query( Link Here
299
# Last record, so no resumption token
299
# Last record, so no resumption token
300
test_query(
300
test_query(
301
    'ListRecords marcxml with resumptionToken 3, response without resumption',
301
    'ListRecords marcxml with resumptionToken 3, response without resumption',
302
    { verb => 'ListRecords', resumptionToken => "marcxml/9/1970-01-01T00:00:00Z/$date_to//0/0" },
302
    { verb => 'ListRecords', resumptionToken => "marcxml/9/1970-01-01T00:00:00Z/$date_to/" },
303
    { ListRecords => {
303
    { ListRecords => {
304
        record => $marcxml[9],
304
        record => $marcxml[9],
305
    },
305
    },
Lines 309-315 test_query('ListRecords oai_dc', {verb => 'ListRecords', metadataPrefix => 'oai_ Link Here
309
    ListRecords => {
309
    ListRecords => {
310
        record => [ @oaidc[0..2] ],
310
        record => [ @oaidc[0..2] ],
311
        resumptionToken => {
311
        resumptionToken => {
312
          content => "oai_dc/3/1970-01-01T00:00:00Z/$date_to//0/0",
312
          content => "oai_dc/3/1970-01-01T00:00:00Z/$date_to/",
313
          cursor  => 3,
313
          cursor  => 3,
314
        },
314
        },
315
    },
315
    },
Lines 317-327 test_query('ListRecords oai_dc', {verb => 'ListRecords', metadataPrefix => 'oai_ Link Here
317
317
318
test_query(
318
test_query(
319
    'ListRecords oai_dc with resumptionToken 1',
319
    'ListRecords oai_dc with resumptionToken 1',
320
    { verb => 'ListRecords', resumptionToken => "oai_dc/3/1970-01-01T00:00:00Z/$date_to//0/0" },
320
    { verb => 'ListRecords', resumptionToken => "oai_dc/3/1970-01-01T00:00:00Z/$date_to/" },
321
    { ListRecords => {
321
    { ListRecords => {
322
        record => [ @oaidc[3..5] ],
322
        record => [ @oaidc[3..5] ],
323
        resumptionToken => {
323
        resumptionToken => {
324
          content => "oai_dc/6/1970-01-01T00:00:00Z/$date_to//0/0",
324
          content => "oai_dc/6/1970-01-01T00:00:00Z/$date_to/",
325
          cursor  => 6,
325
          cursor  => 6,
326
        },
326
        },
327
    },
327
    },
Lines 329-339 test_query( Link Here
329
329
330
test_query(
330
test_query(
331
    'ListRecords oai_dc with resumptionToken 2',
331
    'ListRecords oai_dc with resumptionToken 2',
332
    { verb => 'ListRecords', resumptionToken => "oai_dc/6/1970-01-01T00:00:00Z/$date_to//0/0" },
332
    { verb => 'ListRecords', resumptionToken => "oai_dc/6/1970-01-01T00:00:00Z/$date_to/" },
333
    { ListRecords => {
333
    { ListRecords => {
334
        record => [ @oaidc[6..8] ],
334
        record => [ @oaidc[6..8] ],
335
        resumptionToken => {
335
        resumptionToken => {
336
          content => "oai_dc/9/1970-01-01T00:00:00Z/$date_to//0/0",
336
          content => "oai_dc/9/1970-01-01T00:00:00Z/$date_to/",
337
          cursor  => 9,
337
          cursor  => 9,
338
        },
338
        },
339
    },
339
    },
Lines 342-348 test_query( Link Here
342
# Last record, so no resumption token
342
# Last record, so no resumption token
343
test_query(
343
test_query(
344
    'ListRecords oai_dc with resumptionToken 3, response without resumption',
344
    'ListRecords oai_dc with resumptionToken 3, response without resumption',
345
    { verb => 'ListRecords', resumptionToken => "oai_dc/9/1970-01-01T00:00:00Z/$date_to//0/0" },
345
    { verb => 'ListRecords', resumptionToken => "oai_dc/9/1970-01-01T00:00:00Z/$date_to/" },
346
    { ListRecords => {
346
    { ListRecords => {
347
        record => $oaidc[9],
347
        record => $oaidc[9],
348
    },
348
    },
349
- 

Return to bug 20271