View | Details | Raw Unified | Return to bug 35659
Collapse All | Expand All

(-)a/Koha/OAI/Client/Harvester.pm (-52 / +44 lines)
Lines 30-38 Koha::OAI::Client::Harvester contains OAI-PMH harvester main functions Link Here
30
use utf8;
30
use utf8;
31
use open qw( :std :utf8 );
31
use open qw( :std :utf8 );
32
32
33
use C4::Biblio qw( AddBiblio GetFrameworkCode ModBiblio DelBiblio );
33
use C4::Biblio          qw( AddBiblio GetFrameworkCode ModBiblio DelBiblio );
34
use C4::AuthoritiesMarc qw (AddAuthority GuessAuthTypeCode ModAuthority DelAuthority );
34
use C4::AuthoritiesMarc qw (AddAuthority GuessAuthTypeCode ModAuthority DelAuthority );
35
use C4::Log qw( cronlogaction );
35
use C4::Log             qw( cronlogaction );
36
use HTTP::OAI;
36
use HTTP::OAI;
37
use HTTP::OAI::Metadata::OAI_DC;
37
use HTTP::OAI::Metadata::OAI_DC;
38
use Koha::DateUtils qw( dt_from_string );
38
use Koha::DateUtils qw( dt_from_string );
Lines 47-53 use MARC::Record; Link Here
47
use Modern::Perl;
47
use Modern::Perl;
48
use DateTime;
48
use DateTime;
49
use DateTime::Format::Strptime;
49
use DateTime::Format::Strptime;
50
use Try::Tiny qw( catch try );
50
use Try::Tiny  qw( catch try );
51
use Date::Calc qw(
51
use Date::Calc qw(
52
    Add_Delta_Days
52
    Add_Delta_Days
53
    Today
53
    Today
Lines 58-67 my $strp = DateTime::Format::Strptime->new( Link Here
58
    time_zone => 'floating',
58
    time_zone => 'floating',
59
);
59
);
60
60
61
our $verbose = 0;
62
our $server;
63
our $force       = 0;
64
our $days        = 0;
65
our $xslt_engine = Koha::XSLT::Base->new;
61
our $xslt_engine = Koha::XSLT::Base->new;
66
62
67
=head2 new
63
=head2 new
Lines 81-94 C<$force> force harvesting (ignore records datestamps) Link Here
81
=cut
77
=cut
82
78
83
sub new {
79
sub new {
84
85
    my ( $self, $args ) = @_;
80
    my ( $self, $args ) = @_;
86
81
    return bless $args, $self;
87
    $server  = $args->{'server'};
88
    $verbose = $args->{'verbose'};
89
    $days    = $args->{'days'};
90
    $force   = $args->{'force'};
91
    return $self;
92
}
82
}
93
83
94
=head2 init
84
=head2 init
Lines 100-106 Starts harvesting Link Here
100
sub init {
90
sub init {
101
    my ( $self, $args ) = @_;
91
    my ( $self, $args ) = @_;
102
92
103
    printlog("Starting OAI Harvest from repository");
93
    $self->printlog("Starting OAI Harvest from repository");
94
    my $server = $self->{server};
95
    my $days   = $self->{days};
104
96
105
    my $h = HTTP::OAI::Harvester->new(
97
    my $h = HTTP::OAI::Harvester->new(
106
        repository => HTTP::OAI::Identify->new(
98
        repository => HTTP::OAI::Identify->new(
Lines 109-117 sub init { Link Here
109
        )
101
        )
110
    );
102
    );
111
103
112
    printlog( "Using HTTP::OAI::Harvester version " . $HTTP::OAI::Harvester::VERSION );
104
    $self->printlog( "Using HTTP::OAI::Harvester version " . $HTTP::OAI::Harvester::VERSION );
113
105
114
    printlog( "Using endpoint "
106
    $self->printlog( "Using endpoint "
115
            . $server->endpoint
107
            . $server->endpoint
116
            . ", set "
108
            . ", set "
117
            . $server->oai_set
109
            . $server->oai_set
Lines 126-140 sub init { Link Here
126
        for ( $lmdf->metadataFormat ) {
118
        for ( $lmdf->metadataFormat ) {
127
            $lmdflog .= $_->metadataPrefix . " ";
119
            $lmdflog .= $_->metadataPrefix . " ";
128
        }
120
        }
129
        printlog("Available metadata formats for this repository: $lmdflog");
121
        $self->printlog("Available metadata formats for this repository: $lmdflog");
130
    } catch {
122
    } catch {
131
        printlog("ListMetadataFormats failed");
123
        $self->printlog("ListMetadataFormats failed");
132
    };
124
    };
133
125
134
    if ( $server->add_xslt ) {
126
    if ( $server->add_xslt ) {
135
        printlog( "Using XSLT file " . $server->add_xslt );
127
        $self->printlog( "Using XSLT file " . $server->add_xslt );
136
    } else {
128
    } else {
137
        printlog("Not using an XSLT file");
129
        $self->printlog("Not using an XSLT file");
138
    }
130
    }
139
131
140
    my $start_date = '';
132
    my $start_date = '';
Lines 143-167 sub init { Link Here
143
    if ($days) {
135
    if ($days) {
144
136
145
        # Change this to yyyy-mm-dd
137
        # Change this to yyyy-mm-dd
146
        my $dt_today = dt_from_string();
147
        my ( $nowyear, $nowmonth, $nowday ) = Today();
138
        my ( $nowyear, $nowmonth, $nowday ) = Today();
148
        my @date     = Add_Delta_Days( $nowyear, $nowmonth, $nowday, -$days );
139
        my @date     = Add_Delta_Days( $nowyear, $nowmonth, $nowday, -$days );
149
        my $dt_start = DateTime->new( year => $date[0], month => $date[1], day => $date[2] );
140
        my $dt_start = DateTime->new( year => $date[0], month => $date[1], day => $date[2] );
150
        $start_date = $dt_start->ymd();
141
        $start_date = $dt_start->ymd();
151
        printlog("Harvesting from $start_date");
142
        $self->printlog("Harvesting from $start_date");
152
    }
143
    }
153
144
154
    printlog("Asking for records");
145
    $self->printlog("Asking for records");
155
    my $response = $h->ListRecords(
146
    my $response = $h->ListRecords(
156
        metadataPrefix => $server->dataformat,
147
        metadataPrefix => $server->dataformat,
157
        set            => $server->oai_set,
148
        set            => $server->oai_set,
158
        from           => $start_date,
149
        from           => $start_date,
159
    );
150
    );
160
    if ( $response->is_error ) {
151
    if ( $response->is_error ) {
161
        printlog( "Error requesting ListRecords: " . $response->code . " " . $response->message );
152
        $self->printlog( "Error requesting ListRecords: " . $response->code . " " . $response->message );
162
        exit;
153
        exit;
163
    }
154
    }
164
    printlog( "Request URL: " . $response->requestURL );
155
    $self->printlog( "Request URL: " . $response->requestURL );
165
156
166
    my %stats;
157
    my %stats;
167
    my @statuses = qw(added updated deleted in_error skipped total);
158
    my @statuses = qw(added updated deleted in_error skipped total);
Lines 169-175 sub init { Link Here
169
        $stats{$status} = 0;
160
        $stats{$status} = 0;
170
    }
161
    }
171
162
172
    printlog("Starting processing results");
163
    $self->printlog("Starting processing results");
173
    while ( my $oai_record = $response->next ) {
164
    while ( my $oai_record = $response->next ) {
174
        $stats{'total'}++;
165
        $stats{'total'}++;
175
        my $status = $self->processRecord($oai_record);
166
        my $status = $self->processRecord($oai_record);
Lines 180-186 sub init { Link Here
180
    foreach my $status (@statuses) {
171
    foreach my $status (@statuses) {
181
        $results .= $stats{$status} . " $status\n";
172
        $results .= $stats{$status} . " $status\n";
182
    }
173
    }
183
    printlog( "Harvest results:\n" . $results );
174
    $self->printlog( "Harvest results:\n" . $results );
184
175
185
    if ( C4::Context->preference("OAI-PMH:HarvestEmailReport") ) {
176
    if ( C4::Context->preference("OAI-PMH:HarvestEmailReport") ) {
186
177
Lines 202-208 sub init { Link Here
202
                    total      => $stats{'total'}
193
                    total      => $stats{'total'}
203
                },
194
                },
204
            )
195
            )
205
        ) or printlog("OAI_HARVEST_REPORT letter not found");
196
        ) or $self->printlog("OAI_HARVEST_REPORT letter not found");
206
197
207
        my $success = C4::Letters::EnqueueLetter(
198
        my $success = C4::Letters::EnqueueLetter(
208
            {
199
            {
Lines 211-223 sub init { Link Here
211
            }
202
            }
212
        );
203
        );
213
        if ($success) {
204
        if ($success) {
214
            printlog("Email report enqueued");
205
            $self->printlog("Email report enqueued");
215
        } else {
206
        } else {
216
            printlog("Unable to enqueue report email");
207
            $self->printlog("Unable to enqueue report email");
217
        }
208
        }
218
    }
209
    }
219
210
220
    printlog("Ending OAI Harvest from repository");
211
    $self->printlog("Ending OAI Harvest from repository");
221
}
212
}
222
213
223
=head2 processRecord
214
=head2 processRecord
Lines 230-237 sub processRecord { Link Here
230
    my $self       = shift;
221
    my $self       = shift;
231
    my $oai_record = shift;
222
    my $oai_record = shift;
232
    my $status     = '';
223
    my $status     = '';
224
    my $server     = $self->{server};
225
    my $force      = $self->{force};
233
    unless ( $oai_record->identifier ) {
226
    unless ( $oai_record->identifier ) {
234
        printlog("No identifier found");
227
        $self->printlog("No identifier found");
235
        $status = 'skipped';
228
        $status = 'skipped';
236
        return $status;
229
        return $status;
237
    }
230
    }
Lines 246-252 sub processRecord { Link Here
246
            $outputUnimarc = $xslt_engine->transform( $oai_record->metadata->dom, $server->add_xslt );
239
            $outputUnimarc = $xslt_engine->transform( $oai_record->metadata->dom, $server->add_xslt );
247
            my $err = $xslt_engine->err;
240
            my $err = $xslt_engine->err;
248
            if ($err) {
241
            if ($err) {
249
                printlog("XSLT::Base error: $err");
242
                $self->printlog("XSLT::Base error: $err");
250
                $status = 'in_error';
243
                $status = 'in_error';
251
                return $status;
244
                return $status;
252
            }
245
            }
Lines 280-322 sub processRecord { Link Here
280
                my $biblionumber = $imported_record->biblionumber;
273
                my $biblionumber = $imported_record->biblionumber;
281
                my $error        = DelBiblio($biblionumber);
274
                my $error        = DelBiblio($biblionumber);
282
                if ($error) {
275
                if ($error) {
283
                    printlog(
276
                    $self->printlog(
284
                        "Record " . $oai_record->identifier . " not deleted, biblionumber: $biblionumber ($error)" );
277
                        "Record " . $oai_record->identifier . " not deleted, biblionumber: $biblionumber ($error)" );
285
                    $status = 'in_error';
278
                    $status = 'in_error';
286
                } else {
279
                } else {
287
                    $imported_record->delete;
280
                    $imported_record->delete;
288
                    printlog( "Record " . $oai_record->identifier . " deleted, biblionumber: $biblionumber" );
281
                    $self->printlog( "Record " . $oai_record->identifier . " deleted, biblionumber: $biblionumber" );
289
                    $status = 'deleted';
282
                    $status = 'deleted';
290
                }
283
                }
291
            } else {
284
            } else {
292
                my $authid = $imported_record->authid;
285
                my $authid = $imported_record->authid;
293
                DelAuthority( { authid => $authid } );
286
                DelAuthority( { authid => $authid } );
294
                $imported_record->delete;
287
                $imported_record->delete;
295
                printlog( "Record " . $oai_record->identifier . " deleted, authid: $authid" );
288
                $self->printlog( "Record " . $oai_record->identifier . " deleted, authid: $authid" );
296
                $status = 'deleted';
289
                $status = 'deleted';
297
            }
290
            }
298
        } else {
291
        } else {
299
            my $existing_dt = $strp->parse_datetime( $imported_record->datestamp );
292
            my $existing_dt = dt_from_string( $imported_record->datestamp, 'iso' );
300
            my $incoming_dt = $strp->parse_datetime( $oai_record->datestamp );
293
            my $incoming_dt = $oai_record->datestamp ? dt_from_string( $oai_record->datestamp, 'iso' ) : dt_from_string;
301
294
            if ( $force || $incoming_dt > $existing_dt ) {
302
            if ( $force || !$incoming_dt || !$existing_dt || $incoming_dt > $existing_dt ) {
303
                if ( $server->recordtype eq "biblio" ) {
295
                if ( $server->recordtype eq "biblio" ) {
304
                    my $biblionumber = $imported_record->biblionumber;
296
                    my $biblionumber = $imported_record->biblionumber;
305
                    my $result       = ModBiblio( $marcrecord, $biblionumber, GetFrameworkCode($biblionumber) );
297
                    my $result       = ModBiblio( $marcrecord, $biblionumber, GetFrameworkCode($biblionumber) );
306
                    printlog( "Record " . $oai_record->identifier . " updated, biblionumber: $biblionumber" );
298
                    $self->printlog( "Record " . $oai_record->identifier . " updated, biblionumber: $biblionumber" );
307
                } else {
299
                } else {
308
                    my $authid = $imported_record->authid;
300
                    my $authid = $imported_record->authid;
309
                    my $result = ModAuthority( $authid, $marcrecord, GuessAuthTypeCode($marcrecord) );
301
                    my $result = ModAuthority( $authid, $marcrecord, GuessAuthTypeCode($marcrecord) );
310
                    printlog( "Record " . $oai_record->identifier . " updated, authid: $authid" );
302
                    $self->printlog( "Record " . $oai_record->identifier . " updated, authid: $authid" );
311
                }
303
                }
312
                $imported_record->update(
304
                $imported_record->update(
313
                    {
305
                    {
314
                        datestamp => $imported_record->datestamp,
306
                        datestamp => $incoming_dt,
315
                    }
307
                    }
316
                );
308
                );
317
                $status = 'updated';
309
                $status = 'updated';
318
            } else {
310
            } else {
319
                printlog( "Record "
311
                $self->printlog( "Record "
320
                        . $oai_record->identifier
312
                        . $oai_record->identifier
321
                        . " skipped (incoming record was not newer than existing record)" );
313
                        . " skipped (incoming record was not newer than existing record)" );
322
                $status = 'skipped';
314
                $status = 'skipped';
Lines 325-331 sub processRecord { Link Here
325
    } elsif ( !$to_delete ) {
317
    } elsif ( !$to_delete ) {
326
        if ( $server->recordtype eq "biblio" ) {
318
        if ( $server->recordtype eq "biblio" ) {
327
            my ( $biblionumber, $biblioitemnumber ) = AddBiblio($marcrecord);
319
            my ( $biblionumber, $biblioitemnumber ) = AddBiblio($marcrecord);
328
            printlog( $oai_record->identifier . " added, biblionumber: $biblionumber" );
320
            $self->printlog( $oai_record->identifier . " added, biblionumber: $biblionumber" );
329
            Koha::Import::Oaipmh::Biblio->new(
321
            Koha::Import::Oaipmh::Biblio->new(
330
                {
322
                {
331
                    repository   => $server->endpoint,
323
                    repository   => $server->endpoint,
Lines 337-343 sub processRecord { Link Here
337
            )->store;
329
            )->store;
338
        } else {
330
        } else {
339
            my $authid = AddAuthority( $marcrecord, "", GuessAuthTypeCode($marcrecord) );
331
            my $authid = AddAuthority( $marcrecord, "", GuessAuthTypeCode($marcrecord) );
340
            printlog( $oai_record->identifier . " added, authid: $authid" );
332
            $self->printlog( $oai_record->identifier . " added, authid: $authid" );
341
            Koha::Import::Oaipmh::Authority->new(
333
            Koha::Import::Oaipmh::Authority->new(
342
                {
334
                {
343
                    repository => $server->endpoint,
335
                    repository => $server->endpoint,
Lines 350-371 sub processRecord { Link Here
350
        }
342
        }
351
        $status = 'added';
343
        $status = 'added';
352
    } else {
344
    } else {
353
        printlog( "Record " . $oai_record->identifier . " skipped (record not present or already deleted)" );
345
        $self->printlog( "Record " . $oai_record->identifier . " skipped (record not present or already deleted)" );
354
        $status = 'skipped';
346
        $status = 'skipped';
355
    }
347
    }
356
    return $status;
348
    return $status;
357
}
349
}
358
350
359
=head2 printlog
351
=head2 $self->printlog
360
352
361
This method adds a cronlog and prints to stdout if verbose is enabled
353
This method adds a cronlog and prints to stdout if verbose is enabled
362
354
363
=cut
355
=cut
364
356
365
sub printlog {
357
sub printlog {
366
    my $message = shift;
358
    my ( $self, $message ) = @_;
367
    $message = $server->servername . ": " . $message;
359
    $message = $self->{server}->servername . ": " . $message;
368
    print $message . "\n" if ($verbose);
360
    print $message . "\n" if ( $self->{verbose} );
369
    cronlogaction( { info => $message } );
361
    cronlogaction( { info => $message } );
370
}
362
}
371
363
(-)a/misc/cronjobs/harvest_oai.pl (-1 / +1 lines)
Lines 25-31 use Getopt::Long qw( GetOptions ); Link Here
25
use Koha::Script -cron;
25
use Koha::Script -cron;
26
use Koha::OAI::Client::Harvester;
26
use Koha::OAI::Client::Harvester;
27
use Koha::OaiServers;
27
use Koha::OaiServers;
28
use C4::Log qw( cronlogaction );
28
use C4::Log   qw( cronlogaction );
29
use Try::Tiny qw( catch try );
29
use Try::Tiny qw( catch try );
30
30
31
my $command_line_options = join( " ", @ARGV );
31
my $command_line_options = join( " ", @ARGV );
(-)a/t/db_dependent/Koha/OAIHarvester.t (-3 / +23 lines)
Lines 18-24 Link Here
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
19
20
use Modern::Perl;
20
use Modern::Perl;
21
use Test::More tests => 5;
21
use Test::More tests => 9;
22
use Test::Exception;
22
use Test::Exception;
23
23
24
use t::lib::TestBuilder;
24
use t::lib::TestBuilder;
Lines 53-65 my $record = Link Here
53
    '<metadata xmlns="http://www.openarchives.org/OAI/2.0/"><oai_dc:dc xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd"><dc:title>Pièces diverses </dc:title><dc:identifier>ARCH/0320 [cote]</dc:identifier><dc:relation>FR-920509801 [RCR établissement]</dc:relation><dc:relation>Central obrera boliviana [Fonds ou collection]</dc:relation><dc:format>1 carton</dc:format><dc:date>1971/2000</dc:date><dc:type>Archives et manuscrits</dc:type></oai_dc:dc></metadata>';
53
    '<metadata xmlns="http://www.openarchives.org/OAI/2.0/"><oai_dc:dc xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd"><dc:title>Pièces diverses </dc:title><dc:identifier>ARCH/0320 [cote]</dc:identifier><dc:relation>FR-920509801 [RCR établissement]</dc:relation><dc:relation>Central obrera boliviana [Fonds ou collection]</dc:relation><dc:format>1 carton</dc:format><dc:date>1971/2000</dc:date><dc:type>Archives et manuscrits</dc:type></oai_dc:dc></metadata>';
54
54
55
my $r = HTTP::OAI::Record->new();
55
my $r = HTTP::OAI::Record->new();
56
$r->header->datestamp('2017-05-10T09:18:13Z');
57
$r->metadata( HTTP::OAI::Metadata->new( dom => $record ) );
56
$r->metadata( HTTP::OAI::Metadata->new( dom => $record ) );
58
57
59
my $status = $harvester->processRecord($r);
58
my $status = $harvester->processRecord($r);
60
is( $status, 'skipped', 'Record with no identifier is skipped' );
59
is( $status, 'skipped', 'Record with no identifier is skipped' );
61
60
62
$r->header->identifier('oai:myarchive.org:oid-233');
61
$r->header->identifier('oai:myarchive.org:oid-233');
62
$status = $harvester->processRecord($r);
63
is( $status, 'added', 'Record with no date is added' );
64
65
$status = $harvester->processRecord($r);
66
is( $status, 'updated', 'Record with no date is updated' );
67
68
$status = $harvester->processRecord($r);
69
is( $status, 'updated', 'Record with no date is updated even without force' );
70
71
$r->header->identifier('oai:myarchive.org:oid-234');
72
$r->header->datestamp('2017-05-10T09:18:13Z');
63
73
64
$status = $harvester->processRecord($r);
74
$status = $harvester->processRecord($r);
65
is( $status, 'added', 'Record is added' );
75
is( $status, 'added', 'Record is added' );
Lines 75-78 $r->header->datestamp('2018-05-10T09:18:13Z'); Link Here
75
$status = $harvester->processRecord($r);
85
$status = $harvester->processRecord($r);
76
is( $status, 'updated', 'When force is not used, record is updated if newer' );
86
is( $status, 'updated', 'When force is not used, record is updated if newer' );
77
87
88
my $imported_record = Koha::Import::Oaipmh::Biblios->find( { identifier => 'oai:myarchive.org:oid-234' } );
89
my $added_datestamp = $imported_record->datestamp;
90
$r->header->datestamp(undef);
91
$status          = $harvester->processRecord($r);
92
$imported_record = Koha::Import::Oaipmh::Biblios->find( { identifier => 'oai:myarchive.org:oid-234' } );
93
my $updated_datestamp = $imported_record->datestamp;
94
isnt(
95
    $added_datestamp, $updated_datestamp,
96
    'local datestamp is updated even if there is no datestamp in incoming record'
97
);
98
78
$schema->storage->txn_rollback;
99
$schema->storage->txn_rollback;
79
- 

Return to bug 35659