View | Details | Raw Unified | Return to bug 10662
Collapse All | Expand All

(-)a/C4/OAI/Client/Harvester.pm (+556 lines)
Line 0 Link Here
1
package C4::OAI::Client::Harvester;
2
3
# This file is part of Koha.
4
#
5
# Copyright 2013 Prosentient Systems
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, see <http://www.gnu.org/licenses>.
18
19
=head1 NAME
20
21
C4::OAI::Harvester - OAI-PMH harvester/client which implements the 6 OAI-PMH verbs, record retrieval/harvesting, and import into Koha
22
23
=head1 SYNOPSIS
24
25
use C4::OAI::Harvester;
26
my $oai_repo = C4::OAI::Harvester->new($repository_data);
27
28
my $identify_repository = $oai_repo->Identify;
29
30
my @sets = $oai_repo->ListSets;
31
32
my @formats = $oai_repo->ListMetadataFormats;
33
34
my @headers = $oai_repo->ListIdentifiers;
35
36
my @records = $oai_repo->ListRecords;
37
38
my @records = $oai_repo->GetRecord($oai_unique_identifier);
39
40
my $import_mode = ''; #i.e. not "automatic"
41
42
$oai_repo->ImportRecordsIntoKoha($import_mode,@records);
43
44
=head1 DESCRIPTION
45
46
C4::OAI::Harvester contains functions for querying and harvesting OAI-PMH repositories.
47
48
More information on OAI-PMH can be found L<here|http://www.openarchives.org/OAI/openarchivesprotocol.html>
49
50
=head1 FUNCTIONS
51
52
=head1 AUTHOR
53
54
David Cook <dcook AT prosentient DOT com DOT au>
55
56
=cut
57
58
use Modern::Perl;
59
use HTTP::OAI;
60
use C4::Context;
61
use C4::OAI::Client::Record;
62
use C4::Biblio qw /AddBiblio ModBiblio DelBiblio/;
63
64
use base qw(Class::Accessor);
65
66
67
sub AddOAIRepository {
68
    my ( $database_fields ) = @_;
69
    my $dbh = C4::Context->dbh;
70
    my $sql = "
71
      INSERT INTO oai_harvest_repositories
72
      (baseURL,metadataPrefix,opt_from,opt_until,opt_set,active,XSLT_path,frameworkcode,comments)
73
      VALUES (?,?,?,?,?,?,?,?,?)
74
    ";
75
    my $sth = $dbh->prepare($sql);
76
    $sth->execute(
77
        $database_fields->{baseURL},
78
        $database_fields->{metadataPrefix},
79
        $database_fields->{opt_from},
80
        $database_fields->{opt_until},
81
        $database_fields->{opt_set},
82
        $database_fields->{active},
83
        $database_fields->{XSLT_path},
84
        $database_fields->{frameworkcode},
85
        $database_fields->{comments}
86
     );
87
     if ($sth->err){
88
        die $sth->errstr;
89
    }
90
}
91
92
sub ModOAIRepositoryDateTimeFrom {
93
    my ( $repository_id, $datetime_from ) = @_;
94
    my $dbh = C4::Context->dbh;
95
    my $sql = "
96
        UPDATE oai_harvest_repositories
97
        SET opt_from = ?
98
        WHERE repository_id = ?
99
    ";
100
    my $sth = $dbh->prepare($sql);
101
    $sth->execute($datetime_from,$repository_id);
102
    if ($sth->err){
103
       die $sth->errstr;
104
    }
105
}
106
107
sub ModOAIRepository {
108
    my ( $database_fields ) = @_;
109
    my $dbh = C4::Context->dbh;
110
    my $sql = "
111
        UPDATE oai_harvest_repositories
112
        SET baseURL = ?,
113
        metadataPrefix = ?,
114
        opt_from = ?,
115
        opt_until = ?,
116
        opt_set = ?,
117
        active = ?,
118
        XSLT_path = ?,
119
        frameworkcode = ?,
120
        comments = ?
121
        WHERE repository_id = ?
122
    ";
123
    my $sth = $dbh->prepare($sql);
124
    $sth->execute(
125
        $database_fields->{baseURL},
126
        $database_fields->{metadataPrefix},
127
        $database_fields->{opt_from},
128
        $database_fields->{opt_until},
129
        $database_fields->{opt_set},
130
        $database_fields->{active},
131
        $database_fields->{XSLT_path},
132
        $database_fields->{frameworkcode},
133
        $database_fields->{comments},
134
        $database_fields->{repository_id}
135
     );
136
    if ($sth->err){
137
       die $sth->errstr;
138
    }
139
}
140
141
sub DelOAIRepository {
142
    my ( $repository_id ) = @_;
143
    my $error;
144
    my $dbh = C4::Context->dbh;
145
    my $sql = "
146
        DELETE FROM oai_harvest_repositories
147
        WHERE repository_id = ?
148
    ";
149
    my $sth = $dbh->prepare($sql);
150
    $sth->execute($repository_id);
151
    if ($sth->err){
152
       die $sth->errstr;
153
    }
154
}
155
156
sub GetLatestHistoricalRecordDatestamp {
157
    my ( $repository_id  ) = @_;
158
    my $latest_datestamp;
159
    my $dbh = C4::Context->dbh;
160
    my $sql = "
161
        SELECT datestamp
162
        FROM oai_harvest
163
        WHERE repository_id = ?
164
        ORDER BY datestamp desc
165
        LIMIT 1
166
    ";
167
    my $sth = $dbh->prepare($sql);
168
    $sth->execute($repository_id);
169
    my $row = $sth->fetchrow_hashref;
170
    $latest_datestamp = $row->{datestamp} ? $row->{datestamp} : undef;
171
    return $latest_datestamp;
172
}
173
174
sub GetOAIRepository {
175
    my ( $repository_id ) = @_;
176
    my $dbh = C4::Context->dbh;
177
    my $sql = "
178
      SELECT *
179
      FROM oai_harvest_repositories
180
      WHERE repository_id = ?
181
    ";
182
    my $sth = $dbh->prepare($sql);
183
    $sth->execute($repository_id);
184
    if ($sth->err){
185
       die $sth->errstr;
186
    }
187
    my $row = $sth->fetchrow_hashref;
188
    return $row;
189
}
190
191
=head2 GetOAIRepositoryList
192
193
  my @repositories = C4::OAI::Harvester::GetOAIRepositoryList();
194
  my @repositories = C4::OAI::Harvester::GetOAIRepositoryList($active);
195
196
Returns an array of hashrefs listing all OAI-PMH repositories
197
present in the database.
198
199
If the $active is included, then it will return an array
200
of hashrefs listing all OAI-PMH repositories depending on their
201
active status. $active == 1 shows all active. $active == 0 shows
202
all inactive.
203
204
=cut
205
206
sub GetOAIRepositoryList {
207
    my ( $active ) = @_;
208
    my $dbh = C4::Context->dbh;
209
    my @results;
210
    my $sql = "
211
        SELECT *
212
        FROM oai_harvest_repositories
213
    ";
214
    if (defined $active){
215
        $sql .= " WHERE active = 1 " if $active == 1;
216
        $sql .= " WHERE active = 0 " if $active == 0;
217
    }
218
    my $sth = $dbh->prepare($sql);
219
    $sth->execute;
220
    while (my $row = $sth->fetchrow_hashref){
221
        push(@results,$row);
222
    }
223
    return @results;
224
}
225
226
#TODO: Perhaps create a sub that cleans out the metadata column to keep the table size low?
227
228
sub new {
229
    my($proto, $fields) = @_;
230
    my($class) = ref $proto || $proto;
231
232
    $fields = {} unless defined $fields;
233
234
    if ($fields->{'baseURL'}){
235
        my $h = new HTTP::OAI::Harvester(
236
            baseURL => $fields->{'baseURL'},
237
        );
238
        #If resume is set to 0, automatic token resumption is turned off. This is useful for testing/debugging.
239
        if ($h && exists $fields->{'resume'}){
240
            if ($fields->{'resume'} == 0){
241
                $h->resume(0);
242
            }
243
        }
244
        my $response = $h->repository($h->Identify);
245
        if( $response->is_error ) {
246
            print "Error requesting Identify:\n",
247
                    $response->code . " " . $response->message, "\n";
248
            exit;
249
        }
250
        $fields->{rh} = $h; #Store HTTP::OAI::Harvester object as "repository handle"
251
    }
252
    bless {%$fields}, $class;
253
}
254
255
__PACKAGE__->follow_best_practice; #Use get_ and set_ prefixes for accessors
256
__PACKAGE__->mk_accessors(qw(baseURL opt_from opt_until opt_set metadataPrefix rh repository_id XSLT_path debug frameworkcode));
257
258
=head2 OAI-PMH Verbs
259
260
Koha-specific implementations of the 6 OAI-PMH Verbs.
261
262
The key verbs are "ListRecords" and "GetRecords". These do the actual
263
harvesting of records from a OAI-PMH repository. The others are useful for
264
getting information about a repository and what it has available.
265
266
1) ListRecords
267
268
2) GetRecord
269
270
3) ListIdentifiers
271
272
4) ListMetadataFormats
273
274
5) ListSets
275
276
6) Identify
277
278
=cut
279
280
281
sub ListRecords {
282
    my ( $self ) = @_;
283
    my %args = (
284
        metadataPrefix => $self->{metadataPrefix},
285
        from => $self->{opt_from},
286
        until => $self->{opt_until},
287
        set => $self->{opt_set},
288
    );
289
    if ($self->{debug}){
290
        use Data::Dumper;
291
        print "ListRecords's arguments\n";
292
        print Dumper(\%args);
293
    }
294
    my $response = $self->{rh}->ListRecords(%args);
295
    if( $response->is_error ) {
296
        print "Error requesting ListRecords:\n",
297
                $response->code . " " . $response->message, "\n";
298
        exit;
299
    }
300
    if ($self->{debug}){
301
        print "Successfully retrieved ListRecords response.\n";
302
    }
303
    return $response;
304
}
305
306
sub GetRecord {
307
    my ( $self, $identifier, $harvest ) = @_;
308
    my $response = $self->{rh}->GetRecord(
309
        metadataPrefix => $self->{metadataPrefix},
310
        identifier => $identifier,
311
    );
312
    if( $response->is_error ) {
313
        print "Error requesting GetRecord:\n",
314
                $response->code . " " . $response->message, "\n";
315
        exit;
316
    }
317
    if ($self->{debug}){
318
        print "Successfully retrieved GetRecord response.\n";
319
    }
320
    return $response;
321
}
322
323
sub ListIdentifiers {
324
    my $self = shift;
325
    my @headers;
326
    my $response = $self->{rh}->ListIdentifiers(
327
        metadataPrefix => $self->{metadataPrefix},
328
        from => $self->{opt_from},
329
        until => $self->{opt_until},
330
        set => $self->{opt_set},
331
    );
332
    if( $response->is_error ) {
333
        print "Error requesting ListIdentifiers:\n",
334
                $response->code . " " . $response->message, "\n";
335
        exit;
336
    }
337
    while (my $h = $response->next){
338
        my $header;
339
        #print Dumper($h->dom->toString); #DEBUG: XML representation
340
        $header->{identifier} = $h->identifier;
341
        $header->{datestamp} = $h->datestamp;
342
343
        $header->{status} = $h->status;
344
        $header->{is_deleted} = $h->is_deleted;
345
346
        my @sets = $h->setSpec;
347
        $header->{sets} = \@sets;
348
349
        push (@headers,$header);
350
    }
351
    return \@headers;
352
}
353
354
sub ListMetadataFormats {
355
    my ( $self, $identifier ) = @_;
356
    my @formats;
357
    my $response = $self->{rh}->ListMetadataFormats(
358
        identifier => $identifier,
359
    );
360
    if( $response->is_error ) {
361
        print "Error requesting ListMetadataFormats:\n",
362
                $response->code . " " . $response->message, "\n";
363
        exit;
364
    }
365
    for($response->metadataFormat) {
366
        push(@formats,$_->metadataPrefix);
367
    }
368
    return \@formats;
369
}
370
371
sub ListSets {
372
    my $self = shift;
373
    my @sets;
374
    my $response = $self->{rh}->ListSets();
375
    if( $response->is_error ) {
376
        print "Error requesting ListSets:\n",
377
                $response->code . " " . $response->message, "\n";
378
        exit;
379
    }
380
    while (my $s = $response->next){
381
        my $set;
382
        $set->{setSpec} = $s->setSpec;
383
        $set->{setName} = $s->setName;
384
385
        #TODO: Not really sure what to do with the descriptions as they're XML and not necessarily that easy to parse for GUI views...
386
        #my @temp_setDescriptions = $s->setDescription;
387
        #my @setDescriptions;
388
        #foreach my $temp_setDescription (@temp_setDescriptions){
389
        #    push (@setDescriptions,$temp_setDescription->dom->toString); I think we need to do better than just return the setDescription XML...That's not very useful...
390
        #}
391
        #$set->{setDescription} = \@setDescriptions;
392
        push (@sets,$set);
393
    }
394
    return \@sets;
395
}
396
397
sub Identify {
398
    my $self = shift;
399
    my $response = $self->{rh}->Identify();
400
    if( $response->is_error ) {
401
        print "Error requesting Identify:\n",
402
        $response->code . " " . $response->message, "\n";
403
        exit;
404
    }
405
    my $identify_data;
406
    #DEBUG: View what's in the Identify object
407
    #print Dumper($response->headers);
408
409
    $identify_data->{repositoryName} = $response->repositoryName;
410
    $identify_data->{baseURL} = $response->baseURL;
411
    $identify_data->{protocolVersion} = $response->protocolVersion; #Tim Brody says this will always return 2.0 and that ->version should be used to find the actual version...
412
    #$identify_data->{version} = $response->version;
413
    $identify_data->{earliestDatestamp} = $response->earliestDatestamp;
414
    $identify_data->{deletedRecord} = $response->deletedRecord; #not in the perldoc, but it's in the code and the OAI-PMH spec
415
    $identify_data->{granularity} = $response->granularity;
416
417
    #These methods should be used with an array context so they return all the elements and not just the first one
418
    my @adminEmails = $response->adminEmail;
419
    $identify_data->{adminEmail} = \@adminEmails;
420
    my @compressions = $response->compression;
421
    $identify_data->{compression} = \@compressions;
422
423
    #TODO: Descriptions are encapsulated in XML containers, I believe. Not sure what to do with these at present...
424
    #my @descriptions = $response->description;
425
    #$identify_data->{description} = \@descriptions;
426
    #$response->next
427
428
    return $identify_data;
429
}
430
431
sub ProcessOaipmhRecordResponse {
432
    my ( $self, $response, $harvest, $force ) = @_;
433
    if ($response){
434
        my @records;
435
        while( my $rec = $response->next ){
436
            print "I'm parsing the record..." if $self->{debug};
437
            #Parse from the HTTP::OAI::Record object into our C4::OAI::Client::Record object
438
            my $record_object = _parse_httpoai_records_into_clientoai_records($rec,$self->{repository_id},$self->{metadataPrefix});
439
440
            #If set to harvest mode, we ingest the record
441
            if ($harvest && $record_object){
442
443
                #Retrieve the latest biblionumber associated with this OAI identifier
444
                $record_object->GetBiblionumberFromOaiIdentifier;
445
446
                #Generate a status to be acted upon (e.g. add, update, delete, force_add, force_update, ignore)
447
                $record_object->GenerateStatus($force);
448
449
                print "I'm ingesting the record..." if $self->{debug};
450
                #Based on the above status, act upon the record
451
                my $ingestion_flag  = _ingest_oai_record($record_object, $self->get_XSLT_path, $self->get_frameworkcode);
452
453
                if ($ingestion_flag){
454
                    #Log this OAI-PMH record in the database
455
                    $record_object->AddRecordToLog;
456
                    print $record_object->get_identifier.' -> '.$record_object->get_status.' -> '.$record_object->get_biblionumber." \n ";
457
                }
458
            }
459
            print $rec->identifier."\n" if !$harvest;
460
            push(@records,$record_object) if !$harvest;
461
        }
462
        return \@records;
463
    }
464
}
465
466
sub _ingest_oai_record {
467
    my ( $oai_record, $xslt_path, $frameworkcode ) = @_;
468
469
    if ( $oai_record && $xslt_path ){
470
        my $identifier_status = $oai_record->get_status;
471
        if ($identifier_status){
472
            if ($identifier_status eq 'ignore'){
473
                print $oai_record->get_identifier.' -> '.$identifier_status."\n";
474
                return;
475
            }
476
            elsif ($identifier_status eq 'add' || $identifier_status eq 'force_add'){
477
                $oai_record->TransformMetadata($xslt_path);
478
                if ($oai_record->get_transformed_metadata){
479
                    my $record = $oai_record->GenerateMarcObject($oai_record->get_transformed_metadata->toString);
480
                    my ($biblionumber,$biblioitemnumber) = C4::Biblio::AddBiblio($record,$frameworkcode);
481
                    $oai_record->set_biblionumber($biblionumber) if $biblionumber;
482
                    return 1;
483
                } else {
484
                    return;
485
                }
486
            }
487
            elsif (($identifier_status eq 'update' || $identifier_status eq 'force_update') && $oai_record->get_biblionumber){
488
                $oai_record->TransformMetadata($xslt_path);
489
                if ($oai_record->get_transformed_metadata){
490
                    my $record = $oai_record->GenerateMarcObject($oai_record->get_transformed_metadata->toString);
491
                    if ($oai_record->GetBiblionumberExistence){
492
                        C4::Biblio::ModBiblio($record,$oai_record->get_biblionumber,$frameworkcode);
493
                        return 1;
494
                    } else {
495
                        print "Error modding bib record ".$oai_record->get_biblionumber.", as it does not exist anymore \n";
496
                        $oai_record->set_status($identifier_status."_fail");
497
                        return 1;
498
                    }
499
                } else {
500
                    return;
501
                }
502
            }
503
            elsif ($identifier_status eq 'delete'){
504
                if ($oai_record->get_biblionumber){
505
                    if ($oai_record->GetBiblionumberExistence){
506
                        my $error = C4::Biblio::DelBiblio($oai_record->get_biblionumber);
507
                        if ($error){
508
                            warn "Error deleting biblionumber ".$oai_record->get_biblionumber." -> $error";
509
                            #NOTE: This will fail if there are items attached to the bib record
510
                            #So...just record the failure due to items being attached rather than trying to delete items.
511
                            $oai_record->set_status($identifier_status."_fail");
512
                            return 1;
513
                        }
514
                        else {
515
                            return 1;
516
                        }
517
                    } else {
518
                        print "It looks like biblionumber ".$oai_record->get_biblionumber." has already been deleted \n";
519
                        return;
520
                    }
521
                } else {
522
                    print "There is no biblionumber associated with this deletion record";
523
                    return;
524
                }
525
            }
526
        } else {
527
            print $oai_record->get_identifier ." -> no status (so no action taken)\n";
528
            return;
529
        }
530
    }
531
}
532
533
sub _parse_httpoai_records_into_clientoai_records {
534
    my ( $rec, $repository_id, $metadataPrefix ) = @_;
535
        my $record;
536
        $record->{header} = $rec->header->dom;
537
        $record->{identifier} = $rec->identifier;
538
        $record->{datestamp} = $rec->datestamp;
539
        $record->{metadata} = $rec->metadata ? $rec->metadata->dom : undef; #N.B. there won't be any metadata for deleted records
540
        $record->{about} = $rec->about ? $rec->about->dom : undef;
541
        $record->{deleted} = $rec->is_deleted;
542
        $record->{status} = $rec->status;
543
        $record->{repository_id} = $repository_id;
544
        $record->{metadataPrefix} = $metadataPrefix;
545
        my $record_object = C4::OAI::Client::Record->new($record);
546
        if ($record_object){
547
            return $record_object;
548
        } else {
549
            return;
550
        }
551
}
552
553
554
555
1;
556
__END__
(-)a/C4/OAI/Client/Record.pm (+328 lines)
Line 0 Link Here
1
package C4::OAI::Client::Record;
2
3
=head1 NAME
4
5
C4::OAI::Client::Record - an internal class for handling records retrieved via the OAI-PMH protocol
6
7
=head1 DESCRIPTION
8
9
C4::OAI::Client::Record - Class to handle the management of records retrieved via OAI-PMH
10
11
More information on OAI-PMH can be found L<here|http://www.openarchives.org/OAI/openarchivesprotocol.html>
12
13
=head1 AUTHOR
14
15
David Cook <dcook AT prosentient DOT com DOT au>
16
17
=cut
18
19
use Modern::Perl;
20
use C4::Context;
21
use C4::Templates qw/_current_language/;
22
use C4::Charset qw/StripNonXmlChars/;
23
use C4::XSLT qw/GetURI/;
24
25
use XML::LibXML;
26
use XML::LibXSLT;
27
use MARC::Record;
28
use MARC::File::XML;
29
use base qw(Class::Accessor);
30
31
sub new {
32
    my($proto, $fields) = @_;
33
    my($class) = ref $proto || $proto;
34
    $fields = {} unless defined $fields;
35
    bless {%$fields}, $class;
36
}
37
38
__PACKAGE__->follow_best_practice; #Use get_ and set_ prefixes for accessors
39
__PACKAGE__->mk_accessors(qw(identifier datestamp metadata status deleted header about transformed_metadata repository_id metadataPrefix biblionumber));
40
41
=head2 TransformMetadata
42
43
44
45
=cut
46
47
sub TransformMetadata {
48
    my ( $self, $XSLT_path ) = @_;
49
    my $source = $self->{metadata}; #This is a LibXML object retrieved via HTTP::OAI::Harvester
50
    my $metadataPrefix = $self->get_metadataPrefix;
51
52
    my $no_XSLT_path;
53
    if (!$XSLT_path){
54
        $no_XSLT_path = 1;
55
    }
56
57
    if ($metadataPrefix && $source){
58
        my $xslt = XML::LibXSLT->new();
59
        my $style_doc;
60
61
        #$parser based on equivalent from XSLT.pm
62
        my $parser = XML::LibXML->new();
63
        # don't die when you find &, >, etc
64
        $parser->recover_silently(0);
65
66
        #NOTE: This XSLT essentially copies over all MARC fields except the 952 and 999.
67
        #It also adds a 037 containing the OAI-PMH identifier.
68
        my $default_marc_xslt = C4::Context->config('intrahtdocs') .
69
                                    '/' . C4::Context->preference("template") .
70
                                    '/' . C4::Templates::_current_language() .
71
                                    '/xslt/' .
72
                                    C4::Context->preference('marcflavour') .
73
                                    "slim2" .
74
                                    C4::Context->preference('marcflavour') .
75
                                    "enhanced.xsl";
76
77
        #TODO: This XSLT needs work. In practice, it would be better to create
78
        #customized XSLTs based on the metadata format native to the OAI-PMH source (e.g. DIM for DSpace)
79
        my $default_dc_xslt = C4::Context->config('intrahtdocs') .
80
                                    '/' . C4::Context->preference("template") .
81
                                    '/' . C4::Templates::_current_language() .
82
                                    '/xslt/DC2' .
83
                                    C4::Context->preference('marcflavour') .
84
                                    "slim.xsl";
85
86
        #TODO: As necessary, add more default XSLTs. Perhaps MODS and METS (although METS would be difficult
87
        #since it can contain other metadata formats within itself).
88
89
        if ($XSLT_path){
90
            #This conditional for handling secure remote XSLTs copied from XSLT.pm
91
            if ( $XSLT_path =~ /^https?:\/\// ) {
92
                my $xsltstring = GetURI($XSLT_path);
93
                if ($xsltstring){
94
                    $style_doc = $parser->parse_string($xsltstring);
95
                }
96
                else{
97
                    #If a string isn't retrieved using GetURI, we switch to our default transforms
98
                    $no_XSLT_path  = 1;
99
                }
100
            } elsif ( $XSLT_path eq 'dublincore' ) {
101
                $style_doc = $parser->parse_file($default_dc_xslt); #Use default DC XSLT if XSLT_path is specified as 'dublincore'
102
            } elsif ( $XSLT_path eq 'marc' ) {
103
                $style_doc = $parser->parse_file($default_marc_xslt); #Use default MARC XSLT if XSLT_path is specified as 'marc'
104
            } else {
105
                if ( -e $XSLT_path){
106
                    $style_doc = $parser->parse_file($XSLT_path);
107
                }
108
                else{
109
                    #If the file doesn't actually exist, we switch to our default transforms
110
                    $no_XSLT_path = 1;
111
                }
112
            }
113
        }
114
115
        if ($no_XSLT_path){
116
            if ($metadataPrefix eq 'marcxml' || $metadataPrefix eq 'marc'){
117
                $style_doc = $parser->parse_file($default_marc_xslt);
118
            }
119
            elsif ($metadataPrefix eq 'dc' || $metadataPrefix eq 'oai_dc'){
120
                $style_doc = $parser->parse_file($default_dc_xslt);
121
            }
122
            else{
123
                return -1;
124
            }
125
        }
126
127
        if ($style_doc){
128
            my $stylesheet = $xslt->parse_stylesheet($style_doc);
129
130
            my %xslt_params;
131
            $xslt_params{'identifier'} = $self->{identifier};
132
133
            if ($stylesheet){
134
                #Pass OAI-PMH identifier and matched biblionumber (if there is one) to the XSLT for unique identification/provenance
135
                my $result_xml = $stylesheet->transform($source, XML::LibXSLT::xpath_to_string(%xslt_params));
136
137
                if ($result_xml){
138
                    $self->set_transformed_metadata($result_xml);
139
                    return 1;
140
                }
141
                else{
142
                    return -1;
143
                }
144
            }
145
        }
146
    }
147
    else{
148
        return -1;
149
    }
150
}
151
152
=head2 AddRecordToLog
153
154
155
=cut
156
157
sub AddRecordToLog {
158
    my ( $self ) = @_;
159
160
    my $metadataPrefix = $self->get_metadataPrefix;
161
    my $repository_id = $self->get_repository_id;
162
    my $biblionumber = $self->get_biblionumber;
163
164
    if ($metadataPrefix && $repository_id && $biblionumber){
165
            my $identifier = $self->get_identifier;
166
            my $datestamp = $self->get_datestamp;
167
            my $metadata = $self->get_metadata ? $self->get_metadata->toString : undef;
168
            my $header = $self->get_header ? $self->get_header->toString : undef;
169
            my $status = $self->get_status;
170
171
172
173
            my $dbh = C4::Context->dbh;
174
            my $sql = "
175
              INSERT INTO oai_harvest (identifier,datestamp,metadataPrefix,metadata,header,status,repository_id,biblionumber)
176
              VALUES (?,?,?,?,?,?,?,?)
177
            ";
178
            my $sth = $dbh->prepare($sql);
179
            $sth->execute($identifier, $datestamp, $metadataPrefix, $metadata, $header, $status, $repository_id, $biblionumber);
180
            if ($sth->err){
181
                return "ERROR! return code: " . $sth->err . " error msg: " . $sth->errstr . "\n";
182
            }
183
            else{
184
                return 1;
185
            }
186
    }
187
    else{
188
        return -1;
189
    }
190
}
191
192
=head2 GenerateStatus
193
194
195
196
197
=cut
198
199
sub GenerateStatus {
200
    my ( $self, $force ) = @_;
201
    my $dbh = C4::Context->dbh;
202
    my $repository_id = $self->get_repository_id;
203
    my $identifier_status;
204
205
    #Check if this identifier is found in the database with this repository
206
    my $check_for_id_sql = "
207
      SELECT count(*) as 'count'
208
      FROM oai_harvest
209
      WHERE identifier = ? and repository_id = ?";
210
    my $check_for_id_sth = $dbh->prepare($check_for_id_sql);
211
    $check_for_id_sth->execute($self->get_identifier, $repository_id);
212
    my $check_for_id_row = $check_for_id_sth->fetchrow_hashref;
213
    if ($check_for_id_row->{count} == 0){
214
        #The OAI-PMH unique identifier doesn't exist in this database for this repository == ADD
215
        $identifier_status = "add";
216
    }
217
    else{
218
        #OAI-PMH Unique Identifier does exist in database == UPDATE || IGNORE
219
        my $check_for_id_and_datestamp_sql = "
220
          SELECT count(*) as 'count'
221
          FROM oai_harvest
222
          WHERE identifier = ? and datestamp = ? and repository_id = ?
223
        ";
224
        my $check_for_id_and_datestamp_sth = $dbh->prepare($check_for_id_and_datestamp_sql);
225
        $check_for_id_and_datestamp_sth->execute($self->get_identifier, $self->get_datestamp, $repository_id);
226
        my $check_for_id_and_datestamp_row = $check_for_id_and_datestamp_sth->fetchrow_hashref;
227
        if ($check_for_id_and_datestamp_row->{count} == 0){
228
            #OAI-PMH Unique Identifier and Datestamp combo don't exist in database == UPDATE
229
            #i.e. The identifier exists in the database, but this is an updated datestamp
230
            $identifier_status = "update"
231
        }
232
        else{
233
            #OAI-PMH Unique Identifier and Datestamp combo exist in database but we're forcing an add/update anyway (in accordance with the $force value
234
            if ($force){
235
                if ($self->GetBiblionumberExistence){
236
                    $identifier_status = ( $force eq "only_add" ) ? "ignore" : "force_update";
237
                }
238
                else {
239
                    $identifier_status = ( $force eq "only_update" ) ? "ignore" : "force_add";
240
                }
241
            } else {
242
            #OAI-PMH Unique Identifier and Datestamp combo exist in database == IGNORE
243
                $identifier_status = "ignore";
244
            }
245
        }
246
    }
247
248
    if ($self->get_deleted && $identifier_status ne "ignore"){
249
        $identifier_status = "delete";
250
    }
251
252
    if ($identifier_status){
253
        $self->set_status($identifier_status);
254
    }
255
}
256
257
258
sub GetBiblionumberExistence {
259
    my ( $self ) = @_;
260
    my $biblionumber_to_check = $self->get_biblionumber;
261
    if ($biblionumber_to_check){
262
        my $dbh = C4::Context->dbh;
263
        my $sql = "
264
          SELECT count(biblionumber) as 'count'
265
          FROM biblio
266
          WHERE biblionumber = ?
267
        ";
268
        my $sth = $dbh->prepare($sql);
269
        $sth->execute($biblionumber_to_check);
270
        my $row = $sth->fetchrow_hashref;
271
        if ($row->{count} > 0){
272
            return 1;
273
        } else {
274
            return;
275
        }
276
    } else {
277
        return;
278
    }
279
}
280
281
282
#FIXME: This probably isn't the best way to do this...
283
sub GetBiblionumberFromOaiIdentifier {
284
    my ( $self ) = @_;
285
    my $biblionumber;
286
    my $dbh = C4::Context->dbh;
287
    #Select the newest biblionumber. In theory, there should only be one biblionumber. However, if a record
288
    #was deleted in Koha but force_added again later, there would be multiple biblionumbers in our
289
    #harvesting records. Ergo, we take the highest, so that we're working with the correct number for future
290
    #updates/deletions.
291
    my $sql = "
292
      SELECT DISTINCT biblionumber
293
      FROM oai_harvest
294
      WHERE identifier = ? AND repository_id = ?
295
      ORDER BY timestamp desc
296
      LIMIT 1
297
    ";
298
    my $sth = $dbh->prepare($sql);
299
    $sth->execute($self->get_identifier,$self->get_repository_id);
300
    my $row = $sth->fetchrow_hashref;
301
    $biblionumber = $row->{biblionumber} if $row->{biblionumber};
302
    $self->set_biblionumber($biblionumber) if $biblionumber; #Add biblionumber to the object
303
    return $biblionumber;
304
}
305
306
#FIXME: This should probably be in Biblio.pm and not a method here? (Galen's work might intersect here...)
307
sub GenerateMarcObject {
308
    my ( $self, $marcxml ) = @_;
309
310
    if ($marcxml) {
311
        my $marcxml = StripNonXmlChars( $marcxml ); #Warning as per C4::Charset, this could cause some data loss
312
313
        my $record = eval { MARC::Record::new_from_xml( $marcxml, "utf8", C4::Context->preference('marcflavour') ) };
314
        if ($@) {
315
            warn " problem with : $@ \n$marcxml";
316
            return;
317
        }
318
        return $record;
319
    } else {
320
        return;
321
    }
322
}
323
324
325
326
327
1;
328
__END__
(-)a/admin/oai_repository_targets.pl (+168 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# This file is part of Koha.
4
#
5
# Copyright 2013 Prosentient Systems
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, see <http://www.gnu.org/licenses>.
18
19
=head1 NAME
20
21
oai_repository_targets.pl - script to manage configuration for harvesting external
22
OAI-PMH server/repository targets.
23
24
=head1 DESCRIPTION
25
26
If !$op, then display table listing existing entries
27
28
If $op eq 'add', then display add form OR modify form, if primary key is supplied
29
30
If $op eq 'delete', then delete the entry
31
32
=cut
33
34
use Modern::Perl;
35
use CGI;
36
use C4::Auth;
37
use C4::Output;
38
use C4::Context;
39
40
use C4::Koha qw/getframeworks/;
41
use C4::OAI::Client::Harvester;
42
43
my $input = new CGI;
44
my $script_name = "/cgi-bin/koha/admin/oai_repository_targets.pl";
45
my ($template, $loggedinuser, $cookie)
46
    = get_template_and_user({template_name => "admin/oai_repository_targets.tt",
47
            query => $input,
48
            type => "intranet",
49
            authnotrequired => 0,
50
            flagsrequired => {parameters => 'parameters_remaining_permissions'},
51
            debug => 1,
52
            });
53
54
$template->param(script_name => $script_name);
55
56
my $op = $input->param('op') || '';
57
my $id = $input->param('id') || '';
58
if ($id){
59
    $template->param(id => $id);
60
}
61
62
# get framework list
63
my $frameworks = C4::Koha::getframeworks;
64
my @frameworkcodeloop;
65
foreach my $thisframeworkcode ( sort { uc($frameworks->{$a}->{'frameworktext'}) cmp uc($frameworks->{$b}->{'frameworktext'}) } keys %{$frameworks} ) {
66
    push @frameworkcodeloop, {
67
        value         => $thisframeworkcode,
68
        frameworktext => $frameworks->{$thisframeworkcode}->{'frameworktext'},
69
    };
70
}
71
$template->param(frameworkcodeloop => \@frameworkcodeloop);
72
73
if ($op eq 'add' || $op eq 'edit' || $op eq 'validate'){
74
    #These operations require a form view
75
    $template->param(view => "form");
76
77
    if ($op eq 'edit' && $id){
78
        my $repository_fields = C4::OAI::Client::Harvester::GetOAIRepository($id);
79
        $template->param(form_fields => $repository_fields);
80
    }
81
    elsif ($op eq 'validate'){
82
        my @validation_errors;
83
        my $form_fields;
84
        $form_fields->{baseURL} = $input->param('baseURL') ? $input->param('baseURL') : undef;
85
        if (!$form_fields->{baseURL}){
86
            push(@validation_errors,"baseURL");
87
            $template->param(baseURL_validation_err => "mandatory");
88
        }
89
        $form_fields->{metadataPrefix} = $input->param('metadataPrefix') ? $input->param('metadataPrefix') : undef;
90
        if (!$form_fields->{metadataPrefix}){
91
            push(@validation_errors,"metadataPrefix");
92
            $template->param(metadataPrefix_validation_err => "mandatory");
93
        }
94
        $form_fields->{XSLT_path} = $input->param('XSLT_path') ? $input->param('XSLT_path') : undef;
95
        if (!$form_fields->{XSLT_path}){
96
            push(@validation_errors,"XSLT_path");
97
            $template->param(XSLT_path_validation_err => "mandatory");
98
        }
99
        #Use 'undef' instead of blank so that you take advantage of database level validation
100
        $form_fields->{opt_from} = $input->param('opt_from') ? $input->param('opt_from') : undef;
101
        $form_fields->{opt_until} = $input->param('opt_until') ? $input->param('opt_until') : undef;
102
        $form_fields->{opt_set} = $input->param('opt_set') ? $input->param('opt_set') : undef;
103
        $form_fields->{active} = $input->param('active') == 1 || $input->param('active') == 0 ? $input->param('active') : undef;
104
        $form_fields->{frameworkcode} = $input->param('frameworkcode') ? $input->param('frameworkcode') : '';
105
        $form_fields->{comments} = $input->param('comments') ? $input->param('comments') : undef;
106
        $form_fields->{repository_id} = $id ? $id : undef;
107
108
        if (@validation_errors){
109
            warn "There are ". scalar @validation_errors . " validation errors";
110
            $template->param(
111
                form_fields => $form_fields,
112
            );
113
        }
114
        else {
115
            if ($id){
116
                C4::OAI::Client::Harvester::ModOAIRepository($form_fields);
117
                #Show confirmation view after update
118
                $template->param(
119
                    view => "confirm",
120
                    op => "confirm_update",
121
                );
122
            }
123
            else {
124
                C4::OAI::Client::Harvester::AddOAIRepository($form_fields);
125
                #Show confirmation view after insert/add
126
                $template->param(
127
                    view => "confirm",
128
                    op => "confirm_add",
129
                );
130
            }
131
        }
132
    }
133
}
134
else {
135
    #All other operations require a list view
136
    $template->param(view => "list");
137
138
    my @results;
139
140
    if ($op eq 'delete' && $id){
141
        my $repository_fields = C4::OAI::Client::Harvester::GetOAIRepository($id);
142
        if ($repository_fields){
143
            push(@results,$repository_fields);
144
            $template->param(
145
                op => 'delete'
146
            );
147
        }
148
    }
149
    elsif ($op eq 'perform_delete' && $id){
150
        C4::OAI::Client::Harvester::DelOAIRepository($id);
151
        #Show confirmation view after delete
152
        $template->param(
153
            view => "confirm",
154
            op => "confirm_delete",
155
        );
156
    }
157
    else{
158
        @results = C4::OAI::Client::Harvester::GetOAIRepositoryList();
159
    }
160
161
    if (@results){
162
        $template->param(
163
            result_rows => \@results
164
        );
165
    }
166
}
167
168
output_html_with_http_headers $input, $cookie, $template->output;
(-)a/installer/data/mysql/kohastructure.sql (+41 lines)
Lines 3243-3245 CREATE TABLE IF NOT EXISTS plugin_data ( Link Here
3243
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
3243
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
3244
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
3244
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
3245
3245
3246
--
3247
-- Table structure for table 'oai_harvest_repositories'
3248
--
3249
3250
DROP TABLE IF EXISTS oai_harvest_repositories;
3251
CREATE TABLE  oai_harvest_repositories (
3252
  repository_id int(11) NOT NULL AUTO_INCREMENT, -- primary key identifier
3253
  baseURL text NOT NULL, -- baseURL of the remote OAI-PMH repository (mandatory)
3254
  opt_from varchar(45) DEFAULT NULL, -- "from" time for selective harvesting (optional - gets set automatically by cronjob)
3255
  opt_until varchar(45) DEFAULT NULL, -- "until" time for selective harvesting (optional)
3256
  opt_set varchar(45) DEFAULT NULL, -- the record set to harvest for selective harvesting (optional)
3257
  metadataPrefix varchar(45) NOT NULL, -- metadata format (e.g. oai_dc, dc, marcxml)
3258
  active int(11) NOT NULL DEFAULT '0', -- indicate whether this repo is actively harvested by Koha
3259
  timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, -- date last modified
3260
  XSLT_path text, -- filepath to a local or external XSLT file to use for transforming incoming records into MARCXML
3261
  frameworkcode varchar(4) NOT NULL, -- framework to use when ingesting records
3262
  comments text, -- limited number of characters (controlled by template) to describe the repository (optional - for librarian use rather than system use)
3263
  PRIMARY KEY (repository_id)
3264
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
3265
3266
--
3267
-- Table structure for table 'oai_harvest'
3268
--
3269
3270
DROP TABLE IF EXISTS oai_harvest;
3271
CREATE TABLE oai_harvest (
3272
  oai_harvest_id int(11) NOT NULL AUTO_INCREMENT, -- primary key identifier
3273
  identifier varchar(255) NOT NULL, -- OAI-PMH identifier (unique to its original repo)
3274
  datestamp varchar(45) NOT NULL, -- OAI-PMH datestamp (i.e. date last modified)
3275
  metadataPrefix varchar(45) NOT NULL, -- metadata format of the record
3276
  biblionumber int(11) NOT NULL, -- biblionumber of the biblio record created by this OAI-PMH ingested record
3277
  metadata longtext, -- XML string containing the metadata (this will be null for incoming records marked as deleted)
3278
  header longtext NOT NULL, -- XML string containing the header information (i.e. identifier, datestamp, status)
3279
  about longtext, -- XML string containing explanatory information about a record (optional. Often not included in records)
3280
  timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, -- date last modified
3281
  repository_id int(11) NOT NULL, -- id for this record's OAI-PMH repository (linked to oai_harvest_repositories table)
3282
  status varchar(45) DEFAULT NULL, -- whether this record was an addition, an update, or a deletion
3283
  PRIMARY KEY (oai_harvest_id),
3284
  KEY FK_oai_harvest_1 (repository_id),
3285
  CONSTRAINT FK_oai_harvest_1 FOREIGN KEY (repository_id) REFERENCES oai_harvest_repositories (repository_id) ON UPDATE NO ACTION
3286
) ENGINE=InnoDB AUTO_INCREMENT=2809 DEFAULT CHARSET=utf8;
(-)a/installer/data/mysql/updatedatabase.pl (+40 lines)
Lines 7185-7190 if ( CheckVersion($DBversion) ) { Link Here
7185
    SetVersion($DBversion);
7185
    SetVersion($DBversion);
7186
}
7186
}
7187
7187
7188
$DBversion = "3.13.00.XXX";
7189
if ( CheckVersion($DBversion) ) {
7190
    $dbh->do("
7191
CREATE TABLE IF NOT EXISTS oai_harvest_repositories (
7192
  repository_id int(11) NOT NULL AUTO_INCREMENT,
7193
  baseURL text NOT NULL,
7194
  opt_from varchar(45) DEFAULT NULL,
7195
  opt_until varchar(45) DEFAULT NULL,
7196
  opt_set varchar(45) DEFAULT NULL,
7197
  metadataPrefix varchar(45) NOT NULL,
7198
  active int(11) NOT NULL DEFAULT '0',
7199
  timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
7200
  XSLT_path text,
7201
  frameworkcode varchar(4) NOT NULL,
7202
  comments text,
7203
  PRIMARY KEY (repository_id)
7204
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
7205
    ");
7206
    $dbh->do("
7207
CREATE TABLE IF NOT EXISTS oai_harvest (
7208
  oai_harvest_id int(11) NOT NULL AUTO_INCREMENT,
7209
  identifier varchar(255) NOT NULL,
7210
  datestamp varchar(45) NOT NULL,
7211
  metadataPrefix varchar(45) NOT NULL,
7212
  biblionumber int(11) NOT NULL,
7213
  metadata longtext,
7214
  header longtext NOT NULL,
7215
  about longtext,
7216
  timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
7217
  repository_id int(11) NOT NULL,
7218
  status varchar(45) DEFAULT NULL,
7219
  PRIMARY KEY (oai_harvest_id),
7220
  KEY FK_oai_harvest_1 (repository_id),
7221
  CONSTRAINT FK_oai_harvest_1 FOREIGN KEY (repository_id) REFERENCES oai_harvest_repositories (repository_id) ON UPDATE NO ACTION
7222
) ENGINE=InnoDB AUTO_INCREMENT=2809 DEFAULT CHARSET=utf8;
7223
    ");
7224
    print "Upgrade to $DBversion done (Bug 10662: Build OAI-PMH Harvesting Client)\n";
7225
    SetVersion($DBversion);
7226
}
7227
7188
=head1 FUNCTIONS
7228
=head1 FUNCTIONS
7189
7229
7190
=head2 TableExists($table)
7230
=head2 TableExists($table)
(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc (+1 lines)
Lines 62-67 Link Here
62
<ul>
62
<ul>
63
	<!-- <li><a href="/cgi-bin/koha/admin/printers.pl">Network Printers</a></li> -->
63
	<!-- <li><a href="/cgi-bin/koha/admin/printers.pl">Network Printers</a></li> -->
64
    <li><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50 client targets</a></li>
64
    <li><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50 client targets</a></li>
65
    <li><a href="/cgi-bin/koha/admin/oai_repository_targets.pl">OAI-PMH repository targets</a></li>
65
    <li><a href="/cgi-bin/koha/admin/didyoumean.pl">Did you mean?</a></li>
66
    <li><a href="/cgi-bin/koha/admin/didyoumean.pl">Did you mean?</a></li>
66
</ul>
67
</ul>
67
</div>
68
</div>
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt (+2 lines)
Lines 105-110 Link Here
105
	<dd>Printers (UNIX paths).</dd> -->
105
	<dd>Printers (UNIX paths).</dd> -->
106
    <dt><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50 client targets</a></dt>
106
    <dt><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50 client targets</a></dt>
107
	<dd>Define which servers to query for MARC data in the integrated Z39.50 client.</dd>
107
	<dd>Define which servers to query for MARC data in the integrated Z39.50 client.</dd>
108
    <dt><a href="/cgi-bin/koha/admin/oai_repository_targets.pl">OAI-PMH repository targets</a></dt>
109
    <dd>Define which OAI-PMH repositories from which to automatically harvest metadata records</dd>
108
    <dt><a href="/cgi-bin/koha/admin/didyoumean.pl">Did you mean?</a></dt>
110
    <dt><a href="/cgi-bin/koha/admin/didyoumean.pl">Did you mean?</a></dt>
109
    <dd>Choose which plugins to use to suggest searches to patrons and staff.</dd>
111
    <dd>Choose which plugins to use to suggest searches to patrons and staff.</dd>
110
</dl>
112
</dl>
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_repository_targets.tt (+224 lines)
Line 0 Link Here
1
[% INCLUDE 'doc-head-open.inc' %]
2
<title>Koha &rsaquo; Administration &rsaquo; Manage OAI-PMH repository targets</title>
3
[%# FIXME: change title based on page... each page should have a different title %]
4
[% INCLUDE 'doc-head-close.inc' %]
5
6
[% IF ( view == "list" ) %]
7
<link rel="stylesheet" type="text/css" href="[% themelang %]/css/datatables.css" />
8
<script type="text/javascript" src="[% themelang %]/lib/jquery/plugins/jquery.dataTables.min.js"></script>
9
[% INCLUDE 'datatables-strings.inc' %]
10
<script type="text/javascript" src="[% themelang %]/js/datatables.js"></script>
11
[% END %]
12
[% IF ( view == "list" ) %]
13
    <script type="text/javascript">
14
    //<![CDATA[
15
     $(document).ready(function() {
16
        [% IF ( result_rows ) && ( op != "delete" ) %]$("#repositoriest").dataTable($.extend(true, {}, dataTablesDefaults, {
17
            "aoColumnDefs": [
18
                { "aTargets": [ 9,10 ], "bSortable": false, "bSearchable": false },
19
            ],
20
            "iDisplayLength": 20,
21
            "aLengthMenu": [[10, 20, 50, 100, -1], [10, 20, 50, 100, "All"]],
22
            "sPaginationType": "four_button"
23
        }));[% END %]
24
     });
25
    //]]>
26
    </script>
27
[% END %]
28
29
</head>
30
<body id="admin_oairepositorytargets" class="admin">
31
[% INCLUDE 'header.inc' %]
32
[% INCLUDE 'cat-search.inc' %]
33
<div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/admin/admin-home.pl">Administration</a> &rsaquo; Manage OAI-PMH repository targets</div>
34
35
<div id="doc3" class="yui-t2">
36
    <div id="bd">
37
        <div id="yui-main">
38
            <div class="yui-b">
39
            [% IF ( view == "confirm" ) %]
40
                [% IF ( op == "confirm_add" ) %]
41
                <h1>OAI-PMH repository target added</h1>
42
                <form action="[% script_name %]" method="post">
43
                    <input type="submit" value="OK" />
44
                </form>
45
                [% ELSIF ( op == "confirm_update" ) %]
46
                <h1>OAI-PMH repository target updated</h1>
47
                <form action="[% script_name %]" method="post">
48
                    <input type="submit" value="OK" />
49
                </form>
50
                [% ELSIF ( op == "confirm_delete" ) %]
51
                <h1>OAI-PMH repository target deleted</h1>
52
                <form action="[% script_name %]" method="post">
53
                    <input type="submit" value="OK" />
54
                </form>
55
                [% END %]
56
            [% ELSIF ( view == "form" ) %]
57
                [% IF ( id ) %]
58
                <h1>Edit OAI-PMH repository target</h1>
59
                [% ELSE %]
60
                <h1>New OAI-PMH repository target</h1>
61
                [% END %]
62
                <form action="[% script_name %]" name="repository_add_form" method="post">
63
                <input type="hidden" name="op" value="validate" />
64
                [% IF ( id ) %]
65
                <input type="hidden" name="id" value="[% id %]" />
66
                [% END %]
67
                    <fieldset class="rows">
68
                    <ol>
69
                        [% IF ( baseURL_validation_err ) %]
70
                        <li class="error">
71
                        [% ELSE %]
72
                        <li>
73
                        [% END %]
74
                            <label for="baseURL">Base URL: </label>
75
                            <input type="text" name="baseURL" id="baseURL" size="65" value="[% form_fields.baseURL %]"/>
76
                            [% IF ( baseURL_validation_err == "mandatory") %]
77
                            [Base URL is a mandatory field.]
78
                            [% END %]
79
                        </li>
80
                        <li>
81
                            <label for="opt_from">From: </label>
82
                            <input type="text" name="opt_from" id="opt_from" size="65" value="[% form_fields.opt_from %]"/>
83
                        </li>
84
                        <li>
85
                            <label for="opt_until">Until: </label>
86
                            <input type="text" name="opt_until" id="opt_until" size="65" value="[% form_fields.opt_until %]"/>
87
                        </li>
88
                        <li>
89
                            <label for="opt_set">Set: </label>
90
                            <input type="text" name="opt_set" id="opt_set" size="65" value="[% form_fields.opt_set %]"/>
91
                        </li>
92
                        [% IF ( metadataPrefix_validation_err ) %]
93
                        <li class="error">
94
                        [% ELSE %]
95
                        <li>
96
                        [% END %]
97
                            <label for="metadataPrefix">Metadata prefix: </label>
98
                            <input type="text" name="metadataPrefix" id="metadataPrefix" size="65" value="[% form_fields.metadataPrefix %]"/>
99
                            [% IF ( metadataPrefix_validation_err == "mandatory") %]
100
                            [Metadata prefix is a mandatory field.]
101
                            [% END %]
102
                        </li>
103
                        <li>
104
                            <label for="active">Status: </label>
105
                            <select id="active" name="active">
106
                                [% IF ( form_fields.active == 0 ) %]
107
                                <option value="0" selected="selected">Inactive</option>
108
                                [% ELSE %]
109
                                <option value="0">Inactive</option>
110
                                [% END %]
111
                                [% IF ( form_fields.active == 1 ) %]
112
                                <option value="1" selected="selected">Active</option>
113
                                [% ELSE %]
114
                                <option value="1">Active</option>
115
                                [% END %]
116
                            </select>
117
                        </li>
118
                        <li>
119
                            <label for="frameworkcode">Framework: </label>
120
                            <select name="frameworkcode" id="frameworkcode">
121
                                <option value="">Default framework</option>
122
                            [% FOREACH frameworkcode IN frameworkcodeloop %]
123
                                [% IF ( form_fields.frameworkcode == frameworkcode.value ) %]
124
                                <option value="[% frameworkcode.value %]" selected="selected">[% frameworkcode.frameworktext %]</option>
125
                                [% ELSE %]
126
                                <option value="[% frameworkcode.value %]">[% frameworkcode.frameworktext %]</option>
127
                                [% END %]
128
                            [% END %]
129
                            </select>
130
                        </li>
131
                        [% IF ( XSLT_path_validation_err ) %]
132
                        <li class="error">
133
                        [% ELSE %]
134
                        <li>
135
                        [% END %]
136
                            <label for="XSLT_path">XSLT: </label>
137
                            <input type="text" name="XSLT_path" id="XSLT_path" size="65" value="[% form_fields.XSLT_path %]"/>
138
                            [% IF ( XSLT_path_validation_err == "mandatory") %]
139
                            [XSLT is a mandatory field.]
140
                            [% END %]
141
                            (Please consult your system administrator when choosing your XSLT.)
142
                        </li>
143
                        <li>
144
                            <label for="comments">Comments: </label>
145
                            <textarea id="comments" name="comments" rows="4" cols="60" maxlength="500">[% form_fields.comments %]</textarea>
146
                        </li>
147
                    </ol>
148
                    </fieldset>
149
                    <fieldset class="action"><input type="submit" value="Save" /> <a class="cancel" href="[% script_name %]">Cancel</a></fieldset>
150
                </form>
151
            [% ELSIF ( view == "list" ) %]
152
                [% UNLESS ( op == "delete" ) %]
153
                <div id="toolbar" class="btn-toolbar">
154
                    <a id="newrepositorytarget" class="btn btn-small" href="[% script_name %]?op=add"><i class="icon-plus"></i> New OAI-PMH repository target</a>
155
                </div>
156
                [% END %]
157
                [% IF ( op == "delete" ) %]
158
                <h1>Delete OAI-PMH repository target</h1>
159
                [% ELSE %]
160
                <h1>List OAI-PMH repository targets</h1>
161
                [% END %]
162
                <table id="repositoriest">
163
                    <thead>
164
                        <tr>
165
                            <th>Base URL</th>
166
                            <th>From</th>
167
                            <th>Until</th>
168
                            <th>Set</th>
169
                            <th>Metadata prefix</th>
170
                            <th>Status</th>
171
                            <th>Framework</th>
172
                            <th>XSLT</th>
173
                            <th>Comments</th>
174
                            [% UNLESS ( op == "delete" ) %]
175
                            <th>&nbsp;</th>
176
                            <th>&nbsp;</th>
177
                            [% END %]
178
                        </tr>
179
                    </thead>
180
                    <tbody>
181
                    [% FOREACH result_row IN result_rows %]
182
                        <tr>
183
                            <td><a href="[% result_row.script_name %]?op=edit&amp;id=[% result_row.repository_id |url %]">[% result_row.baseURL %]</a></td>
184
                            <td>[% result_row.opt_from %]</td>
185
                            <td>[% result_row.opt_until %]</td>
186
                            <td>[% result_row.opt_set %]</td>
187
                            <td>[% result_row.metadataPrefix %]</td>
188
                            <td>[% IF ( result_row.active == 1 ) %] Active [% ELSE %] Inactive [% END %]</td>
189
                            <td>
190
                            [% IF ( result_row.frameworkcode == '') %]
191
                            Default
192
                            [% ELSE %]
193
                            [% FOREACH framework IN frameworkcodeloop %][% IF ( result_row.frameworkcode == framework.value )%][% framework.frameworktext %][% END %][% END %]
194
                            [% END %]
195
                            </td>
196
                            <td>[% result_row.XSLT_path %]</td>
197
                            <td>[% IF (result_row.comments.length > 20) %][% result_row.comments.substr(0,20) %]...[% ELSE %][% result_row.comments %][% END %]</td>
198
                            [% UNLESS ( op == "delete" ) %]
199
                            <td><a href="[% result_row.script_name %]?op=edit&amp;id=[% result_row.repository_id |url %]">Edit</a></td>
200
                            <td><a href="[% result_row.script_name %]?op=delete&amp;id=[% result_row.repository_id |url %]">Delete</a></td>
201
                            [% END %]
202
                        </tr>
203
                    [% END %]
204
                    </tbody>
205
                </table>
206
                [% IF ( op == "delete" ) %]
207
                <form action="[% script_name %]" method="post">
208
                    <input type="hidden" name="id" value="[% id %]" />
209
                    <input type="hidden" name="op" value="perform_delete" />
210
                    <fieldset class="action">
211
                        <input type="submit" value="Confirm" />
212
                        <a class="cancel" href="[% script_name %]">Cancel</a>
213
                    </fieldset>
214
                </form>
215
                [% END %]
216
            [% END %]
217
            </div>[%# first yui-b %]
218
        </div>[%# yui-main %]
219
        <div class="yui-b">
220
            [% INCLUDE 'admin-menu.inc' %]
221
        </div> [%# last yui-b %]
222
    </div> [%# bd %]
223
224
[% INCLUDE 'intranet-bottom.inc' %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/xslt/DC2MARC21slim.xsl (+217 lines)
Line 0 Link Here
1
<?xml version="1.0" encoding="UTF-8"?>
2
<xsl:stylesheet version="1.0" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns="http://www.loc.gov/MARC21/slim" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" exclude-result-prefixes="dc">
3
    <xsl:output method="xml" indent="yes"/>
4
5
    <!-- WARNING: This XSLT is a work-in-progress. While it will yield "usable" records. You will be much happier if you ingest MARC records instead of Dublin Core records via OAI-PMH -->
6
7
    <!-- pass in the OAI-PMH identifier for historical purposes -->
8
    <xsl:param name="identifier" select="identifier"/>
9
10
    <xsl:template match="/">
11
        <record xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" >
12
            <xsl:element name="leader">
13
                <xsl:variable name="type" select="//dc:type"/>
14
                <xsl:variable name="leader06">
15
                    <xsl:choose>
16
                        <xsl:when test="$type='collection'">p</xsl:when>
17
                        <xsl:when test="$type='dataset'">m</xsl:when>
18
                        <xsl:when test="$type='event'">r</xsl:when>
19
                        <xsl:when test="$type='image'">k</xsl:when>
20
                        <xsl:when test="$type='interactive resource'">m</xsl:when>
21
                        <xsl:when test="$type='service'">m</xsl:when>
22
                        <xsl:when test="$type='software'">m</xsl:when>
23
                        <xsl:when test="$type='sound'">i</xsl:when>
24
                        <xsl:when test="$type='text'">a</xsl:when>
25
                        <xsl:otherwise>a</xsl:otherwise>
26
                    </xsl:choose>
27
                </xsl:variable>
28
                <xsl:variable name="leader07">
29
                    <xsl:choose>
30
                        <xsl:when test="$type='collection'">c</xsl:when>
31
                        <xsl:otherwise>m</xsl:otherwise>
32
                    </xsl:choose>
33
                </xsl:variable>
34
                <!-- *00-04 Record length : leave blank. Koha will fill this in? -->
35
                <!-- 05* - Record status : should be "n" for new records but should be "c" for updated records. FIXME: How to take this into account? Can we pass in parameters? -->
36
                <!-- 06 - Type of record : handled by variable above -->
37
                <!-- 07 - Bibliographic level : handled by variable above -->
38
                <!-- 08 - Type of control : leave blank for non-archival -->
39
                <!-- 09 - Character coding scheme : "a" for Unicode -->
40
                <!-- 10 - Indicator count : This should always be 2 -->
41
                <!-- 11 - Subfield code count count : This should always be 2 -->
42
                <!-- *12-16 - Base address of data : leave blank. Koha will fill this in? -->
43
                <!-- 17* - Encoding level : "u" for unknown, UNLESS you're reasonably confident of the level used -->
44
                <!-- 18* - Descriptive cataloging form : "u" for unknown, UNLESS you know it is AACR2, ISBD, or neither -->
45
                <!-- 19* - Multipart resource record level : Blank for non-specified or not applicable, UNLESS record describes multiple physical parts bound in a single physical volume. NOT to be used for SERIALS. -->
46
                <!-- 20 - Length of the length-of-field portion : Should always be 4 -->
47
                <!-- 21 - Length of the starting-character-position portion : Should always be 5 -->
48
                <!-- 22 - Length of the implementation-defined portion : Should always be 0 -->
49
                <!-- 23 - Undefined : Should always be 0 -->
50
                <xsl:value-of select="concat('     n',$leader06,$leader07,' a22     uu 4500')"/>
51
            </xsl:element>
52
53
54
            <!-- FIXME: Add a 008 field -->
55
56
            <xsl:for-each select="//dc:identifier">
57
                <xsl:if test="substring(text(),1,4) != 'http'">
58
                <datafield tag="024" ind1="8" ind2=" ">
59
                    <subfield code="a">
60
                        <xsl:value-of select="."/>
61
                    </subfield>
62
                </datafield>
63
                </xsl:if>
64
            </xsl:for-each>
65
66
            <!-- Add record's OAI-PMH identifier for provenance -->
67
            <datafield tag="037" ind1="8" ind2=" ">
68
                <subfield code="a">
69
                    <xsl:value-of select="$identifier"/>
70
                </subfield>
71
                <subfield code="n">
72
                    <xsl:text>OAI-PMH Identifier</xsl:text>
73
                </subfield>
74
            </datafield>
75
76
            <datafield tag="042" ind1=" " ind2=" ">
77
                <subfield code="a">dc</subfield>
78
            </datafield>
79
80
            <xsl:for-each select="//dc:title[1]">
81
                <datafield tag="245" ind1="0" ind2="0">
82
                    <subfield code="a">
83
                        <xsl:value-of select="."/>
84
                    </subfield>
85
                </datafield>
86
            </xsl:for-each>
87
88
            <xsl:for-each select="//dc:title[position()>1]">
89
                <datafield tag="246" ind1="3" ind2="3">
90
                    <subfield code="a">
91
                        <xsl:value-of select="."/>
92
                    </subfield>
93
                </datafield>
94
            </xsl:for-each>
95
96
            <!-- FIXME: Add punctuation handling -->
97
            <xsl:if test="//dc:publisher | //dc:date">
98
                <datafield tag="260" ind1=" " ind2=" ">
99
                <xsl:for-each select="//dc:publisher">
100
                    <subfield code="b">
101
                        <xsl:value-of select="."/>
102
                    </subfield>
103
                </xsl:for-each>
104
                <xsl:for-each select="//dc:date">
105
                    <!-- Strip out ISO8601 extended format datetimes with UTC time designator Z -->
106
                    <xsl:if test="substring(text(),11,1) != 'T' and substring(text(),20,1) != 'Z'">
107
                        <subfield code="c">
108
                            <xsl:value-of select="."/>
109
                        </subfield>
110
                    </xsl:if>
111
                </xsl:for-each>
112
                </datafield>
113
            </xsl:if>
114
115
            <xsl:for-each select="//dc:coverage">
116
                <datafield tag="500" ind1=" " ind2=" ">
117
                    <subfield code="a">
118
                        <xsl:value-of select="."/>
119
                    </subfield>
120
                </datafield>
121
            </xsl:for-each>
122
123
            <xsl:for-each select="//dc:description">
124
                <datafield tag="520" ind1=" " ind2=" ">
125
                    <subfield code="a">
126
                        <xsl:value-of select="."/>
127
                    </subfield>
128
                </datafield>
129
            </xsl:for-each>
130
131
            <xsl:for-each select="//dc:rights">
132
                <datafield tag="540" ind1=" " ind2=" ">
133
                    <subfield code="a">
134
                        <xsl:value-of select="."/>
135
                    </subfield>
136
                </datafield>
137
            </xsl:for-each>
138
139
            <xsl:for-each select="//dc:language">
140
                <datafield tag="546" ind1=" " ind2=" ">
141
                    <subfield code="a">
142
                        <xsl:value-of select="."/>
143
                    </subfield>
144
                </datafield>
145
            </xsl:for-each>
146
147
            <xsl:for-each select="//dc:subject">
148
                <datafield tag="653" ind1=" " ind2=" ">
149
                    <subfield code="a">
150
                        <xsl:value-of select="."/>
151
                    </subfield>
152
                </datafield>
153
            </xsl:for-each>
154
155
            <xsl:for-each select="//dc:type">
156
                <datafield tag="655" ind1="7" ind2=" ">
157
                    <subfield code="a">
158
                        <xsl:value-of select="."/>
159
                    </subfield>
160
                    <subfield code="2">local</subfield>
161
                </datafield>
162
            </xsl:for-each>
163
164
            <xsl:for-each select="//dc:contributor">
165
                <datafield tag="720" ind1="0" ind2="0">
166
                    <subfield code="a">
167
                        <xsl:value-of select="."/>
168
                    </subfield>
169
                    <subfield code="e">collaborator</subfield>
170
                </datafield>
171
            </xsl:for-each>
172
173
            <xsl:for-each select="//dc:creator">
174
                <datafield tag="720" ind1=" " ind2=" ">
175
                    <subfield code="a">
176
                        <xsl:value-of select="."/>
177
                    </subfield>
178
                    <subfield code="e">author</subfield>
179
                </datafield>
180
            </xsl:for-each>
181
182
            <xsl:for-each select="//dc:source">
183
                <datafield tag="786" ind1="0" ind2=" ">
184
                    <subfield code="n">
185
                        <xsl:value-of select="."/>
186
                    </subfield>
187
                </datafield>
188
            </xsl:for-each>
189
190
            <xsl:for-each select="//dc:relation">
191
                <datafield tag="787" ind1="0" ind2=" ">
192
                    <subfield code="n">
193
                        <xsl:value-of select="."/>
194
                    </subfield>
195
                </datafield>
196
            </xsl:for-each>
197
198
            <xsl:for-each select="//dc:format">
199
                <datafield tag="856" ind1=" " ind2=" ">
200
                    <subfield code="q">
201
                        <xsl:value-of select="."/>
202
                    </subfield>
203
                </datafield>
204
            </xsl:for-each>
205
206
            <xsl:for-each select="//dc:identifier">
207
                <xsl:if test="substring(text(),1,4) = 'http'">
208
                <datafield tag="856" ind1=" " ind2=" ">
209
                    <subfield code="u">
210
                        <xsl:value-of select="."/>
211
                    </subfield>
212
                </datafield>
213
                </xsl:if>
214
            </xsl:for-each>
215
        </record>
216
    </xsl:template>
217
</xsl:stylesheet>
(-)a/koha-tmpl/intranet-tmpl/prog/en/xslt/MARC21slim2MARC21enhanced.xsl (+43 lines)
Line 0 Link Here
1
<?xml version="1.0" encoding="UTF-8"?>
2
<xsl:stylesheet version="1.0" xmlns:marc="http://www.loc.gov/MARC21/slim" xmlns:items="http://www.koha-community.org/items" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" >
3
	<xsl:output method="xml" indent="yes"/>
4
5
    <!-- pass in the OAI-PMH identifier for historical purposes -->
6
    <xsl:param name="identifier" select="identifier"/>
7
8
    <!-- this block copies the existing XML verbatim -->
9
	<xsl:template match="@*|node()">
10
      <xsl:copy>
11
        <xsl:apply-templates select="@*|node()"/>
12
      </xsl:copy>
13
    </xsl:template>
14
15
    <!-- Currently, strip out any 952 tags so that we don't have indexing problems in regards to unexpected items... -->
16
    <xsl:template match="marc:datafield[@tag=952]" />
17
18
    <!-- this template matches on the "record" node -->
19
    <xsl:template match="marc:record">
20
      <record>
21
         <!-- Copy all the existing controlfield, datafield, subfield nodes -->
22
         <xsl:apply-templates select="@* | *"/>
23
         <!-- Add a MARC datafield that includes the unique OAI-PMH identifier -->
24
         <xsl:text>
25
      </xsl:text>
26
        <xsl:element name="datafield" xmlns="http://www.loc.gov/MARC21/slim">
27
          <xsl:attribute name="ind1"><xsl:text>8</xsl:text></xsl:attribute>
28
          <xsl:attribute name="ind2"><xsl:text> </xsl:text></xsl:attribute>
29
          <xsl:attribute name="tag">037</xsl:attribute>
30
          <xsl:text>
31
        </xsl:text>
32
          <xsl:element name="subfield">
33
            <xsl:attribute name="code">a</xsl:attribute>
34
            <xsl:value-of select="$identifier"/>
35
          </xsl:element>
36
          <xsl:element name="subfield">
37
            <xsl:attribute name="code">n</xsl:attribute>
38
            <xsl:text>OAI-PMH Identifier</xsl:text>
39
          </xsl:element>
40
        </xsl:element>
41
      </record>
42
    </xsl:template>
43
</xsl:stylesheet>
(-)a/misc/cronjobs/oai_harvester.pl (+95 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# This file is part of Koha.
4
#
5
# Copyright 2013 Prosentient Systems
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, see <http://www.gnu.org/licenses>.
18
19
=head1 NAME
20
21
oai_harvester.pl - cron script to harvest bibliographic records from
22
OAI-PMH repositories.
23
24
=head1 SYNOPSIS
25
26
27
28
=head1 DESCRIPTION
29
30
31
32
=cut
33
34
use Modern::Perl;
35
use Getopt::Long;
36
use Pod::Usage;
37
use C4::Context;
38
use C4::OAI::Client::Harvester;
39
40
my $help = 0;
41
my $debug = 0;
42
my $noresume = 0;
43
my $force = '';
44
my $harvest = 0;
45
46
#TODO: Perhaps create a flag that triggers a delete of all records that have been ingested via OAI-PMH?
47
48
GetOptions(
49
            'h|help|?'         => \$help,
50
            'd'                => \$debug,
51
            'noresume'         => \$noresume,
52
            'force=s'          => \$force,
53
            'harvest'        => \$harvest,
54
55
       );
56
pod2usage(1) if $help;
57
58
my $active = 1; #Get only active repositories
59
my @repositories = C4::OAI::Client::Harvester::GetOAIRepositoryList($active);
60
foreach my $repository_data (@repositories) {
61
62
    #Print extra info for debugging
63
    if ($debug){
64
        $repository_data->{debug} = 1;
65
    }
66
67
    #Turn off automatic token resumption. This is for debugging.
68
    if ($noresume){
69
        $repository_data->{resume} = 0;
70
    }
71
72
    #Create Harvester object
73
    my $repository = C4::OAI::Client::Harvester->new($repository_data);
74
75
    if ($repository){
76
        print "\n\nHarvest ".$repository->{'baseURL'}." in DEBUG MODE\n" if $repository->{debug};
77
        my $response = $repository->ListRecords;
78
        if ($response){
79
            my $records = $repository->ProcessOaipmhRecordResponse($response, $harvest, $force);
80
            #NOTE: No records are returned when in $harvest mode
81
        }
82
83
        if ($harvest){
84
            my $latest_datestamp = C4::OAI::Client::Harvester::GetLatestHistoricalRecordDatestamp($repository->get_repository_id);
85
            if ($latest_datestamp){
86
                if (!$repository->get_opt_from || $latest_datestamp ne $repository->get_opt_from){
87
                    C4::OAI::Client::Harvester::ModOAIRepositoryDateTimeFrom($repository->get_repository_id,$latest_datestamp);
88
                    print "Setting 'from' argument to $latest_datestamp \n";
89
                }
90
            }
91
        }
92
93
    }
94
}
95
print "\n\nOAI-PMH Harvesting Cronjob Complete\n";
(-)a/t/db_dependent/OAI_harvester.t (-1 / +308 lines)
Line 0 Link Here
0
- 
1
#!/usr/bin/perl
2
3
# This file is part of Koha.
4
#
5
# Copyright 2013 Prosentient Systems
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, see <http://www.gnu.org/licenses>.
18
19
use Modern::Perl;
20
use Test::More;
21
use C4::Context;
22
use C4::OAI::Harvester;
23
use Data::Dumper;
24
use Getopt::Long;
25
26
my $verbose = 0;
27
my $show = 0;
28
GetOptions(
29
    'v|verbose' => \$verbose,
30
    's|show:s' => \$show,
31
);
32
33
my $real_dbh = C4::Context->dbh;
34
35
############################################################################################################################################################################################################
36
37
BEGIN {
38
        use_ok('C4::OAI::Harvester');
39
}
40
41
############################################################################################################################################################################################################
42
43
#REAL DATA - We use this for crafting our mock data, making sure the OAI-PMH server is on, and requesting a specific item for the GetRecord method.
44
45
#Initiate Koha baseURLs
46
my $OPACbaseURL = C4::Context->preference("OPACBaseURL");
47
my $staffClientBaseURL = C4::Context->preference("staffClientBaseURL");
48
49
#Check if Koha has its OAI-PMH repository config enabled.
50
#If disabled, re-enable it (we'll reset to the original values
51
#after we've finished our test).
52
my $OAIPMHenabled = C4::Context->preference("OAI-PMH");
53
if (!$OAIPMHenabled){
54
    C4::Context->set_preference("OAI-PMH",1);
55
}
56
my $OAIPMHarchiveID;
57
my $check_OAIPMHarchiveID = C4::Context->preference("OAI-PMH:archiveID");
58
if ($check_OAIPMHarchiveID){
59
    $OAIPMHarchiveID = $check_OAIPMHarchiveID;
60
}
61
else {
62
    C4::Context->set_preference("OAI-PMH:archiveID","KOHAOAITEST");
63
    $OAIPMHarchiveID = C4::Context->preference("OAI-PMH:archiveID");
64
}
65
66
my $check_biblio_table_sql = "
67
    SELECT count(*) as 'count'
68
    FROM biblio
69
";
70
my $check_biblio_table_sth = $real_dbh->prepare($check_biblio_table_sql);
71
$check_biblio_table_sth->execute();
72
my $check_biblio_table_row = $check_biblio_table_sth->fetchrow_hashref;
73
if ($check_biblio_table_row->{count} == 0){
74
    # Generate test biblio records
75
    my $biblio = MARC::Record->new();
76
    $biblio->append_fields(
77
        MARC::Field->new('100', ' ', ' ', a => 'Moffat, Steven'),
78
        MARC::Field->new('245', ' ', ' ', a => 'Silence in the library'),
79
    );
80
    my ($biblionumber, $biblioitemnumber) = AddBiblio($biblio, '');
81
82
    my $biblio2 = MARC::Record->new();
83
    $biblio2->append_fields(
84
        MARC::Field->new('100', ' ', ' ', a => 'Garland, Alex'),
85
        MARC::Field->new('245', ' ', ' ', a => 'The Beach'),
86
    );
87
    my ($biblionumber2, $biblioitemnumber2) = AddBiblio($biblio2, '');
88
89
    my $biblio3 = MARC::Record->new();
90
    $biblio3->append_fields(
91
        MARC::Field->new('100', ' ', ' ', a => 'Michaels, Anne'),
92
        MARC::Field->new('245', ' ', ' ', a => 'The Winter Vault'),
93
    );
94
    my ($biblionumber3, $biblioitemnumber3) = AddBiblio($biblio3, '');
95
}
96
97
my $sql1 = "SELECT biblionumber FROM biblio ORDER BY RAND() LIMIT 1";
98
my $sth1 = $real_dbh->prepare($sql1);
99
$sth1->execute();
100
my $random_biblionumber_row = $sth1->fetchrow_hashref;
101
my $random_biblionumber = $random_biblionumber_row->{biblionumber};
102
103
my $random_OAIPMH_identifier;
104
if ($OAIPMHarchiveID && $random_biblionumber){
105
    $random_OAIPMH_identifier = $OAIPMHarchiveID.":".$random_biblionumber;
106
}
107
108
############################################################################################################################################################################################################
109
110
#MOCK DATA - Create a mock remote OAI-PMH repository using Koha's OAI-PMH server capabilities
111
112
# Start Database Transaction
113
my $dbh = C4::Context->dbh;
114
$dbh->{AutoCommit} = 0;
115
$dbh->{RaiseError} = 1;
116
117
my $record_matcher = C4::Matcher->new;
118
my $record_matcher_id = $record_matcher->store();
119
120
my $data = {};
121
$data->{repository_id} = 1;
122
$data->{baseURL} = "http://$OPACbaseURL/cgi-bin/koha/oai.pl";
123
$data->{opt_from} = undef;
124
$data->{opt_until} = undef;
125
$data->{opt_set} = undef;
126
$data->{metadataPrefix} = "oai_dc"; #or "marcxml"
127
$data->{active} = 1;
128
$data->{XSLT_path} = "http://$staffClientBaseURL/intranet-tmpl/prog/en/xslt/DC2MARC21slim2.xsl"; #"/.../git/koha-tmpl/intranet-tmpl/prog/en/xslt/DC2MARC21slim2.xsl"; #or undef, since this is hard-coded into Harvester.pm anyway
129
$data->{matcher_id} = $record_matcher_id; #NOTE: This is placeholder for testing. In production, you would use a real matching rule. Preferably using biblionumber (i.e. 999$c and Local-Number) for matching.
130
$data->{import_mode} = "automatic"; #Always run "automatic" for test. In production, you might prefer "manual".
131
132
my $insert_mock_data_sql = "
133
    INSERT INTO oai_harvest_repositories
134
    (baseURL, opt_from, opt_until, opt_set, metadataPrefix, active, XSLT_path, matcher_id, import_mode)
135
    VALUES (?,?,?,?,?,?,?,?,?)
136
";
137
my $insert_mock_mock_data_sth = $dbh->prepare($insert_mock_data_sql);
138
$insert_mock_mock_data_sth->execute($data->{baseURL},$data->{opt_from},$data->{opt_until},$data->{opt_set},$data->{metadataPrefix},$data->{active},$data->{XSLT_path},$data->{matcher_id},$data->{import_mode});
139
140
my $get_mock_data_sql = "
141
    SELECT *
142
    FROM oai_harvest_repositories
143
    WHERE active = 1
144
";
145
my $get_mock_data_sth = $dbh->prepare($get_mock_data_sql);
146
$get_mock_data_sth->execute();
147
while (my $get_mock_data_row = $get_mock_data_sth->fetchrow_hashref){
148
149
    #Set 'resume' to 0, if you want to turn off automatic token resumption. This is useful for testing and debugging. Change to 1 for automatic token resumption.
150
    $get_mock_data_row->{resume} = 0;
151
152
############################################################################################################################################################################################################
153
154
    #TEST - OBJECT CREATION
155
    my $harvester = C4::OAI::Harvester->new($get_mock_data_row); # create an object
156
    isa_ok( $harvester, 'C4::OAI::Harvester','Harvester object' );
157
    #END TEST
158
159
    #TEST - OBJECT's "GET" ACCESSOR METHODS
160
    can_ok($harvester, qw(get_baseURL get_from get_until get_set get_metadataPrefix get_rh get_repository_id get_XSLT_path get_matcher_id get_import_mode));
161
    #END TEST
162
163
    #TEST - Check some basic harvester config
164
    my $baseURL = $harvester->get_baseURL;
165
    my $metadataPrefix = $harvester->get_metadataPrefix;
166
    is ($baseURL,"http://$OPACbaseURL/cgi-bin/koha/oai.pl", 'Retrieved baseURL "'.$baseURL.'"');
167
    ok ( defined $metadataPrefix, 'MetadataPrefix is defined as "'.$metadataPrefix.'"');
168
    #END TEST
169
170
    #TESTS - OBJECT's OAI-PMH methods
171
172
    #IDENTIFY
173
    my $identify_repository = $harvester->Identify;
174
    ok ( defined($identify_repository), "Successfully used Identify Verb");
175
    print Dumper($identify_repository) if $verbose or $show eq 'Identify';
176
    #END TEST
177
178
    #LISTSETS
179
    my @sets = $harvester->ListSets;
180
    ok (@sets, "Successfully used ListSets Verb");
181
    print Dumper(\@sets) if $verbose or $show eq 'ListSets';
182
    #END TEST
183
184
    #LISTMETADATAFORMATS
185
    my @formats = $harvester->ListMetadataFormats;
186
    ok (@formats, "Successfully used ListMetadataFormats Verb");
187
    print Dumper(\@formats) if $verbose or $show eq 'ListMetadataFormats';
188
    #END TEST
189
190
    #LISTIDENTIFIERS
191
    my @headers = $harvester->ListIdentifiers;
192
    ok (@headers, "Successfully used ListIdentifiers Verb");
193
    print Dumper(\@headers) if $verbose or $show eq 'ListIdentifiers';
194
    #END TEST
195
196
    #LISTRECORDS
197
    my @list_records = $harvester->ListRecords;
198
    ok (@list_records, "Successfully used ListRecords Verb");
199
    print Dumper(\@list_records) if $verbose or $show eq 'ListRecords';
200
    #END TEST
201
202
    #GETRECORD
203
    my @get_records = $harvester->GetRecord($random_OAIPMH_identifier);
204
    ok (@get_records, "Successfully used GetRecord Verb");
205
    print Dumper(@get_records) if $verbose or $show eq 'GetRecord';
206
    #END TEST
207
208
    my ( $batch_id, $num_with_matches, $num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = $harvester->ImportRecordsIntoKoha(@list_records);
209
210
    #These variables will only be initialized in automatic mode, which is how the test should always be run anyway.
211
    my $batch_total = $num_added + $num_updated;
212
213
    print "Batch id = $batch_id \n";
214
    print "Num added = $num_added \n";
215
    print "Num updated = $num_updated \n";
216
    print "Num items added = $num_items_added \n";
217
    print "Num items errored = $num_items_errored \n";
218
    print "Num ignored = $num_ignored \n";
219
220
221
    #TEST - RETRIEVE IMPORT_BATCH
222
    my @test_batches;
223
    my $batch_sql = "
224
        SELECT *
225
        FROM import_batches
226
        WHERE import_batch_id = ?
227
        ";
228
    my $batch_sth = $dbh->prepare($batch_sql);
229
    $batch_sth->execute($batch_id);
230
    while (my $batch_row = $batch_sth->fetchrow_hashref){
231
        push(@test_batches,$batch_row);
232
    }
233
    cmp_ok(scalar @test_batches,'==',1,"Successfully retrieved batch id ".$batch_id);
234
235
    #END TEST
236
237
    #TEST - RETRIEVE IMPORTED_RECORDS
238
    my $imported_record_sql = "
239
        SELECT count(*) as 'count'
240
        FROM import_records
241
        WHERE import_batch_id = ?
242
        ";
243
    my $imported_record_sth = $dbh->prepare($imported_record_sql);
244
    $imported_record_sth->execute($batch_id);
245
    my $imported_record_row = $imported_record_sth->fetchrow_hashref;
246
    is ($imported_record_row->{count}, $batch_total, "Import_record table count (".$imported_record_row->{count}.") for this batch matches reported import total");
247
    #END TEST
248
249
    #TEST - RETRIEVE IMPORTED_BIBLIOS
250
    my $imported_biblio_sql = "
251
        SELECT count(*) as 'count'
252
        FROM import_biblios
253
        JOIN import_records using (import_record_id)
254
        WHERE import_records.import_batch_id = ?
255
        ";
256
    my $imported_biblio_sth = $dbh->prepare($imported_biblio_sql);
257
    $imported_biblio_sth->execute($batch_id);
258
    my $imported_biblio_row = $imported_biblio_sth->fetchrow_hashref;
259
    is ($imported_biblio_row->{count}, $batch_total, "Import_biblio table count (".$imported_biblio_row->{count}.") for this batch matches reported import total");
260
    #END TEST
261
262
    #TEST - RETRIEVE BIBLIOS
263
    my $biblio_sql = "
264
        SELECT count(*) as 'count'
265
        FROM biblio
266
        JOIN import_biblios on import_biblios.matched_biblionumber = biblio.biblionumber
267
        JOIN import_records using (import_record_id)
268
        WHERE import_records.import_batch_id = ?
269
        ";
270
    my $biblio_sth = $dbh->prepare($biblio_sql);
271
    $biblio_sth->execute($batch_id);
272
    my $biblio_row = $biblio_sth->fetchrow_hashref;
273
    is ($biblio_row->{count}, $batch_total, "Biblio table count (".$biblio_row->{count}.") for this batch matches reported import total. Automatic import successful.");
274
    #END TEST
275
276
    #TEST - RETRIEVE HISTORICAL RECORDS
277
    my $harvest_sql = "
278
        SELECT count(*) as 'count'
279
        FROM oai_harvest
280
        JOIN import_records using (import_record_id)
281
        WHERE import_records.import_batch_id = ?
282
        ";
283
    my $harvest_sth = $dbh->prepare($harvest_sql);
284
    $harvest_sth->execute($batch_id);
285
    my $harvest_row = $harvest_sth->fetchrow_hashref;
286
    is ($harvest_row->{count}, $batch_total, "OAI Harvest table count (".$harvest_row->{count}.") for this batch matches reported import total. Historical records added successfully.");
287
    #NOTE: This ignores deleted records, which will have a historical record but are not part of the batch and thus don't have an import_record_id number.
288
    #END TEST
289
290
    #TEST - RETRIEVE RECORD MATCHES
291
    #There's no way to really automatically test the "replace/update" function, since the first wave of imported records will need to be reindexed before they can be matched on...
292
    #END
293
294
} #End Loop of OAI-PMH Repositories
295
$dbh->rollback();
296
$dbh->{AutoCommit} = 1;
297
298
done_testing();
299
############################################################################################################################################################################################################
300
301
#RESET ORIGINAL VALUES
302
if (!$OAIPMHenabled){
303
    C4::Context->set_preference("OAI-PMH",$OAIPMHenabled);
304
}
305
306
if (!$check_OAIPMHarchiveID){
307
    C4::Context->set_preference("OAI-PMH:archiveID",$check_OAIPMHarchiveID);
308
}

Return to bug 10662