Line 0
Link Here
|
|
|
1 |
package C4::OAI::Client::Harvester; |
2 |
|
3 |
# This file is part of Koha. |
4 |
# |
5 |
# Copyright 2013 Prosentient Systems |
6 |
# |
7 |
# Koha is free software; you can redistribute it and/or modify it under the |
8 |
# terms of the GNU General Public License as published by the Free Software |
9 |
# Foundation; either version 3 of the License, or (at your option) any later |
10 |
# version. |
11 |
# |
12 |
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY |
13 |
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR |
14 |
# A PARTICULAR PURPOSE. See the GNU General Public License for more details. |
15 |
# |
16 |
# You should have received a copy of the GNU General Public License along |
17 |
# with Koha; if not, see <http://www.gnu.org/licenses>. |
18 |
|
19 |
=head1 NAME |
20 |
|
21 |
C4::OAI::Harvester - OAI-PMH harvester/client which implements the 6 OAI-PMH verbs, record retrieval/harvesting, and import into Koha |
22 |
|
23 |
=head1 SYNOPSIS |
24 |
|
25 |
use C4::OAI::Harvester; |
26 |
my $oai_repo = C4::OAI::Harvester->new($repository_data); |
27 |
|
28 |
my $identify_repository = $oai_repo->Identify; |
29 |
|
30 |
my @sets = $oai_repo->ListSets; |
31 |
|
32 |
my @formats = $oai_repo->ListMetadataFormats; |
33 |
|
34 |
my @headers = $oai_repo->ListIdentifiers; |
35 |
|
36 |
my @records = $oai_repo->ListRecords; |
37 |
|
38 |
my @records = $oai_repo->GetRecord($oai_unique_identifier); |
39 |
|
40 |
my $import_mode = ''; #i.e. not "automatic" |
41 |
|
42 |
$oai_repo->ImportRecordsIntoKoha($import_mode,@records); |
43 |
|
44 |
=head1 DESCRIPTION |
45 |
|
46 |
C4::OAI::Harvester contains functions for querying and harvesting OAI-PMH repositories. |
47 |
|
48 |
More information on OAI-PMH can be found L<here|http://www.openarchives.org/OAI/openarchivesprotocol.html> |
49 |
|
50 |
=head1 FUNCTIONS |
51 |
|
52 |
=head1 AUTHOR |
53 |
|
54 |
David Cook <dcook AT prosentient DOT com DOT au> |
55 |
|
56 |
=cut |
57 |
|
58 |
use Modern::Perl; |
59 |
use HTTP::OAI; |
60 |
use C4::Context; |
61 |
use C4::OAI::Client::Record; |
62 |
use C4::Biblio qw /AddBiblio ModBiblio DelBiblio/; |
63 |
|
64 |
use base qw(Class::Accessor); |
65 |
|
66 |
|
67 |
sub AddOAIRepository { |
68 |
my ( $database_fields ) = @_; |
69 |
my $dbh = C4::Context->dbh; |
70 |
my $sql = " |
71 |
INSERT INTO oai_harvest_repositories |
72 |
(baseURL,metadataPrefix,opt_from,opt_until,opt_set,active,XSLT_path,frameworkcode,comments) |
73 |
VALUES (?,?,?,?,?,?,?,?,?) |
74 |
"; |
75 |
my $sth = $dbh->prepare($sql); |
76 |
$sth->execute( |
77 |
$database_fields->{baseURL}, |
78 |
$database_fields->{metadataPrefix}, |
79 |
$database_fields->{opt_from}, |
80 |
$database_fields->{opt_until}, |
81 |
$database_fields->{opt_set}, |
82 |
$database_fields->{active}, |
83 |
$database_fields->{XSLT_path}, |
84 |
$database_fields->{frameworkcode}, |
85 |
$database_fields->{comments} |
86 |
); |
87 |
if ($sth->err){ |
88 |
die $sth->errstr; |
89 |
} |
90 |
} |
91 |
|
92 |
sub ModOAIRepositoryDateTimeFrom { |
93 |
my ( $repository_id, $datetime_from ) = @_; |
94 |
my $dbh = C4::Context->dbh; |
95 |
my $sql = " |
96 |
UPDATE oai_harvest_repositories |
97 |
SET opt_from = ? |
98 |
WHERE repository_id = ? |
99 |
"; |
100 |
my $sth = $dbh->prepare($sql); |
101 |
$sth->execute($datetime_from,$repository_id); |
102 |
if ($sth->err){ |
103 |
die $sth->errstr; |
104 |
} |
105 |
} |
106 |
|
107 |
sub ModOAIRepository { |
108 |
my ( $database_fields ) = @_; |
109 |
my $dbh = C4::Context->dbh; |
110 |
my $sql = " |
111 |
UPDATE oai_harvest_repositories |
112 |
SET baseURL = ?, |
113 |
metadataPrefix = ?, |
114 |
opt_from = ?, |
115 |
opt_until = ?, |
116 |
opt_set = ?, |
117 |
active = ?, |
118 |
XSLT_path = ?, |
119 |
frameworkcode = ?, |
120 |
comments = ? |
121 |
WHERE repository_id = ? |
122 |
"; |
123 |
my $sth = $dbh->prepare($sql); |
124 |
$sth->execute( |
125 |
$database_fields->{baseURL}, |
126 |
$database_fields->{metadataPrefix}, |
127 |
$database_fields->{opt_from}, |
128 |
$database_fields->{opt_until}, |
129 |
$database_fields->{opt_set}, |
130 |
$database_fields->{active}, |
131 |
$database_fields->{XSLT_path}, |
132 |
$database_fields->{frameworkcode}, |
133 |
$database_fields->{comments}, |
134 |
$database_fields->{repository_id} |
135 |
); |
136 |
if ($sth->err){ |
137 |
die $sth->errstr; |
138 |
} |
139 |
} |
140 |
|
141 |
sub DelOAIRepository { |
142 |
my ( $repository_id ) = @_; |
143 |
my $error; |
144 |
my $dbh = C4::Context->dbh; |
145 |
my $sql = " |
146 |
DELETE FROM oai_harvest_repositories |
147 |
WHERE repository_id = ? |
148 |
"; |
149 |
my $sth = $dbh->prepare($sql); |
150 |
$sth->execute($repository_id); |
151 |
if ($sth->err){ |
152 |
die $sth->errstr; |
153 |
} |
154 |
} |
155 |
|
156 |
sub GetLatestHistoricalRecordDatestamp { |
157 |
my ( $repository_id ) = @_; |
158 |
my $latest_datestamp; |
159 |
my $dbh = C4::Context->dbh; |
160 |
my $sql = " |
161 |
SELECT datestamp |
162 |
FROM oai_harvest |
163 |
WHERE repository_id = ? |
164 |
ORDER BY datestamp desc |
165 |
LIMIT 1 |
166 |
"; |
167 |
my $sth = $dbh->prepare($sql); |
168 |
$sth->execute($repository_id); |
169 |
my $row = $sth->fetchrow_hashref; |
170 |
$latest_datestamp = $row->{datestamp} ? $row->{datestamp} : undef; |
171 |
return $latest_datestamp; |
172 |
} |
173 |
|
174 |
sub GetOAIRepository { |
175 |
my ( $repository_id ) = @_; |
176 |
my $dbh = C4::Context->dbh; |
177 |
my $sql = " |
178 |
SELECT * |
179 |
FROM oai_harvest_repositories |
180 |
WHERE repository_id = ? |
181 |
"; |
182 |
my $sth = $dbh->prepare($sql); |
183 |
$sth->execute($repository_id); |
184 |
if ($sth->err){ |
185 |
die $sth->errstr; |
186 |
} |
187 |
my $row = $sth->fetchrow_hashref; |
188 |
return $row; |
189 |
} |
190 |
|
191 |
=head2 GetOAIRepositoryList |
192 |
|
193 |
my @repositories = C4::OAI::Harvester::GetOAIRepositoryList(); |
194 |
my @repositories = C4::OAI::Harvester::GetOAIRepositoryList($active); |
195 |
|
196 |
Returns an array of hashrefs listing all OAI-PMH repositories |
197 |
present in the database. |
198 |
|
199 |
If the $active is included, then it will return an array |
200 |
of hashrefs listing all OAI-PMH repositories depending on their |
201 |
active status. $active == 1 shows all active. $active == 0 shows |
202 |
all inactive. |
203 |
|
204 |
=cut |
205 |
|
206 |
sub GetOAIRepositoryList { |
207 |
my ( $active ) = @_; |
208 |
my $dbh = C4::Context->dbh; |
209 |
my @results; |
210 |
my $sql = " |
211 |
SELECT * |
212 |
FROM oai_harvest_repositories |
213 |
"; |
214 |
if (defined $active){ |
215 |
$sql .= " WHERE active = 1 " if $active == 1; |
216 |
$sql .= " WHERE active = 0 " if $active == 0; |
217 |
} |
218 |
my $sth = $dbh->prepare($sql); |
219 |
$sth->execute; |
220 |
while (my $row = $sth->fetchrow_hashref){ |
221 |
push(@results,$row); |
222 |
} |
223 |
return @results; |
224 |
} |
225 |
|
226 |
#TODO: Perhaps create a sub that cleans out the metadata column to keep the table size low? |
227 |
|
228 |
sub new { |
229 |
my($proto, $fields) = @_; |
230 |
my($class) = ref $proto || $proto; |
231 |
|
232 |
$fields = {} unless defined $fields; |
233 |
|
234 |
if ($fields->{'baseURL'}){ |
235 |
my $h = new HTTP::OAI::Harvester( |
236 |
baseURL => $fields->{'baseURL'}, |
237 |
); |
238 |
#If resume is set to 0, automatic token resumption is turned off. This is useful for testing/debugging. |
239 |
if ($h && exists $fields->{'resume'}){ |
240 |
if ($fields->{'resume'} == 0){ |
241 |
$h->resume(0); |
242 |
} |
243 |
} |
244 |
my $response = $h->repository($h->Identify); |
245 |
if( $response->is_error ) { |
246 |
print "Error requesting Identify:\n", |
247 |
$response->code . " " . $response->message, "\n"; |
248 |
exit; |
249 |
} |
250 |
$fields->{rh} = $h; #Store HTTP::OAI::Harvester object as "repository handle" |
251 |
} |
252 |
bless {%$fields}, $class; |
253 |
} |
254 |
|
255 |
__PACKAGE__->follow_best_practice; #Use get_ and set_ prefixes for accessors |
256 |
__PACKAGE__->mk_accessors(qw(baseURL opt_from opt_until opt_set metadataPrefix rh repository_id XSLT_path debug frameworkcode)); |
257 |
|
258 |
=head2 OAI-PMH Verbs |
259 |
|
260 |
Koha-specific implementations of the 6 OAI-PMH Verbs. |
261 |
|
262 |
The key verbs are "ListRecords" and "GetRecords". These do the actual |
263 |
harvesting of records from a OAI-PMH repository. The others are useful for |
264 |
getting information about a repository and what it has available. |
265 |
|
266 |
1) ListRecords |
267 |
|
268 |
2) GetRecord |
269 |
|
270 |
3) ListIdentifiers |
271 |
|
272 |
4) ListMetadataFormats |
273 |
|
274 |
5) ListSets |
275 |
|
276 |
6) Identify |
277 |
|
278 |
=cut |
279 |
|
280 |
|
281 |
sub ListRecords { |
282 |
my ( $self ) = @_; |
283 |
my %args = ( |
284 |
metadataPrefix => $self->{metadataPrefix}, |
285 |
from => $self->{opt_from}, |
286 |
until => $self->{opt_until}, |
287 |
set => $self->{opt_set}, |
288 |
); |
289 |
if ($self->{debug}){ |
290 |
use Data::Dumper; |
291 |
print "ListRecords's arguments\n"; |
292 |
print Dumper(\%args); |
293 |
} |
294 |
my $response = $self->{rh}->ListRecords(%args); |
295 |
if( $response->is_error ) { |
296 |
print "Error requesting ListRecords:\n", |
297 |
$response->code . " " . $response->message, "\n"; |
298 |
exit; |
299 |
} |
300 |
if ($self->{debug}){ |
301 |
print "Successfully retrieved ListRecords response.\n"; |
302 |
} |
303 |
return $response; |
304 |
} |
305 |
|
306 |
sub GetRecord { |
307 |
my ( $self, $identifier, $harvest ) = @_; |
308 |
my $response = $self->{rh}->GetRecord( |
309 |
metadataPrefix => $self->{metadataPrefix}, |
310 |
identifier => $identifier, |
311 |
); |
312 |
if( $response->is_error ) { |
313 |
print "Error requesting GetRecord:\n", |
314 |
$response->code . " " . $response->message, "\n"; |
315 |
exit; |
316 |
} |
317 |
if ($self->{debug}){ |
318 |
print "Successfully retrieved GetRecord response.\n"; |
319 |
} |
320 |
return $response; |
321 |
} |
322 |
|
323 |
sub ListIdentifiers { |
324 |
my $self = shift; |
325 |
my @headers; |
326 |
my $response = $self->{rh}->ListIdentifiers( |
327 |
metadataPrefix => $self->{metadataPrefix}, |
328 |
from => $self->{opt_from}, |
329 |
until => $self->{opt_until}, |
330 |
set => $self->{opt_set}, |
331 |
); |
332 |
if( $response->is_error ) { |
333 |
print "Error requesting ListIdentifiers:\n", |
334 |
$response->code . " " . $response->message, "\n"; |
335 |
exit; |
336 |
} |
337 |
while (my $h = $response->next){ |
338 |
my $header; |
339 |
#print Dumper($h->dom->toString); #DEBUG: XML representation |
340 |
$header->{identifier} = $h->identifier; |
341 |
$header->{datestamp} = $h->datestamp; |
342 |
|
343 |
$header->{status} = $h->status; |
344 |
$header->{is_deleted} = $h->is_deleted; |
345 |
|
346 |
my @sets = $h->setSpec; |
347 |
$header->{sets} = \@sets; |
348 |
|
349 |
push (@headers,$header); |
350 |
} |
351 |
return \@headers; |
352 |
} |
353 |
|
354 |
sub ListMetadataFormats { |
355 |
my ( $self, $identifier ) = @_; |
356 |
my @formats; |
357 |
my $response = $self->{rh}->ListMetadataFormats( |
358 |
identifier => $identifier, |
359 |
); |
360 |
if( $response->is_error ) { |
361 |
print "Error requesting ListMetadataFormats:\n", |
362 |
$response->code . " " . $response->message, "\n"; |
363 |
exit; |
364 |
} |
365 |
for($response->metadataFormat) { |
366 |
push(@formats,$_->metadataPrefix); |
367 |
} |
368 |
return \@formats; |
369 |
} |
370 |
|
371 |
sub ListSets { |
372 |
my $self = shift; |
373 |
my @sets; |
374 |
my $response = $self->{rh}->ListSets(); |
375 |
if( $response->is_error ) { |
376 |
print "Error requesting ListSets:\n", |
377 |
$response->code . " " . $response->message, "\n"; |
378 |
exit; |
379 |
} |
380 |
while (my $s = $response->next){ |
381 |
my $set; |
382 |
$set->{setSpec} = $s->setSpec; |
383 |
$set->{setName} = $s->setName; |
384 |
|
385 |
#TODO: Not really sure what to do with the descriptions as they're XML and not necessarily that easy to parse for GUI views... |
386 |
#my @temp_setDescriptions = $s->setDescription; |
387 |
#my @setDescriptions; |
388 |
#foreach my $temp_setDescription (@temp_setDescriptions){ |
389 |
# push (@setDescriptions,$temp_setDescription->dom->toString); I think we need to do better than just return the setDescription XML...That's not very useful... |
390 |
#} |
391 |
#$set->{setDescription} = \@setDescriptions; |
392 |
push (@sets,$set); |
393 |
} |
394 |
return \@sets; |
395 |
} |
396 |
|
397 |
sub Identify { |
398 |
my $self = shift; |
399 |
my $response = $self->{rh}->Identify(); |
400 |
if( $response->is_error ) { |
401 |
print "Error requesting Identify:\n", |
402 |
$response->code . " " . $response->message, "\n"; |
403 |
exit; |
404 |
} |
405 |
my $identify_data; |
406 |
#DEBUG: View what's in the Identify object |
407 |
#print Dumper($response->headers); |
408 |
|
409 |
$identify_data->{repositoryName} = $response->repositoryName; |
410 |
$identify_data->{baseURL} = $response->baseURL; |
411 |
$identify_data->{protocolVersion} = $response->protocolVersion; #Tim Brody says this will always return 2.0 and that ->version should be used to find the actual version... |
412 |
#$identify_data->{version} = $response->version; |
413 |
$identify_data->{earliestDatestamp} = $response->earliestDatestamp; |
414 |
$identify_data->{deletedRecord} = $response->deletedRecord; #not in the perldoc, but it's in the code and the OAI-PMH spec |
415 |
$identify_data->{granularity} = $response->granularity; |
416 |
|
417 |
#These methods should be used with an array context so they return all the elements and not just the first one |
418 |
my @adminEmails = $response->adminEmail; |
419 |
$identify_data->{adminEmail} = \@adminEmails; |
420 |
my @compressions = $response->compression; |
421 |
$identify_data->{compression} = \@compressions; |
422 |
|
423 |
#TODO: Descriptions are encapsulated in XML containers, I believe. Not sure what to do with these at present... |
424 |
#my @descriptions = $response->description; |
425 |
#$identify_data->{description} = \@descriptions; |
426 |
#$response->next |
427 |
|
428 |
return $identify_data; |
429 |
} |
430 |
|
431 |
sub ProcessOaipmhRecordResponse { |
432 |
my ( $self, $response, $harvest, $force ) = @_; |
433 |
if ($response){ |
434 |
my @records; |
435 |
while( my $rec = $response->next ){ |
436 |
print "I'm parsing the record..." if $self->{debug}; |
437 |
#Parse from the HTTP::OAI::Record object into our C4::OAI::Client::Record object |
438 |
my $record_object = _parse_httpoai_records_into_clientoai_records($rec,$self->{repository_id},$self->{metadataPrefix}); |
439 |
|
440 |
#If set to harvest mode, we ingest the record |
441 |
if ($harvest && $record_object){ |
442 |
|
443 |
#Retrieve the latest biblionumber associated with this OAI identifier |
444 |
$record_object->GetBiblionumberFromOaiIdentifier; |
445 |
|
446 |
#Generate a status to be acted upon (e.g. add, update, delete, force_add, force_update, ignore) |
447 |
$record_object->GenerateStatus($force); |
448 |
|
449 |
print "I'm ingesting the record..." if $self->{debug}; |
450 |
#Based on the above status, act upon the record |
451 |
my $ingestion_flag = _ingest_oai_record($record_object, $self->get_XSLT_path, $self->get_frameworkcode); |
452 |
|
453 |
if ($ingestion_flag){ |
454 |
#Log this OAI-PMH record in the database |
455 |
$record_object->AddRecordToLog; |
456 |
print $record_object->get_identifier.' -> '.$record_object->get_status.' -> '.$record_object->get_biblionumber." \n "; |
457 |
} |
458 |
} |
459 |
print $rec->identifier."\n" if !$harvest; |
460 |
push(@records,$record_object) if !$harvest; |
461 |
} |
462 |
return \@records; |
463 |
} |
464 |
} |
465 |
|
466 |
sub _ingest_oai_record { |
467 |
my ( $oai_record, $xslt_path, $frameworkcode ) = @_; |
468 |
|
469 |
if ( $oai_record && $xslt_path ){ |
470 |
my $identifier_status = $oai_record->get_status; |
471 |
if ($identifier_status){ |
472 |
if ($identifier_status eq 'ignore'){ |
473 |
print $oai_record->get_identifier.' -> '.$identifier_status."\n"; |
474 |
return; |
475 |
} |
476 |
elsif ($identifier_status eq 'add' || $identifier_status eq 'force_add'){ |
477 |
$oai_record->TransformMetadata($xslt_path); |
478 |
if ($oai_record->get_transformed_metadata){ |
479 |
my $record = $oai_record->GenerateMarcObject($oai_record->get_transformed_metadata->toString); |
480 |
my ($biblionumber,$biblioitemnumber) = C4::Biblio::AddBiblio($record,$frameworkcode); |
481 |
$oai_record->set_biblionumber($biblionumber) if $biblionumber; |
482 |
return 1; |
483 |
} else { |
484 |
return; |
485 |
} |
486 |
} |
487 |
elsif (($identifier_status eq 'update' || $identifier_status eq 'force_update') && $oai_record->get_biblionumber){ |
488 |
$oai_record->TransformMetadata($xslt_path); |
489 |
if ($oai_record->get_transformed_metadata){ |
490 |
my $record = $oai_record->GenerateMarcObject($oai_record->get_transformed_metadata->toString); |
491 |
if ($oai_record->GetBiblionumberExistence){ |
492 |
C4::Biblio::ModBiblio($record,$oai_record->get_biblionumber,$frameworkcode); |
493 |
return 1; |
494 |
} else { |
495 |
print "Error modding bib record ".$oai_record->get_biblionumber.", as it does not exist anymore \n"; |
496 |
$oai_record->set_status($identifier_status."_fail"); |
497 |
return 1; |
498 |
} |
499 |
} else { |
500 |
return; |
501 |
} |
502 |
} |
503 |
elsif ($identifier_status eq 'delete'){ |
504 |
if ($oai_record->get_biblionumber){ |
505 |
if ($oai_record->GetBiblionumberExistence){ |
506 |
my $error = C4::Biblio::DelBiblio($oai_record->get_biblionumber); |
507 |
if ($error){ |
508 |
warn "Error deleting biblionumber ".$oai_record->get_biblionumber." -> $error"; |
509 |
#NOTE: This will fail if there are items attached to the bib record |
510 |
#So...just record the failure due to items being attached rather than trying to delete items. |
511 |
$oai_record->set_status($identifier_status."_fail"); |
512 |
return 1; |
513 |
} |
514 |
else { |
515 |
return 1; |
516 |
} |
517 |
} else { |
518 |
print "It looks like biblionumber ".$oai_record->get_biblionumber." has already been deleted \n"; |
519 |
return; |
520 |
} |
521 |
} else { |
522 |
print "There is no biblionumber associated with this deletion record"; |
523 |
return; |
524 |
} |
525 |
} |
526 |
} else { |
527 |
print $oai_record->get_identifier ." -> no status (so no action taken)\n"; |
528 |
return; |
529 |
} |
530 |
} |
531 |
} |
532 |
|
533 |
sub _parse_httpoai_records_into_clientoai_records { |
534 |
my ( $rec, $repository_id, $metadataPrefix ) = @_; |
535 |
my $record; |
536 |
$record->{header} = $rec->header->dom; |
537 |
$record->{identifier} = $rec->identifier; |
538 |
$record->{datestamp} = $rec->datestamp; |
539 |
$record->{metadata} = $rec->metadata ? $rec->metadata->dom : undef; #N.B. there won't be any metadata for deleted records |
540 |
$record->{about} = $rec->about ? $rec->about->dom : undef; |
541 |
$record->{deleted} = $rec->is_deleted; |
542 |
$record->{status} = $rec->status; |
543 |
$record->{repository_id} = $repository_id; |
544 |
$record->{metadataPrefix} = $metadataPrefix; |
545 |
my $record_object = C4::OAI::Client::Record->new($record); |
546 |
if ($record_object){ |
547 |
return $record_object; |
548 |
} else { |
549 |
return; |
550 |
} |
551 |
} |
552 |
|
553 |
|
554 |
|
555 |
1; |
556 |
__END__ |