From ed6f1a0e18f85a99847ff81fa624bef5d5a2c712 Mon Sep 17 00:00:00 2001 From: David Cook Date: Mon, 5 Aug 2013 15:26:17 +1000 Subject: [PATCH] Bug 10662 - Build OAI-PMH Harvesting Client N.B. This feature is still a work in progress. This commit represents my work to date on the OAI-PMH harvester, but it's not complete... N.B. The test is currently broken (as it was created for an older version of the code). --- C4/OAI/Client/Harvester.pm | 556 ++++++++++++++++++++ C4/OAI/Client/Record.pm | 328 ++++++++++++ admin/oai_repository_targets.pl | 168 ++++++ installer/data/mysql/kohastructure.sql | 41 ++ installer/data/mysql/updatedatabase.pl | 40 ++ .../intranet-tmpl/prog/en/includes/admin-menu.inc | 1 + .../prog/en/modules/admin/admin-home.tt | 2 + .../en/modules/admin/oai_repository_targets.tt | 224 ++++++++ .../intranet-tmpl/prog/en/xslt/DC2MARC21slim.xsl | 217 ++++++++ .../prog/en/xslt/MARC21slim2MARC21enhanced.xsl | 43 ++ misc/cronjobs/oai_harvester.pl | 95 ++++ t/db_dependent/OAI_harvester.t | 308 +++++++++++ 12 files changed, 2023 insertions(+), 0 deletions(-) create mode 100755 C4/OAI/Client/Harvester.pm create mode 100755 C4/OAI/Client/Record.pm create mode 100755 admin/oai_repository_targets.pl create mode 100755 koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_repository_targets.tt create mode 100755 koha-tmpl/intranet-tmpl/prog/en/xslt/DC2MARC21slim.xsl create mode 100755 koha-tmpl/intranet-tmpl/prog/en/xslt/MARC21slim2MARC21enhanced.xsl create mode 100755 misc/cronjobs/oai_harvester.pl create mode 100755 t/db_dependent/OAI_harvester.t diff --git a/C4/OAI/Client/Harvester.pm b/C4/OAI/Client/Harvester.pm new file mode 100755 index 0000000..533adb2 --- /dev/null +++ b/C4/OAI/Client/Harvester.pm @@ -0,0 +1,556 @@ +package C4::OAI::Client::Harvester; + +# This file is part of Koha. +# +# Copyright 2013 Prosentient Systems +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, see . + +=head1 NAME + +C4::OAI::Harvester - OAI-PMH harvester/client which implements the 6 OAI-PMH verbs, record retrieval/harvesting, and import into Koha + +=head1 SYNOPSIS + +use C4::OAI::Harvester; +my $oai_repo = C4::OAI::Harvester->new($repository_data); + +my $identify_repository = $oai_repo->Identify; + +my @sets = $oai_repo->ListSets; + +my @formats = $oai_repo->ListMetadataFormats; + +my @headers = $oai_repo->ListIdentifiers; + +my @records = $oai_repo->ListRecords; + +my @records = $oai_repo->GetRecord($oai_unique_identifier); + +my $import_mode = ''; #i.e. not "automatic" + +$oai_repo->ImportRecordsIntoKoha($import_mode,@records); + +=head1 DESCRIPTION + +C4::OAI::Harvester contains functions for querying and harvesting OAI-PMH repositories. + +More information on OAI-PMH can be found L + +=head1 FUNCTIONS + +=head1 AUTHOR + +David Cook + +=cut + +use Modern::Perl; +use HTTP::OAI; +use C4::Context; +use C4::OAI::Client::Record; +use C4::Biblio qw /AddBiblio ModBiblio DelBiblio/; + +use base qw(Class::Accessor); + + +sub AddOAIRepository { + my ( $database_fields ) = @_; + my $dbh = C4::Context->dbh; + my $sql = " + INSERT INTO oai_harvest_repositories + (baseURL,metadataPrefix,opt_from,opt_until,opt_set,active,XSLT_path,frameworkcode,comments) + VALUES (?,?,?,?,?,?,?,?,?) + "; + my $sth = $dbh->prepare($sql); + $sth->execute( + $database_fields->{baseURL}, + $database_fields->{metadataPrefix}, + $database_fields->{opt_from}, + $database_fields->{opt_until}, + $database_fields->{opt_set}, + $database_fields->{active}, + $database_fields->{XSLT_path}, + $database_fields->{frameworkcode}, + $database_fields->{comments} + ); + if ($sth->err){ + die $sth->errstr; + } +} + +sub ModOAIRepositoryDateTimeFrom { + my ( $repository_id, $datetime_from ) = @_; + my $dbh = C4::Context->dbh; + my $sql = " + UPDATE oai_harvest_repositories + SET opt_from = ? + WHERE repository_id = ? + "; + my $sth = $dbh->prepare($sql); + $sth->execute($datetime_from,$repository_id); + if ($sth->err){ + die $sth->errstr; + } +} + +sub ModOAIRepository { + my ( $database_fields ) = @_; + my $dbh = C4::Context->dbh; + my $sql = " + UPDATE oai_harvest_repositories + SET baseURL = ?, + metadataPrefix = ?, + opt_from = ?, + opt_until = ?, + opt_set = ?, + active = ?, + XSLT_path = ?, + frameworkcode = ?, + comments = ? + WHERE repository_id = ? + "; + my $sth = $dbh->prepare($sql); + $sth->execute( + $database_fields->{baseURL}, + $database_fields->{metadataPrefix}, + $database_fields->{opt_from}, + $database_fields->{opt_until}, + $database_fields->{opt_set}, + $database_fields->{active}, + $database_fields->{XSLT_path}, + $database_fields->{frameworkcode}, + $database_fields->{comments}, + $database_fields->{repository_id} + ); + if ($sth->err){ + die $sth->errstr; + } +} + +sub DelOAIRepository { + my ( $repository_id ) = @_; + my $error; + my $dbh = C4::Context->dbh; + my $sql = " + DELETE FROM oai_harvest_repositories + WHERE repository_id = ? + "; + my $sth = $dbh->prepare($sql); + $sth->execute($repository_id); + if ($sth->err){ + die $sth->errstr; + } +} + +sub GetLatestHistoricalRecordDatestamp { + my ( $repository_id ) = @_; + my $latest_datestamp; + my $dbh = C4::Context->dbh; + my $sql = " + SELECT datestamp + FROM oai_harvest + WHERE repository_id = ? + ORDER BY datestamp desc + LIMIT 1 + "; + my $sth = $dbh->prepare($sql); + $sth->execute($repository_id); + my $row = $sth->fetchrow_hashref; + $latest_datestamp = $row->{datestamp} ? $row->{datestamp} : undef; + return $latest_datestamp; +} + +sub GetOAIRepository { + my ( $repository_id ) = @_; + my $dbh = C4::Context->dbh; + my $sql = " + SELECT * + FROM oai_harvest_repositories + WHERE repository_id = ? + "; + my $sth = $dbh->prepare($sql); + $sth->execute($repository_id); + if ($sth->err){ + die $sth->errstr; + } + my $row = $sth->fetchrow_hashref; + return $row; +} + +=head2 GetOAIRepositoryList + + my @repositories = C4::OAI::Harvester::GetOAIRepositoryList(); + my @repositories = C4::OAI::Harvester::GetOAIRepositoryList($active); + +Returns an array of hashrefs listing all OAI-PMH repositories +present in the database. + +If the $active is included, then it will return an array +of hashrefs listing all OAI-PMH repositories depending on their +active status. $active == 1 shows all active. $active == 0 shows +all inactive. + +=cut + +sub GetOAIRepositoryList { + my ( $active ) = @_; + my $dbh = C4::Context->dbh; + my @results; + my $sql = " + SELECT * + FROM oai_harvest_repositories + "; + if (defined $active){ + $sql .= " WHERE active = 1 " if $active == 1; + $sql .= " WHERE active = 0 " if $active == 0; + } + my $sth = $dbh->prepare($sql); + $sth->execute; + while (my $row = $sth->fetchrow_hashref){ + push(@results,$row); + } + return @results; +} + +#TODO: Perhaps create a sub that cleans out the metadata column to keep the table size low? + +sub new { + my($proto, $fields) = @_; + my($class) = ref $proto || $proto; + + $fields = {} unless defined $fields; + + if ($fields->{'baseURL'}){ + my $h = new HTTP::OAI::Harvester( + baseURL => $fields->{'baseURL'}, + ); + #If resume is set to 0, automatic token resumption is turned off. This is useful for testing/debugging. + if ($h && exists $fields->{'resume'}){ + if ($fields->{'resume'} == 0){ + $h->resume(0); + } + } + my $response = $h->repository($h->Identify); + if( $response->is_error ) { + print "Error requesting Identify:\n", + $response->code . " " . $response->message, "\n"; + exit; + } + $fields->{rh} = $h; #Store HTTP::OAI::Harvester object as "repository handle" + } + bless {%$fields}, $class; +} + +__PACKAGE__->follow_best_practice; #Use get_ and set_ prefixes for accessors +__PACKAGE__->mk_accessors(qw(baseURL opt_from opt_until opt_set metadataPrefix rh repository_id XSLT_path debug frameworkcode)); + +=head2 OAI-PMH Verbs + +Koha-specific implementations of the 6 OAI-PMH Verbs. + +The key verbs are "ListRecords" and "GetRecords". These do the actual +harvesting of records from a OAI-PMH repository. The others are useful for +getting information about a repository and what it has available. + +1) ListRecords + +2) GetRecord + +3) ListIdentifiers + +4) ListMetadataFormats + +5) ListSets + +6) Identify + +=cut + + +sub ListRecords { + my ( $self ) = @_; + my %args = ( + metadataPrefix => $self->{metadataPrefix}, + from => $self->{opt_from}, + until => $self->{opt_until}, + set => $self->{opt_set}, + ); + if ($self->{debug}){ + use Data::Dumper; + print "ListRecords's arguments\n"; + print Dumper(\%args); + } + my $response = $self->{rh}->ListRecords(%args); + if( $response->is_error ) { + print "Error requesting ListRecords:\n", + $response->code . " " . $response->message, "\n"; + exit; + } + if ($self->{debug}){ + print "Successfully retrieved ListRecords response.\n"; + } + return $response; +} + +sub GetRecord { + my ( $self, $identifier, $harvest ) = @_; + my $response = $self->{rh}->GetRecord( + metadataPrefix => $self->{metadataPrefix}, + identifier => $identifier, + ); + if( $response->is_error ) { + print "Error requesting GetRecord:\n", + $response->code . " " . $response->message, "\n"; + exit; + } + if ($self->{debug}){ + print "Successfully retrieved GetRecord response.\n"; + } + return $response; +} + +sub ListIdentifiers { + my $self = shift; + my @headers; + my $response = $self->{rh}->ListIdentifiers( + metadataPrefix => $self->{metadataPrefix}, + from => $self->{opt_from}, + until => $self->{opt_until}, + set => $self->{opt_set}, + ); + if( $response->is_error ) { + print "Error requesting ListIdentifiers:\n", + $response->code . " " . $response->message, "\n"; + exit; + } + while (my $h = $response->next){ + my $header; + #print Dumper($h->dom->toString); #DEBUG: XML representation + $header->{identifier} = $h->identifier; + $header->{datestamp} = $h->datestamp; + + $header->{status} = $h->status; + $header->{is_deleted} = $h->is_deleted; + + my @sets = $h->setSpec; + $header->{sets} = \@sets; + + push (@headers,$header); + } + return \@headers; +} + +sub ListMetadataFormats { + my ( $self, $identifier ) = @_; + my @formats; + my $response = $self->{rh}->ListMetadataFormats( + identifier => $identifier, + ); + if( $response->is_error ) { + print "Error requesting ListMetadataFormats:\n", + $response->code . " " . $response->message, "\n"; + exit; + } + for($response->metadataFormat) { + push(@formats,$_->metadataPrefix); + } + return \@formats; +} + +sub ListSets { + my $self = shift; + my @sets; + my $response = $self->{rh}->ListSets(); + if( $response->is_error ) { + print "Error requesting ListSets:\n", + $response->code . " " . $response->message, "\n"; + exit; + } + while (my $s = $response->next){ + my $set; + $set->{setSpec} = $s->setSpec; + $set->{setName} = $s->setName; + + #TODO: Not really sure what to do with the descriptions as they're XML and not necessarily that easy to parse for GUI views... + #my @temp_setDescriptions = $s->setDescription; + #my @setDescriptions; + #foreach my $temp_setDescription (@temp_setDescriptions){ + # push (@setDescriptions,$temp_setDescription->dom->toString); I think we need to do better than just return the setDescription XML...That's not very useful... + #} + #$set->{setDescription} = \@setDescriptions; + push (@sets,$set); + } + return \@sets; +} + +sub Identify { + my $self = shift; + my $response = $self->{rh}->Identify(); + if( $response->is_error ) { + print "Error requesting Identify:\n", + $response->code . " " . $response->message, "\n"; + exit; + } + my $identify_data; + #DEBUG: View what's in the Identify object + #print Dumper($response->headers); + + $identify_data->{repositoryName} = $response->repositoryName; + $identify_data->{baseURL} = $response->baseURL; + $identify_data->{protocolVersion} = $response->protocolVersion; #Tim Brody says this will always return 2.0 and that ->version should be used to find the actual version... + #$identify_data->{version} = $response->version; + $identify_data->{earliestDatestamp} = $response->earliestDatestamp; + $identify_data->{deletedRecord} = $response->deletedRecord; #not in the perldoc, but it's in the code and the OAI-PMH spec + $identify_data->{granularity} = $response->granularity; + + #These methods should be used with an array context so they return all the elements and not just the first one + my @adminEmails = $response->adminEmail; + $identify_data->{adminEmail} = \@adminEmails; + my @compressions = $response->compression; + $identify_data->{compression} = \@compressions; + + #TODO: Descriptions are encapsulated in XML containers, I believe. Not sure what to do with these at present... + #my @descriptions = $response->description; + #$identify_data->{description} = \@descriptions; + #$response->next + + return $identify_data; +} + +sub ProcessOaipmhRecordResponse { + my ( $self, $response, $harvest, $force ) = @_; + if ($response){ + my @records; + while( my $rec = $response->next ){ + print "I'm parsing the record..." if $self->{debug}; + #Parse from the HTTP::OAI::Record object into our C4::OAI::Client::Record object + my $record_object = _parse_httpoai_records_into_clientoai_records($rec,$self->{repository_id},$self->{metadataPrefix}); + + #If set to harvest mode, we ingest the record + if ($harvest && $record_object){ + + #Retrieve the latest biblionumber associated with this OAI identifier + $record_object->GetBiblionumberFromOaiIdentifier; + + #Generate a status to be acted upon (e.g. add, update, delete, force_add, force_update, ignore) + $record_object->GenerateStatus($force); + + print "I'm ingesting the record..." if $self->{debug}; + #Based on the above status, act upon the record + my $ingestion_flag = _ingest_oai_record($record_object, $self->get_XSLT_path, $self->get_frameworkcode); + + if ($ingestion_flag){ + #Log this OAI-PMH record in the database + $record_object->AddRecordToLog; + print $record_object->get_identifier.' -> '.$record_object->get_status.' -> '.$record_object->get_biblionumber." \n "; + } + } + print $rec->identifier."\n" if !$harvest; + push(@records,$record_object) if !$harvest; + } + return \@records; + } +} + +sub _ingest_oai_record { + my ( $oai_record, $xslt_path, $frameworkcode ) = @_; + + if ( $oai_record && $xslt_path ){ + my $identifier_status = $oai_record->get_status; + if ($identifier_status){ + if ($identifier_status eq 'ignore'){ + print $oai_record->get_identifier.' -> '.$identifier_status."\n"; + return; + } + elsif ($identifier_status eq 'add' || $identifier_status eq 'force_add'){ + $oai_record->TransformMetadata($xslt_path); + if ($oai_record->get_transformed_metadata){ + my $record = $oai_record->GenerateMarcObject($oai_record->get_transformed_metadata->toString); + my ($biblionumber,$biblioitemnumber) = C4::Biblio::AddBiblio($record,$frameworkcode); + $oai_record->set_biblionumber($biblionumber) if $biblionumber; + return 1; + } else { + return; + } + } + elsif (($identifier_status eq 'update' || $identifier_status eq 'force_update') && $oai_record->get_biblionumber){ + $oai_record->TransformMetadata($xslt_path); + if ($oai_record->get_transformed_metadata){ + my $record = $oai_record->GenerateMarcObject($oai_record->get_transformed_metadata->toString); + if ($oai_record->GetBiblionumberExistence){ + C4::Biblio::ModBiblio($record,$oai_record->get_biblionumber,$frameworkcode); + return 1; + } else { + print "Error modding bib record ".$oai_record->get_biblionumber.", as it does not exist anymore \n"; + $oai_record->set_status($identifier_status."_fail"); + return 1; + } + } else { + return; + } + } + elsif ($identifier_status eq 'delete'){ + if ($oai_record->get_biblionumber){ + if ($oai_record->GetBiblionumberExistence){ + my $error = C4::Biblio::DelBiblio($oai_record->get_biblionumber); + if ($error){ + warn "Error deleting biblionumber ".$oai_record->get_biblionumber." -> $error"; + #NOTE: This will fail if there are items attached to the bib record + #So...just record the failure due to items being attached rather than trying to delete items. + $oai_record->set_status($identifier_status."_fail"); + return 1; + } + else { + return 1; + } + } else { + print "It looks like biblionumber ".$oai_record->get_biblionumber." has already been deleted \n"; + return; + } + } else { + print "There is no biblionumber associated with this deletion record"; + return; + } + } + } else { + print $oai_record->get_identifier ." -> no status (so no action taken)\n"; + return; + } + } +} + +sub _parse_httpoai_records_into_clientoai_records { + my ( $rec, $repository_id, $metadataPrefix ) = @_; + my $record; + $record->{header} = $rec->header->dom; + $record->{identifier} = $rec->identifier; + $record->{datestamp} = $rec->datestamp; + $record->{metadata} = $rec->metadata ? $rec->metadata->dom : undef; #N.B. there won't be any metadata for deleted records + $record->{about} = $rec->about ? $rec->about->dom : undef; + $record->{deleted} = $rec->is_deleted; + $record->{status} = $rec->status; + $record->{repository_id} = $repository_id; + $record->{metadataPrefix} = $metadataPrefix; + my $record_object = C4::OAI::Client::Record->new($record); + if ($record_object){ + return $record_object; + } else { + return; + } +} + + + +1; +__END__ \ No newline at end of file diff --git a/C4/OAI/Client/Record.pm b/C4/OAI/Client/Record.pm new file mode 100755 index 0000000..5cabc2a --- /dev/null +++ b/C4/OAI/Client/Record.pm @@ -0,0 +1,328 @@ +package C4::OAI::Client::Record; + +=head1 NAME + +C4::OAI::Client::Record - an internal class for handling records retrieved via the OAI-PMH protocol + +=head1 DESCRIPTION + +C4::OAI::Client::Record - Class to handle the management of records retrieved via OAI-PMH + +More information on OAI-PMH can be found L + +=head1 AUTHOR + +David Cook + +=cut + +use Modern::Perl; +use C4::Context; +use C4::Templates qw/_current_language/; +use C4::Charset qw/StripNonXmlChars/; +use C4::XSLT qw/GetURI/; + +use XML::LibXML; +use XML::LibXSLT; +use MARC::Record; +use MARC::File::XML; +use base qw(Class::Accessor); + +sub new { + my($proto, $fields) = @_; + my($class) = ref $proto || $proto; + $fields = {} unless defined $fields; + bless {%$fields}, $class; +} + +__PACKAGE__->follow_best_practice; #Use get_ and set_ prefixes for accessors +__PACKAGE__->mk_accessors(qw(identifier datestamp metadata status deleted header about transformed_metadata repository_id metadataPrefix biblionumber)); + +=head2 TransformMetadata + + + +=cut + +sub TransformMetadata { + my ( $self, $XSLT_path ) = @_; + my $source = $self->{metadata}; #This is a LibXML object retrieved via HTTP::OAI::Harvester + my $metadataPrefix = $self->get_metadataPrefix; + + my $no_XSLT_path; + if (!$XSLT_path){ + $no_XSLT_path = 1; + } + + if ($metadataPrefix && $source){ + my $xslt = XML::LibXSLT->new(); + my $style_doc; + + #$parser based on equivalent from XSLT.pm + my $parser = XML::LibXML->new(); + # don't die when you find &, >, etc + $parser->recover_silently(0); + + #NOTE: This XSLT essentially copies over all MARC fields except the 952 and 999. + #It also adds a 037 containing the OAI-PMH identifier. + my $default_marc_xslt = C4::Context->config('intrahtdocs') . + '/' . C4::Context->preference("template") . + '/' . C4::Templates::_current_language() . + '/xslt/' . + C4::Context->preference('marcflavour') . + "slim2" . + C4::Context->preference('marcflavour') . + "enhanced.xsl"; + + #TODO: This XSLT needs work. In practice, it would be better to create + #customized XSLTs based on the metadata format native to the OAI-PMH source (e.g. DIM for DSpace) + my $default_dc_xslt = C4::Context->config('intrahtdocs') . + '/' . C4::Context->preference("template") . + '/' . C4::Templates::_current_language() . + '/xslt/DC2' . + C4::Context->preference('marcflavour') . + "slim.xsl"; + + #TODO: As necessary, add more default XSLTs. Perhaps MODS and METS (although METS would be difficult + #since it can contain other metadata formats within itself). + + if ($XSLT_path){ + #This conditional for handling secure remote XSLTs copied from XSLT.pm + if ( $XSLT_path =~ /^https?:\/\// ) { + my $xsltstring = GetURI($XSLT_path); + if ($xsltstring){ + $style_doc = $parser->parse_string($xsltstring); + } + else{ + #If a string isn't retrieved using GetURI, we switch to our default transforms + $no_XSLT_path = 1; + } + } elsif ( $XSLT_path eq 'dublincore' ) { + $style_doc = $parser->parse_file($default_dc_xslt); #Use default DC XSLT if XSLT_path is specified as 'dublincore' + } elsif ( $XSLT_path eq 'marc' ) { + $style_doc = $parser->parse_file($default_marc_xslt); #Use default MARC XSLT if XSLT_path is specified as 'marc' + } else { + if ( -e $XSLT_path){ + $style_doc = $parser->parse_file($XSLT_path); + } + else{ + #If the file doesn't actually exist, we switch to our default transforms + $no_XSLT_path = 1; + } + } + } + + if ($no_XSLT_path){ + if ($metadataPrefix eq 'marcxml' || $metadataPrefix eq 'marc'){ + $style_doc = $parser->parse_file($default_marc_xslt); + } + elsif ($metadataPrefix eq 'dc' || $metadataPrefix eq 'oai_dc'){ + $style_doc = $parser->parse_file($default_dc_xslt); + } + else{ + return -1; + } + } + + if ($style_doc){ + my $stylesheet = $xslt->parse_stylesheet($style_doc); + + my %xslt_params; + $xslt_params{'identifier'} = $self->{identifier}; + + if ($stylesheet){ + #Pass OAI-PMH identifier and matched biblionumber (if there is one) to the XSLT for unique identification/provenance + my $result_xml = $stylesheet->transform($source, XML::LibXSLT::xpath_to_string(%xslt_params)); + + if ($result_xml){ + $self->set_transformed_metadata($result_xml); + return 1; + } + else{ + return -1; + } + } + } + } + else{ + return -1; + } +} + +=head2 AddRecordToLog + + +=cut + +sub AddRecordToLog { + my ( $self ) = @_; + + my $metadataPrefix = $self->get_metadataPrefix; + my $repository_id = $self->get_repository_id; + my $biblionumber = $self->get_biblionumber; + + if ($metadataPrefix && $repository_id && $biblionumber){ + my $identifier = $self->get_identifier; + my $datestamp = $self->get_datestamp; + my $metadata = $self->get_metadata ? $self->get_metadata->toString : undef; + my $header = $self->get_header ? $self->get_header->toString : undef; + my $status = $self->get_status; + + + + my $dbh = C4::Context->dbh; + my $sql = " + INSERT INTO oai_harvest (identifier,datestamp,metadataPrefix,metadata,header,status,repository_id,biblionumber) + VALUES (?,?,?,?,?,?,?,?) + "; + my $sth = $dbh->prepare($sql); + $sth->execute($identifier, $datestamp, $metadataPrefix, $metadata, $header, $status, $repository_id, $biblionumber); + if ($sth->err){ + return "ERROR! return code: " . $sth->err . " error msg: " . $sth->errstr . "\n"; + } + else{ + return 1; + } + } + else{ + return -1; + } +} + +=head2 GenerateStatus + + + + +=cut + +sub GenerateStatus { + my ( $self, $force ) = @_; + my $dbh = C4::Context->dbh; + my $repository_id = $self->get_repository_id; + my $identifier_status; + + #Check if this identifier is found in the database with this repository + my $check_for_id_sql = " + SELECT count(*) as 'count' + FROM oai_harvest + WHERE identifier = ? and repository_id = ?"; + my $check_for_id_sth = $dbh->prepare($check_for_id_sql); + $check_for_id_sth->execute($self->get_identifier, $repository_id); + my $check_for_id_row = $check_for_id_sth->fetchrow_hashref; + if ($check_for_id_row->{count} == 0){ + #The OAI-PMH unique identifier doesn't exist in this database for this repository == ADD + $identifier_status = "add"; + } + else{ + #OAI-PMH Unique Identifier does exist in database == UPDATE || IGNORE + my $check_for_id_and_datestamp_sql = " + SELECT count(*) as 'count' + FROM oai_harvest + WHERE identifier = ? and datestamp = ? and repository_id = ? + "; + my $check_for_id_and_datestamp_sth = $dbh->prepare($check_for_id_and_datestamp_sql); + $check_for_id_and_datestamp_sth->execute($self->get_identifier, $self->get_datestamp, $repository_id); + my $check_for_id_and_datestamp_row = $check_for_id_and_datestamp_sth->fetchrow_hashref; + if ($check_for_id_and_datestamp_row->{count} == 0){ + #OAI-PMH Unique Identifier and Datestamp combo don't exist in database == UPDATE + #i.e. The identifier exists in the database, but this is an updated datestamp + $identifier_status = "update" + } + else{ + #OAI-PMH Unique Identifier and Datestamp combo exist in database but we're forcing an add/update anyway (in accordance with the $force value + if ($force){ + if ($self->GetBiblionumberExistence){ + $identifier_status = ( $force eq "only_add" ) ? "ignore" : "force_update"; + } + else { + $identifier_status = ( $force eq "only_update" ) ? "ignore" : "force_add"; + } + } else { + #OAI-PMH Unique Identifier and Datestamp combo exist in database == IGNORE + $identifier_status = "ignore"; + } + } + } + + if ($self->get_deleted && $identifier_status ne "ignore"){ + $identifier_status = "delete"; + } + + if ($identifier_status){ + $self->set_status($identifier_status); + } +} + + +sub GetBiblionumberExistence { + my ( $self ) = @_; + my $biblionumber_to_check = $self->get_biblionumber; + if ($biblionumber_to_check){ + my $dbh = C4::Context->dbh; + my $sql = " + SELECT count(biblionumber) as 'count' + FROM biblio + WHERE biblionumber = ? + "; + my $sth = $dbh->prepare($sql); + $sth->execute($biblionumber_to_check); + my $row = $sth->fetchrow_hashref; + if ($row->{count} > 0){ + return 1; + } else { + return; + } + } else { + return; + } +} + + +#FIXME: This probably isn't the best way to do this... +sub GetBiblionumberFromOaiIdentifier { + my ( $self ) = @_; + my $biblionumber; + my $dbh = C4::Context->dbh; + #Select the newest biblionumber. In theory, there should only be one biblionumber. However, if a record + #was deleted in Koha but force_added again later, there would be multiple biblionumbers in our + #harvesting records. Ergo, we take the highest, so that we're working with the correct number for future + #updates/deletions. + my $sql = " + SELECT DISTINCT biblionumber + FROM oai_harvest + WHERE identifier = ? AND repository_id = ? + ORDER BY timestamp desc + LIMIT 1 + "; + my $sth = $dbh->prepare($sql); + $sth->execute($self->get_identifier,$self->get_repository_id); + my $row = $sth->fetchrow_hashref; + $biblionumber = $row->{biblionumber} if $row->{biblionumber}; + $self->set_biblionumber($biblionumber) if $biblionumber; #Add biblionumber to the object + return $biblionumber; +} + +#FIXME: This should probably be in Biblio.pm and not a method here? (Galen's work might intersect here...) +sub GenerateMarcObject { + my ( $self, $marcxml ) = @_; + + if ($marcxml) { + my $marcxml = StripNonXmlChars( $marcxml ); #Warning as per C4::Charset, this could cause some data loss + + my $record = eval { MARC::Record::new_from_xml( $marcxml, "utf8", C4::Context->preference('marcflavour') ) }; + if ($@) { + warn " problem with : $@ \n$marcxml"; + return; + } + return $record; + } else { + return; + } +} + + + + +1; +__END__ \ No newline at end of file diff --git a/admin/oai_repository_targets.pl b/admin/oai_repository_targets.pl new file mode 100755 index 0000000..1faa0e6 --- /dev/null +++ b/admin/oai_repository_targets.pl @@ -0,0 +1,168 @@ +#!/usr/bin/perl + +# This file is part of Koha. +# +# Copyright 2013 Prosentient Systems +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, see . + +=head1 NAME + +oai_repository_targets.pl - script to manage configuration for harvesting external +OAI-PMH server/repository targets. + +=head1 DESCRIPTION + +If !$op, then display table listing existing entries + +If $op eq 'add', then display add form OR modify form, if primary key is supplied + +If $op eq 'delete', then delete the entry + +=cut + +use Modern::Perl; +use CGI; +use C4::Auth; +use C4::Output; +use C4::Context; + +use C4::Koha qw/getframeworks/; +use C4::OAI::Client::Harvester; + +my $input = new CGI; +my $script_name = "/cgi-bin/koha/admin/oai_repository_targets.pl"; +my ($template, $loggedinuser, $cookie) + = get_template_and_user({template_name => "admin/oai_repository_targets.tt", + query => $input, + type => "intranet", + authnotrequired => 0, + flagsrequired => {parameters => 'parameters_remaining_permissions'}, + debug => 1, + }); + +$template->param(script_name => $script_name); + +my $op = $input->param('op') || ''; +my $id = $input->param('id') || ''; +if ($id){ + $template->param(id => $id); +} + +# get framework list +my $frameworks = C4::Koha::getframeworks; +my @frameworkcodeloop; +foreach my $thisframeworkcode ( sort { uc($frameworks->{$a}->{'frameworktext'}) cmp uc($frameworks->{$b}->{'frameworktext'}) } keys %{$frameworks} ) { + push @frameworkcodeloop, { + value => $thisframeworkcode, + frameworktext => $frameworks->{$thisframeworkcode}->{'frameworktext'}, + }; +} +$template->param(frameworkcodeloop => \@frameworkcodeloop); + +if ($op eq 'add' || $op eq 'edit' || $op eq 'validate'){ + #These operations require a form view + $template->param(view => "form"); + + if ($op eq 'edit' && $id){ + my $repository_fields = C4::OAI::Client::Harvester::GetOAIRepository($id); + $template->param(form_fields => $repository_fields); + } + elsif ($op eq 'validate'){ + my @validation_errors; + my $form_fields; + $form_fields->{baseURL} = $input->param('baseURL') ? $input->param('baseURL') : undef; + if (!$form_fields->{baseURL}){ + push(@validation_errors,"baseURL"); + $template->param(baseURL_validation_err => "mandatory"); + } + $form_fields->{metadataPrefix} = $input->param('metadataPrefix') ? $input->param('metadataPrefix') : undef; + if (!$form_fields->{metadataPrefix}){ + push(@validation_errors,"metadataPrefix"); + $template->param(metadataPrefix_validation_err => "mandatory"); + } + $form_fields->{XSLT_path} = $input->param('XSLT_path') ? $input->param('XSLT_path') : undef; + if (!$form_fields->{XSLT_path}){ + push(@validation_errors,"XSLT_path"); + $template->param(XSLT_path_validation_err => "mandatory"); + } + #Use 'undef' instead of blank so that you take advantage of database level validation + $form_fields->{opt_from} = $input->param('opt_from') ? $input->param('opt_from') : undef; + $form_fields->{opt_until} = $input->param('opt_until') ? $input->param('opt_until') : undef; + $form_fields->{opt_set} = $input->param('opt_set') ? $input->param('opt_set') : undef; + $form_fields->{active} = $input->param('active') == 1 || $input->param('active') == 0 ? $input->param('active') : undef; + $form_fields->{frameworkcode} = $input->param('frameworkcode') ? $input->param('frameworkcode') : ''; + $form_fields->{comments} = $input->param('comments') ? $input->param('comments') : undef; + $form_fields->{repository_id} = $id ? $id : undef; + + if (@validation_errors){ + warn "There are ". scalar @validation_errors . " validation errors"; + $template->param( + form_fields => $form_fields, + ); + } + else { + if ($id){ + C4::OAI::Client::Harvester::ModOAIRepository($form_fields); + #Show confirmation view after update + $template->param( + view => "confirm", + op => "confirm_update", + ); + } + else { + C4::OAI::Client::Harvester::AddOAIRepository($form_fields); + #Show confirmation view after insert/add + $template->param( + view => "confirm", + op => "confirm_add", + ); + } + } + } +} +else { + #All other operations require a list view + $template->param(view => "list"); + + my @results; + + if ($op eq 'delete' && $id){ + my $repository_fields = C4::OAI::Client::Harvester::GetOAIRepository($id); + if ($repository_fields){ + push(@results,$repository_fields); + $template->param( + op => 'delete' + ); + } + } + elsif ($op eq 'perform_delete' && $id){ + C4::OAI::Client::Harvester::DelOAIRepository($id); + #Show confirmation view after delete + $template->param( + view => "confirm", + op => "confirm_delete", + ); + } + else{ + @results = C4::OAI::Client::Harvester::GetOAIRepositoryList(); + } + + if (@results){ + $template->param( + result_rows => \@results + ); + } +} + +output_html_with_http_headers $input, $cookie, $template->output; \ No newline at end of file diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql index c1a8b8d..1174655 100644 --- a/installer/data/mysql/kohastructure.sql +++ b/installer/data/mysql/kohastructure.sql @@ -3243,3 +3243,44 @@ CREATE TABLE IF NOT EXISTS plugin_data ( /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; +-- +-- Table structure for table 'oai_harvest_repositories' +-- + +DROP TABLE IF EXISTS oai_harvest_repositories; +CREATE TABLE oai_harvest_repositories ( + repository_id int(11) NOT NULL AUTO_INCREMENT, -- primary key identifier + baseURL text NOT NULL, -- baseURL of the remote OAI-PMH repository (mandatory) + opt_from varchar(45) DEFAULT NULL, -- "from" time for selective harvesting (optional - gets set automatically by cronjob) + opt_until varchar(45) DEFAULT NULL, -- "until" time for selective harvesting (optional) + opt_set varchar(45) DEFAULT NULL, -- the record set to harvest for selective harvesting (optional) + metadataPrefix varchar(45) NOT NULL, -- metadata format (e.g. oai_dc, dc, marcxml) + active int(11) NOT NULL DEFAULT '0', -- indicate whether this repo is actively harvested by Koha + timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, -- date last modified + XSLT_path text, -- filepath to a local or external XSLT file to use for transforming incoming records into MARCXML + frameworkcode varchar(4) NOT NULL, -- framework to use when ingesting records + comments text, -- limited number of characters (controlled by template) to describe the repository (optional - for librarian use rather than system use) + PRIMARY KEY (repository_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- +-- Table structure for table 'oai_harvest' +-- + +DROP TABLE IF EXISTS oai_harvest; +CREATE TABLE oai_harvest ( + oai_harvest_id int(11) NOT NULL AUTO_INCREMENT, -- primary key identifier + identifier varchar(255) NOT NULL, -- OAI-PMH identifier (unique to its original repo) + datestamp varchar(45) NOT NULL, -- OAI-PMH datestamp (i.e. date last modified) + metadataPrefix varchar(45) NOT NULL, -- metadata format of the record + biblionumber int(11) NOT NULL, -- biblionumber of the biblio record created by this OAI-PMH ingested record + metadata longtext, -- XML string containing the metadata (this will be null for incoming records marked as deleted) + header longtext NOT NULL, -- XML string containing the header information (i.e. identifier, datestamp, status) + about longtext, -- XML string containing explanatory information about a record (optional. Often not included in records) + timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, -- date last modified + repository_id int(11) NOT NULL, -- id for this record's OAI-PMH repository (linked to oai_harvest_repositories table) + status varchar(45) DEFAULT NULL, -- whether this record was an addition, an update, or a deletion + PRIMARY KEY (oai_harvest_id), + KEY FK_oai_harvest_1 (repository_id), + CONSTRAINT FK_oai_harvest_1 FOREIGN KEY (repository_id) REFERENCES oai_harvest_repositories (repository_id) ON UPDATE NO ACTION +) ENGINE=InnoDB AUTO_INCREMENT=2809 DEFAULT CHARSET=utf8; diff --git a/installer/data/mysql/updatedatabase.pl b/installer/data/mysql/updatedatabase.pl index abd7c8e..c95ed4e 100755 --- a/installer/data/mysql/updatedatabase.pl +++ b/installer/data/mysql/updatedatabase.pl @@ -7185,6 +7185,46 @@ if ( CheckVersion($DBversion) ) { SetVersion($DBversion); } +$DBversion = "3.13.00.XXX"; +if ( CheckVersion($DBversion) ) { + $dbh->do(" +CREATE TABLE IF NOT EXISTS oai_harvest_repositories ( + repository_id int(11) NOT NULL AUTO_INCREMENT, + baseURL text NOT NULL, + opt_from varchar(45) DEFAULT NULL, + opt_until varchar(45) DEFAULT NULL, + opt_set varchar(45) DEFAULT NULL, + metadataPrefix varchar(45) NOT NULL, + active int(11) NOT NULL DEFAULT '0', + timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + XSLT_path text, + frameworkcode varchar(4) NOT NULL, + comments text, + PRIMARY KEY (repository_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + "); + $dbh->do(" +CREATE TABLE IF NOT EXISTS oai_harvest ( + oai_harvest_id int(11) NOT NULL AUTO_INCREMENT, + identifier varchar(255) NOT NULL, + datestamp varchar(45) NOT NULL, + metadataPrefix varchar(45) NOT NULL, + biblionumber int(11) NOT NULL, + metadata longtext, + header longtext NOT NULL, + about longtext, + timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + repository_id int(11) NOT NULL, + status varchar(45) DEFAULT NULL, + PRIMARY KEY (oai_harvest_id), + KEY FK_oai_harvest_1 (repository_id), + CONSTRAINT FK_oai_harvest_1 FOREIGN KEY (repository_id) REFERENCES oai_harvest_repositories (repository_id) ON UPDATE NO ACTION +) ENGINE=InnoDB AUTO_INCREMENT=2809 DEFAULT CHARSET=utf8; + "); + print "Upgrade to $DBversion done (Bug 10662: Build OAI-PMH Harvesting Client)\n"; + SetVersion($DBversion); +} + =head1 FUNCTIONS =head2 TableExists($table) diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc index 91878bf..605a0e0 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc @@ -62,6 +62,7 @@ diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt index a5dbe44..786c865 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt @@ -105,6 +105,8 @@
Printers (UNIX paths).
-->
Z39.50 client targets
Define which servers to query for MARC data in the integrated Z39.50 client.
+
OAI-PMH repository targets
+
Define which OAI-PMH repositories from which to automatically harvest metadata records
Did you mean?
Choose which plugins to use to suggest searches to patrons and staff.
diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_repository_targets.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_repository_targets.tt new file mode 100755 index 0000000..0ad0f5e --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_repository_targets.tt @@ -0,0 +1,224 @@ +[% INCLUDE 'doc-head-open.inc' %] +Koha › Administration › Manage OAI-PMH repository targets +[%# FIXME: change title based on page... each page should have a different title %] +[% INCLUDE 'doc-head-close.inc' %] + +[% IF ( view == "list" ) %] + + +[% INCLUDE 'datatables-strings.inc' %] + +[% END %] +[% IF ( view == "list" ) %] + +[% END %] + + + +[% INCLUDE 'header.inc' %] +[% INCLUDE 'cat-search.inc' %] + + +
+
+
+
+ [% IF ( view == "confirm" ) %] + [% IF ( op == "confirm_add" ) %] +

OAI-PMH repository target added

+
+ +
+ [% ELSIF ( op == "confirm_update" ) %] +

OAI-PMH repository target updated

+
+ +
+ [% ELSIF ( op == "confirm_delete" ) %] +

OAI-PMH repository target deleted

+
+ +
+ [% END %] + [% ELSIF ( view == "form" ) %] + [% IF ( id ) %] +

Edit OAI-PMH repository target

+ [% ELSE %] +

New OAI-PMH repository target

+ [% END %] +
+ + [% IF ( id ) %] + + [% END %] +
+
    + [% IF ( baseURL_validation_err ) %] +
  1. + [% ELSE %] +
  2. + [% END %] + + + [% IF ( baseURL_validation_err == "mandatory") %] + [Base URL is a mandatory field.] + [% END %] +
  3. +
  4. + + +
  5. +
  6. + + +
  7. +
  8. + + +
  9. + [% IF ( metadataPrefix_validation_err ) %] +
  10. + [% ELSE %] +
  11. + [% END %] + + + [% IF ( metadataPrefix_validation_err == "mandatory") %] + [Metadata prefix is a mandatory field.] + [% END %] +
  12. +
  13. + + +
  14. +
  15. + + +
  16. + [% IF ( XSLT_path_validation_err ) %] +
  17. + [% ELSE %] +
  18. + [% END %] + + + [% IF ( XSLT_path_validation_err == "mandatory") %] + [XSLT is a mandatory field.] + [% END %] + (Please consult your system administrator when choosing your XSLT.) +
  19. +
  20. + + +
  21. +
+
+
Cancel
+
+ [% ELSIF ( view == "list" ) %] + [% UNLESS ( op == "delete" ) %] + + [% END %] + [% IF ( op == "delete" ) %] +

Delete OAI-PMH repository target

+ [% ELSE %] +

List OAI-PMH repository targets

+ [% END %] + + + + + + + + + + + + + [% UNLESS ( op == "delete" ) %] + + + [% END %] + + + + [% FOREACH result_row IN result_rows %] + + + + + + + + + + + [% UNLESS ( op == "delete" ) %] + + + [% END %] + + [% END %] + +
Base URLFromUntilSetMetadata prefixStatusFrameworkXSLTComments  
[% result_row.baseURL %][% result_row.opt_from %][% result_row.opt_until %][% result_row.opt_set %][% result_row.metadataPrefix %][% IF ( result_row.active == 1 ) %] Active [% ELSE %] Inactive [% END %] + [% IF ( result_row.frameworkcode == '') %] + Default + [% ELSE %] + [% FOREACH framework IN frameworkcodeloop %][% IF ( result_row.frameworkcode == framework.value )%][% framework.frameworktext %][% END %][% END %] + [% END %] + [% result_row.XSLT_path %][% IF (result_row.comments.length > 20) %][% result_row.comments.substr(0,20) %]...[% ELSE %][% result_row.comments %][% END %]EditDelete
+ [% IF ( op == "delete" ) %] +
+ + +
+ + Cancel +
+
+ [% END %] + [% END %] +
[%# first yui-b %] +
[%# yui-main %] +
+ [% INCLUDE 'admin-menu.inc' %] +
[%# last yui-b %] +
[%# bd %] + +[% INCLUDE 'intranet-bottom.inc' %] \ No newline at end of file diff --git a/koha-tmpl/intranet-tmpl/prog/en/xslt/DC2MARC21slim.xsl b/koha-tmpl/intranet-tmpl/prog/en/xslt/DC2MARC21slim.xsl new file mode 100755 index 0000000..f68cbbe --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/xslt/DC2MARC21slim.xsl @@ -0,0 +1,217 @@ + + + + + + + + + + + + + + + + p + m + r + k + m + m + m + i + a + a + + + + + c + m + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + OAI-PMH Identifier + + + + + dc + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + local + + + + + + + + + collaborator + + + + + + + + + author + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/koha-tmpl/intranet-tmpl/prog/en/xslt/MARC21slim2MARC21enhanced.xsl b/koha-tmpl/intranet-tmpl/prog/en/xslt/MARC21slim2MARC21enhanced.xsl new file mode 100755 index 0000000..0cf9aaa --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/xslt/MARC21slim2MARC21enhanced.xsl @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + 8 + + 037 + + + + a + + + + n + OAI-PMH Identifier + + + + + \ No newline at end of file diff --git a/misc/cronjobs/oai_harvester.pl b/misc/cronjobs/oai_harvester.pl new file mode 100755 index 0000000..fca46fd --- /dev/null +++ b/misc/cronjobs/oai_harvester.pl @@ -0,0 +1,95 @@ +#!/usr/bin/perl + +# This file is part of Koha. +# +# Copyright 2013 Prosentient Systems +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, see . + +=head1 NAME + +oai_harvester.pl - cron script to harvest bibliographic records from +OAI-PMH repositories. + +=head1 SYNOPSIS + + + +=head1 DESCRIPTION + + + +=cut + +use Modern::Perl; +use Getopt::Long; +use Pod::Usage; +use C4::Context; +use C4::OAI::Client::Harvester; + +my $help = 0; +my $debug = 0; +my $noresume = 0; +my $force = ''; +my $harvest = 0; + +#TODO: Perhaps create a flag that triggers a delete of all records that have been ingested via OAI-PMH? + +GetOptions( + 'h|help|?' => \$help, + 'd' => \$debug, + 'noresume' => \$noresume, + 'force=s' => \$force, + 'harvest' => \$harvest, + + ); +pod2usage(1) if $help; + +my $active = 1; #Get only active repositories +my @repositories = C4::OAI::Client::Harvester::GetOAIRepositoryList($active); +foreach my $repository_data (@repositories) { + + #Print extra info for debugging + if ($debug){ + $repository_data->{debug} = 1; + } + + #Turn off automatic token resumption. This is for debugging. + if ($noresume){ + $repository_data->{resume} = 0; + } + + #Create Harvester object + my $repository = C4::OAI::Client::Harvester->new($repository_data); + + if ($repository){ + print "\n\nHarvest ".$repository->{'baseURL'}." in DEBUG MODE\n" if $repository->{debug}; + my $response = $repository->ListRecords; + if ($response){ + my $records = $repository->ProcessOaipmhRecordResponse($response, $harvest, $force); + #NOTE: No records are returned when in $harvest mode + } + + if ($harvest){ + my $latest_datestamp = C4::OAI::Client::Harvester::GetLatestHistoricalRecordDatestamp($repository->get_repository_id); + if ($latest_datestamp){ + if (!$repository->get_opt_from || $latest_datestamp ne $repository->get_opt_from){ + C4::OAI::Client::Harvester::ModOAIRepositoryDateTimeFrom($repository->get_repository_id,$latest_datestamp); + print "Setting 'from' argument to $latest_datestamp \n"; + } + } + } + + } +} +print "\n\nOAI-PMH Harvesting Cronjob Complete\n"; diff --git a/t/db_dependent/OAI_harvester.t b/t/db_dependent/OAI_harvester.t new file mode 100755 index 0000000..2ce7b70 --- /dev/null +++ b/t/db_dependent/OAI_harvester.t @@ -0,0 +1,308 @@ +#!/usr/bin/perl + +# This file is part of Koha. +# +# Copyright 2013 Prosentient Systems +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, see . + +use Modern::Perl; +use Test::More; +use C4::Context; +use C4::OAI::Harvester; +use Data::Dumper; +use Getopt::Long; + +my $verbose = 0; +my $show = 0; +GetOptions( + 'v|verbose' => \$verbose, + 's|show:s' => \$show, +); + +my $real_dbh = C4::Context->dbh; + +############################################################################################################################################################################################################ + +BEGIN { + use_ok('C4::OAI::Harvester'); +} + +############################################################################################################################################################################################################ + +#REAL DATA - We use this for crafting our mock data, making sure the OAI-PMH server is on, and requesting a specific item for the GetRecord method. + +#Initiate Koha baseURLs +my $OPACbaseURL = C4::Context->preference("OPACBaseURL"); +my $staffClientBaseURL = C4::Context->preference("staffClientBaseURL"); + +#Check if Koha has its OAI-PMH repository config enabled. +#If disabled, re-enable it (we'll reset to the original values +#after we've finished our test). +my $OAIPMHenabled = C4::Context->preference("OAI-PMH"); +if (!$OAIPMHenabled){ + C4::Context->set_preference("OAI-PMH",1); +} +my $OAIPMHarchiveID; +my $check_OAIPMHarchiveID = C4::Context->preference("OAI-PMH:archiveID"); +if ($check_OAIPMHarchiveID){ + $OAIPMHarchiveID = $check_OAIPMHarchiveID; +} +else { + C4::Context->set_preference("OAI-PMH:archiveID","KOHAOAITEST"); + $OAIPMHarchiveID = C4::Context->preference("OAI-PMH:archiveID"); +} + +my $check_biblio_table_sql = " + SELECT count(*) as 'count' + FROM biblio +"; +my $check_biblio_table_sth = $real_dbh->prepare($check_biblio_table_sql); +$check_biblio_table_sth->execute(); +my $check_biblio_table_row = $check_biblio_table_sth->fetchrow_hashref; +if ($check_biblio_table_row->{count} == 0){ + # Generate test biblio records + my $biblio = MARC::Record->new(); + $biblio->append_fields( + MARC::Field->new('100', ' ', ' ', a => 'Moffat, Steven'), + MARC::Field->new('245', ' ', ' ', a => 'Silence in the library'), + ); + my ($biblionumber, $biblioitemnumber) = AddBiblio($biblio, ''); + + my $biblio2 = MARC::Record->new(); + $biblio2->append_fields( + MARC::Field->new('100', ' ', ' ', a => 'Garland, Alex'), + MARC::Field->new('245', ' ', ' ', a => 'The Beach'), + ); + my ($biblionumber2, $biblioitemnumber2) = AddBiblio($biblio2, ''); + + my $biblio3 = MARC::Record->new(); + $biblio3->append_fields( + MARC::Field->new('100', ' ', ' ', a => 'Michaels, Anne'), + MARC::Field->new('245', ' ', ' ', a => 'The Winter Vault'), + ); + my ($biblionumber3, $biblioitemnumber3) = AddBiblio($biblio3, ''); +} + +my $sql1 = "SELECT biblionumber FROM biblio ORDER BY RAND() LIMIT 1"; +my $sth1 = $real_dbh->prepare($sql1); +$sth1->execute(); +my $random_biblionumber_row = $sth1->fetchrow_hashref; +my $random_biblionumber = $random_biblionumber_row->{biblionumber}; + +my $random_OAIPMH_identifier; +if ($OAIPMHarchiveID && $random_biblionumber){ + $random_OAIPMH_identifier = $OAIPMHarchiveID.":".$random_biblionumber; +} + +############################################################################################################################################################################################################ + +#MOCK DATA - Create a mock remote OAI-PMH repository using Koha's OAI-PMH server capabilities + +# Start Database Transaction +my $dbh = C4::Context->dbh; +$dbh->{AutoCommit} = 0; +$dbh->{RaiseError} = 1; + +my $record_matcher = C4::Matcher->new; +my $record_matcher_id = $record_matcher->store(); + +my $data = {}; +$data->{repository_id} = 1; +$data->{baseURL} = "http://$OPACbaseURL/cgi-bin/koha/oai.pl"; +$data->{opt_from} = undef; +$data->{opt_until} = undef; +$data->{opt_set} = undef; +$data->{metadataPrefix} = "oai_dc"; #or "marcxml" +$data->{active} = 1; +$data->{XSLT_path} = "http://$staffClientBaseURL/intranet-tmpl/prog/en/xslt/DC2MARC21slim2.xsl"; #"/.../git/koha-tmpl/intranet-tmpl/prog/en/xslt/DC2MARC21slim2.xsl"; #or undef, since this is hard-coded into Harvester.pm anyway +$data->{matcher_id} = $record_matcher_id; #NOTE: This is placeholder for testing. In production, you would use a real matching rule. Preferably using biblionumber (i.e. 999$c and Local-Number) for matching. +$data->{import_mode} = "automatic"; #Always run "automatic" for test. In production, you might prefer "manual". + +my $insert_mock_data_sql = " + INSERT INTO oai_harvest_repositories + (baseURL, opt_from, opt_until, opt_set, metadataPrefix, active, XSLT_path, matcher_id, import_mode) + VALUES (?,?,?,?,?,?,?,?,?) +"; +my $insert_mock_mock_data_sth = $dbh->prepare($insert_mock_data_sql); +$insert_mock_mock_data_sth->execute($data->{baseURL},$data->{opt_from},$data->{opt_until},$data->{opt_set},$data->{metadataPrefix},$data->{active},$data->{XSLT_path},$data->{matcher_id},$data->{import_mode}); + +my $get_mock_data_sql = " + SELECT * + FROM oai_harvest_repositories + WHERE active = 1 +"; +my $get_mock_data_sth = $dbh->prepare($get_mock_data_sql); +$get_mock_data_sth->execute(); +while (my $get_mock_data_row = $get_mock_data_sth->fetchrow_hashref){ + + #Set 'resume' to 0, if you want to turn off automatic token resumption. This is useful for testing and debugging. Change to 1 for automatic token resumption. + $get_mock_data_row->{resume} = 0; + +############################################################################################################################################################################################################ + + #TEST - OBJECT CREATION + my $harvester = C4::OAI::Harvester->new($get_mock_data_row); # create an object + isa_ok( $harvester, 'C4::OAI::Harvester','Harvester object' ); + #END TEST + + #TEST - OBJECT's "GET" ACCESSOR METHODS + can_ok($harvester, qw(get_baseURL get_from get_until get_set get_metadataPrefix get_rh get_repository_id get_XSLT_path get_matcher_id get_import_mode)); + #END TEST + + #TEST - Check some basic harvester config + my $baseURL = $harvester->get_baseURL; + my $metadataPrefix = $harvester->get_metadataPrefix; + is ($baseURL,"http://$OPACbaseURL/cgi-bin/koha/oai.pl", 'Retrieved baseURL "'.$baseURL.'"'); + ok ( defined $metadataPrefix, 'MetadataPrefix is defined as "'.$metadataPrefix.'"'); + #END TEST + + #TESTS - OBJECT's OAI-PMH methods + + #IDENTIFY + my $identify_repository = $harvester->Identify; + ok ( defined($identify_repository), "Successfully used Identify Verb"); + print Dumper($identify_repository) if $verbose or $show eq 'Identify'; + #END TEST + + #LISTSETS + my @sets = $harvester->ListSets; + ok (@sets, "Successfully used ListSets Verb"); + print Dumper(\@sets) if $verbose or $show eq 'ListSets'; + #END TEST + + #LISTMETADATAFORMATS + my @formats = $harvester->ListMetadataFormats; + ok (@formats, "Successfully used ListMetadataFormats Verb"); + print Dumper(\@formats) if $verbose or $show eq 'ListMetadataFormats'; + #END TEST + + #LISTIDENTIFIERS + my @headers = $harvester->ListIdentifiers; + ok (@headers, "Successfully used ListIdentifiers Verb"); + print Dumper(\@headers) if $verbose or $show eq 'ListIdentifiers'; + #END TEST + + #LISTRECORDS + my @list_records = $harvester->ListRecords; + ok (@list_records, "Successfully used ListRecords Verb"); + print Dumper(\@list_records) if $verbose or $show eq 'ListRecords'; + #END TEST + + #GETRECORD + my @get_records = $harvester->GetRecord($random_OAIPMH_identifier); + ok (@get_records, "Successfully used GetRecord Verb"); + print Dumper(@get_records) if $verbose or $show eq 'GetRecord'; + #END TEST + + my ( $batch_id, $num_with_matches, $num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = $harvester->ImportRecordsIntoKoha(@list_records); + + #These variables will only be initialized in automatic mode, which is how the test should always be run anyway. + my $batch_total = $num_added + $num_updated; + + print "Batch id = $batch_id \n"; + print "Num added = $num_added \n"; + print "Num updated = $num_updated \n"; + print "Num items added = $num_items_added \n"; + print "Num items errored = $num_items_errored \n"; + print "Num ignored = $num_ignored \n"; + + + #TEST - RETRIEVE IMPORT_BATCH + my @test_batches; + my $batch_sql = " + SELECT * + FROM import_batches + WHERE import_batch_id = ? + "; + my $batch_sth = $dbh->prepare($batch_sql); + $batch_sth->execute($batch_id); + while (my $batch_row = $batch_sth->fetchrow_hashref){ + push(@test_batches,$batch_row); + } + cmp_ok(scalar @test_batches,'==',1,"Successfully retrieved batch id ".$batch_id); + + #END TEST + + #TEST - RETRIEVE IMPORTED_RECORDS + my $imported_record_sql = " + SELECT count(*) as 'count' + FROM import_records + WHERE import_batch_id = ? + "; + my $imported_record_sth = $dbh->prepare($imported_record_sql); + $imported_record_sth->execute($batch_id); + my $imported_record_row = $imported_record_sth->fetchrow_hashref; + is ($imported_record_row->{count}, $batch_total, "Import_record table count (".$imported_record_row->{count}.") for this batch matches reported import total"); + #END TEST + + #TEST - RETRIEVE IMPORTED_BIBLIOS + my $imported_biblio_sql = " + SELECT count(*) as 'count' + FROM import_biblios + JOIN import_records using (import_record_id) + WHERE import_records.import_batch_id = ? + "; + my $imported_biblio_sth = $dbh->prepare($imported_biblio_sql); + $imported_biblio_sth->execute($batch_id); + my $imported_biblio_row = $imported_biblio_sth->fetchrow_hashref; + is ($imported_biblio_row->{count}, $batch_total, "Import_biblio table count (".$imported_biblio_row->{count}.") for this batch matches reported import total"); + #END TEST + + #TEST - RETRIEVE BIBLIOS + my $biblio_sql = " + SELECT count(*) as 'count' + FROM biblio + JOIN import_biblios on import_biblios.matched_biblionumber = biblio.biblionumber + JOIN import_records using (import_record_id) + WHERE import_records.import_batch_id = ? + "; + my $biblio_sth = $dbh->prepare($biblio_sql); + $biblio_sth->execute($batch_id); + my $biblio_row = $biblio_sth->fetchrow_hashref; + is ($biblio_row->{count}, $batch_total, "Biblio table count (".$biblio_row->{count}.") for this batch matches reported import total. Automatic import successful."); + #END TEST + + #TEST - RETRIEVE HISTORICAL RECORDS + my $harvest_sql = " + SELECT count(*) as 'count' + FROM oai_harvest + JOIN import_records using (import_record_id) + WHERE import_records.import_batch_id = ? + "; + my $harvest_sth = $dbh->prepare($harvest_sql); + $harvest_sth->execute($batch_id); + my $harvest_row = $harvest_sth->fetchrow_hashref; + is ($harvest_row->{count}, $batch_total, "OAI Harvest table count (".$harvest_row->{count}.") for this batch matches reported import total. Historical records added successfully."); + #NOTE: This ignores deleted records, which will have a historical record but are not part of the batch and thus don't have an import_record_id number. + #END TEST + + #TEST - RETRIEVE RECORD MATCHES + #There's no way to really automatically test the "replace/update" function, since the first wave of imported records will need to be reindexed before they can be matched on... + #END + +} #End Loop of OAI-PMH Repositories +$dbh->rollback(); +$dbh->{AutoCommit} = 1; + +done_testing(); +############################################################################################################################################################################################################ + +#RESET ORIGINAL VALUES +if (!$OAIPMHenabled){ + C4::Context->set_preference("OAI-PMH",$OAIPMHenabled); +} + +if (!$check_OAIPMHarchiveID){ + C4::Context->set_preference("OAI-PMH:archiveID",$check_OAIPMHarchiveID); +} -- 1.7.7.4