--- a/C4/OAI/Harvester.pm +++ a/C4/OAI/Harvester.pm @@ -0,0 +1,850 @@ +package C4::OAI::Harvester; + +# This file is part of Koha. +# +# Copyright 2013 Prosentient Systems +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, see . + +=head1 NAME + +C4::OAI::Harvester - OAI-PMH harvester/client which implements the 6 OAI-PMH verbs, record retrieval/harvesting, and import into Koha + +=head1 SYNOPSIS + +use C4::OAI::Harvester; +my $oai_repo = C4::OAI::Harvester->new($repository_data); + +my $identify_repository = $oai_repo->Identify; + +my @sets = $oai_repo->ListSets; + +my @formats = $oai_repo->ListMetadataFormats; + +my @headers = $oai_repo->ListIdentifiers; + +my @records = $oai_repo->ListRecords; + +my @records = $oai_repo->GetRecord($oai_unique_identifier); + +my $import_mode = ''; #i.e. not "automatic" + +$oai_repo->ImportRecordsIntoKoha($import_mode,@records); + +=head1 DESCRIPTION + +C4::OAI::Harvester contains functions for querying and harvesting OAI-PMH repositories. + +More information on OAI-PMH can be found L + +=head1 FUNCTIONS + +=head1 AUTHOR + +David Cook + +=cut + +use Modern::Perl; +use Data::Dumper; +use C4::Context; + +use C4::ImportBatch qw/AddImportBatch SetImportBatchItemAction AddBiblioToBatch SetImportBatchStatus BatchCommitRecords BatchFindDuplicates/; +use C4::Matcher; + +use HTTP::OAI; + +use base qw(Class::Accessor); + +sub new { + my($proto, $fields) = @_; + my($class) = ref $proto || $proto; + + $fields = {} unless defined $fields; + + if ($fields->{'baseURL'}){ + my $h = new HTTP::OAI::Harvester( + baseURL => $fields->{'baseURL'}, + ); + #If resume is set to 0, automatic token resumption is turned off. This is useful for testing/debugging. + if ($h && exists $fields->{'resume'}){ + if ($fields->{'resume'} == 0){ + $h->resume(0); + } + } + my $response = $h->repository($h->Identify); + if( $response->is_error ) { + print "Error requesting Identify:\n", + $response->code . " " . $response->message, "\n"; + exit; + } + $fields->{rh} = $h; #Store HTTP::OAI::Harvester object as "repository handle" + } + bless {%$fields}, $class; +} + +__PACKAGE__->follow_best_practice; #Use get_ and set_ prefixes for accessors +__PACKAGE__->mk_accessors(qw(baseURL from until set metadataPrefix rh repository_id XSLT_path matcher_id import_mode)); + +=head2 OAI-PMH Verbs + +Koha-specific implementations of the 6 OAI-PMH Verbs. + +The key verbs are "ListRecords" and "GetRecords". These do the actual +harvesting of records from a OAI-PMH repository. The others are useful for +getting information about a repository and what it has available. + +1) ListRecords + +2) GetRecord + +3) ListIdentifiers + +4) ListMetadataFormats + +5) ListSets + +6) Identify + +=cut + +sub ListRecords { + my $self = shift; + my $response = $self->{rh}->ListRecords( + metadataPrefix => $self->{metadataPrefix}, + from => $self->{opt_from}, + until => $self->{opt_until}, + set => $self->{opt_set}, + ); + if( $response->is_error ) { + print "Error requesting ListRecords:\n", + $response->code . " " . $response->message, "\n"; + exit; + } + #print Dumper($response->toDOM); #FIXME: dac2013: Fails with this warning "bad ns attribute!" + my @records = _parse_records_into_useful_format($response); + return @records; +} + +sub GetRecord { + my ( $self, $identifier ) = @_; + my $response = $self->{rh}->GetRecord( + metadataPrefix => $self->{metadataPrefix}, + identifier => $identifier, + ); + if( $response->is_error ) { + print "Error requesting GetRecord:\n", + $response->code . " " . $response->message, "\n"; + exit; + } + my @records = _parse_records_into_useful_format($response); + return @records; +} + +sub ListIdentifiers { + my $self = shift; + my @headers; + my $response = $self->{rh}->ListIdentifiers( + metadataPrefix => $self->{metadataPrefix}, + from => $self->{opt_from}, + until => $self->{opt_until}, + set => $self->{opt_set}, + ); + if( $response->is_error ) { + print "Error requesting ListIdentifiers:\n", + $response->code . " " . $response->message, "\n"; + exit; + } + #print Dumper($response->toDOM); #FIXME: dac2013: Fails with this warning "bad ns attribute!" + while (my $h = $response->next){ + my $header; + #print Dumper($h->dom->toString); #DEBUG: dac2013: XML representation + $header->{identifier} = $h->identifier; + $header->{datestamp} = $h->datestamp; + + $header->{status} = $h->status; + $header->{is_deleted} = $h->is_deleted; + + my @sets = $h->setSpec; + $header->{sets} = \@sets; + + push (@headers,$header); + } + return @headers; +} + +sub ListMetadataFormats { + my ( $self, $identifier ) = @_; + my @formats; + my $response = $self->{rh}->ListMetadataFormats( + identifier => $identifier, + ); + if( $response->is_error ) { + print "Error requesting ListMetadataFormats:\n", + $response->code . " " . $response->message, "\n"; + exit; + } + for($response->metadataFormat) { + push(@formats,$_->metadataPrefix); + } + return @formats; +} + +sub ListSets { + my $self = shift; + my @sets; + my $response = $self->{rh}->ListSets(); + if( $response->is_error ) { + print "Error requesting ListSets:\n", + $response->code . " " . $response->message, "\n"; + exit; + } + while (my $s = $response->next){ + my $set; + $set->{setSpec} = $s->setSpec; + $set->{setName} = $s->setName; + + #FIXME: dac2013: Not really sure what to do with the descriptions as they're XML and not necessarily that easy to parse for GUI views... + #my @temp_setDescriptions = $s->setDescription; + #my @setDescriptions; + #foreach my $temp_setDescription (@temp_setDescriptions){ + # push (@setDescriptions,$temp_setDescription->dom->toString); #dac2013: I think I need to do better than just return the setDescription XML...That's not very useful... + #} + #$set->{setDescription} = \@setDescriptions; + push (@sets,$set); + } + return @sets; +} + +sub Identify { + my $self = shift; + my $response = $self->{rh}->Identify(); + if( $response->is_error ) { + print "Error requesting Identify:\n", + $response->code . " " . $response->message, "\n"; + exit; + } + my $identify_data; + #DEBUG: View what's in the Identify object + #print Dumper($response->headers); + + $identify_data->{repositoryName} = $response->repositoryName; + $identify_data->{baseURL} = $response->baseURL; + $identify_data->{protocolVersion} = $response->protocolVersion; #Tim Brody says this will always return 2.0 and that ->version should be used to find the actual version... + #$identify_data->{version} = $response->version; + $identify_data->{earliestDatestamp} = $response->earliestDatestamp; + $identify_data->{deletedRecord} = $response->deletedRecord; #not in the perldoc, but it's in the code and the OAI-PMH spec + $identify_data->{granularity} = $response->granularity; + + #These methods should be used with an array context so they return all the elements and not just the first one + my @adminEmails = $response->adminEmail; + $identify_data->{adminEmail} = \@adminEmails; + my @compressions = $response->compression; + $identify_data->{compression} = \@compressions; + + #FIXME: dac2013: Descriptions are encapsulated in XML containers, I believe. Not sure what to do with these at present... + #my @descriptions = $response->description; + #$identify_data->{description} = \@descriptions; + #$response->next + + return $identify_data; +} + +=head2 _parse_records_into_useful_format + +A C4::OAI::Harvester internal subroutine that parses HTTP::OAI::Record +objects into C4::OAI::Harvester::Record objects which have methods +for transforming incoming XML into MARCXML, saving historical records +in the database, and comparing incoming records against historical +records. + +=cut + +sub _parse_records_into_useful_format { + my ( $response ) = @_; + my @records; + + while( my $rec = $response->next ) { + my $record; + + $record->{header} = $rec->header ? $rec->header->dom : undef; + $record->{identifier} = $rec->identifier; + $record->{datestamp} = $rec->datestamp; + $record->{metadata} = $rec->metadata ? $rec->metadata->dom : undef; #N.B. there won't be any metadata for deleted records + $record->{about} = $rec->about ? $rec->about->dom : undef; + $record->{deleted} = $rec->is_deleted; + $record->{status} = $rec->status; + + #FIXME: Will these always exist for every record? Add error checking. Maybe include an undef or blank if they're not set... + my $record_object = C4::OAI::Harvester::Record->new($record); + + #FIXME: Add error checking + #if( $rec->is_error ) { + # die $response->message; + #} + push(@records,$record_object); + } + return @records; +} + +=head2 ImportRecordsIntoKoha + +$oai_repo->ImportRecordsIntoKoha(@records); + +@records is an array of records that have been retrieved using the +GetRecord or ListRecords verbs. + +$import_mode is used to specify "automatic" or "manual" modes. The +default mode is "manual", but "automatic" can be triggered by +passing the "automatic" string in the variable. Manual mode is +indicated using any other string or even undef. + +This subroutine handles importing our retrieved records into Koha. + + 1. Foreach record: + a) Check its identifier status in the database + b) If its status is "create", "replace", or "error", + we transform it into MARCXML. + + Note: "error" simply indicates that a "replace" couldn't + occur automatically due to multiple biblionumbers being + linked to the unique identifier. + c) If the status is "ignore", we skip to the next record. + 2. Add an import batch + 3. Foreach record: + a) Add the record to the import batch + b) Add a historical record to the oai_harvest table + 4. Check the import batch for duplicates/matches + (i.e. existing records). + + 5. If automatic importing is turned on, batch import the records + (otherwise, leave in staging area for manual import) + +=cut + +sub ImportRecordsIntoKoha { + my ( $self, @records ) = @_; + my @records_to_import; + my ( %ignored, %created, %errored, %replaced, %deleted ); + + + foreach my $record (@records){ + + + #FIXME: Do the delete handling in the function...only use "next" with an "ignore"... + #FIXME: stop returning @biblionumbers with GetIdentifierSatus...it's unnecessary + + #FIXME: Add a way of forcing add/replace despite existing Identifier+datestamp(+metadataPrefix)? + my $identifier_status = $record->AddIdentifierStatusAndLinkedBiblionumbers($self->get_repository_id); + if ($identifier_status){ + if ($identifier_status eq "ignore"){ + #This record's identifier and datestamp already exist in the database, so we don't import this record, or + #this is a record with a deleted status that hasn't been imported into Koha before. + next; + } + } + else { + #We weren't able to get the status of this record, so let's ignore it and skip to the next one + next; + } + my $transform_result_flag = $record->TransformMetadata($self->get_metadataPrefix, $self->get_XSLT_path); + push(@records_to_import,$record); + } + + if (scalar @records_to_import > 0){ + #DEBUGGING ONLY - DUMPER + #print Dumper(@records_to_import); + my $overlay_action = 'replace'; #If an incoming record matches an existing one, we replace the existing one with the incoming record + my $file_name = $self->get_baseURL; + my $comments = "OAI-PMH Harvest"; + my $matcher_id = undef; + my $rec_num = 0; + my $num_with_matches = 0; + + #If a matcher_id is linked to this repository, we'll use that for matching records when applying updates. Otherwise, don't use a matcher. + #FIXME: It might be an idea to make matcher_id a required field for OAI-PMH repositories... + if ($self->get_matcher_id){ + $matcher_id = $self->get_matcher_id; + } + + #FIXME: Have other import options be configurable? + my $batch_id = AddImportBatch( { + overlay_action => $overlay_action, + import_status => 'staging', + batch_type => 'batch', #FIXME: batch|z3950|webservice are the three current options. Despite this being a web service, I that type of batch has special properties that shouldn't be used here? + file_name => $file_name, #This is the baseURL for our remote OAI-PMH repository + comments => $comments, #This contains a mention that this is a OAI-PMH harvest + matcher_id => $matcher_id, #This is defined in our DB config for the repository + } ); + + #N.B. In theory, it would be good if items could be processed. + #However, duplicate items would appear for updated records if + #we turn this on, since there is no way of matching items at the + #moment. + SetImportBatchItemAction($batch_id, 'ignore'); + + foreach my $record_to_import (@records_to_import){ + #Check for transformed_metadata. We do not want to try to + #add a record to a batch without transformed metadata + #(i.e. if the record is deleted). + my $import_record_id = undef; + if ($record_to_import->get_transformed_metadata){ + my $encoding = "utf8"; + my $format = "MARC21"; + my $marc_record = MARC::Record->new_from_xml( $record_to_import->get_transformed_metadata->toString, $encoding, $format ); + $rec_num++; + $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0); + _update_batch_record_counts($batch_id); + } + my $save_result_flag = $record_to_import->AddHistoricalRecord($self->get_metadataPrefix,$self->get_repository_id, $import_record_id); + #if ($save_result_flag != 1){ + #If there was an error saving to the database, we ignore this record + #FIXME: Add debug and print errors to log + #next; + #} + } + + my $matcher = C4::Matcher->fetch($matcher_id); + if ($matcher){ + $num_with_matches = BatchFindDuplicates($batch_id, $matcher); + } + + SetImportBatchStatus($batch_id, 'staged'); + + #When active, this automatically imports records into the catalogue without having to use the "Manage staged MARC records interface", + #although the import can still be reverted using that interface. + if ($self->get_import_mode eq "automatic"){ + my $framework = ''; + my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = + BatchCommitRecords($batch_id, $framework); + return ($batch_id, $num_with_matches, $num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored); + } + return ($batch_id, $num_with_matches); + } + + +} + +#Copied from ImportBatch.pm for convenience...for now +sub _update_batch_record_counts { + my ($batch_id) = @_; + + my $dbh = C4::Context->dbh; + my $sth = $dbh->prepare_cached("UPDATE import_batches SET + -- num_biblios = ( + num_records = ( + SELECT COUNT(*) + FROM import_records + WHERE import_batch_id = import_batches.import_batch_id + AND record_type = 'biblio'), + num_items = ( + SELECT COUNT(*) + FROM import_records + JOIN import_items USING (import_record_id) + WHERE import_batch_id = import_batches.import_batch_id + AND record_type = 'biblio') + WHERE import_batch_id = ?"); + $sth->bind_param(1, $batch_id); + $sth->execute(); + $sth->finish(); +} + + + + + +package C4::OAI::Harvester::Record; + +=head1 NAME + +C4::OAI::Harvester::Record - an internal class for handling records retrieved via the OAI-PMH protocol + +=head1 DESCRIPTION + +C4::OAI::Harvester::Record - Class to handle the management of records retrieved via OAI-PMH + +More information on OAI-PMH can be found L + +=head1 AUTHOR + +David Cook + +=cut + +use Modern::Perl; +use C4::Context; +use C4::Templates; +use C4::XSLT qw/GetURI/; + +use XML::LibXML; +use XML::LibXSLT; + +use base qw(Class::Accessor); + +sub new { + my($proto, $fields) = @_; + my($class) = ref $proto || $proto; + + $fields = {} unless defined $fields; + + $fields->{transformed_metadata} = undef; + $fields->{linked_biblionumbers} = ''; + + bless {%$fields}, $class; +} + +__PACKAGE__->follow_best_practice; #Use get_ and set_ prefixes for accessors +__PACKAGE__->mk_accessors(qw(identifier datestamp metadata status deleted header about transformed_metadata linked_biblionumbers)); + +=head2 TransformMetadata + +A C4::OAI::Harvester::Record method for transforming incoming metadata +into a format that Koha can use (e.g. MARC21 MARCXML). + +my $transform_result_flag = $record->TransformMetadata( + $self->get_metadataPrefix, + $self->get_XSLT_path, + $biblionumber_to_use_in_XSLT); + +$self->get_metadataPrefix is the metadataPrefix for the record, +which comes from the database config for this OAI-PMH repository. It +is a required parameter. + +$self->get_XSLT_path is the filepath to the XSLT to be used in +the metadata transformation. This can be a local filepath or a remote +URL (like in XSLT.pm). Technically, this is an optional parameter if +the metadataPrefix is "marcxml", "dc", or "oai_dc". However, it is +recommended to always specify a XSLT path. You will almost certainly +have problems down the road if you do not use one, especially +for marcxml (as the biblionumber for any existing imported records +will not be added, which means you'll have great difficulty managing +updates via OAI-PMH). + +$biblionumber_to_use_in_XSLT is the biblionumber linked to this +record's unique OAI-PMH identifier. Typically, this will only be used +if this record has always been ingested/added to Koha before. This +biblionumber is added during the transform process so that Koha +can match the incoming and existing records for merging/replacing. + +=cut + +sub TransformMetadata { + my ( $self, $metadataPrefix, $XSLT_path ) = @_; + my $source = $self->{metadata}; #LibXML object retrieved via HTTP::OAI::Harvester + + my $biblionumber; + my $no_XSLT_path; + if (!$XSLT_path){ + $no_XSLT_path = 1; + } + + #If we have a single linked_biblionumber and our status is replace, + #then we initialize $biblionumber with that biblionumber, so that + #we can create a 999$c or 999$d for matching duringn the import. + my $linked_biblionumbers = $self->get_linked_biblionumbers; + if (ref $linked_biblionumbers eq 'ARRAY' && $self->get_status eq 'replace'){ + if (scalar @$linked_biblionumbers == 1){ + $biblionumber = @$linked_biblionumbers[0]; + } + } + + if ($metadataPrefix && $source){ + my $xslt = XML::LibXSLT->new(); + my $style_doc; + + #$parser based on equivalent from XSLT.pm + my $parser = XML::LibXML->new(); + # don't die when you find &, >, etc + $parser->recover_silently(0); + + if ($XSLT_path){ + #This conditional for handling secure remote XSLTs copied from XSLT.pm + if ( $XSLT_path =~ /^https?:\/\// ) { + my $xsltstring = GetURI($XSLT_path); + if ($xsltstring){ + $style_doc = $parser->parse_string($xsltstring); + } + else{ + #If a string isn't retrieved using GetURI, we switch to our default transforms + $no_XSLT_path = 1; + } + } else { + if ( -e $XSLT_path){ + $style_doc = $parser->parse_file($XSLT_path); + } + else{ + #If the file doesn't actually exist, we switch to our default transforms + $no_XSLT_path = 1; + } + } + } + + if ($no_XSLT_path){ + if ($metadataPrefix eq 'marcxml'){ + $self->set_transformed_metadata($source); + return 1; + } + elsif ($metadataPrefix eq 'dc' || $metadataPrefix eq 'oai_dc'){ + my $xsl = C4::Context->config('intrahtdocs') . + '/' . C4::Context->preference("template") . + '/' . C4::Templates::_current_language() . + '/xslt/DC2MARC21slim.xsl'; + #'/xslt/' . + #C4::Context->preference('marcflavour') . + #"slim2intranetDetail.xsl"; + $style_doc = $parser->parse_file($xsl); + } + else{ + #FIXME: Use more robust error handling than this + return -1; + } + } + + my $stylesheet = $xslt->parse_stylesheet($style_doc); + #FIXME: Add error handling here... + + my %xslt_params; + $xslt_params{'identifier'} = $self->{identifier}; + $xslt_params{'biblionumber'} = $biblionumber; + + #Pass OAI-PMH identifier and matched biblionumber (if there is one) to the XSLT for unique identification/provenance + my $results = $stylesheet->transform($source, XML::LibXSLT::xpath_to_string(%xslt_params)); + + if ($results){ + $self->set_transformed_metadata($results); + return 1; + } + else{ + return -1; + } + + #FIXME: Add some better error checking here...throw an exception if a record fails? + + } + else{ + return -1; + } +} + +=head2 AddHistoricalRecord + +A C4::OAI::Harvester::Record method that saves an entry into the +database for historical and importing purposes. + +my $save_result_flag = $record_to_import->AddHistoricalRecord($self->get_metadataPrefix,$self->get_repository_id, $import_record_id); + +$self->get_metadataPrefix is the record's metadataPrefix (i.e. metadata +format), as defined in the database config, since records do not +self-identify their metadataPrefix. + +$self->get_repository_id is the local DB primary key for the record's +repository. + +$import_record_id is the key in the import_records and import_biblios +tables which show that the record has been staged and if it has been +imported. This is ESSENTIAL for performing matching between existing +and incoming records. + +=cut + +sub AddHistoricalRecord { + my ( $self, $metadataPrefix, $repository_id, $import_record_id ) = @_; + #FIXME: Error/variable checking should be better in this sub... + + #Only check for these two, since deleted records won't have an $import_record_id + if ($metadataPrefix && $repository_id){ + my $identifier = $self->get_identifier; + my $datestamp = $self->get_datestamp; + my $metadata = $self->get_metadata ? $self->get_metadata->toString : undef; + my $header = $self->get_header ? $self->get_header->toString : undef; + my $status = $self->get_status; + my $string_of_linked_biblionumbers = undef; + + my $linked_biblionumbers = $self->get_linked_biblionumbers; + if (ref $linked_biblionumbers eq 'ARRAY'){ + $string_of_linked_biblionumbers = join(" | ",@$linked_biblionumbers); + } + + my $dbh = C4::Context->dbh; + my $sql = "INSERT INTO oai_harvest (identifier,datestamp,metadataPrefix,import_record_id,metadata,header,repository_id,status, linked_biblionumbers) VALUES (?,?,?,?,?,?,?,?,?)"; + my $sth = $dbh->prepare($sql); + $sth->execute($identifier, $datestamp, $metadataPrefix, $import_record_id, $metadata, $header, $repository_id, $status, $string_of_linked_biblionumbers); + if ($sth->err){ + return "ERROR! return code: " . $sth->err . " error msg: " . $sth->errstr . "\n"; + } + else{ + return 1; + } + } + else{ + return -1; + } +} + +=head2 AddIdentifierStatusAndLinkedBiblionumbers + +A C4::OAI::Harvester::Record method that compares the status of +the record with any matching identifiers in the database. It sets +and returns a status saying "ignore", "deleted", "create", +"replace", or "ambiguous". It also checks if the incoming record is +linked with any existing biblionumbers. If it is, it sets the +linked_biblionumbers property to any linkages it finds. + +"Ignore" means that this identifier already exists with the same +datestamp, or the incoming record has a deleted status and isn't +linked to any existing bib records in Koha. + +"Deleted" means that the incoming record has a status of "deleted" +and contains no metadata, but shares an identifier with an imported +record in the Koha database. We record this status as evidence +that there has been a deletion in the remote OAI-PMH repository. + +"Create" means that the identifier hasn't been logged before, or it +it has been logged and its record staged, but it has not been +imported, so we are free to create a new HistoricalRecord and +stage a new bib record. + +"Replace" means that this identifier already exists, but the +datestamp is different, so we assume that this is an updated +version of the record. We check to see if there is a biblionumber +linked to this identifier (using import_record_id). If there is +a biblionumber, we store it so that we can use it for matching +during import. + +However, if there is more than one biblionumber, we're in an +"Ambiguous" situation where we can not reliably determine which +bib record we should be replacing. In this case, we store all +the bib numbers linked to this identifier, and treat it as a +new record. + +Ideally, we signal staff to perform manual merging +or deleting to remedy this one identifier to many biblionumber +scenario. + +In theory, the "ambiguous" situation should never occur. However, +it's better to be safe than sorry. We don't want to accidentally +overwrite the wrong record during a "replace" update. + +=cut + +sub AddIdentifierStatusAndLinkedBiblionumbers { + my ( $self, $repository_id ) = @_; + my $dbh = C4::Context->dbh; + my $identifier_status; + my @linked_biblionumbers; + + my $check_for_id_sql = " + SELECT count(*) as count + FROM oai_harvest + WHERE identifier = ? and repository_id = ?"; + my $check_for_id_sth = $dbh->prepare($check_for_id_sql); + $check_for_id_sth->execute($self->get_identifier, $repository_id); + my $check_for_id_row = $check_for_id_sth->fetchrow_hashref; + if ($check_for_id_row->{count} == 0){ + #OAI-PMH Unique Identifier doesn't exist in database == CREATE + + $identifier_status = "create"; + } + else{ + #OAI-PMH Unique Identifier does exist in database == IGNORE || REPLACE + #FIXME: System preference to govern whether you match on identifier+datestamp || identifier+datestamp+metadataPrefix? + my $check_for_id_and_datestamp_sql = "SELECT count(*) as count FROM oai_harvest WHERE identifier = ? and datestamp = ? and repository_id = ?"; + my $check_for_id_and_datestamp_sth = $dbh->prepare($check_for_id_and_datestamp_sql); + $check_for_id_and_datestamp_sth->execute($self->get_identifier, $self->get_datestamp, $repository_id); + my $check_for_id_and_datestamp_row = $check_for_id_and_datestamp_sth->fetchrow_hashref; + + if ($check_for_id_and_datestamp_row->{count} > 0){ + #OAI-PMH Unique Identifier and Datestamp combo exist in database == IGNORE + #FIXME: What happens if the metadataPrefix is different? Perhaps do a REPLACE in this case? It would need to be tied to a system preference... + + $identifier_status = "ignore"; + } + else{ + #OAI-PMH Unique Identifier and Datestamp combo don't exist in database == REPLACE + #i.e. The identifier exists in the database, but this is an updated datestamp + + + my $check_for_biblionumber_sql = "SELECT * + FROM oai_harvest + JOIN import_biblios using (import_record_id) + WHERE identifier = ? and repository_id = ?"; + my $check_for_biblionumber_sth = $dbh->prepare($check_for_biblionumber_sql); + $check_for_biblionumber_sth->execute($self->get_identifier, $repository_id); + while ( my $check_for_biblionumber_row = $check_for_biblionumber_sth->fetchrow_hashref ) { + if ($check_for_biblionumber_row->{matched_biblionumber}){ + if (!grep(/^$check_for_biblionumber_row->{matched_biblionumber}$/, @linked_biblionumbers)){ + push(@linked_biblionumbers,$check_for_biblionumber_row->{matched_biblionumber}); + } + } + } + if (scalar @linked_biblionumbers == 1){ + #If there is only one matching bib number for this record, we'll add it to our incoming + #record for import matching + + $identifier_status = "replace"; + + } + elsif (scalar @linked_biblionumbers == 0){ + #In this case, we've staged this OAI-PMH record before, but haven't imported it. + #Treat this as a CREATE + + $identifier_status = "create"; + } + elsif (scalar @linked_biblionumbers > 1){ + #EXCEPTIONAL CASE - This means that we have imported this record as two different bibs before + + #As per wizzyrea's suggestion, email the administrator/librarian to tell them about this exceptional case + #We can give them the bib numbers, the OAI-PMH identifier, and other information so they can manually + #merge these bibs together... + + $identifier_status = "ambiguous"; + } + } + } + + if ($self->get_deleted){ + #If the incoming record has a deleted status, we'll want to + #ignore it during import, since deleted records do not + #have any metadata attached. + $identifier_status = "ignore"; + } + + if (scalar @linked_biblionumbers > 0){ + $self->set_linked_biblionumbers(\@linked_biblionumbers); + + #If the incoming record has a deleted status AND it is linked to + #existing bib records that we've imported, we will want to keep + #evidence of this deleted status, so that we can alert staff to + #delete these records. + + #We will commit an AddHistoricalRecord entry, but we won't import + #this record into Koha (since there won't be any metadata to + #import anyway) + + #FIXME: Email the librarian/administrator mentioning that the record associated with XXX bibs have been deleted in the OAI-PMH repo? + #FIXME: Suggest that they change the LEADER position 05 to "d" for deleted? How does Koha handle records with a LDR05 of "d"? + + if ($self->get_deleted){ + $identifier_status = "deleted"; + } + } + + if ($identifier_status){ + $self->set_status($identifier_status); + } + + return ($identifier_status); +} + +1; +__END__ --- a/installer/data/mysql/kohastructure.sql +++ a/installer/data/mysql/kohastructure.sql @@ -3223,3 +3223,45 @@ CREATE TABLE IF NOT EXISTS plugin_data ( /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; +-- +-- Table structure for table 'oai_harvest_repositories' +-- + +DROP TABLE IF EXISTS oai_harvest_repositories; +CREATE TABLE oai_harvest_repositories ( + repository_id int(11) NOT NULL AUTO_INCREMENT, -- primary key identifier + baseURL text NOT NULL, -- baseURL of the remote OAI-PMH repository (mandatory) + opt_from varchar(45) DEFAULT NULL, -- "from" time for selective harvesting (optional) + opt_until varchar(45) DEFAULT NULL, -- "until" time for selective harvesting (optional) + opt_set varchar(45) DEFAULT NULL, -- the record set to harvest for selective harvesting (optional) + metadataPrefix varchar(45) NOT NULL, -- metadata format (e.g. oai_dc, dc, marcxml) + active int(11) NOT NULL DEFAULT '0', -- indicate whether this repo is actively harvested by Koha + timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, -- date last modified/created + XSLT_path text, -- filepath to an XSLT file to use for transforming incoming records into a format Koha can use + matcher_id int(11) DEFAULT NULL, -- id of the record matching rule to use for incoming records + import_mode enum('manual','automatic') NOT NULL DEFAULT 'manual', -- automatically import records or leave in staging for manual import + PRIMARY KEY (repository_id) USING BTREE + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- +-- Table structure for table 'oai_harvest' +-- + +DROP TABLE IF EXISTS oai_harvest; +CREATE TABLE oai_harvest ( + oai_harvest_id int(11) NOT NULL AUTO_INCREMENT, -- primary key identifier + identifier varchar(255) NOT NULL, -- OAI-PMH identifier (unique to its original repo) + datestamp varchar(45) NOT NULL, -- OAI-PMH datestamp (i.e. date created/modified/deleted) + metadataPrefix varchar(45) NOT NULL, -- metadata format the record is in + import_record_id int(11) DEFAULT NULL, -- the id in the import_records table. Used for import matching. + linked_biblionumbers varchar(255) DEFAULT NULL, -- biblionumbers linked to this record. Usually only one, but multiple are separated with a pipe (i.e. |) + metadata longtext, -- XML string containing the metadata (this will be null for incoming records marked as deleted) + header longtext NOT NULL, -- XML string containing the header information (i.e. identifier, datestamp, status) + about longtext, -- XML string containing explanatory information about a record (optional. Often not included in records) + timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, -- timestamp when added/updated + repository_id int(11) NOT NULL, -- id for this record's OAI-PMH repository (linked to oai_harvest_repositories table) + status varchar(45) DEFAULT NULL, -- whether or not the record has a status of deleted + PRIMARY KEY (oai_harvest_id) USING BTREE, + KEY FK_oai_harvest_1 (repository_id), + CONSTRAINT FK_oai_harvest_1 FOREIGN KEY (repository_id) REFERENCES oai_harvest_repositories (repository_id) ON UPDATE NO ACTION +) ENGINE=InnoDB DEFAULT CHARSET=utf8; --- a/installer/data/mysql/updatedatabase.pl +++ a/installer/data/mysql/updatedatabase.pl @@ -7067,6 +7067,47 @@ if ( CheckVersion($DBversion) ) { SetVersion($DBversion); } +$DBversion = "3.13.00.XXX"; +if ( CheckVersion($DBversion) ) { + $dbh->do(" +CREATE TABLE IF NOT EXISTS oai_harvest_repositories ( + repository_id int(11) NOT NULL AUTO_INCREMENT, + baseURL text NOT NULL, + opt_from varchar(45) DEFAULT NULL, + opt_until varchar(45) DEFAULT NULL, + opt_set varchar(45) DEFAULT NULL, + metadataPrefix varchar(45) NOT NULL, + active int(11) NOT NULL DEFAULT '0', + timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + XSLT_path text, + matcher_id int(11) DEFAULT NULL, + import_mode enum('manual','automatic') NOT NULL DEFAULT 'manual', + PRIMARY KEY (repository_id) USING BTREE + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + "); + $dbh->do(" +CREATE TABLE IF NOT EXISTS oai_harvest ( + oai_harvest_id int(11) NOT NULL AUTO_INCREMENT, + identifier varchar(255) NOT NULL, + datestamp varchar(45) NOT NULL, + metadataPrefix varchar(45) NOT NULL, + import_record_id int(11) DEFAULT NULL, + linked_biblionumbers varchar(255) DEFAULT NULL, + metadata longtext, + header longtext NOT NULL, + about longtext, + timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + repository_id int(11) NOT NULL, + status varchar(45) DEFAULT NULL, + PRIMARY KEY (oai_harvest_id) USING BTREE, + KEY FK_oai_harvest_1 (repository_id), + CONSTRAINT FK_oai_harvest_1 FOREIGN KEY (repository_id) REFERENCES oai_harvest_repositories (repository_id) ON UPDATE NO ACTION +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + "); + print "Upgrade to $DBversion done (Bug 7494: Add itemBarcodeFallbackSearch syspref)\n"; + SetVersion($DBversion); +} + =head1 FUNCTIONS =head2 TableExists($table) --- a/koha-tmpl/intranet-tmpl/prog/en/xslt/DC2MARC21slim.xsl +++ a/koha-tmpl/intranet-tmpl/prog/en/xslt/DC2MARC21slim.xsl @@ -0,0 +1,213 @@ + + + + + + + + + + + + + + + + p + m + r + k + m + m + m + i + a + a + + + + + c + m + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + dc + + + + + + + + collaborator + + + + + + + + + + + + + + + + + author + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + local + + + + + --- a/koha-tmpl/intranet-tmpl/prog/en/xslt/MARC21slim2MARC21enhanced.xsl +++ a/koha-tmpl/intranet-tmpl/prog/en/xslt/MARC21slim2MARC21enhanced.xsl @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 8 + + 037 + + + + a + + + + + + + + + + + 999 + + + + c + + + + d + + + + + + + --- a/misc/cronjobs/oai_harvester.pl +++ a/misc/cronjobs/oai_harvester.pl @@ -0,0 +1,64 @@ +#!/usr/bin/perl + +# This file is part of Koha. +# +# Copyright 2013 Prosentient Systems +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, see . + +use Modern::Perl; + +use C4::Context; +use Data::Dumper; + +use C4::OAI::Harvester; + +#DATA - Retrieve data from the database +my $dbh1 = C4::Context->dbh; +my $sql1 = "SELECT * FROM oai_harvest_repositories WHERE repository_id = ?"; +my $sth1 = $dbh1->prepare($sql1); +$sth1->execute(1); +my $data = $sth1->fetchrow_hashref; + +$data->{resume} = 0; #Comment this out or change to 1 if you want automatic token resumption + +#OBJECT CREATION +my $test = C4::OAI::Harvester->new($data); + + +#OBJECT METHODS +#List data about the OAI-PMH repository +my $identify_repository = $test->Identify; +print Dumper($identify_repository); + +#List available sets +my @sets = $test->ListSets; +print Dumper(@sets); + +#List available metadata formats +my @formats = $test->ListMetadataFormats; +print Dumper(@formats); + +#List available identifiers/headers +#my @headers = $test->ListIdentifiers; +#print Dumper(@headers); + +#Retrieve all records +my @records = $test->ListRecords; +print Dumper(@records); + +#Retrieve a particular record +#my @records = $test->GetRecord('KOHAOAITEST:286'); +#print Dumper(@records); + +$test->ImportRecordsIntoKoha(@records); --- a/t/db_dependent/OAI_harvester.t +++ a/t/db_dependent/OAI_harvester.t @@ -0,0 +1,303 @@ +#!/usr/bin/perl + +# This file is part of Koha. +# +# Copyright 2013 Prosentient Systems +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, see . + +use Modern::Perl; +use Test::More; #tests => 4; +use C4::Context; +use C4::OAI::Harvester; +use Data::Dumper; +use Getopt::Long; + +my $verbose = 0; +my $show = 0; +GetOptions( + 'v|verbose' => \$verbose, + 's|show:s' => \$show, +); + +my $real_dbh = C4::Context->dbh; + +############################################################################################################################################################################################################ + +###FIXME: use_ok or require_ok test up here for C4::OAI::Harvester? + +############################################################################################################################################################################################################ + +#REAL DATA - We use this for crafting our mock data, making sure the OAI-PMH server is on, and requesting a specific item for the GetRecord method. + +#Initiate Koha baseURLs +my $OPACbaseURL = C4::Context->preference("OPACBaseURL"); +my $staffClientBaseURL = C4::Context->preference("staffClientBaseURL"); + +#Check if Koha has its OAI-PMH repository config enabled. +#If disabled, re-enable it (we'll reset to the original values +#after we've finished our test). +my $OAIPMHenabled = C4::Context->preference("OAI-PMH"); +if (!$OAIPMHenabled){ + C4::Context->set_preference("OAI-PMH",1); +} +my $OAIPMHarchiveID; +my $check_OAIPMHarchiveID = C4::Context->preference("OAI-PMH:archiveID"); +if ($check_OAIPMHarchiveID){ + $OAIPMHarchiveID = $check_OAIPMHarchiveID; +} +else { + C4::Context->set_preference("OAI-PMH:archiveID","KOHAOAITEST"); + $OAIPMHarchiveID = C4::Context->preference("OAI-PMH:archiveID"); +} + +my $check_biblio_table_sql = " + SELECT count(*) as 'count' + FROM biblio +"; +my $check_biblio_table_sth = $real_dbh->prepare($check_biblio_table_sql); +$check_biblio_table_sth->execute(); +my $check_biblio_table_row = $check_biblio_table_sth->fetchrow_hashref; +if ($check_biblio_table_row->{count} == 0){ + # Generate test biblio records + my $biblio = MARC::Record->new(); + $biblio->append_fields( + MARC::Field->new('100', ' ', ' ', a => 'Moffat, Steven'), + MARC::Field->new('245', ' ', ' ', a => 'Silence in the library'), + ); + my ($biblionumber, $biblioitemnumber) = AddBiblio($biblio, ''); + + my $biblio2 = MARC::Record->new(); + $biblio2->append_fields( + MARC::Field->new('100', ' ', ' ', a => 'Garland, Alex'), + MARC::Field->new('245', ' ', ' ', a => 'The Beach'), + ); + my ($biblionumber2, $biblioitemnumber2) = AddBiblio($biblio2, ''); + + my $biblio3 = MARC::Record->new(); + $biblio3->append_fields( + MARC::Field->new('100', ' ', ' ', a => 'Michaels, Anne'), + MARC::Field->new('245', ' ', ' ', a => 'The Winter Vault'), + ); + my ($biblionumber3, $biblioitemnumber3) = AddBiblio($biblio3, ''); +} + +my $sql1 = "SELECT biblionumber FROM biblio ORDER BY RAND() LIMIT 1"; +my $sth1 = $real_dbh->prepare($sql1); +$sth1->execute(); +my $random_biblionumber_row = $sth1->fetchrow_hashref; +my $random_biblionumber = $random_biblionumber_row->{biblionumber}; + +my $random_OAIPMH_identifier; +if ($OAIPMHarchiveID && $random_biblionumber){ + $random_OAIPMH_identifier = $OAIPMHarchiveID.":".$random_biblionumber; +} + +############################################################################################################################################################################################################ + +#MOCK DATA - Create a mock remote OAI-PMH repository using Koha's OAI-PMH server capabilities + +# Start Database Transaction +my $dbh = C4::Context->dbh; +$dbh->{AutoCommit} = 0; +$dbh->{RaiseError} = 1; + +my $data = {}; +$data->{repository_id} = 1; +$data->{baseURL} = "http://$OPACbaseURL/cgi-bin/koha/oai.pl"; +$data->{opt_from} = undef; +$data->{opt_until} = undef; +$data->{opt_set} = undef; +$data->{metadataPrefix} = "oai_dc"; #or "marcxml" +$data->{active} = 1; +$data->{XSLT_path} = "http://$staffClientBaseURL/intranet-tmpl/prog/en/xslt/DC2MARC21slim2.xsl"; #"/.../git/koha-tmpl/intranet-tmpl/prog/en/xslt/DC2MARC21slim2.xsl"; #or undef, since this is hard-coded into Harvester.pm anyway +$data->{matcher_id} = 4; +$data->{import_mode} = "automatic"; #Always run "automatic" for test. In production, you might prefer "manual". + +my $insert_mock_data_sql = " + INSERT INTO oai_harvest_repositories + (baseURL, opt_from, opt_until, opt_set, metadataPrefix, active, XSLT_path, matcher_id, import_mode) + VALUES (?,?,?,?,?,?,?,?,?) +"; +my $insert_mock_mock_data_sth = $dbh->prepare($insert_mock_data_sql); +$insert_mock_mock_data_sth->execute($data->{baseURL},$data->{opt_from},$data->{opt_until},$data->{opt_set},$data->{metadataPrefix},$data->{active},$data->{XSLT_path},$data->{matcher_id},$data->{import_mode}); + +my $get_mock_data_sql = " + SELECT * + FROM oai_harvest_repositories + WHERE active = 1 +"; +my $get_mock_data_sth = $dbh->prepare($get_mock_data_sql); +$get_mock_data_sth->execute(); +while (my $get_mock_data_row = $get_mock_data_sth->fetchrow_hashref){ + + #Set 'resume' to 0, if you want to turn off automatic token resumption. This is useful for testing and debugging. Change to 1 for automatic token resumption. + $get_mock_data_row->{resume} = 0; + +############################################################################################################################################################################################################ + + #TEST - OBJECT CREATION + my $harvester = C4::OAI::Harvester->new($get_mock_data_row); # create an object + isa_ok( $harvester, 'C4::OAI::Harvester','Harvester object' ); + #END TEST + + #TEST - OBJECT's "GET" ACCESSOR METHODS + can_ok($harvester, qw(get_baseURL get_from get_until get_set get_metadataPrefix get_rh get_repository_id get_XSLT_path get_matcher_id get_import_mode)); + #END TEST + + #TEST - Check some basic harvester config + my $baseURL = $harvester->get_baseURL; + my $metadataPrefix = $harvester->get_metadataPrefix; + is ($baseURL,"http://$OPACbaseURL/cgi-bin/koha/oai.pl", 'Retrieved baseURL "'.$baseURL.'"'); + ok ( defined $metadataPrefix, 'MetadataPrefix is defined as "'.$metadataPrefix.'"'); + #END TEST + + #TESTS - OBJECT's OAI-PMH methods + + #IDENTIFY + my $identify_repository = $harvester->Identify; + ok ( defined($identify_repository), "Successfully used Identify Verb"); + print Dumper($identify_repository) if $verbose or $show eq 'Identify'; + #END TEST + + #LISTSETS + my @sets = $harvester->ListSets; + ok (@sets, "Successfully used ListSets Verb"); + print Dumper(\@sets) if $verbose or $show eq 'ListSets'; + #END TEST + + #LISTMETADATAFORMATS + my @formats = $harvester->ListMetadataFormats; + ok (@formats, "Successfully used ListMetadataFormats Verb"); + print Dumper(\@formats) if $verbose or $show eq 'ListMetadataFormats'; + #END TEST + + #LISTIDENTIFIERS + my @headers = $harvester->ListIdentifiers; + ok (@headers, "Successfully used ListIdentifiers Verb"); + print Dumper(\@headers) if $verbose or $show eq 'ListIdentifiers'; + #END TEST + + #LISTRECORDS + my @list_records = $harvester->ListRecords; + ok (@list_records, "Successfully used ListRecords Verb"); + print Dumper(\@list_records) if $verbose or $show eq 'ListRecords'; + #END TEST + + #GETRECORD + my @get_records = $harvester->GetRecord($random_OAIPMH_identifier); + ok (@get_records, "Successfully used GetRecord Verb"); + print Dumper(@get_records) if $verbose or $show eq 'GetRecord'; + #END TEST + + my ( $batch_id, $num_with_matches, $num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = $harvester->ImportRecordsIntoKoha(@list_records); + + #These variables will only be initialized in automatic mode, which is how the test should always be run anyway. + my $batch_total = $num_added + $num_updated; + + print "Batch id = $batch_id \n"; + print "Num added = $num_added \n"; + print "Num updated = $num_updated \n"; + print "Num items added = $num_items_added \n"; + print "Num items errored = $num_items_errored \n"; + print "Num ignored = $num_ignored \n"; + + + #TEST - RETRIEVE IMPORT_BATCH + my @test_batches; + my $batch_sql = " + SELECT * + FROM import_batches + WHERE import_batch_id = ? + "; + my $batch_sth = $dbh->prepare($batch_sql); + $batch_sth->execute($batch_id); + while (my $batch_row = $batch_sth->fetchrow_hashref){ + push(@test_batches,$batch_row); + } + cmp_ok(scalar @test_batches,'==',1,"Successfully retrieved batch id ".$batch_id); + + #END TEST + + #TEST - RETRIEVE IMPORTED_RECORDS + my $imported_record_sql = " + SELECT count(*) as 'count' + FROM import_records + WHERE import_batch_id = ? + "; + my $imported_record_sth = $dbh->prepare($imported_record_sql); + $imported_record_sth->execute($batch_id); + my $imported_record_row = $imported_record_sth->fetchrow_hashref; + is ($imported_record_row->{count}, $batch_total, "Import_record table count (".$imported_record_row->{count}.") for this batch matches reported import total"); + #END TEST + + #TEST - RETRIEVE IMPORTED_BIBLIOS + my $imported_biblio_sql = " + SELECT count(*) as 'count' + FROM import_biblios + JOIN import_records using (import_record_id) + WHERE import_records.import_batch_id = ? + "; + my $imported_biblio_sth = $dbh->prepare($imported_biblio_sql); + $imported_biblio_sth->execute($batch_id); + my $imported_biblio_row = $imported_biblio_sth->fetchrow_hashref; + is ($imported_biblio_row->{count}, $batch_total, "Import_biblio table count (".$imported_biblio_row->{count}.") for this batch matches reported import total"); + #END TEST + + #TEST - RETRIEVE BIBLIOS + my $biblio_sql = " + SELECT count(*) as 'count' + FROM biblio + JOIN import_biblios on import_biblios.matched_biblionumber = biblio.biblionumber + JOIN import_records using (import_record_id) + WHERE import_records.import_batch_id = ? + "; + my $biblio_sth = $dbh->prepare($biblio_sql); + $biblio_sth->execute($batch_id); + my $biblio_row = $biblio_sth->fetchrow_hashref; + is ($biblio_row->{count}, $batch_total, "Biblio table count (".$biblio_row->{count}.") for this batch matches reported import total. Automatic import successful."); + #END TEST + + #TEST - RETRIEVE HISTORICAL RECORDS + my $harvest_sql = " + SELECT count(*) as 'count' + FROM oai_harvest + JOIN import_records using (import_record_id) + WHERE import_records.import_batch_id = ? + "; + my $harvest_sth = $dbh->prepare($harvest_sql); + $harvest_sth->execute($batch_id); + my $harvest_row = $harvest_sth->fetchrow_hashref; + is ($harvest_row->{count}, $batch_total, "OAI Harvest table count (".$harvest_row->{count}.") for this batch matches reported import total. Historical records added successfully."); + #NOTE: This ignores deleted records, which will have a historical record but are not part of the batch and thus don't have an import_record_id number. + #END TEST + + #TEST - RETRIEVE RECORD MATCHES + #There's no way to really automatically test the "replace/update" function, since the first wave of imported records will need to be reindexed before they can be matched on... + #END + +} #End Loop of OAI-PMH Repositories +$dbh->rollback(); +$dbh->{AutoCommit} = 1; + +done_testing(); +############################################################################################################################################################################################################ + +#RESET ORIGINAL VALUES +if (!$OAIPMHenabled){ + C4::Context->set_preference("OAI-PMH",$OAIPMHenabled); +} + +if (!$check_OAIPMHarchiveID){ + C4::Context->set_preference("OAI-PMH:archiveID",$check_OAIPMHarchiveID); +} --