package C4::OAI::Harvester;

# This file is part of Koha.

#

# Copyright 2013 Prosentient Systems

#

# Koha is free software; you can redistribute it and/or modify it under the

# terms of the GNU General Public License as published by the Free Software

# Foundation; either version 3 of the License, or (at your option) any later

# version.

#

# Koha is distributed in the hope that it will be useful, but WITHOUT ANY

# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR

# A PARTICULAR PURPOSE. See the GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License along

# with Koha; if not, see <http://www.gnu.org/licenses>.

=head1 NAME

C4::OAI::Harvester - OAI-PMH harvester/client which implements the 6 OAI-PMH verbs, record retrieval/harvesting, and import into Koha 

=head1 SYNOPSIS

use C4::OAI::Harvester;

my $oai_repo = C4::OAI::Harvester->new($repository_data);

my $identify_repository = $oai_repo->Identify;

my @sets = $oai_repo->ListSets;

my @formats = $oai_repo->ListMetadataFormats;

my @headers = $oai_repo->ListIdentifiers;

my @records = $oai_repo->ListRecords;

my @records = $oai_repo->GetRecord($oai_unique_identifier);

my $import_mode = ''; #i.e. not "automatic"

$oai_repo->ImportRecordsIntoKoha($import_mode,@records);

=head1 DESCRIPTION

C4::OAI::Harvester contains functions for querying and harvesting OAI-PMH repositories.

More information on OAI-PMH can be found L<here|http://www.openarchives.org/OAI/openarchivesprotocol.html>

=head1 FUNCTIONS

=head1 AUTHOR

David Cook <dcook AT prosentient DOT com DOT au>

=cut

use Modern::Perl;

use Data::Dumper;

use C4::Context;

use C4::ImportBatch qw/AddImportBatch SetImportBatchItemAction AddBiblioToBatch SetImportBatchStatus BatchCommitRecords BatchFindDuplicates/;

use C4::Matcher;

use HTTP::OAI;

use base qw(Class::Accessor);

sub new {

    my($proto, $fields) = @_;

    my($class) = ref $proto || $proto;

    $fields = {} unless defined $fields;

    if ($fields->{'baseURL'}){

        my $h = new HTTP::OAI::Harvester(

            baseURL => $fields->{'baseURL'},

        );

        #If resume is set to 0, automatic token resumption is turned off. This is useful for testing/debugging.

        if ($h && exists $fields->{'resume'}){

            if ($fields->{'resume'} == 0){

                $h->resume(0);

            }

        }

        my $response = $h->repository($h->Identify);

        if( $response->is_error ) {

            print "Error requesting Identify:\n",

                    $response->code . " " . $response->message, "\n";

            exit;

        }

        $fields->{rh} = $h; #Store HTTP::OAI::Harvester object as "repository handle"

    }

    bless {%$fields}, $class;

}

__PACKAGE__->follow_best_practice; #Use get_ and set_ prefixes for accessors

__PACKAGE__->mk_accessors(qw(baseURL from until set metadataPrefix rh repository_id XSLT_path matcher_id import_mode));

=head2 OAI-PMH Verbs

Koha-specific implementations of the 6 OAI-PMH Verbs.

The key verbs are "ListRecords" and "GetRecords". These do the actual

harvesting of records from a OAI-PMH repository. The others are useful for

getting information about a repository and what it has available.

1) ListRecords

2) GetRecord

3) ListIdentifiers

4) ListMetadataFormats

5) ListSets

6) Identify 

=cut

sub ListRecords {

    my $self = shift;

    my $response = $self->{rh}->ListRecords(

        metadataPrefix => $self->{metadataPrefix},

        from => $self->{opt_from},

        until => $self->{opt_until},

        set => $self->{opt_set},

    );

    if( $response->is_error ) {

        print "Error requesting ListRecords:\n",

                $response->code . " " . $response->message, "\n";

        exit;

    }

    #print Dumper($response->toDOM); #FIXME: dac2013: Fails with this warning "bad ns attribute!"

    my @records = _parse_records_into_useful_format($response);

    return @records;

}

sub GetRecord {

    my ( $self, $identifier ) = @_;

    my $response = $self->{rh}->GetRecord(

        metadataPrefix => $self->{metadataPrefix},

        identifier => $identifier,

    );

    if( $response->is_error ) {

        print "Error requesting GetRecord:\n",

                $response->code . " " . $response->message, "\n";

        exit;

    }

    my @records = _parse_records_into_useful_format($response);

    return @records;

}

sub ListIdentifiers {

    my $self = shift;

    my @headers;

    my $response = $self->{rh}->ListIdentifiers(

        metadataPrefix => $self->{metadataPrefix},

        from => $self->{opt_from},

        until => $self->{opt_until},

        set => $self->{opt_set},

    );

    if( $response->is_error ) {

        print "Error requesting ListIdentifiers:\n",

                $response->code . " " . $response->message, "\n";

        exit;

    }

    #print Dumper($response->toDOM); #FIXME: dac2013: Fails with this warning "bad ns attribute!"

    while (my $h = $response->next){

        my $header;

        #print Dumper($h->dom->toString); #DEBUG: dac2013: XML representation 

        $header->{identifier} = $h->identifier;

        $header->{datestamp} = $h->datestamp;

        $header->{status} = $h->status;

        $header->{is_deleted} = $h->is_deleted;

        my @sets = $h->setSpec;

        $header->{sets} = \@sets;

        push (@headers,$header);

    }

    return @headers;

}

sub ListMetadataFormats {

    my ( $self, $identifier ) = @_;

    my @formats;

    my $response = $self->{rh}->ListMetadataFormats(

        identifier => $identifier,

    );

    if( $response->is_error ) {

        print "Error requesting ListMetadataFormats:\n",

                $response->code . " " . $response->message, "\n";

        exit;

    }

    for($response->metadataFormat) {

        push(@formats,$_->metadataPrefix);

    }

    return @formats;

}

sub ListSets {

    my $self = shift;

    my @sets;

    my $response = $self->{rh}->ListSets();

    if( $response->is_error ) {

        print "Error requesting ListSets:\n",

                $response->code . " " . $response->message, "\n";

        exit;

    }

    while (my $s = $response->next){

        my $set;

        $set->{setSpec} = $s->setSpec;

        $set->{setName} = $s->setName;

        #FIXME: dac2013: Not really sure what to do with the descriptions as they're XML and not necessarily that easy to parse for GUI views...

        #my @temp_setDescriptions = $s->setDescription; 

        #my @setDescriptions;

        #foreach my $temp_setDescription (@temp_setDescriptions){

        #    push (@setDescriptions,$temp_setDescription->dom->toString); #dac2013: I think I need to do better than just return the setDescription XML...That's not very useful...

        #}

        #$set->{setDescription} = \@setDescriptions;

        push (@sets,$set);

    }

    return @sets;

}

sub Identify {

    my $self = shift;

    my $response = $self->{rh}->Identify();

    if( $response->is_error ) {

        print "Error requesting Identify:\n",

        $response->code . " " . $response->message, "\n";

        exit;

    }

    my $identify_data;

    #DEBUG: View what's in the Identify object

    #print Dumper($response->headers);

    $identify_data->{repositoryName} = $response->repositoryName;

    $identify_data->{baseURL} = $response->baseURL;

    $identify_data->{protocolVersion} = $response->protocolVersion; #Tim Brody says this will always return 2.0 and that ->version should be used to find the actual version...

    #$identify_data->{version} = $response->version;

    $identify_data->{earliestDatestamp} = $response->earliestDatestamp;

    $identify_data->{deletedRecord} = $response->deletedRecord; #not in the perldoc, but it's in the code and the OAI-PMH spec

    $identify_data->{granularity} = $response->granularity;

    #These methods should be used with an array context so they return all the elements and not just the first one

    my @adminEmails = $response->adminEmail;

    $identify_data->{adminEmail} = \@adminEmails; 

    my @compressions = $response->compression;

    $identify_data->{compression} = \@compressions;

    #FIXME: dac2013: Descriptions are encapsulated in XML containers, I believe. Not sure what to do with these at present...

    #my @descriptions = $response->description;

    #$identify_data->{description} = \@descriptions;

    #$response->next

    return $identify_data;

}

=head2 _parse_records_into_useful_format

A C4::OAI::Harvester internal subroutine that parses HTTP::OAI::Record 

objects into C4::OAI::Harvester::Record objects which have methods

for transforming incoming XML into MARCXML, saving historical records

in the database, and comparing incoming records against historical

records. 

=cut

sub _parse_records_into_useful_format {

    my ( $response ) = @_;

    my @records;

    while( my $rec = $response->next ) {

        my $record;

        $record->{header} = $rec->header ? $rec->header->dom : undef;

        $record->{identifier} = $rec->identifier;

        $record->{datestamp} = $rec->datestamp;

        $record->{metadata} = $rec->metadata ? $rec->metadata->dom : undef; #N.B. there won't be any metadata for deleted records

        $record->{about} = $rec->about ? $rec->about->dom : undef;

        $record->{deleted} = $rec->is_deleted;

        $record->{status} = $rec->status;

        #FIXME: Will these always exist for every record? Add error checking. Maybe include an undef or blank if they're not set...

        my $record_object = C4::OAI::Harvester::Record->new($record);

        #FIXME: Add error checking

        #if( $rec->is_error ) {

        #    die $response->message;

        #}

        push(@records,$record_object);

    }

    return @records;

}

=head2 ImportRecordsIntoKoha

$oai_repo->ImportRecordsIntoKoha(@records);

@records is an array of records that have been retrieved using the

GetRecord or ListRecords verbs. 

$import_mode is used to specify "automatic" or "manual" modes. The 

default mode is "manual", but "automatic" can be triggered by 

passing the "automatic" string in the variable. Manual mode is

indicated using any other string or even undef. 

This subroutine handles importing our retrieved records into Koha.

  1. Foreach record:

     a) Check its identifier status in the database

     b) If its status is "create", "replace", or "error",

        we transform it into MARCXML. 

        Note: "error" simply indicates that a "replace" couldn't

        occur automatically due to multiple biblionumbers being

        linked to the unique identifier. 

     c) If the status is "ignore", we skip to the next record.

  2. Add an import batch

  3. Foreach record:

     a) Add the record to the import batch

     b) Add a historical record to the oai_harvest table

  4. Check the import batch for duplicates/matches 

    (i.e. existing records).

  5. If automatic importing is turned on, batch import the records

     (otherwise, leave in staging area for manual import)

=cut

sub ImportRecordsIntoKoha {

    my ( $self, @records ) = @_;

    my @records_to_import;

    my ( %ignored, %created, %errored, %replaced, %deleted );

    foreach my $record (@records){

        #FIXME: Do the delete handling in the function...only use "next" with an "ignore"...

        #FIXME: stop returning @biblionumbers with GetIdentifierSatus...it's unnecessary

        #FIXME: Add a way of forcing add/replace despite existing Identifier+datestamp(+metadataPrefix)?

        my $identifier_status = $record->AddIdentifierStatusAndLinkedBiblionumbers($self->get_repository_id);

        if ($identifier_status){

            if ($identifier_status eq "ignore"){

                #This record's identifier and datestamp already exist in the database, so we don't import this record, or

                #this is a record with a deleted status that hasn't been imported into Koha before.

                next;

            }

        }

        else {

            #We weren't able to get the status of this record, so let's ignore it and skip to the next one

            next;

        }

        my $transform_result_flag = $record->TransformMetadata($self->get_metadataPrefix, $self->get_XSLT_path);

        push(@records_to_import,$record);

    }

    if (scalar @records_to_import > 0){

        #DEBUGGING ONLY - DUMPER

        #print Dumper(@records_to_import);

        my $overlay_action = 'replace'; #If an incoming record matches an existing one, we replace the existing one with the incoming record

        my $file_name = $self->get_baseURL;

        my $comments = "OAI-PMH Harvest";

        my $matcher_id = undef;

        my $rec_num = 0;

        my $num_with_matches = 0;

        #If a matcher_id is linked to this repository, we'll use that for matching records when applying updates. Otherwise, don't use a matcher.

        #FIXME: It might be an idea to make matcher_id a required field for OAI-PMH repositories...

        if ($self->get_matcher_id){

            $matcher_id = $self->get_matcher_id;

        }

        #FIXME: Have other import options be configurable?

        my $batch_id = AddImportBatch( {

                overlay_action => $overlay_action,

                import_status => 'staging',

                batch_type => 'batch', #FIXME: batch|z3950|webservice are the three current options. Despite this being a web service, I that type of batch has special properties that shouldn't be used here?

                file_name => $file_name, #This is the baseURL for our remote OAI-PMH repository

                comments => $comments, #This contains a mention that this is a OAI-PMH harvest

                matcher_id => $matcher_id, #This is defined in our DB config for the repository

            } );

        #N.B. In theory, it would be good if items could be processed. 

        #However, duplicate items would appear for updated records if

        #we turn this on, since there is no way of matching items at the

        #moment.

        SetImportBatchItemAction($batch_id, 'ignore');

        foreach my $record_to_import (@records_to_import){

            #Check for transformed_metadata. We do not want to try to

            #add a record to a batch without transformed metadata

            #(i.e. if the record is deleted). 

            my $import_record_id = undef;

            if ($record_to_import->get_transformed_metadata){

                my $encoding = "utf8";

                my $format = "MARC21";

                my $marc_record = MARC::Record->new_from_xml( $record_to_import->get_transformed_metadata->toString, $encoding, $format );

                $rec_num++;

                $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0);

                _update_batch_record_counts($batch_id);

            }

            my $save_result_flag = $record_to_import->AddHistoricalRecord($self->get_metadataPrefix,$self->get_repository_id, $import_record_id);

            #if ($save_result_flag != 1){

                #If there was an error saving to the database, we ignore this record

                #FIXME: Add debug and print errors to log

               #next;

            #}

        }

        my $matcher = C4::Matcher->fetch($matcher_id);

        if ($matcher){

            $num_with_matches = BatchFindDuplicates($batch_id, $matcher);

        }

        SetImportBatchStatus($batch_id, 'staged');

        #When active, this automatically imports records into the catalogue without having to use the "Manage staged MARC records interface",

        #although the import can still be reverted using that interface.

        if ($self->get_import_mode eq "automatic"){

            my $framework = '';

            my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) =

                    BatchCommitRecords($batch_id, $framework);

            return ($batch_id, $num_with_matches, $num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored);

        }

        return ($batch_id, $num_with_matches);

    }

}

#Copied from ImportBatch.pm for convenience...for now

sub _update_batch_record_counts {

    my ($batch_id) = @_;

    my $dbh = C4::Context->dbh;

    my $sth = $dbh->prepare_cached("UPDATE import_batches SET

                                        -- num_biblios = (

                                           num_records = (

                                            SELECT COUNT(*)

                                            FROM import_records

                                            WHERE import_batch_id = import_batches.import_batch_id

                                            AND record_type = 'biblio'),

                                        num_items = (

                                            SELECT COUNT(*)

                                            FROM import_records

                                            JOIN import_items USING (import_record_id)

                                            WHERE import_batch_id = import_batches.import_batch_id

                                            AND record_type = 'biblio')

                                    WHERE import_batch_id = ?");

    $sth->bind_param(1, $batch_id);

    $sth->execute();

    $sth->finish();

}

package C4::OAI::Harvester::Record;

=head1 NAME

C4::OAI::Harvester::Record - an internal class for handling records retrieved via the OAI-PMH protocol

=head1 DESCRIPTION

C4::OAI::Harvester::Record - Class to handle the management of records retrieved via OAI-PMH

More information on OAI-PMH can be found L<here|http://www.openarchives.org/OAI/openarchivesprotocol.html>

=head1 AUTHOR

David Cook <dcook AT prosentient DOT com DOT au>

=cut

use Modern::Perl;

use C4::Context;

use C4::Templates;

use C4::XSLT qw/GetURI/;

use XML::LibXML;

use XML::LibXSLT;

use base qw(Class::Accessor);

sub new {

    my($proto, $fields) = @_;

    my($class) = ref $proto || $proto;

    $fields = {} unless defined $fields;

    $fields->{transformed_metadata} = undef;

    $fields->{linked_biblionumbers} = '';

    bless {%$fields}, $class;

}

__PACKAGE__->follow_best_practice; #Use get_ and set_ prefixes for accessors

__PACKAGE__->mk_accessors(qw(identifier datestamp metadata status deleted header about transformed_metadata linked_biblionumbers));

=head2 TransformMetadata

A C4::OAI::Harvester::Record method for transforming incoming metadata

into a format that Koha can use (e.g. MARC21 MARCXML).

my $transform_result_flag = $record->TransformMetadata(

    $self->get_metadataPrefix, 

    $self->get_XSLT_path, 

    $biblionumber_to_use_in_XSLT);

$self->get_metadataPrefix is the metadataPrefix for the record,

which comes from the database config for this OAI-PMH repository. It

is a required parameter.

$self->get_XSLT_path is the filepath to the XSLT to be used in

the metadata transformation. This can be a local filepath or a remote

URL (like in XSLT.pm). Technically, this is an optional parameter if

the metadataPrefix is "marcxml", "dc", or "oai_dc". However, it is

recommended to always specify a XSLT path. You will almost certainly

have problems down the road if you do not use one, especially

for marcxml (as the biblionumber for any existing imported records

will not be added, which means you'll have great difficulty managing 

updates via OAI-PMH).

$biblionumber_to_use_in_XSLT is the biblionumber linked to this 

record's unique OAI-PMH identifier. Typically, this will only be used

if this record has always been ingested/added to Koha before. This

biblionumber is added during the transform process so that Koha

can match the incoming and existing records for merging/replacing.

=cut

sub TransformMetadata {

    my ( $self, $metadataPrefix, $XSLT_path ) = @_;

    my $source = $self->{metadata}; #LibXML object retrieved via HTTP::OAI::Harvester

    my $biblionumber;

    my $no_XSLT_path;

    if (!$XSLT_path){

        $no_XSLT_path = 1;

    }

    #If we have a single linked_biblionumber and our status is replace, 

    #then we initialize $biblionumber with that biblionumber, so that

    #we can create a 999$c or 999$d for matching duringn the import.

    my $linked_biblionumbers = $self->get_linked_biblionumbers;

    if (ref $linked_biblionumbers eq 'ARRAY' && $self->get_status eq 'replace'){

        if (scalar @$linked_biblionumbers == 1){

            $biblionumber = @$linked_biblionumbers[0];

        }

    }

    if ($metadataPrefix && $source){

        my $xslt = XML::LibXSLT->new();

        my $style_doc;

        #$parser based on equivalent from XSLT.pm

        my $parser = XML::LibXML->new();

        # don't die when you find &, >, etc

        $parser->recover_silently(0);

        if ($XSLT_path){

            #This conditional for handling secure remote XSLTs copied from XSLT.pm

            if ( $XSLT_path =~ /^https?:\/\// ) {

                my $xsltstring = GetURI($XSLT_path);

                if ($xsltstring){

                    $style_doc = $parser->parse_string($xsltstring);

                }

                else{

                    #If a string isn't retrieved using GetURI, we switch to our default transforms

                    $no_XSLT_path  = 1;

                }

            } else {

                if ( -e $XSLT_path){

                    $style_doc = $parser->parse_file($XSLT_path);

                }

                else{

                    #If the file doesn't actually exist, we switch to our default transforms

                    $no_XSLT_path = 1;

                }

            }

        }

        if ($no_XSLT_path){

            if ($metadataPrefix eq 'marcxml'){

                $self->set_transformed_metadata($source);

                return 1;

            }

            elsif ($metadataPrefix eq 'dc' || $metadataPrefix eq 'oai_dc'){

                my $xsl = C4::Context->config('intrahtdocs') .

                                    '/' . C4::Context->preference("template") .

                                    '/' . C4::Templates::_current_language() .

                                    '/xslt/DC2MARC21slim.xsl';

                                    #'/xslt/' .

                                    #C4::Context->preference('marcflavour') .

                                    #"slim2intranetDetail.xsl";

                $style_doc = $parser->parse_file($xsl);

            }

            else{

                #FIXME: Use more robust error handling than this

                return -1;

            }

        }

        my $stylesheet = $xslt->parse_stylesheet($style_doc);

        #FIXME: Add error handling here...

        my %xslt_params;

        $xslt_params{'identifier'} = $self->{identifier};

        $xslt_params{'biblionumber'} = $biblionumber;

        #Pass OAI-PMH identifier and matched biblionumber (if there is one) to the XSLT for unique identification/provenance

        my $results = $stylesheet->transform($source, XML::LibXSLT::xpath_to_string(%xslt_params));

        if ($results){

            $self->set_transformed_metadata($results);

            return 1;

        }

        else{

            return -1;

        }

        #FIXME: Add some better error checking here...throw an exception if a record fails?

    }

    else{

        return -1;

    }

}

=head2 AddHistoricalRecord

A C4::OAI::Harvester::Record method that saves an entry into the

database for historical and importing purposes.

my $save_result_flag = $record_to_import->AddHistoricalRecord($self->get_metadataPrefix,$self->get_repository_id, $import_record_id);

$self->get_metadataPrefix is the record's metadataPrefix (i.e. metadata

format), as defined in the database config, since records do not

self-identify their metadataPrefix.

$self->get_repository_id is the local DB primary key for the record's

repository. 

$import_record_id is the key in the import_records and import_biblios

tables which show that the record has been staged and if it has been

imported. This is ESSENTIAL for performing matching between existing

and incoming records. 

=cut

sub AddHistoricalRecord {

    my ( $self, $metadataPrefix, $repository_id, $import_record_id ) = @_;

    #FIXME: Error/variable checking should be better in this sub...

    #Only check for these two, since deleted records won't have an $import_record_id

    if ($metadataPrefix && $repository_id){

            my $identifier = $self->get_identifier;

            my $datestamp = $self->get_datestamp;

            my $metadata = $self->get_metadata ? $self->get_metadata->toString : undef;

            my $header = $self->get_header ? $self->get_header->toString : undef;

            my $status = $self->get_status;

            my $string_of_linked_biblionumbers = undef;

            my $linked_biblionumbers = $self->get_linked_biblionumbers;

            if (ref $linked_biblionumbers eq 'ARRAY'){

                $string_of_linked_biblionumbers = join(" | ",@$linked_biblionumbers);

            }

            my $dbh = C4::Context->dbh;

            my $sql = "INSERT INTO oai_harvest (identifier,datestamp,metadataPrefix,import_record_id,metadata,header,repository_id,status, linked_biblionumbers) VALUES (?,?,?,?,?,?,?,?,?)";

            my $sth = $dbh->prepare($sql);

            $sth->execute($identifier, $datestamp, $metadataPrefix, $import_record_id, $metadata, $header, $repository_id, $status, $string_of_linked_biblionumbers);

            if ($sth->err){

                return "ERROR! return code: " . $sth->err . " error msg: " . $sth->errstr . "\n";

            }

            else{

                return 1;

            }

    }

    else{

        return -1;

    }

}

=head2 AddIdentifierStatusAndLinkedBiblionumbers

A C4::OAI::Harvester::Record method that compares the status of

the record with any matching identifiers in the database. It sets

and returns a status saying "ignore", "deleted", "create",

"replace", or "ambiguous". It also checks if the incoming record is

linked with any existing biblionumbers. If it is, it sets the

linked_biblionumbers property to any linkages it finds.

"Ignore" means that this identifier already exists with the same

datestamp, or the incoming record has a deleted status and isn't

linked to any existing bib records in Koha.

"Deleted" means that the incoming record has a status of "deleted"

and contains no metadata, but shares an identifier with an imported

record in the Koha database. We record this status as evidence

that there has been a deletion in the remote OAI-PMH repository.

"Create" means that the identifier hasn't been logged before, or it

it has been logged and its record staged, but it has not been

imported, so we are free to create a new HistoricalRecord and

stage a new bib record.

"Replace" means that this identifier already exists, but the

datestamp is different, so we assume that this is an updated

version of the record. We check to see if there is a biblionumber

linked to this identifier (using import_record_id). If there is

a biblionumber, we store it so that we can use it for matching

during import.

However, if there is more than one biblionumber, we're in an 

"Ambiguous" situation where we can not reliably determine which

bib record we should be replacing. In this case, we store all

the bib numbers linked to this identifier, and treat it as a

new record. 

Ideally, we signal staff to perform manual merging

or deleting to remedy this one identifier to many biblionumber

scenario.

In theory, the "ambiguous" situation should never occur. However,

it's better to be safe than sorry. We don't want to accidentally

overwrite the wrong record during a "replace" update. 

=cut

sub AddIdentifierStatusAndLinkedBiblionumbers {

    my ( $self, $repository_id ) = @_;

    my $dbh = C4::Context->dbh;

    my $identifier_status;

    my @linked_biblionumbers;

    my $check_for_id_sql = "

    SELECT count(*) as count 

    FROM oai_harvest 

    WHERE identifier = ? and repository_id = ?";

    my $check_for_id_sth = $dbh->prepare($check_for_id_sql);

    $check_for_id_sth->execute($self->get_identifier, $repository_id);

    my $check_for_id_row = $check_for_id_sth->fetchrow_hashref;

    if ($check_for_id_row->{count} == 0){

        #OAI-PMH Unique Identifier doesn't exist in database == CREATE

        $identifier_status = "create";

    }

    else{

    #OAI-PMH Unique Identifier does exist in database == IGNORE || REPLACE 

        #FIXME: System preference to govern whether you match on identifier+datestamp || identifier+datestamp+metadataPrefix?

        my $check_for_id_and_datestamp_sql = "SELECT count(*) as count FROM oai_harvest WHERE identifier = ? and datestamp = ? and repository_id = ?";

        my $check_for_id_and_datestamp_sth = $dbh->prepare($check_for_id_and_datestamp_sql);

        $check_for_id_and_datestamp_sth->execute($self->get_identifier, $self->get_datestamp, $repository_id);

        my $check_for_id_and_datestamp_row = $check_for_id_and_datestamp_sth->fetchrow_hashref;

        if ($check_for_id_and_datestamp_row->{count} > 0){

            #OAI-PMH Unique Identifier and Datestamp combo exist in database == IGNORE

            #FIXME: What happens if the metadataPrefix is different? Perhaps do a REPLACE in this case? It would need to be tied to a system preference...

            $identifier_status = "ignore";

        }

        else{

            #OAI-PMH Unique Identifier and Datestamp combo don't exist in database == REPLACE

            #i.e. The identifier exists in the database, but this is an updated datestamp

            my $check_for_biblionumber_sql = "SELECT *

                FROM oai_harvest

                JOIN import_biblios using (import_record_id)

                WHERE identifier = ? and repository_id = ?";

            my $check_for_biblionumber_sth = $dbh->prepare($check_for_biblionumber_sql);

            $check_for_biblionumber_sth->execute($self->get_identifier, $repository_id);

            while ( my $check_for_biblionumber_row = $check_for_biblionumber_sth->fetchrow_hashref ) {

                if ($check_for_biblionumber_row->{matched_biblionumber}){

                    if (!grep(/^$check_for_biblionumber_row->{matched_biblionumber}$/, @linked_biblionumbers)){

                        push(@linked_biblionumbers,$check_for_biblionumber_row->{matched_biblionumber});

                    }

                }

            }

            if (scalar @linked_biblionumbers == 1){

                #If there is only one matching bib number for this record, we'll add it to our incoming 

                #record for import matching

                $identifier_status = "replace";

            }

            elsif (scalar @linked_biblionumbers == 0){

                #In this case, we've staged this OAI-PMH record before, but haven't imported it. 

                #Treat this as a CREATE

                $identifier_status = "create";

            }

            elsif (scalar @linked_biblionumbers > 1){

                #EXCEPTIONAL CASE - This means that we have imported this record as two different bibs before

                #As per wizzyrea's suggestion, email the administrator/librarian to tell them about this exceptional case

                #We can give them the bib numbers, the OAI-PMH identifier, and other information so they can manually

                #merge these bibs together...

                $identifier_status = "ambiguous";

            }

        }

    }

    if ($self->get_deleted){

        #If the incoming record has a deleted status, we'll want to

        #ignore it during import, since deleted records do not 

        #have any metadata attached.

        $identifier_status = "ignore";

    }

    if (scalar @linked_biblionumbers > 0){

        $self->set_linked_biblionumbers(\@linked_biblionumbers);

        #If the incoming record has a deleted status AND it is linked to

        #existing bib records that we've imported, we will want to keep 

        #evidence of this deleted status, so that we can alert staff to

        #delete these records.

        #We will commit an AddHistoricalRecord entry, but we won't import

        #this record into Koha (since there won't be any metadata to 

        #import anyway)

        #FIXME: Email the librarian/administrator mentioning that the record associated with XXX bibs have been deleted in the OAI-PMH repo?

        #FIXME: Suggest that they change the LEADER position 05 to "d" for deleted? How does Koha handle records with a LDR05 of "d"?

        if ($self->get_deleted){

            $identifier_status = "deleted";

        }

    }

    if ($identifier_status){

        $self->set_status($identifier_status);

    }

    return ($identifier_status);

}

1;

__END__

--

-- Table structure for table 'oai_harvest_repositories'

--

DROP TABLE IF EXISTS oai_harvest_repositories;

CREATE TABLE oai_harvest_repositories (

  repository_id int(11) NOT NULL AUTO_INCREMENT, -- primary key identifier

  baseURL text NOT NULL, -- baseURL of the remote OAI-PMH repository (mandatory)

  opt_from varchar(45) DEFAULT NULL, -- "from" time for selective harvesting (optional)

  opt_until varchar(45) DEFAULT NULL, -- "until" time for selective harvesting (optional)

  opt_set varchar(45) DEFAULT NULL, -- the record set to harvest for selective harvesting (optional)

  metadataPrefix varchar(45) NOT NULL, -- metadata format (e.g. oai_dc, dc, marcxml)

  active int(11) NOT NULL DEFAULT '0', -- indicate whether this repo is actively harvested by Koha

  timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, -- date last modified/created

  XSLT_path text, -- filepath to an XSLT file to use for transforming incoming records into a format Koha can use

  matcher_id int(11) DEFAULT NULL, -- id of the record matching rule to use for incoming records

  import_mode enum('manual','automatic') NOT NULL DEFAULT 'manual', -- automatically import records or leave in staging for manual import

  PRIMARY KEY (repository_id) USING BTREE

  ) ENGINE=InnoDB DEFAULT CHARSET=utf8;

--

-- Table structure for table 'oai_harvest'

--

DROP TABLE IF EXISTS oai_harvest;

CREATE TABLE oai_harvest (

  oai_harvest_id int(11) NOT NULL AUTO_INCREMENT, -- primary key identifier

  identifier varchar(255) NOT NULL, -- OAI-PMH identifier (unique to its original repo)

  datestamp varchar(45) NOT NULL, -- OAI-PMH datestamp (i.e. date created/modified/deleted)

  metadataPrefix varchar(45) NOT NULL, -- metadata format the record is in

  import_record_id int(11) DEFAULT NULL, -- the id in the import_records table. Used for import matching.

  linked_biblionumbers varchar(255) DEFAULT NULL, -- biblionumbers linked to this record. Usually only one, but multiple are separated with a pipe (i.e. |)

  metadata longtext, -- XML string containing the metadata (this will be null for incoming records marked as deleted)

  header longtext NOT NULL, -- XML string containing the header information (i.e. identifier, datestamp, status)

  about longtext, -- XML string containing explanatory information about a record (optional. Often not included in records)

  timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, -- timestamp when added/updated

  repository_id int(11) NOT NULL,  -- id for this record's OAI-PMH repository (linked to oai_harvest_repositories table)

  status varchar(45) DEFAULT NULL, -- whether or not the record has a status of deleted

  PRIMARY KEY (oai_harvest_id) USING BTREE,

  KEY FK_oai_harvest_1 (repository_id),

  CONSTRAINT FK_oai_harvest_1 FOREIGN KEY (repository_id) REFERENCES oai_harvest_repositories (repository_id) ON UPDATE NO ACTION

) ENGINE=InnoDB DEFAULT CHARSET=utf8;

$DBversion = "3.13.00.XXX";

if ( CheckVersion($DBversion) ) {

    $dbh->do("

CREATE TABLE IF NOT EXISTS oai_harvest_repositories (

  repository_id int(11) NOT NULL AUTO_INCREMENT,

  baseURL text NOT NULL,

  opt_from varchar(45) DEFAULT NULL,

  opt_until varchar(45) DEFAULT NULL,

  opt_set varchar(45) DEFAULT NULL,

  metadataPrefix varchar(45) NOT NULL,

  active int(11) NOT NULL DEFAULT '0',

  timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,

  XSLT_path text,

  matcher_id int(11) DEFAULT NULL,

  import_mode enum('manual','automatic') NOT NULL DEFAULT 'manual',

  PRIMARY KEY (repository_id) USING BTREE

  ) ENGINE=InnoDB DEFAULT CHARSET=utf8;

    ");

    $dbh->do("

CREATE TABLE IF NOT EXISTS oai_harvest (

  oai_harvest_id int(11) NOT NULL AUTO_INCREMENT,

  identifier varchar(255) NOT NULL,

  datestamp varchar(45) NOT NULL,

  metadataPrefix varchar(45) NOT NULL,

  import_record_id int(11) DEFAULT NULL,

  linked_biblionumbers varchar(255) DEFAULT NULL,

  metadata longtext,

  header longtext NOT NULL,

  about longtext,

  timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,

  repository_id int(11) NOT NULL,

  status varchar(45) DEFAULT NULL,

  PRIMARY KEY (oai_harvest_id) USING BTREE,

  KEY FK_oai_harvest_1 (repository_id),

  CONSTRAINT FK_oai_harvest_1 FOREIGN KEY (repository_id) REFERENCES oai_harvest_repositories (repository_id) ON UPDATE NO ACTION

) ENGINE=InnoDB DEFAULT CHARSET=utf8;

    ");

    print "Upgrade to $DBversion done (Bug 7494: Add itemBarcodeFallbackSearch syspref)\n";

    SetVersion($DBversion);

}

<?xml version="1.0" encoding="UTF-8"?>

<xsl:stylesheet version="1.0" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns="http://www.loc.gov/MARC21/slim" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" exclude-result-prefixes="dc">

	<xsl:output method="xml" indent="yes"/>

    <!-- pass in the OAI-PMH identifier to use for later matching -->

    <xsl:param name="identifier" select="identifier"/>

    <!-- pass in the Koha biblionumber associated with this OAI-PMH identifier to use for import matching -->

    <xsl:param name="biblionumber" select="biblionumber"/>

	<xsl:template match="/">

		<record xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" >

			<xsl:element name="leader">

				<xsl:variable name="type" select="//dc:type"/>

				<xsl:variable name="leader06">

					<xsl:choose>

						<xsl:when test="$type='collection'">p</xsl:when>

						<xsl:when test="$type='dataset'">m</xsl:when>

						<xsl:when test="$type='event'">r</xsl:when>

						<xsl:when test="$type='image'">k</xsl:when>

						<xsl:when test="$type='interactive resource'">m</xsl:when>

						<xsl:when test="$type='service'">m</xsl:when>

						<xsl:when test="$type='software'">m</xsl:when>

						<xsl:when test="$type='sound'">i</xsl:when>

						<xsl:when test="$type='text'">a</xsl:when>

						<xsl:otherwise>a</xsl:otherwise>

					</xsl:choose>

				</xsl:variable>

				<xsl:variable name="leader07">

					<xsl:choose>

						<xsl:when test="$type='collection'">c</xsl:when>

						<xsl:otherwise>m</xsl:otherwise>

					</xsl:choose>

				</xsl:variable>

                <!-- *00-04 Record length : leave blank. Koha will fill this in? -->

                <!-- 05* - Record status : should be "n" for new records but should be "c" for updated records. FIXME: How to take this into account? Can we pass in parameters? -->

                <!-- 06 - Type of record : handled by variable above -->

                <!-- 07 - Bibliographic level : handled by variable above -->

                <!-- 08 - Type of control : leave blank for non-archival -->

                <!-- 09 - Character coding scheme : "a" for Unicode -->

                <!-- 10 - Indicator count : This should always be 2 -->

                <!-- 11 - Subfield code count count : This should always be 2 -->

                <!-- *12-16 - Base address of data : leave blank. Koha will fill this in? -->

                <!-- 17* - Encoding level : "u" for unknown, UNLESS you're reasonably confident of the level used -->

                <!-- 18* - Descriptive cataloging form : "u" for unknown, UNLESS you know it is AACR2, ISBD, or neither -->

                <!-- 19* - Multipart resource record level : Blank for non-specified or not applicable, UNLESS record describes multiple physical parts bound in a single physical volume. NOT to be used for SERIALS. -->

                <!-- 20 - Length of the length-of-field portion : Should always be 4 -->

                <!-- 21 - Length of the starting-character-position portion : Should always be 5 -->

                <!-- 22 - Length of the implementation-defined portion : Should always be 0 -->

                <!-- 23 - Undefined : Should always be 0 -->

				<xsl:value-of select="concat('     n',$leader06,$leader07,' a22     uu 4500')"/>

			</xsl:element>

            <!-- dac2013: FIXME: Add a 008 field -->

            <!-- dac2013: Add record's OAI-PMH identifier for matching/provenance -->

            <!-- should this be a 035 or 9xx? It doesn't necessarily follow the form of the 035... -->

            <datafield tag="037" ind1="8" ind2=" ">

                <subfield code="a">

                    <xsl:value-of select="$identifier"/>

                </subfield>

            </datafield>

            <!-- dac2013: Add the biblionumber associated with this record's OAI-PMH identifier for matching -->

            <xsl:if test="$biblionumber != ''">

            <datafield tag="999" ind1=" " ind2=" ">

                <subfield code="c">

                    <xsl:value-of select="$biblionumber"/>

                </subfield>

                <subfield code="d">

                    <xsl:value-of select="$biblionumber"/>

                </subfield>

            </datafield>

            </xsl:if>

			<datafield tag="042" ind1=" " ind2=" ">

				<subfield code="a">dc</subfield>

			</datafield>

			<xsl:for-each select="//dc:contributor">

				<datafield tag="720" ind1="0" ind2="0">

					<subfield code="a">

						<xsl:value-of select="."/>

					</subfield>

					<subfield code="e">collaborator</subfield>

				</datafield>

			</xsl:for-each>

			<xsl:for-each select="//dc:coverage">

				<datafield tag="500" ind1=" " ind2=" ">

					<subfield code="a">

						<xsl:value-of select="."/>

					</subfield>

				</datafield>

			</xsl:for-each>

			<xsl:for-each select="//dc:creator">

				<datafield tag="720" ind1=" " ind2=" ">

					<subfield code="a">

						<xsl:value-of select="."/>

					</subfield>

					<subfield code="e">author</subfield>

				</datafield>

			</xsl:for-each>

			<xsl:for-each select="//dc:date">

				<datafield tag="260" ind1=" " ind2=" ">

					<subfield code="c">

						<xsl:value-of select="."/>

					</subfield>

				</datafield>

			</xsl:for-each>

			<xsl:for-each select="//dc:publisher">

				<datafield tag="260" ind1=" " ind2=" ">

					<subfield code="b">

						<xsl:value-of select="."/>

					</subfield>

				</datafield>

			</xsl:for-each>

			<xsl:for-each select="//dc:description">

				<datafield tag="520" ind1=" " ind2=" ">

					<subfield code="a">

						<xsl:value-of select="."/>

					</subfield>

				</datafield>

			</xsl:for-each>

			<xsl:for-each select="//dc:format">

				<datafield tag="856" ind1=" " ind2=" ">

					<subfield code="q">

						<xsl:value-of select="."/>

					</subfield>

				</datafield>

			</xsl:for-each>

			<xsl:for-each select="//dc:identifier">

				<datafield tag="024" ind1="8" ind2=" ">

					<subfield code="a">

						<xsl:value-of select="."/>

					</subfield>

				</datafield>

			</xsl:for-each>

			<xsl:for-each select="//dc:language">

				<datafield tag="546" ind1=" " ind2=" ">

					<subfield code="a">

						<xsl:value-of select="."/>

					</subfield>

				</datafield>

			</xsl:for-each>

			<xsl:for-each select="//dc:relation">

				<datafield tag="787" ind1="0" ind2=" ">

					<subfield code="n">

						<xsl:value-of select="."/>

					</subfield>

				</datafield>

			</xsl:for-each>

			<xsl:for-each select="//dc:rights">

				<datafield tag="540" ind1=" " ind2=" ">

					<subfield code="a">

						<xsl:value-of select="."/>

					</subfield>

				</datafield>

			</xsl:for-each>

			<xsl:for-each select="//dc:source">

				<datafield tag="786" ind1="0" ind2=" ">

					<subfield code="n">

						<xsl:value-of select="."/>

					</subfield>

				</datafield>

			</xsl:for-each>

			<xsl:for-each select="//dc:subject">

				<datafield tag="653" ind1=" " ind2=" ">

					<subfield code="a">

						<xsl:value-of select="."/>

					</subfield>

				</datafield>

			</xsl:for-each>

			<xsl:for-each select="//dc:title[1]">

				<datafield tag="245" ind1="0" ind2="0">

					<subfield code="a">

						<xsl:value-of select="."/>

					</subfield>

				</datafield>

			</xsl:for-each>

			<xsl:for-each select="//dc:title[position()>1]">

				<datafield tag="246" ind1="3" ind2="3">

					<subfield code="a">

						<xsl:value-of select="."/>

					</subfield>

				</datafield>

			</xsl:for-each>

			<xsl:for-each select="//dc:type">

				<datafield tag="655" ind1="7" ind2=" ">

					<subfield code="a">

						<xsl:value-of select="."/>

					</subfield>

					<subfield code="2">local</subfield>

				</datafield>

			</xsl:for-each>

		</record>

	</xsl:template>

</xsl:stylesheet>

<?xml version="1.0" encoding="UTF-8"?>

<xsl:stylesheet version="1.0" xmlns:marc="http://www.loc.gov/MARC21/slim" xmlns:items="http://www.koha-community.org/items" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" >

	<xsl:output method="xml" indent="yes"/>

    <!-- pass in the OAI-PMH identifier to use for later matching -->

    <xsl:param name="identifier" select="identifier"/>

    <!-- pass in the Koha biblionumber associated with this OAI-PMH identifier to use for import matching -->

    <xsl:param name="biblionumber" select="biblionumber"/>

    <!-- this block copies the existing XML verbatim -->

	<xsl:template match="@*|node()">

      <xsl:copy>

        <xsl:apply-templates select="@*|node()"/>

      </xsl:copy>

    </xsl:template>

    <!-- dac2013: Strip out any existing 999 datafields, since these will have biblionumbers from a different Koha (or another system) and not this one -->

    <xsl:template match="marc:datafield[@tag=999]" />

    <!-- this template matches on the "record" node -->

    <xsl:template match="marc:record">

      <record>

         <!-- Copy all the existing controlfield, datafield, subfield nodes -->

         <xsl:apply-templates select="@* | *"/>

         <!-- Add a MARC datafield that includes the unique OAI-PMH identifier -->

         <xsl:text>

      </xsl:text>

        <xsl:element name="datafield" xmlns="http://www.loc.gov/MARC21/slim">

          <xsl:attribute name="ind1"><xsl:text>8</xsl:text></xsl:attribute>

          <xsl:attribute name="ind2"><xsl:text> </xsl:text></xsl:attribute>

          <xsl:attribute name="tag">037</xsl:attribute>

          <xsl:text>

        </xsl:text>

          <xsl:element name="subfield">

            <xsl:attribute name="code">a</xsl:attribute>

            <xsl:value-of select="$identifier"/>

          </xsl:element>

        </xsl:element>

      <!-- dac2013: Add the biblionumber associated with this identifier, if there is one -->

      <xsl:if test="$biblionumber != ''">

        <xsl:text>

      </xsl:text>

        <xsl:element name="datafield" xmlns="http://www.loc.gov/MARC21/slim">

          <xsl:attribute name="ind1"><xsl:text> </xsl:text></xsl:attribute>

          <xsl:attribute name="ind2"><xsl:text> </xsl:text></xsl:attribute>

          <xsl:attribute name="tag">999</xsl:attribute>

          <xsl:text>

        </xsl:text>

          <xsl:element name="subfield">

            <xsl:attribute name="code">c</xsl:attribute>

            <xsl:value-of select="$biblionumber"/>

          </xsl:element>

          <xsl:element name="subfield">

            <xsl:attribute name="code">d</xsl:attribute>

            <xsl:value-of select="$biblionumber"/>

          </xsl:element>

        </xsl:element>

        </xsl:if>

      </record>

    </xsl:template>

</xsl:stylesheet>

#!/usr/bin/perl

# This file is part of Koha.

#

# Copyright 2013 Prosentient Systems

#

# Koha is free software; you can redistribute it and/or modify it under the

# terms of the GNU General Public License as published by the Free Software

# Foundation; either version 3 of the License, or (at your option) any later

# version.

#

# Koha is distributed in the hope that it will be useful, but WITHOUT ANY

# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR

# A PARTICULAR PURPOSE. See the GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License along

# with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;

use C4::Context;

use Data::Dumper;

use C4::OAI::Harvester;

#DATA - Retrieve data from the database

my $dbh1 = C4::Context->dbh;

my $sql1 = "SELECT * FROM oai_harvest_repositories WHERE repository_id = ?";

my $sth1 = $dbh1->prepare($sql1);

$sth1->execute(1);

my $data = $sth1->fetchrow_hashref;

$data->{resume} = 0; #Comment this out or change to 1 if you want automatic token resumption

#OBJECT CREATION

my $test = C4::OAI::Harvester->new($data);

#OBJECT METHODS

#List data about the OAI-PMH repository

my $identify_repository = $test->Identify;

print Dumper($identify_repository);

#List available sets

my @sets = $test->ListSets;

print Dumper(@sets);

#List available metadata formats

my @formats = $test->ListMetadataFormats;

print Dumper(@formats);

#List available identifiers/headers

#my @headers = $test->ListIdentifiers;

#print Dumper(@headers);

#Retrieve all records

my @records = $test->ListRecords;

print Dumper(@records);

#Retrieve a particular record

#my @records = $test->GetRecord('KOHAOAITEST:286');

#print Dumper(@records);

$test->ImportRecordsIntoKoha(@records);