Attachment 21954 Details for Bug 10662 – Bug 10662 - Build OAI-PMH Harvesting Client

[patch] Bug 10662 - Build OAI-PMH Harvesting Client

Bug-10662---Build-OAI-PMH-Harvesting-Client.patch (text/plain), 85.28 KB, created by David Cook on 2013-10-11 01:11:44 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: David Cook

Created: 2013-10-11 01:11:44 UTC

Size: 85.28 KB

patch

obsolete

>From ed6f1a0e18f85a99847ff81fa624bef5d5a2c712 Mon Sep 17 00:00:00 2001
>From: David Cook <dcook@prosentient.com.au>
>Date: Mon, 5 Aug 2013 15:26:17 +1000
>Subject: [PATCH] Bug 10662 - Build OAI-PMH Harvesting Client
>
>N.B. This feature is still a work in progress. This commit represents
>my work to date on the OAI-PMH harvester, but it's not complete...
>
>N.B. The test is currently broken (as it was created for an older version
>of the code).
>---
> C4/OAI/Client/Harvester.pm                         |  556 ++++++++++++++++++++
> C4/OAI/Client/Record.pm                            |  328 ++++++++++++
> admin/oai_repository_targets.pl                    |  168 ++++++
> installer/data/mysql/kohastructure.sql             |   41 ++
> installer/data/mysql/updatedatabase.pl             |   40 ++
> .../intranet-tmpl/prog/en/includes/admin-menu.inc  |    1 +
> .../prog/en/modules/admin/admin-home.tt            |    2 +
> .../en/modules/admin/oai_repository_targets.tt     |  224 ++++++++
> .../intranet-tmpl/prog/en/xslt/DC2MARC21slim.xsl   |  217 ++++++++
> .../prog/en/xslt/MARC21slim2MARC21enhanced.xsl     |   43 ++
> misc/cronjobs/oai_harvester.pl                     |   95 ++++
> t/db_dependent/OAI_harvester.t                     |  308 +++++++++++
> 12 files changed, 2023 insertions(+), 0 deletions(-)
> create mode 100755 C4/OAI/Client/Harvester.pm
> create mode 100755 C4/OAI/Client/Record.pm
> create mode 100755 admin/oai_repository_targets.pl
> create mode 100755 koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_repository_targets.tt
> create mode 100755 koha-tmpl/intranet-tmpl/prog/en/xslt/DC2MARC21slim.xsl
> create mode 100755 koha-tmpl/intranet-tmpl/prog/en/xslt/MARC21slim2MARC21enhanced.xsl
> create mode 100755 misc/cronjobs/oai_harvester.pl
> create mode 100755 t/db_dependent/OAI_harvester.t
>
>diff --git a/C4/OAI/Client/Harvester.pm b/C4/OAI/Client/Harvester.pm
>new file mode 100755
>index 0000000..533adb2
>--- /dev/null
>+++ b/C4/OAI/Client/Harvester.pm
>@@ -0,0 +1,556 @@
>+package C4::OAI::Client::Harvester;
>+
>+# This file is part of Koha.
>+#
>+# Copyright 2013 Prosentient Systems
>+#
>+# Koha is free software; you can redistribute it and/or modify it under the
>+# terms of the GNU General Public License as published by the Free Software
>+# Foundation; either version 3 of the License, or (at your option) any later
>+# version.
>+#
>+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
>+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
>+# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
>+#
>+# You should have received a copy of the GNU General Public License along
>+# with Koha; if not, see <http://www.gnu.org/licenses>.
>+
>+=head1 NAME
>+
>+C4::OAI::Harvester - OAI-PMH harvester/client which implements the 6 OAI-PMH verbs, record retrieval/harvesting, and import into Koha
>+
>+=head1 SYNOPSIS
>+
>+use C4::OAI::Harvester;
>+my $oai_repo = C4::OAI::Harvester->new($repository_data);
>+
>+my $identify_repository = $oai_repo->Identify;
>+
>+my @sets = $oai_repo->ListSets;
>+
>+my @formats = $oai_repo->ListMetadataFormats;
>+
>+my @headers = $oai_repo->ListIdentifiers;
>+
>+my @records = $oai_repo->ListRecords;
>+
>+my @records = $oai_repo->GetRecord($oai_unique_identifier);
>+
>+my $import_mode = ''; #i.e. not "automatic"
>+
>+$oai_repo->ImportRecordsIntoKoha($import_mode,@records);
>+
>+=head1 DESCRIPTION
>+
>+C4::OAI::Harvester contains functions for querying and harvesting OAI-PMH repositories.
>+
>+More information on OAI-PMH can be found L<here|http://www.openarchives.org/OAI/openarchivesprotocol.html>
>+
>+=head1 FUNCTIONS
>+
>+=head1 AUTHOR
>+
>+David Cook <dcook AT prosentient DOT com DOT au>
>+
>+=cut
>+
>+use Modern::Perl;
>+use HTTP::OAI;
>+use C4::Context;
>+use C4::OAI::Client::Record;
>+use C4::Biblio qw /AddBiblio ModBiblio DelBiblio/;
>+
>+use base qw(Class::Accessor);
>+
>+
>+sub AddOAIRepository {
>+    my ( $database_fields ) = @_;
>+    my $dbh = C4::Context->dbh;
>+    my $sql = "
>+      INSERT INTO oai_harvest_repositories
>+      (baseURL,metadataPrefix,opt_from,opt_until,opt_set,active,XSLT_path,frameworkcode,comments)
>+      VALUES (?,?,?,?,?,?,?,?,?)
>+    ";
>+    my $sth = $dbh->prepare($sql);
>+    $sth->execute(
>+        $database_fields->{baseURL},
>+        $database_fields->{metadataPrefix},
>+        $database_fields->{opt_from},
>+        $database_fields->{opt_until},
>+        $database_fields->{opt_set},
>+        $database_fields->{active},
>+        $database_fields->{XSLT_path},
>+        $database_fields->{frameworkcode},
>+        $database_fields->{comments}
>+     );
>+     if ($sth->err){
>+        die $sth->errstr;
>+    }
>+}
>+
>+sub ModOAIRepositoryDateTimeFrom {
>+    my ( $repository_id, $datetime_from ) = @_;
>+    my $dbh = C4::Context->dbh;
>+    my $sql = "
>+        UPDATE oai_harvest_repositories
>+        SET opt_from = ?
>+        WHERE repository_id = ?
>+    ";
>+    my $sth = $dbh->prepare($sql);
>+    $sth->execute($datetime_from,$repository_id);
>+    if ($sth->err){
>+       die $sth->errstr;
>+    }
>+}
>+
>+sub ModOAIRepository {
>+    my ( $database_fields ) = @_;
>+    my $dbh = C4::Context->dbh;
>+    my $sql = "
>+        UPDATE oai_harvest_repositories
>+        SET baseURL = ?,
>+        metadataPrefix = ?,
>+        opt_from = ?,
>+        opt_until = ?,
>+        opt_set = ?,
>+        active = ?,
>+        XSLT_path = ?,
>+        frameworkcode = ?,
>+        comments = ?
>+        WHERE repository_id = ?
>+    ";
>+    my $sth = $dbh->prepare($sql);
>+    $sth->execute(
>+        $database_fields->{baseURL},
>+        $database_fields->{metadataPrefix},
>+        $database_fields->{opt_from},
>+        $database_fields->{opt_until},
>+        $database_fields->{opt_set},
>+        $database_fields->{active},
>+        $database_fields->{XSLT_path},
>+        $database_fields->{frameworkcode},
>+        $database_fields->{comments},
>+        $database_fields->{repository_id}
>+     );
>+    if ($sth->err){
>+       die $sth->errstr;
>+    }
>+}
>+
>+sub DelOAIRepository {
>+    my ( $repository_id ) = @_;
>+    my $error;
>+    my $dbh = C4::Context->dbh;
>+    my $sql = "
>+        DELETE FROM oai_harvest_repositories
>+        WHERE repository_id = ?
>+    ";
>+    my $sth = $dbh->prepare($sql);
>+    $sth->execute($repository_id);
>+    if ($sth->err){
>+       die $sth->errstr;
>+    }
>+}
>+
>+sub GetLatestHistoricalRecordDatestamp {
>+    my ( $repository_id  ) = @_;
>+    my $latest_datestamp;
>+    my $dbh = C4::Context->dbh;
>+    my $sql = "
>+        SELECT datestamp
>+        FROM oai_harvest
>+        WHERE repository_id = ?
>+        ORDER BY datestamp desc
>+        LIMIT 1
>+    ";
>+    my $sth = $dbh->prepare($sql);
>+    $sth->execute($repository_id);
>+    my $row = $sth->fetchrow_hashref;
>+    $latest_datestamp = $row->{datestamp} ? $row->{datestamp} : undef;
>+    return $latest_datestamp;
>+}
>+
>+sub GetOAIRepository {
>+    my ( $repository_id ) = @_;
>+    my $dbh = C4::Context->dbh;
>+    my $sql = "
>+      SELECT *
>+      FROM oai_harvest_repositories
>+      WHERE repository_id = ?
>+    ";
>+    my $sth = $dbh->prepare($sql);
>+    $sth->execute($repository_id);
>+    if ($sth->err){
>+       die $sth->errstr;
>+    }
>+    my $row = $sth->fetchrow_hashref;
>+    return $row;
>+}
>+
>+=head2 GetOAIRepositoryList
>+
>+  my @repositories = C4::OAI::Harvester::GetOAIRepositoryList();
>+  my @repositories = C4::OAI::Harvester::GetOAIRepositoryList($active);
>+
>+Returns an array of hashrefs listing all OAI-PMH repositories
>+present in the database.
>+
>+If the $active is included, then it will return an array
>+of hashrefs listing all OAI-PMH repositories depending on their
>+active status. $active == 1 shows all active. $active == 0 shows
>+all inactive.
>+
>+=cut
>+
>+sub GetOAIRepositoryList {
>+    my ( $active ) = @_;
>+    my $dbh = C4::Context->dbh;
>+    my @results;
>+    my $sql = "
>+        SELECT *
>+        FROM oai_harvest_repositories
>+    ";
>+    if (defined $active){
>+        $sql .= " WHERE active = 1 " if $active == 1;
>+        $sql .= " WHERE active = 0 " if $active == 0;
>+    }
>+    my $sth = $dbh->prepare($sql);
>+    $sth->execute;
>+    while (my $row = $sth->fetchrow_hashref){
>+        push(@results,$row);
>+    }
>+    return @results;
>+}
>+
>+#TODO: Perhaps create a sub that cleans out the metadata column to keep the table size low?
>+
>+sub new {
>+    my($proto, $fields) = @_;
>+    my($class) = ref $proto || $proto;
>+
>+    $fields = {} unless defined $fields;
>+
>+    if ($fields->{'baseURL'}){
>+        my $h = new HTTP::OAI::Harvester(
>+            baseURL => $fields->{'baseURL'},
>+        );
>+        #If resume is set to 0, automatic token resumption is turned off. This is useful for testing/debugging.
>+        if ($h && exists $fields->{'resume'}){
>+            if ($fields->{'resume'} == 0){
>+                $h->resume(0);
>+            }
>+        }
>+        my $response = $h->repository($h->Identify);
>+        if( $response->is_error ) {
>+            print "Error requesting Identify:\n",
>+                    $response->code . " " . $response->message, "\n";
>+            exit;
>+        }
>+        $fields->{rh} = $h; #Store HTTP::OAI::Harvester object as "repository handle"
>+    }
>+    bless {%$fields}, $class;
>+}
>+
>+__PACKAGE__->follow_best_practice; #Use get_ and set_ prefixes for accessors
>+__PACKAGE__->mk_accessors(qw(baseURL opt_from opt_until opt_set metadataPrefix rh repository_id XSLT_path debug frameworkcode));
>+
>+=head2 OAI-PMH Verbs
>+
>+Koha-specific implementations of the 6 OAI-PMH Verbs.
>+
>+The key verbs are "ListRecords" and "GetRecords". These do the actual
>+harvesting of records from a OAI-PMH repository. The others are useful for
>+getting information about a repository and what it has available.
>+
>+1) ListRecords
>+
>+2) GetRecord
>+
>+3) ListIdentifiers
>+
>+4) ListMetadataFormats
>+
>+5) ListSets
>+
>+6) Identify
>+
>+=cut
>+
>+
>+sub ListRecords {
>+    my ( $self ) = @_;
>+    my %args = (
>+        metadataPrefix => $self->{metadataPrefix},
>+        from => $self->{opt_from},
>+        until => $self->{opt_until},
>+        set => $self->{opt_set},
>+    );
>+    if ($self->{debug}){
>+        use Data::Dumper;
>+        print "ListRecords's arguments\n";
>+        print Dumper(\%args);
>+    }
>+    my $response = $self->{rh}->ListRecords(%args);
>+    if( $response->is_error ) {
>+        print "Error requesting ListRecords:\n",
>+                $response->code . " " . $response->message, "\n";
>+        exit;
>+    }
>+    if ($self->{debug}){
>+        print "Successfully retrieved ListRecords response.\n";
>+    }
>+    return $response;
>+}
>+
>+sub GetRecord {
>+    my ( $self, $identifier, $harvest ) = @_;
>+    my $response = $self->{rh}->GetRecord(
>+        metadataPrefix => $self->{metadataPrefix},
>+        identifier => $identifier,
>+    );
>+    if( $response->is_error ) {
>+        print "Error requesting GetRecord:\n",
>+                $response->code . " " . $response->message, "\n";
>+        exit;
>+    }
>+    if ($self->{debug}){
>+        print "Successfully retrieved GetRecord response.\n";
>+    }
>+    return $response;
>+}
>+
>+sub ListIdentifiers {
>+    my $self = shift;
>+    my @headers;
>+    my $response = $self->{rh}->ListIdentifiers(
>+        metadataPrefix => $self->{metadataPrefix},
>+        from => $self->{opt_from},
>+        until => $self->{opt_until},
>+        set => $self->{opt_set},
>+    );
>+    if( $response->is_error ) {
>+        print "Error requesting ListIdentifiers:\n",
>+                $response->code . " " . $response->message, "\n";
>+        exit;
>+    }
>+    while (my $h = $response->next){
>+        my $header;
>+        #print Dumper($h->dom->toString); #DEBUG: XML representation
>+        $header->{identifier} = $h->identifier;
>+        $header->{datestamp} = $h->datestamp;
>+
>+        $header->{status} = $h->status;
>+        $header->{is_deleted} = $h->is_deleted;
>+
>+        my @sets = $h->setSpec;
>+        $header->{sets} = \@sets;
>+
>+        push (@headers,$header);
>+    }
>+    return \@headers;
>+}
>+
>+sub ListMetadataFormats {
>+    my ( $self, $identifier ) = @_;
>+    my @formats;
>+    my $response = $self->{rh}->ListMetadataFormats(
>+        identifier => $identifier,
>+    );
>+    if( $response->is_error ) {
>+        print "Error requesting ListMetadataFormats:\n",
>+                $response->code . " " . $response->message, "\n";
>+        exit;
>+    }
>+    for($response->metadataFormat) {
>+        push(@formats,$_->metadataPrefix);
>+    }
>+    return \@formats;
>+}
>+
>+sub ListSets {
>+    my $self = shift;
>+    my @sets;
>+    my $response = $self->{rh}->ListSets();
>+    if( $response->is_error ) {
>+        print "Error requesting ListSets:\n",
>+                $response->code . " " . $response->message, "\n";
>+        exit;
>+    }
>+    while (my $s = $response->next){
>+        my $set;
>+        $set->{setSpec} = $s->setSpec;
>+        $set->{setName} = $s->setName;
>+
>+        #TODO: Not really sure what to do with the descriptions as they're XML and not necessarily that easy to parse for GUI views...
>+        #my @temp_setDescriptions = $s->setDescription;
>+        #my @setDescriptions;
>+        #foreach my $temp_setDescription (@temp_setDescriptions){
>+        #    push (@setDescriptions,$temp_setDescription->dom->toString); I think we need to do better than just return the setDescription XML...That's not very useful...
>+        #}
>+        #$set->{setDescription} = \@setDescriptions;
>+        push (@sets,$set);
>+    }
>+    return \@sets;
>+}
>+
>+sub Identify {
>+    my $self = shift;
>+    my $response = $self->{rh}->Identify();
>+    if( $response->is_error ) {
>+        print "Error requesting Identify:\n",
>+        $response->code . " " . $response->message, "\n";
>+        exit;
>+    }
>+    my $identify_data;
>+    #DEBUG: View what's in the Identify object
>+    #print Dumper($response->headers);
>+
>+    $identify_data->{repositoryName} = $response->repositoryName;
>+    $identify_data->{baseURL} = $response->baseURL;
>+    $identify_data->{protocolVersion} = $response->protocolVersion; #Tim Brody says this will always return 2.0 and that ->version should be used to find the actual version...
>+    #$identify_data->{version} = $response->version;
>+    $identify_data->{earliestDatestamp} = $response->earliestDatestamp;
>+    $identify_data->{deletedRecord} = $response->deletedRecord; #not in the perldoc, but it's in the code and the OAI-PMH spec
>+    $identify_data->{granularity} = $response->granularity;
>+
>+    #These methods should be used with an array context so they return all the elements and not just the first one
>+    my @adminEmails = $response->adminEmail;
>+    $identify_data->{adminEmail} = \@adminEmails;
>+    my @compressions = $response->compression;
>+    $identify_data->{compression} = \@compressions;
>+
>+    #TODO: Descriptions are encapsulated in XML containers, I believe. Not sure what to do with these at present...
>+    #my @descriptions = $response->description;
>+    #$identify_data->{description} = \@descriptions;
>+    #$response->next
>+
>+    return $identify_data;
>+}
>+
>+sub ProcessOaipmhRecordResponse {
>+    my ( $self, $response, $harvest, $force ) = @_;
>+    if ($response){
>+        my @records;
>+        while( my $rec = $response->next ){
>+            print "I'm parsing the record..." if $self->{debug};
>+            #Parse from the HTTP::OAI::Record object into our C4::OAI::Client::Record object
>+            my $record_object = _parse_httpoai_records_into_clientoai_records($rec,$self->{repository_id},$self->{metadataPrefix});
>+
>+            #If set to harvest mode, we ingest the record
>+            if ($harvest && $record_object){
>+
>+                #Retrieve the latest biblionumber associated with this OAI identifier
>+                $record_object->GetBiblionumberFromOaiIdentifier;
>+
>+                #Generate a status to be acted upon (e.g. add, update, delete, force_add, force_update, ignore)
>+                $record_object->GenerateStatus($force);
>+
>+                print "I'm ingesting the record..." if $self->{debug};
>+                #Based on the above status, act upon the record
>+                my $ingestion_flag  = _ingest_oai_record($record_object, $self->get_XSLT_path, $self->get_frameworkcode);
>+
>+                if ($ingestion_flag){
>+                    #Log this OAI-PMH record in the database
>+                    $record_object->AddRecordToLog;
>+                    print $record_object->get_identifier.' -> '.$record_object->get_status.' -> '.$record_object->get_biblionumber." \n ";
>+                }
>+            }
>+            print $rec->identifier."\n" if !$harvest;
>+            push(@records,$record_object) if !$harvest;
>+        }
>+        return \@records;
>+    }
>+}
>+
>+sub _ingest_oai_record {
>+    my ( $oai_record, $xslt_path, $frameworkcode ) = @_;
>+
>+    if ( $oai_record && $xslt_path ){
>+        my $identifier_status = $oai_record->get_status;
>+        if ($identifier_status){
>+            if ($identifier_status eq 'ignore'){
>+                print $oai_record->get_identifier.' -> '.$identifier_status."\n";
>+                return;
>+            }
>+            elsif ($identifier_status eq 'add' || $identifier_status eq 'force_add'){
>+                $oai_record->TransformMetadata($xslt_path);
>+                if ($oai_record->get_transformed_metadata){
>+                    my $record = $oai_record->GenerateMarcObject($oai_record->get_transformed_metadata->toString);
>+                    my ($biblionumber,$biblioitemnumber) = C4::Biblio::AddBiblio($record,$frameworkcode);
>+                    $oai_record->set_biblionumber($biblionumber) if $biblionumber;
>+                    return 1;
>+                } else {
>+                    return;
>+                }
>+            }
>+            elsif (($identifier_status eq 'update' || $identifier_status eq 'force_update') && $oai_record->get_biblionumber){
>+                $oai_record->TransformMetadata($xslt_path);
>+                if ($oai_record->get_transformed_metadata){
>+                    my $record = $oai_record->GenerateMarcObject($oai_record->get_transformed_metadata->toString);
>+                    if ($oai_record->GetBiblionumberExistence){
>+                        C4::Biblio::ModBiblio($record,$oai_record->get_biblionumber,$frameworkcode);
>+                        return 1;
>+                    } else {
>+                        print "Error modding bib record ".$oai_record->get_biblionumber.", as it does not exist anymore \n";
>+                        $oai_record->set_status($identifier_status."_fail");
>+                        return 1;
>+                    }
>+                } else {
>+                    return;
>+                }
>+            }
>+            elsif ($identifier_status eq 'delete'){
>+                if ($oai_record->get_biblionumber){
>+                    if ($oai_record->GetBiblionumberExistence){
>+                        my $error = C4::Biblio::DelBiblio($oai_record->get_biblionumber);
>+                        if ($error){
>+                            warn "Error deleting biblionumber ".$oai_record->get_biblionumber." -> $error";
>+                            #NOTE: This will fail if there are items attached to the bib record
>+                            #So...just record the failure due to items being attached rather than trying to delete items.
>+                            $oai_record->set_status($identifier_status."_fail");
>+                            return 1;
>+                        }
>+                        else {
>+                            return 1;
>+                        }
>+                    } else {
>+                        print "It looks like biblionumber ".$oai_record->get_biblionumber." has already been deleted \n";
>+                        return;
>+                    }
>+                } else {
>+                    print "There is no biblionumber associated with this deletion record";
>+                    return;
>+                }
>+            }
>+        } else {
>+            print $oai_record->get_identifier ." -> no status (so no action taken)\n";
>+            return;
>+        }
>+    }
>+}
>+
>+sub _parse_httpoai_records_into_clientoai_records {
>+    my ( $rec, $repository_id, $metadataPrefix ) = @_;
>+        my $record;
>+        $record->{header} = $rec->header->dom;
>+        $record->{identifier} = $rec->identifier;
>+        $record->{datestamp} = $rec->datestamp;
>+        $record->{metadata} = $rec->metadata ? $rec->metadata->dom : undef; #N.B. there won't be any metadata for deleted records
>+        $record->{about} = $rec->about ? $rec->about->dom : undef;
>+        $record->{deleted} = $rec->is_deleted;
>+        $record->{status} = $rec->status;
>+        $record->{repository_id} = $repository_id;
>+        $record->{metadataPrefix} = $metadataPrefix;
>+        my $record_object = C4::OAI::Client::Record->new($record);
>+        if ($record_object){
>+            return $record_object;
>+        } else {
>+            return;
>+        }
>+}
>+
>+
>+
>+1;
>+__END__
>\ No newline at end of file
>diff --git a/C4/OAI/Client/Record.pm b/C4/OAI/Client/Record.pm
>new file mode 100755
>index 0000000..5cabc2a
>--- /dev/null
>+++ b/C4/OAI/Client/Record.pm
>@@ -0,0 +1,328 @@
>+package C4::OAI::Client::Record;
>+
>+=head1 NAME
>+
>+C4::OAI::Client::Record - an internal class for handling records retrieved via the OAI-PMH protocol
>+
>+=head1 DESCRIPTION
>+
>+C4::OAI::Client::Record - Class to handle the management of records retrieved via OAI-PMH
>+
>+More information on OAI-PMH can be found L<here|http://www.openarchives.org/OAI/openarchivesprotocol.html>
>+
>+=head1 AUTHOR
>+
>+David Cook <dcook AT prosentient DOT com DOT au>
>+
>+=cut
>+
>+use Modern::Perl;
>+use C4::Context;
>+use C4::Templates qw/_current_language/;
>+use C4::Charset qw/StripNonXmlChars/;
>+use C4::XSLT qw/GetURI/;
>+
>+use XML::LibXML;
>+use XML::LibXSLT;
>+use MARC::Record;
>+use MARC::File::XML;
>+use base qw(Class::Accessor);
>+
>+sub new {
>+    my($proto, $fields) = @_;
>+    my($class) = ref $proto || $proto;
>+    $fields = {} unless defined $fields;
>+    bless {%$fields}, $class;
>+}
>+
>+__PACKAGE__->follow_best_practice; #Use get_ and set_ prefixes for accessors
>+__PACKAGE__->mk_accessors(qw(identifier datestamp metadata status deleted header about transformed_metadata repository_id metadataPrefix biblionumber));
>+
>+=head2 TransformMetadata
>+
>+
>+
>+=cut
>+
>+sub TransformMetadata {
>+    my ( $self, $XSLT_path ) = @_;
>+    my $source = $self->{metadata}; #This is a LibXML object retrieved via HTTP::OAI::Harvester
>+    my $metadataPrefix = $self->get_metadataPrefix;
>+
>+    my $no_XSLT_path;
>+    if (!$XSLT_path){
>+        $no_XSLT_path = 1;
>+    }
>+
>+    if ($metadataPrefix && $source){
>+        my $xslt = XML::LibXSLT->new();
>+        my $style_doc;
>+
>+        #$parser based on equivalent from XSLT.pm
>+        my $parser = XML::LibXML->new();
>+        # don't die when you find &, >, etc
>+        $parser->recover_silently(0);
>+
>+        #NOTE: This XSLT essentially copies over all MARC fields except the 952 and 999.
>+        #It also adds a 037 containing the OAI-PMH identifier.
>+        my $default_marc_xslt = C4::Context->config('intrahtdocs') .
>+                                    '/' . C4::Context->preference("template") .
>+                                    '/' . C4::Templates::_current_language() .
>+                                    '/xslt/' .
>+                                    C4::Context->preference('marcflavour') .
>+                                    "slim2" .
>+                                    C4::Context->preference('marcflavour') .
>+                                    "enhanced.xsl";
>+
>+        #TODO: This XSLT needs work. In practice, it would be better to create
>+        #customized XSLTs based on the metadata format native to the OAI-PMH source (e.g. DIM for DSpace)
>+        my $default_dc_xslt = C4::Context->config('intrahtdocs') .
>+                                    '/' . C4::Context->preference("template") .
>+                                    '/' . C4::Templates::_current_language() .
>+                                    '/xslt/DC2' .
>+                                    C4::Context->preference('marcflavour') .
>+                                    "slim.xsl";
>+
>+        #TODO: As necessary, add more default XSLTs. Perhaps MODS and METS (although METS would be difficult
>+        #since it can contain other metadata formats within itself).
>+
>+        if ($XSLT_path){
>+            #This conditional for handling secure remote XSLTs copied from XSLT.pm
>+            if ( $XSLT_path =~ /^https?:\/\// ) {
>+                my $xsltstring = GetURI($XSLT_path);
>+                if ($xsltstring){
>+                    $style_doc = $parser->parse_string($xsltstring);
>+                }
>+                else{
>+                    #If a string isn't retrieved using GetURI, we switch to our default transforms
>+                    $no_XSLT_path  = 1;
>+                }
>+            } elsif ( $XSLT_path eq 'dublincore' ) {
>+                $style_doc = $parser->parse_file($default_dc_xslt); #Use default DC XSLT if XSLT_path is specified as 'dublincore'
>+            } elsif ( $XSLT_path eq 'marc' ) {
>+                $style_doc = $parser->parse_file($default_marc_xslt); #Use default MARC XSLT if XSLT_path is specified as 'marc'
>+            } else {
>+                if ( -e $XSLT_path){
>+                    $style_doc = $parser->parse_file($XSLT_path);
>+                }
>+                else{
>+                    #If the file doesn't actually exist, we switch to our default transforms
>+                    $no_XSLT_path = 1;
>+                }
>+            }
>+        }
>+
>+        if ($no_XSLT_path){
>+            if ($metadataPrefix eq 'marcxml' || $metadataPrefix eq 'marc'){
>+                $style_doc = $parser->parse_file($default_marc_xslt);
>+            }
>+            elsif ($metadataPrefix eq 'dc' || $metadataPrefix eq 'oai_dc'){
>+                $style_doc = $parser->parse_file($default_dc_xslt);
>+            }
>+            else{
>+                return -1;
>+            }
>+        }
>+
>+        if ($style_doc){
>+            my $stylesheet = $xslt->parse_stylesheet($style_doc);
>+
>+            my %xslt_params;
>+            $xslt_params{'identifier'} = $self->{identifier};
>+
>+            if ($stylesheet){
>+                #Pass OAI-PMH identifier and matched biblionumber (if there is one) to the XSLT for unique identification/provenance
>+                my $result_xml = $stylesheet->transform($source, XML::LibXSLT::xpath_to_string(%xslt_params));
>+
>+                if ($result_xml){
>+                    $self->set_transformed_metadata($result_xml);
>+                    return 1;
>+                }
>+                else{
>+                    return -1;
>+                }
>+            }
>+        }
>+    }
>+    else{
>+        return -1;
>+    }
>+}
>+
>+=head2 AddRecordToLog
>+
>+
>+=cut
>+
>+sub AddRecordToLog {
>+    my ( $self ) = @_;
>+
>+    my $metadataPrefix = $self->get_metadataPrefix;
>+    my $repository_id = $self->get_repository_id;
>+    my $biblionumber = $self->get_biblionumber;
>+
>+    if ($metadataPrefix && $repository_id && $biblionumber){
>+            my $identifier = $self->get_identifier;
>+            my $datestamp = $self->get_datestamp;
>+            my $metadata = $self->get_metadata ? $self->get_metadata->toString : undef;
>+            my $header = $self->get_header ? $self->get_header->toString : undef;
>+            my $status = $self->get_status;
>+
>+
>+
>+            my $dbh = C4::Context->dbh;
>+            my $sql = "
>+              INSERT INTO oai_harvest (identifier,datestamp,metadataPrefix,metadata,header,status,repository_id,biblionumber)
>+              VALUES (?,?,?,?,?,?,?,?)
>+            ";
>+            my $sth = $dbh->prepare($sql);
>+            $sth->execute($identifier, $datestamp, $metadataPrefix, $metadata, $header, $status, $repository_id, $biblionumber);
>+            if ($sth->err){
>+                return "ERROR! return code: " . $sth->err . " error msg: " . $sth->errstr . "\n";
>+            }
>+            else{
>+                return 1;
>+            }
>+    }
>+    else{
>+        return -1;
>+    }
>+}
>+
>+=head2 GenerateStatus
>+
>+
>+
>+
>+=cut
>+
>+sub GenerateStatus {
>+    my ( $self, $force ) = @_;
>+    my $dbh = C4::Context->dbh;
>+    my $repository_id = $self->get_repository_id;
>+    my $identifier_status;
>+
>+    #Check if this identifier is found in the database with this repository
>+    my $check_for_id_sql = "
>+      SELECT count(*) as 'count'
>+      FROM oai_harvest
>+      WHERE identifier = ? and repository_id = ?";
>+    my $check_for_id_sth = $dbh->prepare($check_for_id_sql);
>+    $check_for_id_sth->execute($self->get_identifier, $repository_id);
>+    my $check_for_id_row = $check_for_id_sth->fetchrow_hashref;
>+    if ($check_for_id_row->{count} == 0){
>+        #The OAI-PMH unique identifier doesn't exist in this database for this repository == ADD
>+        $identifier_status = "add";
>+    }
>+    else{
>+        #OAI-PMH Unique Identifier does exist in database == UPDATE || IGNORE
>+        my $check_for_id_and_datestamp_sql = "
>+          SELECT count(*) as 'count'
>+          FROM oai_harvest
>+          WHERE identifier = ? and datestamp = ? and repository_id = ?
>+        ";
>+        my $check_for_id_and_datestamp_sth = $dbh->prepare($check_for_id_and_datestamp_sql);
>+        $check_for_id_and_datestamp_sth->execute($self->get_identifier, $self->get_datestamp, $repository_id);
>+        my $check_for_id_and_datestamp_row = $check_for_id_and_datestamp_sth->fetchrow_hashref;
>+        if ($check_for_id_and_datestamp_row->{count} == 0){
>+            #OAI-PMH Unique Identifier and Datestamp combo don't exist in database == UPDATE
>+            #i.e. The identifier exists in the database, but this is an updated datestamp
>+            $identifier_status = "update"
>+        }
>+        else{
>+            #OAI-PMH Unique Identifier and Datestamp combo exist in database but we're forcing an add/update anyway (in accordance with the $force value
>+            if ($force){
>+                if ($self->GetBiblionumberExistence){
>+                    $identifier_status = ( $force eq "only_add" ) ? "ignore" : "force_update";
>+                }
>+                else {
>+                    $identifier_status = ( $force eq "only_update" ) ? "ignore" : "force_add";
>+                }
>+            } else {
>+            #OAI-PMH Unique Identifier and Datestamp combo exist in database == IGNORE
>+                $identifier_status = "ignore";
>+            }
>+        }
>+    }
>+
>+    if ($self->get_deleted && $identifier_status ne "ignore"){
>+        $identifier_status = "delete";
>+    }
>+
>+    if ($identifier_status){
>+        $self->set_status($identifier_status);
>+    }
>+}
>+
>+
>+sub GetBiblionumberExistence {
>+    my ( $self ) = @_;
>+    my $biblionumber_to_check = $self->get_biblionumber;
>+    if ($biblionumber_to_check){
>+        my $dbh = C4::Context->dbh;
>+        my $sql = "
>+          SELECT count(biblionumber) as 'count'
>+          FROM biblio
>+          WHERE biblionumber = ?
>+        ";
>+        my $sth = $dbh->prepare($sql);
>+        $sth->execute($biblionumber_to_check);
>+        my $row = $sth->fetchrow_hashref;
>+        if ($row->{count} > 0){
>+            return 1;
>+        } else {
>+            return;
>+        }
>+    } else {
>+        return;
>+    }
>+}
>+
>+
>+#FIXME: This probably isn't the best way to do this...
>+sub GetBiblionumberFromOaiIdentifier {
>+    my ( $self ) = @_;
>+    my $biblionumber;
>+    my $dbh = C4::Context->dbh;
>+    #Select the newest biblionumber. In theory, there should only be one biblionumber. However, if a record
>+    #was deleted in Koha but force_added again later, there would be multiple biblionumbers in our
>+    #harvesting records. Ergo, we take the highest, so that we're working with the correct number for future
>+    #updates/deletions.
>+    my $sql = "
>+      SELECT DISTINCT biblionumber
>+      FROM oai_harvest
>+      WHERE identifier = ? AND repository_id = ?
>+      ORDER BY timestamp desc
>+      LIMIT 1
>+    ";
>+    my $sth = $dbh->prepare($sql);
>+    $sth->execute($self->get_identifier,$self->get_repository_id);
>+    my $row = $sth->fetchrow_hashref;
>+    $biblionumber = $row->{biblionumber} if $row->{biblionumber};
>+    $self->set_biblionumber($biblionumber) if $biblionumber; #Add biblionumber to the object
>+    return $biblionumber;
>+}
>+
>+#FIXME: This should probably be in Biblio.pm and not a method here? (Galen's work might intersect here...)
>+sub GenerateMarcObject {
>+    my ( $self, $marcxml ) = @_;
>+
>+    if ($marcxml) {
>+        my $marcxml = StripNonXmlChars( $marcxml ); #Warning as per C4::Charset, this could cause some data loss
>+
>+        my $record = eval { MARC::Record::new_from_xml( $marcxml, "utf8", C4::Context->preference('marcflavour') ) };
>+        if ($@) {
>+            warn " problem with : $@ \n$marcxml";
>+            return;
>+        }
>+        return $record;
>+    } else {
>+        return;
>+    }
>+}
>+
>+
>+
>+
>+1;
>+__END__
>\ No newline at end of file
>diff --git a/admin/oai_repository_targets.pl b/admin/oai_repository_targets.pl
>new file mode 100755
>index 0000000..1faa0e6
>--- /dev/null
>+++ b/admin/oai_repository_targets.pl
>@@ -0,0 +1,168 @@
>+#!/usr/bin/perl
>+
>+# This file is part of Koha.
>+#
>+# Copyright 2013 Prosentient Systems
>+#
>+# Koha is free software; you can redistribute it and/or modify it under the
>+# terms of the GNU General Public License as published by the Free Software
>+# Foundation; either version 3 of the License, or (at your option) any later
>+# version.
>+#
>+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
>+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
>+# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
>+#
>+# You should have received a copy of the GNU General Public License along
>+# with Koha; if not, see <http://www.gnu.org/licenses>.
>+
>+=head1 NAME
>+
>+oai_repository_targets.pl - script to manage configuration for harvesting external
>+OAI-PMH server/repository targets.
>+
>+=head1 DESCRIPTION
>+
>+If !$op, then display table listing existing entries
>+
>+If $op eq 'add', then display add form OR modify form, if primary key is supplied
>+
>+If $op eq 'delete', then delete the entry
>+
>+=cut
>+
>+use Modern::Perl;
>+use CGI;
>+use C4::Auth;
>+use C4::Output;
>+use C4::Context;
>+
>+use C4::Koha qw/getframeworks/;
>+use C4::OAI::Client::Harvester;
>+
>+my $input = new CGI;
>+my $script_name = "/cgi-bin/koha/admin/oai_repository_targets.pl";
>+my ($template, $loggedinuser, $cookie)
>+    = get_template_and_user({template_name => "admin/oai_repository_targets.tt",
>+            query => $input,
>+            type => "intranet",
>+            authnotrequired => 0,
>+            flagsrequired => {parameters => 'parameters_remaining_permissions'},
>+            debug => 1,
>+            });
>+
>+$template->param(script_name => $script_name);
>+
>+my $op = $input->param('op') || '';
>+my $id = $input->param('id') || '';
>+if ($id){
>+    $template->param(id => $id);
>+}
>+
>+# get framework list
>+my $frameworks = C4::Koha::getframeworks;
>+my @frameworkcodeloop;
>+foreach my $thisframeworkcode ( sort { uc($frameworks->{$a}->{'frameworktext'}) cmp uc($frameworks->{$b}->{'frameworktext'}) } keys %{$frameworks} ) {
>+    push @frameworkcodeloop, {
>+        value         => $thisframeworkcode,
>+        frameworktext => $frameworks->{$thisframeworkcode}->{'frameworktext'},
>+    };
>+}
>+$template->param(frameworkcodeloop => \@frameworkcodeloop);
>+
>+if ($op eq 'add' || $op eq 'edit' || $op eq 'validate'){
>+    #These operations require a form view
>+    $template->param(view => "form");
>+
>+    if ($op eq 'edit' && $id){
>+        my $repository_fields = C4::OAI::Client::Harvester::GetOAIRepository($id);
>+        $template->param(form_fields => $repository_fields);
>+    }
>+    elsif ($op eq 'validate'){
>+        my @validation_errors;
>+        my $form_fields;
>+        $form_fields->{baseURL} = $input->param('baseURL') ? $input->param('baseURL') : undef;
>+        if (!$form_fields->{baseURL}){
>+            push(@validation_errors,"baseURL");
>+            $template->param(baseURL_validation_err => "mandatory");
>+        }
>+        $form_fields->{metadataPrefix} = $input->param('metadataPrefix') ? $input->param('metadataPrefix') : undef;
>+        if (!$form_fields->{metadataPrefix}){
>+            push(@validation_errors,"metadataPrefix");
>+            $template->param(metadataPrefix_validation_err => "mandatory");
>+        }
>+        $form_fields->{XSLT_path} = $input->param('XSLT_path') ? $input->param('XSLT_path') : undef;
>+        if (!$form_fields->{XSLT_path}){
>+            push(@validation_errors,"XSLT_path");
>+            $template->param(XSLT_path_validation_err => "mandatory");
>+        }
>+        #Use 'undef' instead of blank so that you take advantage of database level validation
>+        $form_fields->{opt_from} = $input->param('opt_from') ? $input->param('opt_from') : undef;
>+        $form_fields->{opt_until} = $input->param('opt_until') ? $input->param('opt_until') : undef;
>+        $form_fields->{opt_set} = $input->param('opt_set') ? $input->param('opt_set') : undef;
>+        $form_fields->{active} = $input->param('active') == 1 || $input->param('active') == 0 ? $input->param('active') : undef;
>+        $form_fields->{frameworkcode} = $input->param('frameworkcode') ? $input->param('frameworkcode') : '';
>+        $form_fields->{comments} = $input->param('comments') ? $input->param('comments') : undef;
>+        $form_fields->{repository_id} = $id ? $id : undef;
>+
>+        if (@validation_errors){
>+            warn "There are ". scalar @validation_errors . " validation errors";
>+            $template->param(
>+                form_fields => $form_fields,
>+            );
>+        }
>+        else {
>+            if ($id){
>+                C4::OAI::Client::Harvester::ModOAIRepository($form_fields);
>+                #Show confirmation view after update
>+                $template->param(
>+                    view => "confirm",
>+                    op => "confirm_update",
>+                );
>+            }
>+            else {
>+                C4::OAI::Client::Harvester::AddOAIRepository($form_fields);
>+                #Show confirmation view after insert/add
>+                $template->param(
>+                    view => "confirm",
>+                    op => "confirm_add",
>+                );
>+            }
>+        }
>+    }
>+}
>+else {
>+    #All other operations require a list view
>+    $template->param(view => "list");
>+
>+    my @results;
>+
>+    if ($op eq 'delete' && $id){
>+        my $repository_fields = C4::OAI::Client::Harvester::GetOAIRepository($id);
>+        if ($repository_fields){
>+            push(@results,$repository_fields);
>+            $template->param(
>+                op => 'delete'
>+            );
>+        }
>+    }
>+    elsif ($op eq 'perform_delete' && $id){
>+        C4::OAI::Client::Harvester::DelOAIRepository($id);
>+        #Show confirmation view after delete
>+        $template->param(
>+            view => "confirm",
>+            op => "confirm_delete",
>+        );
>+    }
>+    else{
>+        @results = C4::OAI::Client::Harvester::GetOAIRepositoryList();
>+    }
>+
>+    if (@results){
>+        $template->param(
>+            result_rows => \@results
>+        );
>+    }
>+}
>+
>+output_html_with_http_headers $input, $cookie, $template->output;
>\ No newline at end of file
>diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql
>index c1a8b8d..1174655 100644
>--- a/installer/data/mysql/kohastructure.sql
>+++ b/installer/data/mysql/kohastructure.sql
>@@ -3243,3 +3243,44 @@ CREATE TABLE IF NOT EXISTS plugin_data (
> /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
> /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
> 
>+--
>+-- Table structure for table 'oai_harvest_repositories'
>+--
>+
>+DROP TABLE IF EXISTS oai_harvest_repositories;
>+CREATE TABLE  oai_harvest_repositories (
>+  repository_id int(11) NOT NULL AUTO_INCREMENT, -- primary key identifier
>+  baseURL text NOT NULL, -- baseURL of the remote OAI-PMH repository (mandatory)
>+  opt_from varchar(45) DEFAULT NULL, -- "from" time for selective harvesting (optional - gets set automatically by cronjob)
>+  opt_until varchar(45) DEFAULT NULL, -- "until" time for selective harvesting (optional)
>+  opt_set varchar(45) DEFAULT NULL, -- the record set to harvest for selective harvesting (optional)
>+  metadataPrefix varchar(45) NOT NULL, -- metadata format (e.g. oai_dc, dc, marcxml)
>+  active int(11) NOT NULL DEFAULT '0', -- indicate whether this repo is actively harvested by Koha
>+  timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, -- date last modified
>+  XSLT_path text, -- filepath to a local or external XSLT file to use for transforming incoming records into MARCXML
>+  frameworkcode varchar(4) NOT NULL, -- framework to use when ingesting records
>+  comments text, -- limited number of characters (controlled by template) to describe the repository (optional - for librarian use rather than system use)
>+  PRIMARY KEY (repository_id)
>+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
>+
>+--
>+-- Table structure for table 'oai_harvest'
>+--
>+
>+DROP TABLE IF EXISTS oai_harvest;
>+CREATE TABLE oai_harvest (
>+  oai_harvest_id int(11) NOT NULL AUTO_INCREMENT, -- primary key identifier
>+  identifier varchar(255) NOT NULL, -- OAI-PMH identifier (unique to its original repo)
>+  datestamp varchar(45) NOT NULL, -- OAI-PMH datestamp (i.e. date last modified)
>+  metadataPrefix varchar(45) NOT NULL, -- metadata format of the record
>+  biblionumber int(11) NOT NULL, -- biblionumber of the biblio record created by this OAI-PMH ingested record
>+  metadata longtext, -- XML string containing the metadata (this will be null for incoming records marked as deleted)
>+  header longtext NOT NULL, -- XML string containing the header information (i.e. identifier, datestamp, status)
>+  about longtext, -- XML string containing explanatory information about a record (optional. Often not included in records)
>+  timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, -- date last modified
>+  repository_id int(11) NOT NULL, -- id for this record's OAI-PMH repository (linked to oai_harvest_repositories table)
>+  status varchar(45) DEFAULT NULL, -- whether this record was an addition, an update, or a deletion
>+  PRIMARY KEY (oai_harvest_id),
>+  KEY FK_oai_harvest_1 (repository_id),
>+  CONSTRAINT FK_oai_harvest_1 FOREIGN KEY (repository_id) REFERENCES oai_harvest_repositories (repository_id) ON UPDATE NO ACTION
>+) ENGINE=InnoDB AUTO_INCREMENT=2809 DEFAULT CHARSET=utf8;
>diff --git a/installer/data/mysql/updatedatabase.pl b/installer/data/mysql/updatedatabase.pl
>index abd7c8e..c95ed4e 100755
>--- a/installer/data/mysql/updatedatabase.pl
>+++ b/installer/data/mysql/updatedatabase.pl
>@@ -7185,6 +7185,46 @@ if ( CheckVersion($DBversion) ) {
>     SetVersion($DBversion);
> }
> 
>+$DBversion = "3.13.00.XXX";
>+if ( CheckVersion($DBversion) ) {
>+    $dbh->do("
>+CREATE TABLE IF NOT EXISTS oai_harvest_repositories (
>+  repository_id int(11) NOT NULL AUTO_INCREMENT,
>+  baseURL text NOT NULL,
>+  opt_from varchar(45) DEFAULT NULL,
>+  opt_until varchar(45) DEFAULT NULL,
>+  opt_set varchar(45) DEFAULT NULL,
>+  metadataPrefix varchar(45) NOT NULL,
>+  active int(11) NOT NULL DEFAULT '0',
>+  timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
>+  XSLT_path text,
>+  frameworkcode varchar(4) NOT NULL,
>+  comments text,
>+  PRIMARY KEY (repository_id)
>+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
>+    ");
>+    $dbh->do("
>+CREATE TABLE IF NOT EXISTS oai_harvest (
>+  oai_harvest_id int(11) NOT NULL AUTO_INCREMENT,
>+  identifier varchar(255) NOT NULL,
>+  datestamp varchar(45) NOT NULL,
>+  metadataPrefix varchar(45) NOT NULL,
>+  biblionumber int(11) NOT NULL,
>+  metadata longtext,
>+  header longtext NOT NULL,
>+  about longtext,
>+  timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
>+  repository_id int(11) NOT NULL,
>+  status varchar(45) DEFAULT NULL,
>+  PRIMARY KEY (oai_harvest_id),
>+  KEY FK_oai_harvest_1 (repository_id),
>+  CONSTRAINT FK_oai_harvest_1 FOREIGN KEY (repository_id) REFERENCES oai_harvest_repositories (repository_id) ON UPDATE NO ACTION
>+) ENGINE=InnoDB AUTO_INCREMENT=2809 DEFAULT CHARSET=utf8;
>+    ");
>+    print "Upgrade to $DBversion done (Bug 10662: Build OAI-PMH Harvesting Client)\n";
>+    SetVersion($DBversion);
>+}
>+
> =head1 FUNCTIONS
> 
> =head2 TableExists($table)
>diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc
>index 91878bf..605a0e0 100644
>--- a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc
>+++ b/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc
>@@ -62,6 +62,7 @@
> <ul>
> 	
>     <li><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50 client targets</a></li>
>+    <li><a href="/cgi-bin/koha/admin/oai_repository_targets.pl">OAI-PMH repository targets</a></li>
>     <li><a href="/cgi-bin/koha/admin/didyoumean.pl">Did you mean?</a></li>
> </ul>
> </div>
>diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt
>index a5dbe44..786c865 100644
>--- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt
>+++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt
>@@ -105,6 +105,8 @@
> 	<dd>Printers (UNIX paths).</dd> -->
>     <dt><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50 client targets</a></dt>
> 	<dd>Define which servers to query for MARC data in the integrated Z39.50 client.</dd>
>+    <dt><a href="/cgi-bin/koha/admin/oai_repository_targets.pl">OAI-PMH repository targets</a></dt>
>+    <dd>Define which OAI-PMH repositories from which to automatically harvest metadata records</dd>
>     <dt><a href="/cgi-bin/koha/admin/didyoumean.pl">Did you mean?</a></dt>
>     <dd>Choose which plugins to use to suggest searches to patrons and staff.</dd>
> </dl>
>diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_repository_targets.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_repository_targets.tt
>new file mode 100755
>index 0000000..0ad0f5e
>--- /dev/null
>+++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_repository_targets.tt
>@@ -0,0 +1,224 @@
>+[% INCLUDE 'doc-head-open.inc' %]
>+<title>Koha &rsaquo; Administration &rsaquo; Manage OAI-PMH repository targets</title>
>+[%# FIXME: change title based on page... each page should have a different title %]
>+[% INCLUDE 'doc-head-close.inc' %]
>+
>+[% IF ( view == "list" ) %]
>+<link rel="stylesheet" type="text/css" href="[% themelang %]/css/datatables.css" />
>+<script type="text/javascript" src="[% themelang %]/lib/jquery/plugins/jquery.dataTables.min.js"></script>
>+[% INCLUDE 'datatables-strings.inc' %]
>+<script type="text/javascript" src="[% themelang %]/js/datatables.js"></script>
>+[% END %]
>+[% IF ( view == "list" ) %]
>+    <script type="text/javascript">
>+    //<![CDATA[
>+     $(document).ready(function() {
>+        [% IF ( result_rows ) && ( op != "delete" ) %]$("#repositoriest").dataTable($.extend(true, {}, dataTablesDefaults, {
>+            "aoColumnDefs": [
>+                { "aTargets": [ 9,10 ], "bSortable": false, "bSearchable": false },
>+            ],
>+            "iDisplayLength": 20,
>+            "aLengthMenu": [[10, 20, 50, 100, -1], [10, 20, 50, 100, "All"]],
>+            "sPaginationType": "four_button"
>+        }));[% END %]
>+     });
>+    //]]>
>+    </script>
>+[% END %]
>+
>+</head>
>+<body id="admin_oairepositorytargets" class="admin">
>+[% INCLUDE 'header.inc' %]
>+[% INCLUDE 'cat-search.inc' %]
>+<div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/admin/admin-home.pl">Administration</a> &rsaquo; Manage OAI-PMH repository targets</div>
>+
>+<div id="doc3" class="yui-t2">
>+    <div id="bd">
>+        <div id="yui-main">
>+            <div class="yui-b">
>+            [% IF ( view == "confirm" ) %]
>+                [% IF ( op == "confirm_add" ) %]
>+                <h1>OAI-PMH repository target added</h1>
>+                <form action="[% script_name %]" method="post">
>+                    <input type="submit" value="OK" />
>+                </form>
>+                [% ELSIF ( op == "confirm_update" ) %]
>+                <h1>OAI-PMH repository target updated</h1>
>+                <form action="[% script_name %]" method="post">
>+                    <input type="submit" value="OK" />
>+                </form>
>+                [% ELSIF ( op == "confirm_delete" ) %]
>+                <h1>OAI-PMH repository target deleted</h1>
>+                <form action="[% script_name %]" method="post">
>+                    <input type="submit" value="OK" />
>+                </form>
>+                [% END %]
>+            [% ELSIF ( view == "form" ) %]
>+                [% IF ( id ) %]
>+                <h1>Edit OAI-PMH repository target</h1>
>+                [% ELSE %]
>+                <h1>New OAI-PMH repository target</h1>
>+                [% END %]
>+                <form action="[% script_name %]" name="repository_add_form" method="post">
>+                <input type="hidden" name="op" value="validate" />
>+                [% IF ( id ) %]
>+                <input type="hidden" name="id" value="[% id %]" />
>+                [% END %]
>+                    <fieldset class="rows">
>+                    <ol>
>+                        [% IF ( baseURL_validation_err ) %]
>+                        <li class="error">
>+                        [% ELSE %]
>+                        <li>
>+                        [% END %]
>+                            <label for="baseURL">Base URL: </label>
>+                            <input type="text" name="baseURL" id="baseURL" size="65" value="[% form_fields.baseURL %]"/>
>+                            [% IF ( baseURL_validation_err == "mandatory") %]
>+                            [Base URL is a mandatory field.]
>+                            [% END %]
>+                        </li>
>+                        <li>
>+                            <label for="opt_from">From: </label>
>+                            <input type="text" name="opt_from" id="opt_from" size="65" value="[% form_fields.opt_from %]"/>
>+                        </li>
>+                        <li>
>+                            <label for="opt_until">Until: </label>
>+                            <input type="text" name="opt_until" id="opt_until" size="65" value="[% form_fields.opt_until %]"/>
>+                        </li>
>+                        <li>
>+                            <label for="opt_set">Set: </label>
>+                            <input type="text" name="opt_set" id="opt_set" size="65" value="[% form_fields.opt_set %]"/>
>+                        </li>
>+                        [% IF ( metadataPrefix_validation_err ) %]
>+                        <li class="error">
>+                        [% ELSE %]
>+                        <li>
>+                        [% END %]
>+                            <label for="metadataPrefix">Metadata prefix: </label>
>+                            <input type="text" name="metadataPrefix" id="metadataPrefix" size="65" value="[% form_fields.metadataPrefix %]"/>
>+                            [% IF ( metadataPrefix_validation_err == "mandatory") %]
>+                            [Metadata prefix is a mandatory field.]
>+                            [% END %]
>+                        </li>
>+                        <li>
>+                            <label for="active">Status: </label>
>+                            <select id="active" name="active">
>+                                [% IF ( form_fields.active == 0 ) %]
>+                                <option value="0" selected="selected">Inactive</option>
>+                                [% ELSE %]
>+                                <option value="0">Inactive</option>
>+                                [% END %]
>+                                [% IF ( form_fields.active == 1 ) %]
>+                                <option value="1" selected="selected">Active</option>
>+                                [% ELSE %]
>+                                <option value="1">Active</option>
>+                                [% END %]
>+                            </select>
>+                        </li>
>+                        <li>
>+                            <label for="frameworkcode">Framework: </label>
>+                            <select name="frameworkcode" id="frameworkcode">
>+                                <option value="">Default framework</option>
>+                            [% FOREACH frameworkcode IN frameworkcodeloop %]
>+                                [% IF ( form_fields.frameworkcode == frameworkcode.value ) %]
>+                                <option value="[% frameworkcode.value %]" selected="selected">[% frameworkcode.frameworktext %]</option>
>+                                [% ELSE %]
>+                                <option value="[% frameworkcode.value %]">[% frameworkcode.frameworktext %]</option>
>+                                [% END %]
>+                            [% END %]
>+                            </select>
>+                        </li>
>+                        [% IF ( XSLT_path_validation_err ) %]
>+                        <li class="error">
>+                        [% ELSE %]
>+                        <li>
>+                        [% END %]
>+                            <label for="XSLT_path">XSLT: </label>
>+                            <input type="text" name="XSLT_path" id="XSLT_path" size="65" value="[% form_fields.XSLT_path %]"/>
>+                            [% IF ( XSLT_path_validation_err == "mandatory") %]
>+                            [XSLT is a mandatory field.]
>+                            [% END %]
>+                            (Please consult your system administrator when choosing your XSLT.)
>+                        </li>
>+                        <li>
>+                            <label for="comments">Comments: </label>
>+                            <textarea id="comments" name="comments" rows="4" cols="60" maxlength="500">[% form_fields.comments %]</textarea>
>+                        </li>
>+                    </ol>
>+                    </fieldset>
>+                    <fieldset class="action"><input type="submit" value="Save" /> <a class="cancel" href="[% script_name %]">Cancel</a></fieldset>
>+                </form>
>+            [% ELSIF ( view == "list" ) %]
>+                [% UNLESS ( op == "delete" ) %]
>+                <div id="toolbar" class="btn-toolbar">
>+                    <a id="newrepositorytarget" class="btn btn-small" href="[% script_name %]?op=add"><i class="icon-plus"></i> New OAI-PMH repository target</a>
>+                </div>
>+                [% END %]
>+                [% IF ( op == "delete" ) %]
>+                <h1>Delete OAI-PMH repository target</h1>
>+                [% ELSE %]
>+                <h1>List OAI-PMH repository targets</h1>
>+                [% END %]
>+                <table id="repositoriest">
>+                    <thead>
>+                        <tr>
>+                            <th>Base URL</th>
>+                            <th>From</th>
>+                            <th>Until</th>
>+                            <th>Set</th>
>+                            <th>Metadata prefix</th>
>+                            <th>Status</th>
>+                            <th>Framework</th>
>+                            <th>XSLT</th>
>+                            <th>Comments</th>
>+                            [% UNLESS ( op == "delete" ) %]
>+                            <th>&nbsp;</th>
>+                            <th>&nbsp;</th>
>+                            [% END %]
>+                        </tr>
>+                    </thead>
>+                    <tbody>
>+                    [% FOREACH result_row IN result_rows %]
>+                        <tr>
>+                            <td><a href="[% result_row.script_name %]?op=edit&amp;id=[% result_row.repository_id |url %]">[% result_row.baseURL %]</a></td>
>+                            <td>[% result_row.opt_from %]</td>
>+                            <td>[% result_row.opt_until %]</td>
>+                            <td>[% result_row.opt_set %]</td>
>+                            <td>[% result_row.metadataPrefix %]</td>
>+                            <td>[% IF ( result_row.active == 1 ) %] Active [% ELSE %] Inactive [% END %]</td>
>+                            <td>
>+                            [% IF ( result_row.frameworkcode == '') %]
>+                            Default
>+                            [% ELSE %]
>+                            [% FOREACH framework IN frameworkcodeloop %][% IF ( result_row.frameworkcode == framework.value )%][% framework.frameworktext %][% END %][% END %]
>+                            [% END %]
>+                            </td>
>+                            <td>[% result_row.XSLT_path %]</td>
>+                            <td>[% IF (result_row.comments.length > 20) %][% result_row.comments.substr(0,20) %]...[% ELSE %][% result_row.comments %][% END %]</td>
>+                            [% UNLESS ( op == "delete" ) %]
>+                            <td><a href="[% result_row.script_name %]?op=edit&amp;id=[% result_row.repository_id |url %]">Edit</a></td>
>+                            <td><a href="[% result_row.script_name %]?op=delete&amp;id=[% result_row.repository_id |url %]">Delete</a></td>
>+                            [% END %]
>+                        </tr>
>+                    [% END %]
>+                    </tbody>
>+                </table>
>+                [% IF ( op == "delete" ) %]
>+                <form action="[% script_name %]" method="post">
>+                    <input type="hidden" name="id" value="[% id %]" />
>+                    <input type="hidden" name="op" value="perform_delete" />
>+                    <fieldset class="action">
>+                        <input type="submit" value="Confirm" />
>+                        <a class="cancel" href="[% script_name %]">Cancel</a>
>+                    </fieldset>
>+                </form>
>+                [% END %]
>+            [% END %]
>+            </div>[%# first yui-b %]
>+        </div>[%# yui-main %]
>+        <div class="yui-b">
>+            [% INCLUDE 'admin-menu.inc' %]
>+        </div> [%# last yui-b %]
>+    </div> [%# bd %]
>+
>+[% INCLUDE 'intranet-bottom.inc' %]
>\ No newline at end of file
>diff --git a/koha-tmpl/intranet-tmpl/prog/en/xslt/DC2MARC21slim.xsl b/koha-tmpl/intranet-tmpl/prog/en/xslt/DC2MARC21slim.xsl
>new file mode 100755
>index 0000000..f68cbbe
>--- /dev/null
>+++ b/koha-tmpl/intranet-tmpl/prog/en/xslt/DC2MARC21slim.xsl
>@@ -0,0 +1,217 @@
>+<?xml version="1.0" encoding="UTF-8"?>
>+<xsl:stylesheet version="1.0" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns="http://www.loc.gov/MARC21/slim" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" exclude-result-prefixes="dc">
>+    <xsl:output method="xml" indent="yes"/>
>+
>+    
>+
>+    
>+    <xsl:param name="identifier" select="identifier"/>
>+
>+    <xsl:template match="/">
>+        <record xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" >
>+            <xsl:element name="leader">
>+                <xsl:variable name="type" select="//dc:type"/>
>+                <xsl:variable name="leader06">
>+                    <xsl:choose>
>+                        <xsl:when test="$type='collection'">p</xsl:when>
>+                        <xsl:when test="$type='dataset'">m</xsl:when>
>+                        <xsl:when test="$type='event'">r</xsl:when>
>+                        <xsl:when test="$type='image'">k</xsl:when>
>+                        <xsl:when test="$type='interactive resource'">m</xsl:when>
>+                        <xsl:when test="$type='service'">m</xsl:when>
>+                        <xsl:when test="$type='software'">m</xsl:when>
>+                        <xsl:when test="$type='sound'">i</xsl:when>
>+                        <xsl:when test="$type='text'">a</xsl:when>
>+                        <xsl:otherwise>a</xsl:otherwise>
>+                    </xsl:choose>
>+                </xsl:variable>
>+                <xsl:variable name="leader07">
>+                    <xsl:choose>
>+                        <xsl:when test="$type='collection'">c</xsl:when>
>+                        <xsl:otherwise>m</xsl:otherwise>
>+                    </xsl:choose>
>+                </xsl:variable>
>+                
>+                
>+                
>+                
>+                
>+                
>+                
>+                
>+                
>+                
>+                
>+                
>+                
>+                
>+                
>+                
>+                <xsl:value-of select="concat('     n',$leader06,$leader07,' a22     uu 4500')"/>
>+            </xsl:element>
>+
>+
>+            
>+
>+            <xsl:for-each select="//dc:identifier">
>+                <xsl:if test="substring(text(),1,4) != 'http'">
>+                <datafield tag="024" ind1="8" ind2=" ">
>+                    <subfield code="a">
>+                        <xsl:value-of select="."/>
>+                    </subfield>
>+                </datafield>
>+                </xsl:if>
>+            </xsl:for-each>
>+
>+            
>+            <datafield tag="037" ind1="8" ind2=" ">
>+                <subfield code="a">
>+                    <xsl:value-of select="$identifier"/>
>+                </subfield>
>+                <subfield code="n">
>+                    <xsl:text>OAI-PMH Identifier</xsl:text>
>+                </subfield>
>+            </datafield>
>+
>+            <datafield tag="042" ind1=" " ind2=" ">
>+                <subfield code="a">dc</subfield>
>+            </datafield>
>+
>+            <xsl:for-each select="//dc:title[1]">
>+                <datafield tag="245" ind1="0" ind2="0">
>+                    <subfield code="a">
>+                        <xsl:value-of select="."/>
>+                    </subfield>
>+                </datafield>
>+            </xsl:for-each>
>+
>+            <xsl:for-each select="//dc:title[position()>1]">
>+                <datafield tag="246" ind1="3" ind2="3">
>+                    <subfield code="a">
>+                        <xsl:value-of select="."/>
>+                    </subfield>
>+                </datafield>
>+            </xsl:for-each>
>+
>+            
>+            <xsl:if test="//dc:publisher | //dc:date">
>+                <datafield tag="260" ind1=" " ind2=" ">
>+                <xsl:for-each select="//dc:publisher">
>+                    <subfield code="b">
>+                        <xsl:value-of select="."/>
>+                    </subfield>
>+                </xsl:for-each>
>+                <xsl:for-each select="//dc:date">
>+                    
>+                    <xsl:if test="substring(text(),11,1) != 'T' and substring(text(),20,1) != 'Z'">
>+                        <subfield code="c">
>+                            <xsl:value-of select="."/>
>+                        </subfield>
>+                    </xsl:if>
>+                </xsl:for-each>
>+                </datafield>
>+            </xsl:if>
>+
>+            <xsl:for-each select="//dc:coverage">
>+                <datafield tag="500" ind1=" " ind2=" ">
>+                    <subfield code="a">
>+                        <xsl:value-of select="."/>
>+                    </subfield>
>+                </datafield>
>+            </xsl:for-each>
>+
>+            <xsl:for-each select="//dc:description">
>+                <datafield tag="520" ind1=" " ind2=" ">
>+                    <subfield code="a">
>+                        <xsl:value-of select="."/>
>+                    </subfield>
>+                </datafield>
>+            </xsl:for-each>
>+
>+            <xsl:for-each select="//dc:rights">
>+                <datafield tag="540" ind1=" " ind2=" ">
>+                    <subfield code="a">
>+                        <xsl:value-of select="."/>
>+                    </subfield>
>+                </datafield>
>+            </xsl:for-each>
>+
>+            <xsl:for-each select="//dc:language">
>+                <datafield tag="546" ind1=" " ind2=" ">
>+                    <subfield code="a">
>+                        <xsl:value-of select="."/>
>+                    </subfield>
>+                </datafield>
>+            </xsl:for-each>
>+
>+            <xsl:for-each select="//dc:subject">
>+                <datafield tag="653" ind1=" " ind2=" ">
>+                    <subfield code="a">
>+                        <xsl:value-of select="."/>
>+                    </subfield>
>+                </datafield>
>+            </xsl:for-each>
>+
>+            <xsl:for-each select="//dc:type">
>+                <datafield tag="655" ind1="7" ind2=" ">
>+                    <subfield code="a">
>+                        <xsl:value-of select="."/>
>+                    </subfield>
>+                    <subfield code="2">local</subfield>
>+                </datafield>
>+            </xsl:for-each>
>+
>+            <xsl:for-each select="//dc:contributor">
>+                <datafield tag="720" ind1="0" ind2="0">
>+                    <subfield code="a">
>+                        <xsl:value-of select="."/>
>+                    </subfield>
>+                    <subfield code="e">collaborator</subfield>
>+                </datafield>
>+            </xsl:for-each>
>+
>+            <xsl:for-each select="//dc:creator">
>+                <datafield tag="720" ind1=" " ind2=" ">
>+                    <subfield code="a">
>+                        <xsl:value-of select="."/>
>+                    </subfield>
>+                    <subfield code="e">author</subfield>
>+                </datafield>
>+            </xsl:for-each>
>+
>+            <xsl:for-each select="//dc:source">
>+                <datafield tag="786" ind1="0" ind2=" ">
>+                    <subfield code="n">
>+                        <xsl:value-of select="."/>
>+                    </subfield>
>+                </datafield>
>+            </xsl:for-each>
>+
>+            <xsl:for-each select="//dc:relation">
>+                <datafield tag="787" ind1="0" ind2=" ">
>+                    <subfield code="n">
>+                        <xsl:value-of select="."/>
>+                    </subfield>
>+                </datafield>
>+            </xsl:for-each>
>+
>+            <xsl:for-each select="//dc:format">
>+                <datafield tag="856" ind1=" " ind2=" ">
>+                    <subfield code="q">
>+                        <xsl:value-of select="."/>
>+                    </subfield>
>+                </datafield>
>+            </xsl:for-each>
>+
>+            <xsl:for-each select="//dc:identifier">
>+                <xsl:if test="substring(text(),1,4) = 'http'">
>+                <datafield tag="856" ind1=" " ind2=" ">
>+                    <subfield code="u">
>+                        <xsl:value-of select="."/>
>+                    </subfield>
>+                </datafield>
>+                </xsl:if>
>+            </xsl:for-each>
>+        </record>
>+    </xsl:template>
>+</xsl:stylesheet>
>\ No newline at end of file
>diff --git a/koha-tmpl/intranet-tmpl/prog/en/xslt/MARC21slim2MARC21enhanced.xsl b/koha-tmpl/intranet-tmpl/prog/en/xslt/MARC21slim2MARC21enhanced.xsl
>new file mode 100755
>index 0000000..0cf9aaa
>--- /dev/null
>+++ b/koha-tmpl/intranet-tmpl/prog/en/xslt/MARC21slim2MARC21enhanced.xsl
>@@ -0,0 +1,43 @@
>+<?xml version="1.0" encoding="UTF-8"?>
>+<xsl:stylesheet version="1.0" xmlns:marc="http://www.loc.gov/MARC21/slim" xmlns:items="http://www.koha-community.org/items" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" >
>+	<xsl:output method="xml" indent="yes"/>
>+
>+    
>+    <xsl:param name="identifier" select="identifier"/>
>+
>+    
>+	<xsl:template match="@*|node()">
>+      <xsl:copy>
>+        <xsl:apply-templates select="@*|node()"/>
>+      </xsl:copy>
>+    </xsl:template>
>+
>+    
>+    <xsl:template match="marc:datafield[@tag=952]" />
>+
>+    
>+    <xsl:template match="marc:record">
>+      <record>
>+         
>+         <xsl:apply-templates select="@* | *"/>
>+         
>+         <xsl:text>
>+      </xsl:text>
>+        <xsl:element name="datafield" xmlns="http://www.loc.gov/MARC21/slim">
>+          <xsl:attribute name="ind1"><xsl:text>8</xsl:text></xsl:attribute>
>+          <xsl:attribute name="ind2"><xsl:text> </xsl:text></xsl:attribute>
>+          <xsl:attribute name="tag">037</xsl:attribute>
>+          <xsl:text>
>+        </xsl:text>
>+          <xsl:element name="subfield">
>+            <xsl:attribute name="code">a</xsl:attribute>
>+            <xsl:value-of select="$identifier"/>
>+          </xsl:element>
>+          <xsl:element name="subfield">
>+            <xsl:attribute name="code">n</xsl:attribute>
>+            <xsl:text>OAI-PMH Identifier</xsl:text>
>+          </xsl:element>
>+        </xsl:element>
>+      </record>
>+    </xsl:template>
>+</xsl:stylesheet>
>\ No newline at end of file
>diff --git a/misc/cronjobs/oai_harvester.pl b/misc/cronjobs/oai_harvester.pl
>new file mode 100755
>index 0000000..fca46fd
>--- /dev/null
>+++ b/misc/cronjobs/oai_harvester.pl
>@@ -0,0 +1,95 @@
>+#!/usr/bin/perl
>+
>+# This file is part of Koha.
>+#
>+# Copyright 2013 Prosentient Systems
>+#
>+# Koha is free software; you can redistribute it and/or modify it under the
>+# terms of the GNU General Public License as published by the Free Software
>+# Foundation; either version 3 of the License, or (at your option) any later
>+# version.
>+#
>+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
>+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
>+# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
>+#
>+# You should have received a copy of the GNU General Public License along
>+# with Koha; if not, see <http://www.gnu.org/licenses>.
>+
>+=head1 NAME
>+
>+oai_harvester.pl - cron script to harvest bibliographic records from
>+OAI-PMH repositories.
>+
>+=head1 SYNOPSIS
>+
>+
>+
>+=head1 DESCRIPTION
>+
>+
>+
>+=cut
>+
>+use Modern::Perl;
>+use Getopt::Long;
>+use Pod::Usage;
>+use C4::Context;
>+use C4::OAI::Client::Harvester;
>+
>+my $help = 0;
>+my $debug = 0;
>+my $noresume = 0;
>+my $force = '';
>+my $harvest = 0;
>+
>+#TODO: Perhaps create a flag that triggers a delete of all records that have been ingested via OAI-PMH?
>+
>+GetOptions(
>+            'h|help|?'         => \$help,
>+            'd'                => \$debug,
>+            'noresume'         => \$noresume,
>+            'force=s'          => \$force,
>+            'harvest'        => \$harvest,
>+
>+       );
>+pod2usage(1) if $help;
>+
>+my $active = 1; #Get only active repositories
>+my @repositories = C4::OAI::Client::Harvester::GetOAIRepositoryList($active);
>+foreach my $repository_data (@repositories) {
>+
>+    #Print extra info for debugging
>+    if ($debug){
>+        $repository_data->{debug} = 1;
>+    }
>+
>+    #Turn off automatic token resumption. This is for debugging.
>+    if ($noresume){
>+        $repository_data->{resume} = 0;
>+    }
>+
>+    #Create Harvester object
>+    my $repository = C4::OAI::Client::Harvester->new($repository_data);
>+
>+    if ($repository){
>+        print "\n\nHarvest ".$repository->{'baseURL'}." in DEBUG MODE\n" if $repository->{debug};
>+        my $response = $repository->ListRecords;
>+        if ($response){
>+            my $records = $repository->ProcessOaipmhRecordResponse($response, $harvest, $force);
>+            #NOTE: No records are returned when in $harvest mode
>+        }
>+
>+        if ($harvest){
>+            my $latest_datestamp = C4::OAI::Client::Harvester::GetLatestHistoricalRecordDatestamp($repository->get_repository_id);
>+            if ($latest_datestamp){
>+                if (!$repository->get_opt_from || $latest_datestamp ne $repository->get_opt_from){
>+                    C4::OAI::Client::Harvester::ModOAIRepositoryDateTimeFrom($repository->get_repository_id,$latest_datestamp);
>+                    print "Setting 'from' argument to $latest_datestamp \n";
>+                }
>+            }
>+        }
>+
>+    }
>+}
>+print "\n\nOAI-PMH Harvesting Cronjob Complete\n";
>diff --git a/t/db_dependent/OAI_harvester.t b/t/db_dependent/OAI_harvester.t
>new file mode 100755
>index 0000000..2ce7b70
>--- /dev/null
>+++ b/t/db_dependent/OAI_harvester.t
>@@ -0,0 +1,308 @@
>+#!/usr/bin/perl
>+
>+# This file is part of Koha.
>+#
>+# Copyright 2013 Prosentient Systems
>+#
>+# Koha is free software; you can redistribute it and/or modify it under the
>+# terms of the GNU General Public License as published by the Free Software
>+# Foundation; either version 3 of the License, or (at your option) any later
>+# version.
>+#
>+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
>+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
>+# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
>+#
>+# You should have received a copy of the GNU General Public License along
>+# with Koha; if not, see <http://www.gnu.org/licenses>.
>+
>+use Modern::Perl;
>+use Test::More;
>+use C4::Context;
>+use C4::OAI::Harvester;
>+use Data::Dumper;
>+use Getopt::Long;
>+
>+my $verbose = 0;
>+my $show = 0;
>+GetOptions(
>+    'v|verbose' => \$verbose,
>+    's|show:s' => \$show,
>+);
>+
>+my $real_dbh = C4::Context->dbh;
>+
>+############################################################################################################################################################################################################
>+
>+BEGIN {
>+        use_ok('C4::OAI::Harvester');
>+}
>+
>+############################################################################################################################################################################################################
>+
>+#REAL DATA - We use this for crafting our mock data, making sure the OAI-PMH server is on, and requesting a specific item for the GetRecord method.
>+
>+#Initiate Koha baseURLs
>+my $OPACbaseURL = C4::Context->preference("OPACBaseURL");
>+my $staffClientBaseURL = C4::Context->preference("staffClientBaseURL");
>+
>+#Check if Koha has its OAI-PMH repository config enabled.
>+#If disabled, re-enable it (we'll reset to the original values
>+#after we've finished our test).
>+my $OAIPMHenabled = C4::Context->preference("OAI-PMH");
>+if (!$OAIPMHenabled){
>+    C4::Context->set_preference("OAI-PMH",1);
>+}
>+my $OAIPMHarchiveID;
>+my $check_OAIPMHarchiveID = C4::Context->preference("OAI-PMH:archiveID");
>+if ($check_OAIPMHarchiveID){
>+    $OAIPMHarchiveID = $check_OAIPMHarchiveID;
>+}
>+else {
>+    C4::Context->set_preference("OAI-PMH:archiveID","KOHAOAITEST");
>+    $OAIPMHarchiveID = C4::Context->preference("OAI-PMH:archiveID");
>+}
>+
>+my $check_biblio_table_sql = "
>+    SELECT count(*) as 'count'
>+    FROM biblio
>+";
>+my $check_biblio_table_sth = $real_dbh->prepare($check_biblio_table_sql);
>+$check_biblio_table_sth->execute();
>+my $check_biblio_table_row = $check_biblio_table_sth->fetchrow_hashref;
>+if ($check_biblio_table_row->{count} == 0){
>+    # Generate test biblio records
>+    my $biblio = MARC::Record->new();
>+    $biblio->append_fields(
>+        MARC::Field->new('100', ' ', ' ', a => 'Moffat, Steven'),
>+        MARC::Field->new('245', ' ', ' ', a => 'Silence in the library'),
>+    );
>+    my ($biblionumber, $biblioitemnumber) = AddBiblio($biblio, '');
>+
>+    my $biblio2 = MARC::Record->new();
>+    $biblio2->append_fields(
>+        MARC::Field->new('100', ' ', ' ', a => 'Garland, Alex'),
>+        MARC::Field->new('245', ' ', ' ', a => 'The Beach'),
>+    );
>+    my ($biblionumber2, $biblioitemnumber2) = AddBiblio($biblio2, '');
>+
>+    my $biblio3 = MARC::Record->new();
>+    $biblio3->append_fields(
>+        MARC::Field->new('100', ' ', ' ', a => 'Michaels, Anne'),
>+        MARC::Field->new('245', ' ', ' ', a => 'The Winter Vault'),
>+    );
>+    my ($biblionumber3, $biblioitemnumber3) = AddBiblio($biblio3, '');
>+}
>+
>+my $sql1 = "SELECT biblionumber FROM biblio ORDER BY RAND() LIMIT 1";
>+my $sth1 = $real_dbh->prepare($sql1);
>+$sth1->execute();
>+my $random_biblionumber_row = $sth1->fetchrow_hashref;
>+my $random_biblionumber = $random_biblionumber_row->{biblionumber};
>+
>+my $random_OAIPMH_identifier;
>+if ($OAIPMHarchiveID && $random_biblionumber){
>+    $random_OAIPMH_identifier = $OAIPMHarchiveID.":".$random_biblionumber;
>+}
>+
>+############################################################################################################################################################################################################
>+
>+#MOCK DATA - Create a mock remote OAI-PMH repository using Koha's OAI-PMH server capabilities
>+
>+# Start Database Transaction
>+my $dbh = C4::Context->dbh;
>+$dbh->{AutoCommit} = 0;
>+$dbh->{RaiseError} = 1;
>+
>+my $record_matcher = C4::Matcher->new;
>+my $record_matcher_id = $record_matcher->store();
>+
>+my $data = {};
>+$data->{repository_id} = 1;
>+$data->{baseURL} = "http://$OPACbaseURL/cgi-bin/koha/oai.pl";
>+$data->{opt_from} = undef;
>+$data->{opt_until} = undef;
>+$data->{opt_set} = undef;
>+$data->{metadataPrefix} = "oai_dc"; #or "marcxml"
>+$data->{active} = 1;
>+$data->{XSLT_path} = "http://$staffClientBaseURL/intranet-tmpl/prog/en/xslt/DC2MARC21slim2.xsl"; #"/.../git/koha-tmpl/intranet-tmpl/prog/en/xslt/DC2MARC21slim2.xsl"; #or undef, since this is hard-coded into Harvester.pm anyway
>+$data->{matcher_id} = $record_matcher_id; #NOTE: This is placeholder for testing. In production, you would use a real matching rule. Preferably using biblionumber (i.e. 999$c and Local-Number) for matching.
>+$data->{import_mode} = "automatic"; #Always run "automatic" for test. In production, you might prefer "manual".
>+
>+my $insert_mock_data_sql = "
>+    INSERT INTO oai_harvest_repositories
>+    (baseURL, opt_from, opt_until, opt_set, metadataPrefix, active, XSLT_path, matcher_id, import_mode)
>+    VALUES (?,?,?,?,?,?,?,?,?)
>+";
>+my $insert_mock_mock_data_sth = $dbh->prepare($insert_mock_data_sql);
>+$insert_mock_mock_data_sth->execute($data->{baseURL},$data->{opt_from},$data->{opt_until},$data->{opt_set},$data->{metadataPrefix},$data->{active},$data->{XSLT_path},$data->{matcher_id},$data->{import_mode});
>+
>+my $get_mock_data_sql = "
>+    SELECT *
>+    FROM oai_harvest_repositories
>+    WHERE active = 1
>+";
>+my $get_mock_data_sth = $dbh->prepare($get_mock_data_sql);
>+$get_mock_data_sth->execute();
>+while (my $get_mock_data_row = $get_mock_data_sth->fetchrow_hashref){
>+
>+    #Set 'resume' to 0, if you want to turn off automatic token resumption. This is useful for testing and debugging. Change to 1 for automatic token resumption.
>+    $get_mock_data_row->{resume} = 0;
>+
>+############################################################################################################################################################################################################
>+
>+    #TEST - OBJECT CREATION
>+    my $harvester = C4::OAI::Harvester->new($get_mock_data_row); # create an object
>+    isa_ok( $harvester, 'C4::OAI::Harvester','Harvester object' );
>+    #END TEST
>+
>+    #TEST - OBJECT's "GET" ACCESSOR METHODS
>+    can_ok($harvester, qw(get_baseURL get_from get_until get_set get_metadataPrefix get_rh get_repository_id get_XSLT_path get_matcher_id get_import_mode));
>+    #END TEST
>+
>+    #TEST - Check some basic harvester config
>+    my $baseURL = $harvester->get_baseURL;
>+    my $metadataPrefix = $harvester->get_metadataPrefix;
>+    is ($baseURL,"http://$OPACbaseURL/cgi-bin/koha/oai.pl", 'Retrieved baseURL "'.$baseURL.'"');
>+    ok ( defined $metadataPrefix, 'MetadataPrefix is defined as "'.$metadataPrefix.'"');
>+    #END TEST
>+
>+    #TESTS - OBJECT's OAI-PMH methods
>+
>+    #IDENTIFY
>+    my $identify_repository = $harvester->Identify;
>+    ok ( defined($identify_repository), "Successfully used Identify Verb");
>+    print Dumper($identify_repository) if $verbose or $show eq 'Identify';
>+    #END TEST
>+
>+    #LISTSETS
>+    my @sets = $harvester->ListSets;
>+    ok (@sets, "Successfully used ListSets Verb");
>+    print Dumper(\@sets) if $verbose or $show eq 'ListSets';
>+    #END TEST
>+
>+    #LISTMETADATAFORMATS
>+    my @formats = $harvester->ListMetadataFormats;
>+    ok (@formats, "Successfully used ListMetadataFormats Verb");
>+    print Dumper(\@formats) if $verbose or $show eq 'ListMetadataFormats';
>+    #END TEST
>+
>+    #LISTIDENTIFIERS
>+    my @headers = $harvester->ListIdentifiers;
>+    ok (@headers, "Successfully used ListIdentifiers Verb");
>+    print Dumper(\@headers) if $verbose or $show eq 'ListIdentifiers';
>+    #END TEST
>+
>+    #LISTRECORDS
>+    my @list_records = $harvester->ListRecords;
>+    ok (@list_records, "Successfully used ListRecords Verb");
>+    print Dumper(\@list_records) if $verbose or $show eq 'ListRecords';
>+    #END TEST
>+
>+    #GETRECORD
>+    my @get_records = $harvester->GetRecord($random_OAIPMH_identifier);
>+    ok (@get_records, "Successfully used GetRecord Verb");
>+    print Dumper(@get_records) if $verbose or $show eq 'GetRecord';
>+    #END TEST
>+
>+    my ( $batch_id, $num_with_matches, $num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = $harvester->ImportRecordsIntoKoha(@list_records);
>+
>+    #These variables will only be initialized in automatic mode, which is how the test should always be run anyway.
>+    my $batch_total = $num_added + $num_updated;
>+
>+    print "Batch id = $batch_id \n";
>+    print "Num added = $num_added \n";
>+    print "Num updated = $num_updated \n";
>+    print "Num items added = $num_items_added \n";
>+    print "Num items errored = $num_items_errored \n";
>+    print "Num ignored = $num_ignored \n";
>+
>+
>+    #TEST - RETRIEVE IMPORT_BATCH
>+    my @test_batches;
>+    my $batch_sql = "
>+        SELECT *
>+        FROM import_batches
>+        WHERE import_batch_id = ?
>+        ";
>+    my $batch_sth = $dbh->prepare($batch_sql);
>+    $batch_sth->execute($batch_id);
>+    while (my $batch_row = $batch_sth->fetchrow_hashref){
>+        push(@test_batches,$batch_row);
>+    }
>+    cmp_ok(scalar @test_batches,'==',1,"Successfully retrieved batch id ".$batch_id);
>+
>+    #END TEST
>+
>+    #TEST - RETRIEVE IMPORTED_RECORDS
>+    my $imported_record_sql = "
>+        SELECT count(*) as 'count'
>+        FROM import_records
>+        WHERE import_batch_id = ?
>+        ";
>+    my $imported_record_sth = $dbh->prepare($imported_record_sql);
>+    $imported_record_sth->execute($batch_id);
>+    my $imported_record_row = $imported_record_sth->fetchrow_hashref;
>+    is ($imported_record_row->{count}, $batch_total, "Import_record table count (".$imported_record_row->{count}.") for this batch matches reported import total");
>+    #END TEST
>+
>+    #TEST - RETRIEVE IMPORTED_BIBLIOS
>+    my $imported_biblio_sql = "
>+        SELECT count(*) as 'count'
>+        FROM import_biblios
>+        JOIN import_records using (import_record_id)
>+        WHERE import_records.import_batch_id = ?
>+        ";
>+    my $imported_biblio_sth = $dbh->prepare($imported_biblio_sql);
>+    $imported_biblio_sth->execute($batch_id);
>+    my $imported_biblio_row = $imported_biblio_sth->fetchrow_hashref;
>+    is ($imported_biblio_row->{count}, $batch_total, "Import_biblio table count (".$imported_biblio_row->{count}.") for this batch matches reported import total");
>+    #END TEST
>+
>+    #TEST - RETRIEVE BIBLIOS
>+    my $biblio_sql = "
>+        SELECT count(*) as 'count'
>+        FROM biblio
>+        JOIN import_biblios on import_biblios.matched_biblionumber = biblio.biblionumber
>+        JOIN import_records using (import_record_id)
>+        WHERE import_records.import_batch_id = ?
>+        ";
>+    my $biblio_sth = $dbh->prepare($biblio_sql);
>+    $biblio_sth->execute($batch_id);
>+    my $biblio_row = $biblio_sth->fetchrow_hashref;
>+    is ($biblio_row->{count}, $batch_total, "Biblio table count (".$biblio_row->{count}.") for this batch matches reported import total. Automatic import successful.");
>+    #END TEST
>+
>+    #TEST - RETRIEVE HISTORICAL RECORDS
>+    my $harvest_sql = "
>+        SELECT count(*) as 'count'
>+        FROM oai_harvest
>+        JOIN import_records using (import_record_id)
>+        WHERE import_records.import_batch_id = ?
>+        ";
>+    my $harvest_sth = $dbh->prepare($harvest_sql);
>+    $harvest_sth->execute($batch_id);
>+    my $harvest_row = $harvest_sth->fetchrow_hashref;
>+    is ($harvest_row->{count}, $batch_total, "OAI Harvest table count (".$harvest_row->{count}.") for this batch matches reported import total. Historical records added successfully.");
>+    #NOTE: This ignores deleted records, which will have a historical record but are not part of the batch and thus don't have an import_record_id number.
>+    #END TEST
>+
>+    #TEST - RETRIEVE RECORD MATCHES
>+    #There's no way to really automatically test the "replace/update" function, since the first wave of imported records will need to be reindexed before they can be matched on...
>+    #END
>+
>+} #End Loop of OAI-PMH Repositories
>+$dbh->rollback();
>+$dbh->{AutoCommit} = 1;
>+
>+done_testing();
>+############################################################################################################################################################################################################
>+
>+#RESET ORIGINAL VALUES
>+if (!$OAIPMHenabled){
>+    C4::Context->set_preference("OAI-PMH",$OAIPMHenabled);
>+}
>+
>+if (!$check_OAIPMHarchiveID){
>+    C4::Context->set_preference("OAI-PMH:archiveID",$check_OAIPMHarchiveID);
>+}
>-- 
>1.7.7.4

Actions: View | Diff | Splinter Review

Attachments on bug 10662: 20757 | 20758 | 21954 | 42446 | 42447 | 42448 | 44792 | 44793 | 47675 | 48096 | 49832 | 50254 | 50980 | 51510 | 51511 | 51512 | 51514 | 51562 | 51563 | 51564 | 51565 | 51705 | 53258 | 53259 | 53260 | 53361 | 53362 | 64444 | 64445 | 64446 | 64479 | 64480 | 64481 | 65016 | 65017 | 65018 | 65019 | 68508 | 71008 | 71009 | 71010 | 71011 | 71012 | 71013 | 71014 | 71015 | 78569 | 78570 | 78571 | 78572 | 78573 | 78574 | 78575 | 78576 | 78577 | 78583 | 78601 | 78617 | 78754 | 78756 | 78758 | 78760 | 78762 | 78764 | 78767 | 78769 | 78771 | 78859 | 78860 | 78914 | 78956 | 79039 | 79045 | 81783 | 81784 | 81786 | 81789 | 81790 | 81855 | 81856 | 81861 | 81862 | 81863 | 81864 | 81902 | 82219 | 84318 | 84319 | 84320 | 84321 | 84322 | 85152 | 85224 | 85225 | 85226 | 85227 | 85228 | 85229 | 99739 | 99740 | 99741 | 99742 | 99743 | 99744 | 99745