Attachment #42447 for bug #10662

View | Details | Raw Unified | Return to bug 10662
Collapse All | Expand All




package Koha::OAI::Client::Importer;

# Copyright Prosentient Systems 2015
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;

use Koha::Database;
use Koha::XSLT_Handler;
use C4::Context;
use C4::Languages;
use C4::XSLT;
use C4::Biblio;

use parent qw(Class::Accessor);
__PACKAGE__->mk_accessors( qw (xslt_handler verbosity) );

=head1 NAME

Koha::OAI::Importer - Imports records, previously harvested via OAI-PMH, into Koha

=head1 SYNOPSIS

  my $oai_importer = Koha::OAI::Importer->new({
    verbosity => $verbose,
  });
  $oai_importer->import_records();

=head1 DESCRIPTION

An OAI Importer is an object which retrieves records, previously harvested
via OAI-PMH, and determines whether they should be added as new records to the
catalogue or if they should update existing catalogue records.

After this determination, the record's metadata is run through a XSLT, which
is configured on a per server basis, and then handed over to Koha's internal APIs.

The record in the OAI harvesting queue is then updated to reflect the
action taken and the biblionumber to which it is linked.

=head1 METHODS

=head2 new

The Importer constructor which sets the 'verbosity' and 'xslt_handler'
for the object.

The 'verbosity' should be set by calling script.

The 'xslt_handler' could also be set by a calling script, but defaults
to a new Koha::XSLT_Handler object with silenced warnings, so it is
not too noisy.

=cut

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;
    $args->{xslt_handler} = Koha::XSLT_Handler->new({print_warns => 0,}) unless defined $args->{xslt_handler};
    $args->{verbosity} = 0 unless defined $args->{verbosity};
    return bless ($args, $class);
}

sub delete_record {
    my ( $self, $record ) = @_;

    my ($action, $system_number) = $self->determine_action_and_system_number({
        identifier => $record->identifier,
        repository_id => $record->repository_id,
    });
    my $error = C4::Biblio::DelBiblio($system_number);
    if ($error){
        die $error;
    } else {
        $record->set({
            system_number => $system_number,
            action => "deleted",
        });
        $record->store();
    }
}

=head2 import_record

This is the method doing the heavy lifting. It runs the other
methods for determining what to do with the record and if it is
linked to an existing record, it transforms the record via XSLT,
and it ultimately adds or updates the record as need be.

=cut

sub import_record {
    my ( $self, $record ) = @_;

    #Step one: determine action (i.e. determine if a record with this identifier from this repository has been successfully added before)
    my ($action, $system_number) = $self->determine_action_and_system_number({
        identifier => $record->identifier,
        repository_id => $record->repository_id,
    });

    #Step two: transform metadata (if necessary) && add/update in Koha
    if ( ($action eq 'add') || ($action eq 'update') ){
        my $schema = Koha::Database->new()->schema();

        #Lookup the repository so we can get the xslt_path and frameworkcode
        my $repository = $schema->resultset('OaiHarvestRepository')->find($record->repository_id);

        my $metadata = $record->metadata;
        if ($metadata && $repository->xslt_path){
            eval {
                #If there is a XSLT path, try to transform metadata
                my $transformed_metadata = $self->transform_metadata({
                    xslt_path => $repository->xslt_path,
                    metadata => $metadata,
                    metadata_prefix => $record->metadata_prefix,
                    identifier => $record->identifier,
                });
                if ($transformed_metadata){
                    $metadata = $transformed_metadata;
                }
            };
            if ($@){
                warn "There was a non-fatal problem transforming the metadata during import: $@";
            }
        }
        if ($metadata){
            if ($record->record_type eq 'biblio'){
                my ( $biblionumber, $biblioitemnumber );
                #create MARC::Record object
                my $marc_record = MARC::Record::new_from_xml( $metadata, "utf8", C4::Context->preference('marcflavour') );
                if ($marc_record){
                    if ($action eq 'add'){
                        #Add record to Koha
                        ($biblionumber,$biblioitemnumber) = C4::Biblio::AddBiblio($marc_record,$repository->frameworkcode);
                    } elsif ($action eq 'update'){
                        #Update record in Koha
                        eval {C4::Biblio::ModBiblio( $marc_record,$system_number,$repository->frameworkcode)};
                        if ($@){
                            my $deleted_biblio = $schema->resultset('Deletedbiblio')->find($system_number);
                            my $existing_biblio = $schema->resultset('Biblio')->find($system_number);
                            if ( ( $deleted_biblio ) || (!$deleted_biblio && !$existing_biblio) ){
                                die "It appears the original record referenced by biblionumber $system_number has been deleted.";
                                #FIXME: What should happen in this situation? You can't re-add the record without first removing
                                #the previous oai_harvest entries or there will be errors...
                                #($biblionumber,$biblioitemnumber) = C4::Biblio::AddBiblio($marc_record,$repository->frameworkcode);
                                #warn "Re-adding the incoming record as a new bibliographic record (biblionumber $system_number).";
                                #$action = "re-add";
                            } else {
                                die $@;
                            }
                        }
                    }
                }
                if ($biblionumber){
                    $system_number = $biblionumber;
                }
            }
            #TODO: Add handling for 'holdings' and 'auth'
        }
        if ($action && $system_number){
            #Update 'action' and 'system_number' columns for this record row
            $record->set({
                action => $action,
                system_number => $system_number,
            });
            $record->store();
        } else {
            die "Error importing record! action: ($action);";
        }
    }
}

=head 2 determine_action_and_system_number

This method checks the database to see if a record from the same
OAI-PMH server with the same identifier has already been added
into Koha. If it hasn't, it should be added. If it has, it should
update the existing record.

=cut

sub determine_action_and_system_number {
    my ( $self, $args ) = @_;
    my ( $action, $system_number );
    if ( $args->{identifier} && $args->{repository_id} ){
        my $schema = Koha::Database->new()->schema();

        #NOTE: $search_results is a DBIx::Class::ResultSet object
        my $search_results = $schema->resultset('OaiHarvest')->search(
            {
                -and => [
                            identifier => $args->{identifier},
                            repository_id => $args->{repository_id},
                            system_number => {'!=', undef},
                            -or => [
                                action => 'add',
                                action => 'update',
                            ],
                        ],
            },
            {
              columns => [ qw/system_number/ ], #Only retrieve system_number
              group_by => [ qw/system_number/ ], #Group results by system_number so you only get 1 result if you have many rows sharing the same system_number
            }
        );

        #NOTE: The DBIx::Class::ResultSet class has overloaded operators,
        #which means a "count()" method is run whenever an object is called in a
        #numerical context (like below). It will also always return 1 in a boolean context.
        if ($search_results == 0){
            $action = 'add';
        } elsif ($search_results == 1){
            $action = 'update';
            while( my $result = $search_results->next ) {
                $system_number = $result->system_number;
            }
        } else {
            if ($search_results && $search_results > 1){
                die "More than one system_number matched the identifier " . $args->{identifier} . " with a repository_id of " . $args->{repository_id} . ".";
            } else {
                die "Unknown error scanning database for previous entries.";
            }
        }
    }
    return ($action,$system_number);
}

=head2 transform_metadata

This method takes a record's XML metadata and runs it through a XSLT
(using a Koha::XSLT_Handler object). The purpose of this is to remove
undesirable fields or to add desired ones.

=cut

sub transform_metadata {
    my ( $self, $args ) = @_;
    my $transformed_metadata;

    if ($self->xslt_handler){
        if ( ($args->{xslt_path}) && ($args->{metadata_prefix}) && ($args->{metadata}) && ($args->{identifier}) ){

            my $xslt = _resolve_xslt_path($args->{xslt_path}, $args->{metadata_prefix});

            #If there is a XSLT file, we try to transform the data
            if ($xslt){
                require XML::LibXSLT;
                #NOTE: You can use "register_function()" here to provide extension functions to the XSLT
                #NOTE: For parameters, you must wrap XML::LibXSLT::xpath_to_string() around each key => value pair within a hashref
                #{ XML::LibXSLT::xpath_to_string(identifier => $args->{identifier}), XML::LibXSLT::xpath_to_string(test => "tester"), }
                $transformed_metadata = $self->xslt_handler->transform({
                    xml => $args->{metadata},
                    file => $xslt,
                    parameters => { XML::LibXSLT::xpath_to_string(identifier => $args->{identifier}), },
                });
                if ( $self->xslt_handler->err ){
                    die $self->xslt_handler->errstr;
                }
                if (! $transformed_metadata){
                    die "Unable to transform metadata";
                }
            }
        }
    }

    return $transformed_metadata;
}

=head2 _resolve_xslt_path

An internal sub for turning magical values into actual filepaths to a XSLT.

=cut

sub _resolve_xslt_path {
    my ( $xslfilename, $metadata_prefix ) = @_;
    if ( $xslfilename ){
        if ( $xslfilename =~ /^\s*"?default"?\s*$/i ) {
            if ($metadata_prefix =~ "marc"){
                #If the repository has a default XSLT and harvests MARCXML:
                my $htdocs  = C4::Context->config('intrahtdocs');
                my $theme   = C4::Context->preference("template");
                my $lang = C4::Languages::getlanguage();
                my $xslfile = C4::Context->preference('marcflavour') .
                                   "slimFromOAI.xsl";

                $xslfilename = C4::XSLT::_get_best_default_xslt_filename($htdocs, $theme, $lang, $xslfile);
                if ( ! -f $xslfilename ){
                    #If the default XSLT doesn't exist, nullify this variable
                    $xslfilename = undef;
                }
            } else {
                #If the repository has a default XSLT and doesn't harvest marc:
                #We return a null because currently we only provide default XSLTs for MARC21 marcxml
                $xslfilename = undef;
            }
        }
    }
    return $xslfilename;
}


1;




package Koha::OAI::Client::Record;

# Copyright Prosentient Systems 2015
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;
use Carp;

use base qw(Koha::Object);
use C4::Biblio;

=head1 NAME

Koha::OAI::Client::Record -

=head1 API

=head2 Class Methods

=cut

sub delete {
    my ( $self ) = @_;
    if ($self->record_type eq 'biblio' && $self->system_number){
        my $error = C4::Biblio::DelBiblio($self->system_number);
        if ($error){
            die $error;
        }
    }
    $self->SUPER::delete();
}

=head3 type

=cut

sub type {
    return 'OaiHarvest';
}

=head1 AUTHOR

David Cook <dcook@prosentient.com.au>

=cut

1;




package Koha::OAI::Client::Records;

# Copyright Prosentient Systems 2015
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;
use Carp;

use base qw(Koha::Objects);
use Koha::OAI::Client::Record;

=head1 NAME

Koha::OAI::Client::Records -

=head1 API

=head2 Class Methods

=cut

=head3 type

=cut

sub type {
    return 'OaiHarvest';
}

=head3 object_class

=cut

sub object_class {
    return 'Koha::OAI::Client::Record';
}

=head1 AUTHOR

David Cook <dcook@prosentient.com.au>

=cut

1;




package Koha::OAI::Client::Repositories;

# Copyright Prosentient Systems 2015
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;

use Carp;

use Koha::Database;

use Koha::OAI::Client::Repository;

use base qw(Koha::Objects);

=head1 NAME

Koha::OAI::Client::Repositories -

=head1 API

=head2 Class Methods

=cut

=head3 type

=cut

sub type {
    return 'OaiHarvestRepository';
}

=head3 object_class

=cut

sub object_class {
    return 'Koha::OAI::Client::Repository';
}

=head1 AUTHOR

David Cook <dcook@prosentient.com.au>

=cut

1;




package Koha::OAI::Client::Repository;

# Copyright Prosentient Systems 2015
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;
use Carp;
use POSIX qw(strftime);
use HTTP::OAI;
use URI;
use DateTime;
use DateTime::Format::Strptime;
use base qw(Koha::Object);

use Koha::OAI::Client::Records;

=head1 NAME

Koha::OAI::Client::Repository -

=head1 API

=head2 Class Methods

=cut

=head3 test_settings

=cut

sub test_settings {
    my ( $self ) = @_;
    my $errors = {};
    my $harvester = $self->harvester();
    if ($harvester){
        my $identify = $harvester->Identify;
        if ($identify->is_success){

            #Test Granularity
            my $granularity_setting = $self->datetime_granularity;
            if ($granularity_setting eq "YYYY-MM-DDThh:mm:ssZ"){
                my $actual_granularity = $identify->granularity;
                if ($granularity_setting ne $actual_granularity){
                    $errors->{second_granularity_not_supported} = 1;
                }
            }

            #Test Set
            my $set_setting = $self->opt_set;
            if ($set_setting){
                my $sets = $harvester->ListSets();
                my $matched_set;
                while ( my $set = $sets->next ){
                    if ($set_setting eq $set->setSpec){
                        $matched_set = 1;
                        last;
                    }
                }
                if ( ! $matched_set ){
                    $errors->{set_does_not_exist} = 1;
                }
            }

            #Test Metadata prefix
            my $metadata_prefix_setting = $self->metadata_prefix;
            if ($metadata_prefix_setting){
                my $metadata_formats = $harvester->ListMetadataFormats();
                my $matched_format;
                while ( my $metadata_format = $metadata_formats->next ){
                    if ($metadata_prefix_setting eq $metadata_format->metadataPrefix){
                        $matched_format = 1;
                        last;
                    }
                }
                if ( ! $matched_format ){
                    $errors->{metadata_prefix_does_not_exist} = 1;
                }
            }
        } else {
            if ($identify->is_error()){
                foreach my $error ($identify->errors){
                    if ($error->code =~ /^404$/){
                        $errors->{url_not_found} = 1;
                    } elsif ($error->code =~ /^401$/){
                        $errors->{failed_authentication} = 1;
                    } else {
                        $errors->{generic_identify_error} = 1;
                    }
                }
            } else {
                #This should never happen, but you never know...
                $errors->{generic_identify_error} = 1;
            }
        }
    } else {
        $errors->{no_harvester} = 1;
    }
    return $errors;
}


=head3 validate

=cut

sub validate {
    my ( $self ) = @_;
    my $errors = {};
    if ($self->base_url){
        my $uri = URI->new($self->base_url);
        if ($uri){
            my $reference = ref $uri;
            if ( ! grep { $reference eq $_ } ("URI::http","URI::https")){
                $errors->{base_url}->{invalid} = 1;
            }
        }
    } elsif (! defined $self->base_url){
        #base_url is undefined
        $errors->{base_url}->{required} = 1;
    }
    if ( ! defined $self->metadata_prefix ){
        $errors->{metadata_prefix}->{required} = 1;
    }
    return $errors;
}

=head3 find_original_system_number

=cut

sub find_original_system_number {
    my ( $self, $args ) = @_;

    my $original_system_number;
    my $metadata = $args->{metadata};
    if ($metadata){
        my $original_system_field = $self->original_system_field; #e.g. 001 or 999$c
        if ($original_system_field){
            #Select root element (e.g. <metadata>)
            my $root = $metadata->documentElement();
            #Select the first child of root (ie the original metadata record) (eg <record>)
            my $original_metadata = $root->firstChild;

            #Check to see if there is a MARC namespace
            my $marcxml_prefix = $original_metadata->lookupNamespacePrefix("http://www.loc.gov/MARC21/slim");
            if (defined $marcxml_prefix){
                my ($tag,$code) = split(/\$/,$original_system_field); #e.g. split 999$c into 999 and c || split 001 into 001
                my $xpc = XML::LibXML::XPathContext->new;
                $xpc->registerNs('marc', 'http://www.loc.gov/MARC21/slim');
                my $fields = $xpc->find(qq(//marc:record/node()[\@tag="$tag"]),$original_metadata);
                if ($fields){
                    #Use the first field found
                    my $field = $fields->get_node(1);
                    if ($field){
                        my $node_name = $field->nodeName;
                        if ($node_name eq 'controlfield'){
                            #If it's a controlfield, we can use the text without drilling down
                            $original_system_number = $field->textContent;
                        } elsif ($node_name eq 'datafield'){
                            #If it's a datafield, we need to drill down into the subfields
                            if ($code){
                                my $subfields = $field->find(qq(node()[\@code="$code"]));
                                if ($subfields){
                                    #Use the first subfield found
                                    my $subfield = $subfields->get_node(1);
                                    if ($subfield){
                                        $original_system_number = $subfield->textContent;
                                    }

                                }
                            }
                        }
                    }
                }
            }
        }
    }
    return $original_system_number;
}

=head3 queue_record

=cut

sub queue_record {
    my ( $self, $args ) = @_;
    my $record = $args->{record};
    my $stylesheet = $args->{stylesheet};

    my $action = "to_import";
    if ( $record->is_deleted() ){
        my $already_deleted_rows = Koha::OAI::Client::Records->search({ identifier => $record->identifier, action => "deleted" });
        my $identified_rows = Koha::OAI::Client::Records->search({ identifier => $record->identifier });
        if ($identified_rows->count() > 0 && $already_deleted_rows->count() == 0){
            #Only queue records "to_delete", if the record has been previously harvested
            $action = "to_delete";
        } else {
            return;
        }
    }

    my $metadata = $record->metadata ? $record->metadata->dom : '';
    my $original_system_number;
    if ($metadata){
        if ($stylesheet){
            #Use a stylesheet to strip the OAI-PMH metadata XML wrapper
            eval {
                my $result = $stylesheet->transform($metadata);
                if ($result){
                    $metadata = $result;
                }
            };
            if ($@){
                warn "Problem transforming harvested metadata with XSLT: $@";
            }
        }

        $original_system_number = $self->find_original_system_number({metadata => $metadata});
    }

    my $entry = Koha::OAI::Client::Records->_resultset->find_or_new({
        repository_id => $self->repository_id, #Internal ID for the OAI repository
        action => $action,
        identifier => $record->identifier, #External ID for the OAI record
        datestamp => $record->datestamp, #External datestamp for the OAI record
        metadata_prefix => $self->metadata_prefix, #External metadataPrefix for the OAI record
        metadata => $metadata ? $metadata->toString(1) : undef,
        record_type => $self->record_type,
        original_system_number => $original_system_number,
    },
    { key => "harvest_identifier_datestamp" }
    );
    if( !$entry->in_storage ) {
      $entry->insert;
    }
}

sub _format_datetime {
    my ( $self, $args ) = @_;
    my $datetime_string = $args->{datetime_string};
    my $formatted_string;
    if ($datetime_string && $datetime_string ne "0000-00-00 00:00:00"){
        my $strp = DateTime::Format::Strptime->new(
            pattern   => '%F %T',
        );
        my $dt = $strp->parse_datetime($datetime_string);
        if ($dt){
            if ($self->datetime_granularity && $self->datetime_granularity eq 'YYYY-MM-DDThh:mm:ssZ'){
                $formatted_string = $dt->strftime("%FT%TZ");
            } else {
                $formatted_string = $dt->strftime("%F");
            }
        }
    }
    return $formatted_string;
}

sub harvester {
    my ( $self ) = @_;
    my $harvester;
    if ($self->base_url){
        $harvester = new HTTP::OAI::Harvester( baseURL => $self->base_url );
        #Use basic http authentication credentials if they're set
       # if ($self->basic_realm && $self->basic_username && $self->basic_password){
            #Decompose the URI in order to register the basic authentication credentials
            my $uri = URI->new($self->base_url);
            my $host = $uri->host;
            my $port = $uri->port;
            $harvester->credentials($host.":".$port, $self->basic_realm, $self->basic_username, $self->basic_password);
       # }
    }
    return $harvester;
}

=head3 query_repository

=cut

sub query_repository {
    my ( $self ) = @_;
    my $response;
    my $h = $self->harvester();
    if ($h){
        my $opt_from = $self->opt_from;
        my $opt_until = $self->opt_until;
        foreach my $datetime ($opt_from,$opt_until){
            #Format the datetime strings from the database into the format and granularity
            #expected by the target OAI-PMH server
            $datetime = $self->_format_datetime({datetime_string => $datetime,});
        }

        #Issue the request to the OAI-PMH server
        $response = $h->ListRecords(
            metadataPrefix => $self->metadata_prefix,
            from => $opt_from,
            until => $opt_until,
            set => $self->opt_set,
        );
    }
    return $response;
}

=head3 harvest

=cut

sub harvest {
    my ( $self, $args ) = @_;

    my $verbose = $args->{verbose} // '';
    my $stylesheet = $args->{stylesheet} // '';
    $verbose && print "Harvesting records from ".$self->base_url."\n";

    #Record the exact time before sending the first OAI-PMH request, and set "opt_from" to this after the harvest is finished
    my $new_from_date = strftime "%Y-%m-%d %H:%M:%S", localtime;
    #Send a request to the OAI-PMH repository
    my $oai_response = $self->query_repository();
    if ($oai_response){
        if( $oai_response->is_error ){
            warn "responseDate => " . $oai_response->responseDate . "\n";
            warn "requestURL => " . $oai_response->requestURL . "\n";
            warn "Error harvesting: " . $oai_response->message . "\n";
        } else {
            while( my $rec = $oai_response->next ) {
                $verbose && print $rec->identifier."\n";
                #Queue each record, so a different script can process them in Koha
                $self->queue_record({record => $rec, stylesheet => $stylesheet, });
            }
            #Update opt_from after a successful harvest
            $self->opt_from($new_from_date);
            $verbose && print "Updating `opt_from` to $new_from_date \n";
            $self->store();
        }
    }
}

#FIXME
=head3 reharvest

=cut

sub reharvest {
    my ( $self, $args ) = @_;
    my $verbose = $args->{verbose} // '';
    eval {
        $self->delete_harvest({ verbose => $verbose, });
    };
    if ($@){
        warn $@;
    } else {
        $self->harvest({ verbose => $verbose, });
    }
}

#FIXME
=head3 delete_harvest

=cut

sub delete_harvest {
    my ( $self ) = @_;

    my $count_of_records = 0;
    my $count_of_deleted_records = 0;
    my $problem_records = [];

    if ($self->repository_id){
        my $records = Koha::OAI::Client::Records->search({ "repository_id" => $self->repository_id });
        while (my $record = $records->next){
            #Increment overall count
            $count_of_records++;
            eval {
                my $is_deleted = $record->delete();
                if ($is_deleted){
                    $count_of_deleted_records++;
                }
            };
            if ($@){
                push(@$problem_records,$record);
            }
        }
    }

    if ($count_of_deleted_records < $count_of_records){
        #Not all of this repository's records could be deleted. Manual intervention may be required to remove items
        die $problem_records;
    }
}

=head3 type

=cut

sub type {
    return 'OaiHarvestRepository';
}

=head1 AUTHOR

David Cook <dcook@prosentient.com.au>

=cut

1;




#!/usr/bin/perl

# Copyright Prosentient Systems 2015
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

=head1 NAME

oai_client.pl

=head1 DESCRIPTION

Admin page to manage Koha's target OAI-PMH servers

=cut

use Modern::Perl;
use CGI qw ( -utf8 );
use C4::Auth;
use C4::Output;
use C4::Koha qw/GetFrameworksLoop/;
use Koha::OAI::Client::Repository;
use Koha::OAI::Client::Repositories;
use Koha::OAI::Client::Records;

my $input = new CGI;

my ($template, $loggedinuser, $cookie, $flags) = get_template_and_user( {
    template_name   => 'admin/oai_client.tt',
    query           => $input,
    type            => 'intranet',
    authnotrequired => 0,
    flagsrequired   => { 'parameters' => 'parameters_remaining_permissions' },
} );

my $op = $input->param('op');
my $test_settings = $input->param('test_settings');
my $repository_id = $input->param('repository_id');

my $base_url = $input->param('base_url') || undef;
my $metadata_prefix = $input->param('metadata_prefix') || undef;
my $opt_from = $input->param('opt_from') || undef;
my $opt_until = $input->param('opt_until') || undef;
my $opt_set = $input->param('opt_set') || undef;
my $active = $input->param('active');
my $xslt_path = $input->param('xslt_path') || undef;
my $frameworkcode = $input->param('frameworkcode'); #This can be empty as the default framework is an empty value
my $comments = $input->param('comments') || undef;
my $basic_username = $input->param('basic_username') || undef;
my $basic_password = $input->param('basic_password') || undef;
my $basic_realm = $input->param('basic_realm') || undef;
my $datetime_granularity = $input->param('datetime_granularity');
my $record_type = $input->param('record_type');
my $original_system_field = $input->param('original_system_field') || undef;

if ( $op ){
    my $repository;
    if ( $repository_id ){
        #Fetch repository from database...
        $repository = Koha::OAI::Client::Repositories->find($repository_id);
    }

    if ( $op eq "new" ){
        $template->param(
            frameworkloop => C4::Koha::GetFrameworksLoop(),
        );
    } elsif ( $op eq "create" ){
        my $new_repository = Koha::OAI::Client::Repository->new({
            base_url => $base_url,
            metadata_prefix => $metadata_prefix,
            opt_from => $opt_from,
            opt_until => $opt_until,
            opt_set => $opt_set,
            active => $active,
            xslt_path => $xslt_path,
            frameworkcode => $frameworkcode,
            comments => $comments,
            basic_username => $basic_username,
            basic_password => $basic_password,
            basic_realm => $basic_realm,
            datetime_granularity => $datetime_granularity,
            record_type => $record_type,
            original_system_field => $original_system_field,
        });
        my $errors = $new_repository->validate();
        if (%$errors){
            $op = "new";
            $template->param(
                repository => $new_repository,
                frameworkloop => C4::Koha::GetFrameworksLoop($new_repository->frameworkcode),
                errors => $errors,
            );
        } else {
            $new_repository->store();
            $op = "list";
        }
    } elsif ( $op eq "edit" ){
        if ($repository){
            $template->param(
                repository => $repository,
                frameworkloop => C4::Koha::GetFrameworksLoop($repository->frameworkcode),
            );
        }
    } elsif ( ($op eq "view") || ($op eq "reset") ){
        if ($repository){

            #If op eq reset, then delete the entire harvest for this repository
            if ($op eq "reset"){
                eval {
                    $repository->delete_harvest();
                    $template->param( reset_success => 1 );
                };
                if ($@){
                    my $problem_records = $@;
                    $template->param( problem_records => $problem_records );
                }
            }

            #"add", "update"
            my $imported_records = Koha::OAI::Client::Records->search({
                repository_id => $repository->repository_id,
                system_number => {'!=', undef},
            }, {
              group_by => [ qw/system_number/ ],
              #Group results by system_number so you only get 1 result if you have many rows sharing the same system_number
              #Also, group by system number because you might have imported record 1, deleted it, and then it gets re-imported later as a different Koha record
            });
            my $waiting_records = Koha::OAI::Client::Records->search({ repository_id => $repository->repository_id, action => 'to_import',});
            my $to_delete_records = Koha::OAI::Client::Records->search({ repository_id => $repository->repository_id, action => 'to_delete',});
            my $deleted_records = Koha::OAI::Client::Records->search({ repository_id => $repository->repository_id, action => 'deleted',});
            my $error_records = Koha::OAI::Client::Records->search({ repository_id => $repository->repository_id, action => 'error',});

            $template->param(
                repository => $repository,
                imported_records_count => $imported_records->count(),
                waiting_records_count => $waiting_records->count(),
                to_delete_records_count => $to_delete_records->count(),
                deleted_records_count => $deleted_records->count(),
                error_records_count => $error_records->count(),
            );
        }
        if ($op eq "reset"){
            $op = "view";
        }
    } elsif ( $op eq "update" ){
        if ($repository){
            $repository->set({
                base_url => $base_url,
                metadata_prefix => $metadata_prefix,
                opt_from => $opt_from,
                opt_until => $opt_until,
                opt_set => $opt_set,
                active => $active,
                xslt_path => $xslt_path,
                frameworkcode => $frameworkcode,
                comments => $comments,
                basic_username => $basic_username,
                basic_password => $basic_password,
                basic_realm => $basic_realm,
                datetime_granularity => $datetime_granularity,
                record_type => $record_type,
                original_system_field => $original_system_field,
            });
            my $errors = $repository->validate();
            if (%$errors || $test_settings){
                if (! %$errors && $test_settings){
                    my $test_errors = $repository->test_settings();
                    if (%$test_errors){
                        $template->param( test_errors => $test_errors );
                    } else {
                        $template->param( test_success => 1 );
                    }
                }
                $op = "edit";
                $template->param(
                    repository => $repository,
                    frameworkloop => C4::Koha::GetFrameworksLoop($repository->frameworkcode),
                    errors => $errors,
                );
            } else {
                $repository->store();
                $op = "list";
            }
        }
    } elsif ( $op eq "delete" ){
        if ($repository){
            $repository->delete;
        }
        $op = "list";
    }
} else {
    $op = "list";
}

if ( $op eq "list" ){
        my @repositories = Koha::OAI::Client::Repositories->as_list();
        $template->param(
            repositories => \@repositories,
            frameworkloop => C4::Koha::GetFrameworksLoop(),
        );
}

$template->param(
    op => $op,
);

output_html_with_http_headers $input, $cookie, $template->output;




DROP TABLE IF EXISTS oai_harvest;
DROP TABLE IF EXISTS oai_harvest_repositories;

--
-- Table structure for table `oai_harvest_repositories`
--

CREATE TABLE IF NOT EXISTS oai_harvest_repositories (
  repository_id int(11) NOT NULL AUTO_INCREMENT, -- primary key identifier
  base_url text NOT NULL, -- baseURL of the remote OAI-PMH repository (mandatory)
  basic_username text, -- username for basic http authentication
  basic_password text, -- password for basic http authentication
  basic_realm text, -- realm for basic http authentication
  datetime_granularity enum('YYYY-MM-DDThh:mm:ssZ','YYYY-MM-DD') NOT NULL DEFAULT 'YYYY-MM-DD', -- granularity to use in harvest request
  opt_from DATETIME DEFAULT NULL, -- "from" DATETIME for selective harvesting (optional - automatically updated by cronjob)
  opt_until DATETIME DEFAULT NULL, -- "until" DATETIME for selective harvesting (optional)
  opt_set varchar(45) DEFAULT NULL, -- the record set to harvest for selective harvesting (optional)
  metadata_prefix varchar(45) NOT NULL, -- metadata format (e.g. oai_dc, dc, marcxml)
  active int(1) NOT NULL DEFAULT '0', -- indicate whether this repo is actively harvested by Koha
  timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- date last modified. Maybe `datelastmodified` would be a better column name.
  xslt_path text, -- filepath to a local or external XSLT file to use for transforming incoming records into MARCXML
  frameworkcode varchar(4) NOT NULL DEFAULT '', -- framework to use when ingesting records
  comments text, -- limited number of characters (controlled by template) to describe the repository (optional - for librarian use rather than system use)
  record_type enum('biblio','auth','holdings') NOT NULL DEFAULT 'biblio',
  original_system_field varchar(45), -- Field where you can a find a record's original system number (e.g. 001 or 999$c)
  -- NOTE: In future, may need to add fields for holdings-biblio relationship (e.g. field to designate 004 in holdings record as link to a biblio's original system number)
  PRIMARY KEY (repository_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

--
-- Table structure for table `oai_harvest`
--

CREATE TABLE IF NOT EXISTS oai_harvest (
  oai_harvest_id int(11) NOT NULL AUTO_INCREMENT, -- unique internal id for the OAI record
  repository_id int(11) NOT NULL, -- OAI repository from which the record was harvested
  system_number int(11) DEFAULT NULL, -- indicates the Koha record created from this record
  action varchar(45) DEFAULT NULL, -- indicates whether this record has been processed and how it's been processed (e.g. add, update, ignore)
  timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- last modified date in Koha. Maybe `datelastmodified` would be a better column name.
  identifier varchar(255) NOT NULL, -- the record's OAI ID from the OAI repository
  datestamp varchar(45) NOT NULL, -- the record's OAI datestamp from the OAI repository (e.g. YYYY-MM-DDTHH:MM:SSZ).
  metadata_prefix varchar(45) NOT NULL, -- the type of metadata (e.g. marcxml, oai_dc, etc.)
  metadata longtext, -- original metadata as transmitted via OAI-PMH
  record_type enum('biblio','auth','holdings') NOT NULL DEFAULT 'biblio',
  original_system_number varchar(45), -- original system number (e.g. the unique number in the 001 of the incoming record)
  -- NOTE: In future, may need to add fields for holdings-biblio relationship
  PRIMARY KEY (oai_harvest_id),
  UNIQUE KEY harvest_identifier_datestamp (identifier,datestamp),
  KEY FK_oai_harvest_1 (repository_id),
  CONSTRAINT FK_oai_harvest_1 FOREIGN KEY (repository_id) REFERENCES oai_harvest_repositories (repository_id) ON UPDATE NO ACTION
) ENGINE=InnoDB DEFAULT CHARSET=utf8;




  CONSTRAINT borrower_discharges_ibfk1 FOREIGN KEY (borrower) REFERENCES borrowers (borrowernumber) ON DELETE CASCADE ON UPDATE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;

--
-- Table structure for table `oai_harvest_repositories`
--

CREATE TABLE IF NOT EXISTS oai_harvest_repositories (
  repository_id int(11) NOT NULL AUTO_INCREMENT, -- primary key identifier
  base_url text NOT NULL, -- baseURL of the remote OAI-PMH repository (mandatory)
  basic_username text, -- username for basic http authentication
  basic_password text, -- password for basic http authentication
  basic_realm text, -- realm for basic http authentication
  datetime_granularity enum('YYYY-MM-DDThh:mm:ssZ','YYYY-MM-DD') NOT NULL DEFAULT 'YYYY-MM-DD', -- granularity to use in harvest request
  opt_from DATETIME DEFAULT NULL, -- "from" DATETIME for selective harvesting (optional - automatically updated by cronjob)
  opt_until DATETIME DEFAULT NULL, -- "until" DATETIME for selective harvesting (optional)
  opt_set varchar(45) DEFAULT NULL, -- the record set to harvest for selective harvesting (optional)
  metadata_prefix varchar(45) NOT NULL, -- metadata format (e.g. oai_dc, dc, marcxml)
  active int(1) NOT NULL DEFAULT '0', -- indicate whether this repo is actively harvested by Koha
  timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- date last modified. Maybe `datelastmodified` would be a better column name.
  xslt_path text, -- filepath to a local or external XSLT file to use for transforming incoming records into MARCXML
  frameworkcode varchar(4) NOT NULL DEFAULT '', -- framework to use when ingesting records
  comments text, -- limited number of characters (controlled by template) to describe the repository (optional - for librarian use rather than system use)
  record_type enum('biblio','auth','holdings') NOT NULL DEFAULT 'biblio',
  original_system_field varchar(45), -- Field where you can a find a record's original system number (e.g. 001 or 999$c)
  -- NOTE: In future, may need to add fields for holdings-biblio relationship (e.g. field to designate 004 in holdings record as link to a biblio's original system number)
  PRIMARY KEY (repository_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

--
-- Table structure for table `oai_harvest`
--

CREATE TABLE IF NOT EXISTS oai_harvest (
  oai_harvest_id int(11) NOT NULL AUTO_INCREMENT, -- unique internal id for the OAI record
  repository_id int(11) NOT NULL, -- OAI repository from which the record was harvested
  system_number int(11) DEFAULT NULL, -- indicates the Koha record created from this record
  action varchar(45) DEFAULT NULL, -- indicates whether this record has been processed and how it's been processed (e.g. add, update, ignore)
  timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- last modified date in Koha. Maybe `datelastmodified` would be a better column name.
  identifier varchar(255) NOT NULL, -- the record's OAI ID from the OAI repository
  datestamp varchar(45) NOT NULL, -- the record's OAI datestamp from the OAI repository (e.g. YYYY-MM-DDTHH:MM:SSZ).
  metadata_prefix varchar(45) NOT NULL, -- the type of metadata (e.g. marcxml, oai_dc, etc.)
  metadata longtext, -- original metadata as transmitted via OAI-PMH
  record_type enum('biblio','auth','holdings') NOT NULL DEFAULT 'biblio',
  original_system_number varchar(45), -- original system number (e.g. the unique number in the 001 of the incoming record)
  -- NOTE: In future, may need to add fields for holdings-biblio relationship
  PRIMARY KEY (oai_harvest_id),
  UNIQUE KEY harvest_identifier_datestamp (identifier,datestamp),
  KEY FK_oai_harvest_1 (repository_id),
  CONSTRAINT FK_oai_harvest_1 FOREIGN KEY (repository_id) REFERENCES oai_harvest_repositories (repository_id) ON UPDATE NO ACTION
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;

Lines 62-67 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc (+1 lines)
62	<ul>	62	<ul>
63	<!-- <li><a href="/cgi-bin/koha/admin/printers.pl">Network Printers</a></li> -->	63	<!-- <li><a href="/cgi-bin/koha/admin/printers.pl">Network Printers</a></li> -->
64	<li><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50/SRU servers</a></li>	64	<li><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50/SRU servers</a></li>
		65	<li><a href="/cgi-bin/koha/admin/oai_client.pl">OAI-PMH servers</a></li>
65	<li><a href="/cgi-bin/koha/admin/didyoumean.pl">Did you mean?</a></li>	66	<li><a href="/cgi-bin/koha/admin/didyoumean.pl">Did you mean?</a></li>
66	<li><a href="/cgi-bin/koha/admin/columns_settings.pl">Columns settings</a></li>	67	<li><a href="/cgi-bin/koha/admin/columns_settings.pl">Columns settings</a></li>
67	</ul>	68	</ul>

Lines 107-112 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt (+2 lines)
107	<dd>Printers (UNIX paths).</dd> -->	107	<dd>Printers (UNIX paths).</dd> -->
108	<dt><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50/SRU servers</a></dt>	108	<dt><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50/SRU servers</a></dt>
109	<dd>Define which external servers to query for MARC data.</dd>	109	<dd>Define which external servers to query for MARC data.</dd>
		110	<dt><a href="/cgi-bin/koha/admin/oai_client.pl">OAI-PMH servers</a></dt>
		111	<dd>Define external servers from which to harvest records via OAI-PMH.</dd>
110	<dt><a href="/cgi-bin/koha/admin/didyoumean.pl">Did you mean?</a></dt>	112	<dt><a href="/cgi-bin/koha/admin/didyoumean.pl">Did you mean?</a></dt>
111	<dd>Choose which plugins to use to suggest searches to patrons and staff.</dd>	113	<dd>Choose which plugins to use to suggest searches to patrons and staff.</dd>
112	<dt><a href="/cgi-bin/koha/admin/columns_settings.pl">Configure columns</a></dt>	114	<dt><a href="/cgi-bin/koha/admin/columns_settings.pl">Configure columns</a></dt>




[% INCLUDE 'doc-head-open.inc' %]
<title>Koha &rsaquo; Administration &rsaquo; OAI-PMH server targets</title>
[% INCLUDE 'doc-head-close.inc' %]
[% INCLUDE 'calendar.inc' %]
<script type="text/javascript" src="[% interface %]/lib/jquery/plugins/jquery-ui-timepicker-addon.min.js"></script>
[% INCLUDE 'timepicker.inc' %]
[% IF ( op == "list" ) %]
    <link rel="stylesheet" type="text/css" href="[% themelang %]/css/datatables.css" />
    [% INCLUDE 'datatables.inc' %]
    <script type="text/javascript">
    //<![CDATA[
        $(document).ready(function() {
            $("#serverst").dataTable($.extend(true, {}, dataTablesDefaults, {
                "aoColumnDefs": [
                    { "aTargets": [8,9], "bSortable": false },
                ],
                "sPaginationType": "four_button"
            }));
        });
    //]]>
    </script>
[% ELSIF ( op == "edit" ) || ( op == "new" ) %]
    <script type="text/javascript">
    //<![CDATA[
        $(document).ready(function() {
            $(".opt_datetime").datetimepicker({
                dateFormat: "yy-mm-dd",
                timeFormat: "HH:mm:ss",
                hour: 0,
                minute: 0,
                second: 0,
                showSecond: 1,
            });
        });
    //]]>
    </script>
    <style type="text/css">
        /* Override staff-global.css which hides second, millisecond, and microsecond sliders */
        .ui_tpicker_second {
            display: block;
        }
        .test-success {
            /* same color as .text-success in Bootstrap 2.2.2 */
            color:#468847;
        }
    </style>
[% END %]
</head>

<body id="admin_oai_server_targets" class="admin">
[% INCLUDE 'header.inc' %]
[% INCLUDE 'cat-search.inc' %]
<div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/admin/admin-home.pl">Administration</a> &rsaquo; OAI-PMH server targets</div>

<div id="doc3" class="yui-t2">

<div id="bd">
  <div id="yui-main">
    <div class="yui-b">
        [% IF ( op ) %]
            [% IF ( op == "list" ) %]
                <div id="toolbar" class="btn-toolbar">
                    <a id="newserver" class="btn btn-small" href="/cgi-bin/koha/admin/oai_client.pl?op=new"><i class="icon-plus"></i> New OAI-PMH server target</a>
                </div>
                <h1>OAI-PMH server targets</h1>
                <table id="serverst">
                    <thead>
                        <tr>
                            <th>Base URL</th>
                            <th>Metadata Prefix</th>
                            <th>From</th>
                            <th>Until</th>
                            <th>Set</th>
                            <th>Active</th>
                            <th>Path to XSLT</th>
                            <th>Record type</th>
                            <th>MARC framework</th>
                            <th>Original system field</th>
                            <th>Comments</th>
                            <th>Options</th>
                        </tr>
                    </thead>
                    <tbody>
                    [% FOREACH repository IN repositories %]
                        <tr>
                            <td>[% repository.base_url %]</td>
                            <td>[% repository.metadata_prefix %]</td>
                            <td>[% IF ( repository.opt_from ) != "0000-00-00 00:00:00"; repository.opt_from; END; %]</td>
                            <td>[% IF ( repository.opt_until ) != "0000-00-00 00:00:00"; repository.opt_until; END; %]</td>
                            <td>[% repository.opt_set %]</td>
                            <td>
                                [% IF ( repository.active == 1 ) %]
                                    Active
                                [% ELSE %]
                                    Inactive
                                [% END %]
                            </td>
                            <td>[% repository.xslt_path %]</td>
                            <td>[% repository.record_type %]</td>
                            <td>
                                [%
                                    framework_name = '';
                                    FOREACH framework IN frameworkloop;
                                        IF ( framework.value == repository.frameworkcode );
                                            framework_name = framework.description;
                                        END;
                                    END;
                                %]
                                [% IF ( framework_name ) %]
                                    [% framework_name %]
                                [% ELSE %]
                                    Default
                                [% END %]
                            </td>
                            <td>[% repository.original_system_field %]</td>
                            <td>[% repository.comments %]</td>
                            <td>
                                <a href="/cgi-bin/koha/admin/oai_client.pl?op=view&repository_id=[% repository.repository_id %]">View</a>
                                <a href="/cgi-bin/koha/admin/oai_client.pl?op=edit&repository_id=[% repository.repository_id %]">Edit</a>
                                <a href="/cgi-bin/koha/admin/oai_client.pl?op=delete&repository_id=[% repository.repository_id %]">Delete</a>
                            </td>
                        </tr>
                    [% END %]
                    </tbody>
                </table>
            [% ELSIF ( op == "view" ) %]
                <h1>View OAI-PMH server target</h1>
                <h2>[% repository.base_url %]</h2>
                <fieldset class="rows">
                    <ol>
                        <li>Koha records created from harvested records: [% imported_records_count %]</li>
                        <li>Koha records deleted by harvested records: [% deleted_records_count %]</li>
                        <li>Harvested records waiting to be imported: [% waiting_records_count %]</li>
                        <li>Harvested records waiting to be deleted from Koha: [% to_delete_records_count %]</li>
                        <li>Harvested records in an error state: [% error_records_count %]</li>
                    </ol>
                </fieldset>
                <form action="/cgi-bin/koha/admin/oai_client.pl" name="reset-form" method="get" id="oai-server-reset" novalidate="novalidate">
                    <fieldset class="action">
                        <input type="hidden" name="op" value="reset" />
                        <input type="hidden" name="repository_id" value="[% repository.repository_id %]" />
                        <button type="submit">Reset repository harvest</button>
                    </fieldset>
                </form>
                [% IF ( problem_records ) %]
                    <p>Not all of this repository's records could be deleted. Manual intervention may be required to remove items. Review the following records:</p>
                    <ol>
                    [% FOREACH problem_record IN problem_records %]
                        [% IF ( problem_record.record_type == "biblio" ) %]
                        <li><a target="_blank" href="/cgi-bin/koha/catalogue/detail.pl?biblionumber=[% problem_record.system_number %]">[% problem_record.record_type %] - [% problem_record.system_number %]</a></li>
                        [% ELSE %]
                        <li>[% problem_record.record_type %] - [% problem_record.system_number %]</li>
                        [% END %]
                    [% END %]
                    </ol>
                [% END %]
                [% IF ( reset_success ) %]
                    <p>This repository no longer has any harvested records. Modify the "From" and "Until" dates to prepare the repository for the next harvest.</p>
                [% END %]
            [% ELSIF ( op == "edit" ) || ( op == "new" ) %]
                [% IF ( op == "new" ) %]
                    <h1>New OAI-PMH server target</h1>
                [% ELSIF ( op == "edit" ) %]
                    <h1>Modify OAI-PMH server target</h1>
                [% END %]
                <form action="/cgi-bin/koha/admin/oai_client.pl" name="detail-form" method="post" id="oai-server-target-details" novalidate="novalidate">
                    [% IF ( op == "new" ) %]
                        <input type="hidden" name="op" value="create" />
                    [% ELSIF ( op == "edit" ) %]
                        <input type="hidden" name="repository_id" value="[% repository.repository_id %]" />
                        <input type="hidden" name="op" value="update" />
                    [% END %]
                    <fieldset class="rows">
                        <legend>HTTP parameters:</legend>
                        <ol>
                            <li>
                                <label for="base_url">Base URL: </label>
                                <input type="text" id="base_url" name="base_url" value="[% repository.base_url %]" size="60" />
                                [% IF ( errors.base_url.required ) %]<span class="required">Required</span>[% END %]
                                <span class="error">[% IF ( errors.base_url.invalid ) %](Invalid URL)[% END %]</span>
                                <span class="error">[% IF ( test_errors.url_not_found ) %](404 - URL Not Found)[% END %]</span>

                            </li>
                            <li>
                                <label for="basic_username">Basic auth username: </label>
                                <input type="text" id="basic_username" name="basic_username" value="[% repository.basic_username %]" size="60" />
                                <span class="error">[% IF ( test_errors.failed_authentication ) %](Authentication failed)[% END %]</span>
                            </li>
                            <li>
                                <label for="basic_password">Basic auth password: </label>
                                <input type="text" id="basic_password" name="basic_password" value="[% repository.basic_password %]" size="60" />
                                <span class="error">[% IF ( test_errors.failed_authentication ) %](Authentication failed)[% END %]</span>
                            </li>
                            <li>
                                <label for="basic_realm">Basic auth realm: </label>
                                <input type="text" id="basic_realm" name="basic_realm" value="[% repository.basic_realm %]" size="60" />
                                <span class="error">[% IF ( test_errors.failed_authentication ) %](Authentication failed)[% END %]</span>
                            </li>
                        </ol>
                    </fieldset>
                    <fieldset class="rows">
                        <legend>OAI-PMH parameters:</legend>
                        <ol>
                            <li>
                                <label for="opt_set">Set: </label>
                                <input type="text" name="opt_set" value="[% repository.opt_set %]" size="30" />
                                <span class="error">[% IF ( test_errors.set_does_not_exist ) %](Set unavailable)[% END %]</span>
                            </li>
                            <li>
                                <label for="metadata_prefix">Metadata prefix: </label>
                                <input type="text" name="metadata_prefix" value="[% repository.metadata_prefix %]" size="30" />
                                [% IF ( errors.metadata_prefix.required ) %]<span class="required">Required</span>[% END %]
                                [% IF ( test_errors.metadata_prefix_does_not_exist ) %]<span class="error">(Metadata prefix unavailable)</span>[% END %]

                            </li>
                            <li>
                                <label for="datetime_granularity">Granularity: </label>
                                <select name="datetime_granularity">
                                    [% FOREACH granularity IN ['YYYY-MM-DD', 'YYYY-MM-DDThh:mm:ssZ'] %]
                                        [% IF ( granularity == repository.datetime_granularity ) %]
                                        <option value="[% granularity %]" selected="selected">[% granularity %]</option>
                                        [% ELSE %]
                                        <option value="[% granularity %]">[% granularity %]</option>
                                        [% END %]
                                    [% END %]
                                </select>
                                <!-- <span class="required">Required</span> -->
                                [% IF ( test_errors.second_granularity_not_supported ) %]<span class="error">(Second granularity is not supported)</span>[% END %]
                            </li>
                            <li>
                                <label for="opt_from">From: </label>
                                <input type="text" class="opt_datetime" name="opt_from" value="[% IF ( repository.opt_from ) != "0000-00-00 00:00:00"; repository.opt_from; END; %]" size="30" />
                            </li>
                            <li>
                                <label for="opt_until">Until: </label>
                                <input type="text" class="opt_datetime" name="opt_until" value="[% IF ( repository.opt_until ) != "0000-00-00 00:00:00"; repository.opt_until; END; %]" size="30" />
                            </li>
                        </ol>
                    </fieldset>
                    <fieldset class="action">
                       [% IF ( test_success ) %]
                            <p class="test-success">HTTP and OAI-PMH parameter tests completed successfully.</p>
                       [% END %]
                       [% IF (test_errors.no_harvester) %]
                            <p>Unable to create a OAI-PMH harvester with these settings</p>
                       [% END %]
                       [% IF (test_errors.generic_identify_error) %]
                            <p>An error was encountered using the "Identify" verb with the server target.</p>
                       [% END %]
                        <input type="submit" name="test_settings" value="Test HTTP and OAI-PMH parameters">
                    </fieldset>
                    <fieldset class="rows">
                        <legend>Import parameters:</legend>
                        <ol>
                            <li>
                                <label for="active">Active: </label>
                                [% IF ( repository.active == 1) %]
                                    <input type="radio" name="active" value="1" checked="checked" />
                                [% ELSE %]
                                    <input type="radio" name="active" value="1" />
                                [% END %]
                                 Active
                                [% IF ( repository.active != 1) %]
                                    <input type="radio" name="active" value="0" checked="checked" />
                                [% ELSE %]
                                    <input type="radio" name="active" value="0" />
                                [% END %]
                                 Inactive
                            </li>
                            <li>
                                <label for="xslt_path">Path to XSLT: </label>
                                <input type="text" name="xslt_path" value="[% repository.xslt_path %]" size="60" />
                            </li>
                            <li>
                                <label for="record_type">Record type:</label>
                                <select name="record_type">
                                    [% record_types = [ 'biblio' ] #TODO: In future, add 'auth' and 'holdings' support here %]
                                    [% FOREACH type IN record_types %]
                                        [% IF type == repository.record_type %]
                                            <option value="[% type %]" selected="selected">[% type %]</option>
                                        [% ELSE %]
                                            <option value="[% type %]">[% type %]</option>
                                        [% END %]
                                    [% END %]
                                </select>
                                <!-- <span class="required">Required</span> -->
                            </li>
                            [%# Only need framework for 'biblio' and 'holdings'. This will need to change for 'auth'. %]
                            <li>
                                <label for="frameworkcode">MARC framework: </label>
                                <select name="frameworkcode">
                                    <option value="">Default</option>
                                [% FOREACH framework IN frameworkloop %]
                                    [% IF ( framework.selected ) %]
                                        <option value="[% framework.value %]" selected="selected">[% framework.description %]</option>
                                    [% ELSE %]
                                        <option value="[% framework.value %]">[% framework.description %]</option>
                                    [% END %]
                                [% END %]
                                </select>
                                <!-- <span class="required">Required</span> -->
                            </li>
                            <li>
                                <label for="original_system_field">Original system field</label>
                                <input type="text" name="original_system_field" value="[% repository.original_system_field %]" size="60%" />
                            </li>
                            <li>
                                <label for="comments">Comments: </label>
                                <textarea name="comments">[% repository.comments %]</textarea>
                            </li>
                        </ol>
                    </fieldset>
                    <fieldset class="action">
                        <input type="submit" value="Save">
                        <a class="cancel" href="/cgi-bin/koha/admin/oai_client.pl">Cancel</a>
                    </fieldset>
                </form>
            [% END %]
        [% END #/op %]
    </div>
  </div>
  <div class="yui-b">
    [% INCLUDE 'admin-menu.inc' %]
  </div>
</div>
[% INCLUDE 'intranet-bottom.inc' %]




<?xml version="1.0" encoding="UTF-8"?>
<xsl:stylesheet version="1.0"
    xmlns:marc="http://www.loc.gov/MARC21/slim"
    xmlns:oai="http://www.openarchives.org/OAI/2.0/"
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>

    <!-- pass in the OAI-PMH identifier for archival purposes -->
    <xsl:param name="identifier"/>

    <!-- This is the first template which matches the incoming metadata -->
    <!-- We strip out the OAI metadata wrapper by doing nothing but applying the templates for marc:record -->
    <xsl:template match="oai:metadata">
        <xsl:apply-templates />
    </xsl:template>

    <xsl:template match="marc:record">
        <xsl:copy>
            <!-- Apply all relevant templates for all attributes and elements -->
            <xsl:apply-templates select="@* | *" mode="copy"/>

            <!-- Add new node (or whatever else you want to do after copying the existing record) -->
            <xsl:if test="$identifier">
                <xsl:element name="datafield" xmlns="http://www.loc.gov/MARC21/slim">
                    <xsl:attribute name="ind1"><xsl:text>8</xsl:text></xsl:attribute>
                    <xsl:attribute name="ind2"><xsl:text> </xsl:text></xsl:attribute>
                    <xsl:attribute name="tag">037</xsl:attribute>

                    <xsl:element name="subfield">
                        <xsl:attribute name="code">a</xsl:attribute>
                        <xsl:value-of select="$identifier"/>
                    </xsl:element>

                    <xsl:element name="subfield">
                        <xsl:attribute name="code">b</xsl:attribute>
                        <xsl:text>OAI-PMH</xsl:text>
                    </xsl:element>
                </xsl:element>
            </xsl:if>
        </xsl:copy>
    </xsl:template>


    <!-- Identity transformation: this template is the workhorse that copies attributes and nodes -->
    <!-- In terms of nodes, it'll apply to the leader, controlfield, and subfields. It won't apply to datafields, as we have a more specific template for those. -->
    <xsl:template match="@* | node()" mode="copy">
        <!-- Create a copy of this attribute or node -->
        <xsl:copy>
            <!-- Recursively apply this template to the attributes and child nodes of this element -->
            <xsl:apply-templates select="@* | node()" mode="copy"/>
        </xsl:copy>
    </xsl:template>

     <xsl:template match="marc:datafield" mode="copy">
        <!-- Add subfields by changing the predicate in the select attribute (e.g. [@code='9' or @code='a']) -->
        <xsl:variable name="child_subfields_to_remove" select="child::*[@code='9']"/>
        <!-- Strip out all $9 subfields, as these provide links to authority records. These will nearly never be correct linkages, so strip them out. -->
        <xsl:choose>
            <xsl:when test="self::node()[@tag = '952' or @tag = '942' or @tag = '999']">
            <!-- STRIP DATAFIELDS -->
            <!-- Add datafields to strip by changing the predicate in the test attribute -->
            <!-- Strip out any 952 tags so that we don't have indexing problems in regards to unexpected items... -->
            <!-- Strip out any 942 tags. They'll contain local data (e.g. 942$c item type) which is local and thus won't correspond with the Koha that is importing these records -->
            <!-- Strip all 999 fields; this isn't strictly necessary though, as "C4::Biblio::_koha_add_biblio" and "C4::Biblio::_koha_add_biblioitem" will typically fix the 999$c and 999$d fields in MARC21 -->
            <!-- NOTE: If you don't strip the 942 field, you'll need to make sure that the item type specified matches one that already exists in Koha or you'll have problems -->
            </xsl:when>
            <xsl:when test="count(child::*) = count($child_subfields_to_remove)">
            <!-- STRIP DATAFIELDS WHICH WILL BECOME EMPTY -->
            <!-- We don't want to output a datafield if we're going to remove all its children, as empty datafields cause fatal errors in MARC::Record -->
            </xsl:when>
            <xsl:otherwise>
                <!-- Create a copy of the datafield element (without attributes or child nodes) -->
                <xsl:copy>
                    <!-- Apply copy templates for datafield attributes -->
                    <xsl:apply-templates select="@*" mode="copy"/>
                    <!-- Apply copy templates for subfield nodes -->
                    <xsl:apply-templates select="marc:subfield[not(self::node() = $child_subfields_to_remove)]" mode="copy"/>
                </xsl:copy>
            </xsl:otherwise>
        </xsl:choose>
    </xsl:template>
</xsl:stylesheet>




<?xml version="1.0" encoding="UTF-8" ?>
<xsl:stylesheet version="1.0"
    xmlns:oai="http://www.openarchives.org/OAI/2.0/"
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>

    <!-- This is the first template that matches the OAI-PMH root metadata wrapper -->
    <!-- We strip out the OAI-PMH metadata wrapper by doing nothing but applying the remaining templates to its child nodes -->
    <!-- NOTE: We need to strip the wrapper, so that we can access the actual record using modules like MARC::Record-->
    <xsl:template match="oai:metadata">
        <xsl:apply-templates />
    </xsl:template>

    <!-- Identity transformation: this template copies attributes and nodes -->
    <xsl:template match="@* | node()">
        <!-- Create a copy of this attribute or node -->
        <xsl:copy>
            <!-- Recursively apply this template to the attributes and child nodes of this element -->
            <xsl:apply-templates select="@* | node()" />
        </xsl:copy>
    </xsl:template>
</xsl:stylesheet>




#!/usr/bin/perl

# Copyright Prosentient Systems 2015
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;
use Getopt::Long;
use Pod::Usage;


=head1 NAME

oai_harvester.pl - cron script to download records via OAI-PMH, and import
into Koha.

=head1 SYNOPSIS

perl oai_harvester.pl [ -r ][ -d ][ -i ]

=head1 OPTIONS

=over

=item B<--help>

Print a brief help message and exits.

=item B<--man>

Prints the manual page and exits.

=item B<-v>

Verbose. Without this flag set, only errors are reported.

=item B<-r>

Re-harvest the OAI server by resetting the "from" parameter to null.

=item B<-d>

Download records from the OAI-PMH server, and queue for importing.

=item B<-i>

Import queued records into Koha.

=back

=head1 DESCRIPTION

With the "-d" option, this script iterates through a list of
OAI servers (ie repositories), issues a ListRecords OAI-PMH request, and
downloads the results into the Koha database to await further
processing (ie adding new records and updating existing ones).

With the "-i" option, this script iterates through a list of
harvested OAI-PMH metadata records. Each record is checked
to see whether it needs to be added as a new record in Koha,
or if it needs to update a record that has previously been
harvested. After this checking, it is optionally transformed
using a user-defined XSLT. Finally, it is passed to
Koha's internal APIs to be added/updated in the database.


=cut

my $help    = 0;
my $man     = 0;
my $verbose = 0;
my $reset = 0;
my $download = 0;
my $import = 0;

GetOptions(
            'h|help|?'          => \$help,
            'm|man'             => \$man,
            'v'                 => \$verbose,
            'd'                 => \$download,
            'i'                 => \$import,
            'r'                 => \$reset,
       )or pod2usage(2);
pod2usage(1) if $help;
pod2usage( -verbose => 2 ) if $man;

if ($download){
    require C4::Context;
    require XML::LibXSLT;
    require Koha::OAI::Client::Repositories;

    my $stylesheet;
    #Build path to XSLT filename
    #FIXME: It would make more sense to have a utility XSLT directory outside of htdocs which is language independent...
    my $htdocs  = C4::Context->config('intrahtdocs');
    my $theme   = C4::Context->preference("template");
    my $xslfilename = "$htdocs/$theme/en/xslt/StripOAIWrapper.xsl";
    if ( -f $xslfilename ){
        #FIXME: Ideally, it would be good to use Koha::XSLT_Handler here, but
        #we want to output a XML::LibXML::Document instead of output_as_chars.
        my $xslt = XML::LibXSLT->new();
        my $style_doc = XML::LibXML->load_xml(location => $xslfilename);
        $stylesheet = $xslt->parse_stylesheet($style_doc);
    }

    my $repositories = Koha::OAI::Client::Repositories->search({
        active => 1,
        base_url => {'!=', undef},
    });
    while (my $repository = $repositories->next){
        if ($reset){
            $verbose && print "Resetting opt_from to undefined...\n";
            $repository->opt_from(undef);
        }
        $repository->harvest({ verbose => $verbose, stylesheet => $stylesheet, });
    }
}

if ($import){
    require Koha::OAI::Client::Importer;
    require Koha::OAI::Client::Records;

    #Import records, previously harvested via OAI-PMH, into Koha
    my $oai_importer = Koha::OAI::Client::Importer->new();
    if ($oai_importer){
        my $records = Koha::OAI::Client::Records->search({
            system_number => undef,
            action => ["to_import","to_delete"],
        });

        $verbose && print $records->count." records to process\n";
        if ($records){

            my $number_imported = 0;
            my $number_deleted = 0;

            #Iterate through the records and import (i.e. add/update/re-add) or delete them in Koha
            while( my $record = $records->next ) {
                if ($record->action eq "to_import"){
                    eval {
                        $oai_importer->import_record($record);
                    };
                    if ($@){
                        warn "Error importing record (oai_harvest_id ".$record->oai_harvest_id.") into Koha:\n$@";
                        $record->action("error");
                        $record->store();
                    } else {
                        $verbose && print $record->action . " identifier (".$record->identifier.") (".$record->record_type.") as system number (".$record->system_number.")\n";
                        $number_imported++;
                    }
                } elsif ($record->action eq "to_delete") {
                    eval {
                        $oai_importer->delete_record($record);
                    };
                    if ($@){
                        warn "Error deleting record (oai_harvest_id ".$record->oai_harvest_id.") from Koha:\n$@";
                    } else {
                        $number_deleted++;
                    }
                }
            }
            $verbose && print "$number_imported records imported (i.e. added/updated)\n";
            $verbose && print "$number_deleted records deleted \n";


        }
    }
}

Return to bug 10662

Lines 3549-3554 CREATE TABLE discharges ( Link Here

(-)a/installer/data/mysql/kohastructure.sql (+49 lines)
3549	CONSTRAINT borrower_discharges_ibfk1 FOREIGN KEY (borrower) REFERENCES borrowers (borrowernumber) ON DELETE CASCADE ON UPDATE CASCADE	3549	CONSTRAINT borrower_discharges_ibfk1 FOREIGN KEY (borrower) REFERENCES borrowers (borrowernumber) ON DELETE CASCADE ON UPDATE CASCADE
3550	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;	3550	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
3551		3551
		3552	--
		3553	-- Table structure for table `oai_harvest_repositories`
		3554	--
		3555
		3556	CREATE TABLE IF NOT EXISTS oai_harvest_repositories (
		3557	repository_id int(11) NOT NULL AUTO_INCREMENT, -- primary key identifier
		3558	base_url text NOT NULL, -- baseURL of the remote OAI-PMH repository (mandatory)
		3559	basic_username text, -- username for basic http authentication
		3560	basic_password text, -- password for basic http authentication
		3561	basic_realm text, -- realm for basic http authentication
		3562	datetime_granularity enum('YYYY-MM-DDThh:mm:ssZ','YYYY-MM-DD') NOT NULL DEFAULT 'YYYY-MM-DD', -- granularity to use in harvest request
		3563	opt_from DATETIME DEFAULT NULL, -- "from" DATETIME for selective harvesting (optional - automatically updated by cronjob)
		3564	opt_until DATETIME DEFAULT NULL, -- "until" DATETIME for selective harvesting (optional)
		3565	opt_set varchar(45) DEFAULT NULL, -- the record set to harvest for selective harvesting (optional)
		3566	metadata_prefix varchar(45) NOT NULL, -- metadata format (e.g. oai_dc, dc, marcxml)
		3567	active int(1) NOT NULL DEFAULT '0', -- indicate whether this repo is actively harvested by Koha
		3568	timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- date last modified. Maybe `datelastmodified` would be a better column name.
		3569	xslt_path text, -- filepath to a local or external XSLT file to use for transforming incoming records into MARCXML
		3570	frameworkcode varchar(4) NOT NULL DEFAULT '', -- framework to use when ingesting records
		3571	comments text, -- limited number of characters (controlled by template) to describe the repository (optional - for librarian use rather than system use)
		3572	record_type enum('biblio','auth','holdings') NOT NULL DEFAULT 'biblio',
		3573	original_system_field varchar(45), -- Field where you can a find a record's original system number (e.g. 001 or 999$c)
		3574	-- NOTE: In future, may need to add fields for holdings-biblio relationship (e.g. field to designate 004 in holdings record as link to a biblio's original system number)
		3575	PRIMARY KEY (repository_id)
		3576	) ENGINE=InnoDB DEFAULT CHARSET=utf8;
		3577
		3578	--
		3579	-- Table structure for table `oai_harvest`
		3580	--
		3581
		3582	CREATE TABLE IF NOT EXISTS oai_harvest (
		3583	oai_harvest_id int(11) NOT NULL AUTO_INCREMENT, -- unique internal id for the OAI record
		3584	repository_id int(11) NOT NULL, -- OAI repository from which the record was harvested
		3585	system_number int(11) DEFAULT NULL, -- indicates the Koha record created from this record
		3586	action varchar(45) DEFAULT NULL, -- indicates whether this record has been processed and how it's been processed (e.g. add, update, ignore)
		3587	timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- last modified date in Koha. Maybe `datelastmodified` would be a better column name.
		3588	identifier varchar(255) NOT NULL, -- the record's OAI ID from the OAI repository
		3589	datestamp varchar(45) NOT NULL, -- the record's OAI datestamp from the OAI repository (e.g. YYYY-MM-DDTHH:MM:SSZ).
		3590	metadata_prefix varchar(45) NOT NULL, -- the type of metadata (e.g. marcxml, oai_dc, etc.)
		3591	metadata longtext, -- original metadata as transmitted via OAI-PMH
		3592	record_type enum('biblio','auth','holdings') NOT NULL DEFAULT 'biblio',
		3593	original_system_number varchar(45), -- original system number (e.g. the unique number in the 001 of the incoming record)
		3594	-- NOTE: In future, may need to add fields for holdings-biblio relationship
		3595	PRIMARY KEY (oai_harvest_id),
		3596	UNIQUE KEY harvest_identifier_datestamp (identifier,datestamp),
		3597	KEY FK_oai_harvest_1 (repository_id),
		3598	CONSTRAINT FK_oai_harvest_1 FOREIGN KEY (repository_id) REFERENCES oai_harvest_repositories (repository_id) ON UPDATE NO ACTION
		3599	) ENGINE=InnoDB DEFAULT CHARSET=utf8;
		3600
3552	/!40103 SET TIME_ZONE=@OLD_TIME_ZONE /;	3601	/!40103 SET TIME_ZONE=@OLD_TIME_ZONE /;
3553	/!40101 SET SQL_MODE=@OLD_SQL_MODE /;	3602	/!40101 SET SQL_MODE=@OLD_SQL_MODE /;
3554	/!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS /;	3603	/!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS /;

Line 0 Link Here

(-)a/Koha/OAI/Client/Importer.pm (+316 lines)
		1	package Koha::OAI::Client::Importer;
		2
		3	# Copyright Prosentient Systems 2015
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, see <http://www.gnu.org/licenses>.
		18
		19	use Modern::Perl;
		20
		21	use Koha::Database;
		22	use Koha::XSLT_Handler;
		23	use C4::Context;
		24	use C4::Languages;
		25	use C4::XSLT;
		26	use C4::Biblio;
		27
		28	use parent qw(Class::Accessor);
		29	__PACKAGE__->mk_accessors( qw (xslt_handler verbosity) );
		30
		31	=head1 NAME
		32
		33	Koha::OAI::Importer - Imports records, previously harvested via OAI-PMH, into Koha
		34
		35	=head1 SYNOPSIS
		36
		37	my $oai_importer = Koha::OAI::Importer->new({
		38	verbosity => $verbose,
		39	});
		40	$oai_importer->import_records();
		41
		42	=head1 DESCRIPTION
		43
		44	An OAI Importer is an object which retrieves records, previously harvested
		45	via OAI-PMH, and determines whether they should be added as new records to the
		46	catalogue or if they should update existing catalogue records.
		47
		48	After this determination, the record's metadata is run through a XSLT, which
		49	is configured on a per server basis, and then handed over to Koha's internal APIs.
		50
		51	The record in the OAI harvesting queue is then updated to reflect the
		52	action taken and the biblionumber to which it is linked.
		53
		54	=head1 METHODS
		55
		56	=head2 new
		57
		58	The Importer constructor which sets the 'verbosity' and 'xslt_handler'
		59	for the object.
		60
		61	The 'verbosity' should be set by calling script.
		62
		63	The 'xslt_handler' could also be set by a calling script, but defaults
		64	to a new Koha::XSLT_Handler object with silenced warnings, so it is
65	not too noisy.
66
67	=cut
68
69	sub new {
70	my ($class, $args) = @_;
71	$args = {} unless defined $args;
72	$args->{xslt_handler} = Koha::XSLT_Handler->new({print_warns => 0,}) unless defined $args->{xslt_handler};
73	$args->{verbosity} = 0 unless defined $args->{verbosity};
74	return bless ($args, $class);
75	}
76
77	sub delete_record {
78	my ( $self, $record ) = @_;
79
80	my ($action, $system_number) = $self->determine_action_and_system_number({
81	identifier => $record->identifier,
82	repository_id => $record->repository_id,
83	});
84	my $error = C4::Biblio::DelBiblio($system_number);
85	if ($error){
86	die $error;
87	} else {
88	$record->set({
89	system_number => $system_number,
90	action => "deleted",
91	});
92	$record->store();
93	}
94	}
95
96	=head2 import_record
97
98	This is the method doing the heavy lifting. It runs the other
99	methods for determining what to do with the record and if it is
100	linked to an existing record, it transforms the record via XSLT,
101	and it ultimately adds or updates the record as need be.
102
103	=cut
104
105	sub import_record {
106	my ( $self, $record ) = @_;
107
108	#Step one: determine action (i.e. determine if a record with this identifier from this repository has been successfully added before)
109	my ($action, $system_number) = $self->determine_action_and_system_number({
110	identifier => $record->identifier,
111	repository_id => $record->repository_id,
112	});
113
114	#Step two: transform metadata (if necessary) && add/update in Koha
115	if ( ($action eq 'add') \|\| ($action eq 'update') ){
116	my $schema = Koha::Database->new()->schema();
117
118	#Lookup the repository so we can get the xslt_path and frameworkcode
119	my $repository = $schema->resultset('OaiHarvestRepository')->find($record->repository_id);
120
121	my $metadata = $record->metadata;
122	if ($metadata && $repository->xslt_path){
123	eval {
124	#If there is a XSLT path, try to transform metadata
125	my $transformed_metadata = $self->transform_metadata({
126	xslt_path => $repository->xslt_path,
127	metadata => $metadata,
128	metadata_prefix => $record->metadata_prefix,
129	identifier => $record->identifier,
130	});
131	if ($transformed_metadata){
132	$metadata = $transformed_metadata;
133	}
134	};
135	if ($@){
136	warn "There was a non-fatal problem transforming the metadata during import: $@";
137	}
138	}
139	if ($metadata){
140	if ($record->record_type eq 'biblio'){
141	my ( $biblionumber, $biblioitemnumber );
142	#create MARC::Record object
143	my $marc_record = MARC::Record::new_from_xml( $metadata, "utf8", C4::Context->preference('marcflavour') );
144	if ($marc_record){
145	if ($action eq 'add'){
146	#Add record to Koha
147	($biblionumber,$biblioitemnumber) = C4::Biblio::AddBiblio($marc_record,$repository->frameworkcode);
148	} elsif ($action eq 'update'){
149	#Update record in Koha
150	eval {C4::Biblio::ModBiblio( $marc_record,$system_number,$repository->frameworkcode)};
151	if ($@){
152	my $deleted_biblio = $schema->resultset('Deletedbiblio')->find($system_number);
153	my $existing_biblio = $schema->resultset('Biblio')->find($system_number);
154	if ( ( $deleted_biblio ) \|\| (!$deleted_biblio && !$existing_biblio) ){
155	die "It appears the original record referenced by biblionumber $system_number has been deleted.";
156	#FIXME: What should happen in this situation? You can't re-add the record without first removing
157	#the previous oai_harvest entries or there will be errors...
158	#($biblionumber,$biblioitemnumber) = C4::Biblio::AddBiblio($marc_record,$repository->frameworkcode);
159	#warn "Re-adding the incoming record as a new bibliographic record (biblionumber $system_number).";
160	#$action = "re-add";
161	} else {
162	die $@;
163	}
164	}
165	}
166	}
167	if ($biblionumber){
168	$system_number = $biblionumber;
169	}
170	}
171	#TODO: Add handling for 'holdings' and 'auth'
172	}
173	if ($action && $system_number){
174	#Update 'action' and 'system_number' columns for this record row
175	$record->set({
176	action => $action,
177	system_number => $system_number,
178	});
179	$record->store();
180	} else {
181	die "Error importing record! action: ($action);";
182	}
183	}
184	}
185
186	=head 2 determine_action_and_system_number
187
188	This method checks the database to see if a record from the same
189	OAI-PMH server with the same identifier has already been added
190	into Koha. If it hasn't, it should be added. If it has, it should
191	update the existing record.
192
193	=cut
194
195	sub determine_action_and_system_number {
196	my ( $self, $args ) = @_;
197	my ( $action, $system_number );
198	if ( $args->{identifier} && $args->{repository_id} ){
199	my $schema = Koha::Database->new()->schema();
200
201	#NOTE: $search_results is a DBIx::Class::ResultSet object
202	my $search_results = $schema->resultset('OaiHarvest')->search(
203	{
204	-and => [
205	identifier => $args->{identifier},
206	repository_id => $args->{repository_id},
207	system_number => {'!=', undef},
208	-or => [
209	action => 'add',
210	action => 'update',
211	],
212	],
213	},
214	{
215	columns => [ qw/system_number/ ], #Only retrieve system_number
216	group_by => [ qw/system_number/ ], #Group results by system_number so you only get 1 result if you have many rows sharing the same system_number
217	}
218	);
219
220	#NOTE: The DBIx::Class::ResultSet class has overloaded operators,
221	#which means a "count()" method is run whenever an object is called in a
222	#numerical context (like below). It will also always return 1 in a boolean context.
223	if ($search_results == 0){
224	$action = 'add';
225	} elsif ($search_results == 1){
226	$action = 'update';
227	while( my $result = $search_results->next ) {
228	$system_number = $result->system_number;
229	}
230	} else {
231	if ($search_results && $search_results > 1){
232	die "More than one system_number matched the identifier " . $args->{identifier} . " with a repository_id of " . $args->{repository_id} . ".";
233	} else {
234	die "Unknown error scanning database for previous entries.";
235	}
236	}
237	}
238	return ($action,$system_number);
239	}
240
241	=head2 transform_metadata
242
243	This method takes a record's XML metadata and runs it through a XSLT
244	(using a Koha::XSLT_Handler object). The purpose of this is to remove
245	undesirable fields or to add desired ones.
246
247	=cut
248
249	sub transform_metadata {
250	my ( $self, $args ) = @_;
251	my $transformed_metadata;
252
253	if ($self->xslt_handler){
254	if ( ($args->{xslt_path}) && ($args->{metadata_prefix}) && ($args->{metadata}) && ($args->{identifier}) ){
255
256	my $xslt = _resolve_xslt_path($args->{xslt_path}, $args->{metadata_prefix});
257
258	#If there is a XSLT file, we try to transform the data
259	if ($xslt){
260	require XML::LibXSLT;
261	#NOTE: You can use "register_function()" here to provide extension functions to the XSLT
262	#NOTE: For parameters, you must wrap XML::LibXSLT::xpath_to_string() around each key => value pair within a hashref
263	#{ XML::LibXSLT::xpath_to_string(identifier => $args->{identifier}), XML::LibXSLT::xpath_to_string(test => "tester"), }
264	$transformed_metadata = $self->xslt_handler->transform({
265	xml => $args->{metadata},
266	file => $xslt,
267	parameters => { XML::LibXSLT::xpath_to_string(identifier => $args->{identifier}), },
268	});
269	if ( $self->xslt_handler->err ){
270	die $self->xslt_handler->errstr;
271	}
272	if (! $transformed_metadata){
273	die "Unable to transform metadata";
274	}
275	}
276	}
277	}
278
279	return $transformed_metadata;
280	}
281
282	=head2 _resolve_xslt_path
283
284	An internal sub for turning magical values into actual filepaths to a XSLT.
285
286	=cut
287
288	sub _resolve_xslt_path {
289	my ( $xslfilename, $metadata_prefix ) = @_;
290	if ( $xslfilename ){
291	if ( $xslfilename =~ /^\s"?default"?\s$/i ) {
292	if ($metadata_prefix =~ "marc"){
293	#If the repository has a default XSLT and harvests MARCXML:
294	my $htdocs = C4::Context->config('intrahtdocs');
295	my $theme = C4::Context->preference("template");
296	my $lang = C4::Languages::getlanguage();
297	my $xslfile = C4::Context->preference('marcflavour') .
298	"slimFromOAI.xsl";
299
300	$xslfilename = C4::XSLT::_get_best_default_xslt_filename($htdocs, $theme, $lang, $xslfile);
301	if ( ! -f $xslfilename ){
302	#If the default XSLT doesn't exist, nullify this variable
303	$xslfilename = undef;
304	}
305	} else {
306	#If the repository has a default XSLT and doesn't harvest marc:
307	#We return a null because currently we only provide default XSLTs for MARC21 marcxml
308	$xslfilename = undef;
309	}
310	}
311	}
312	return $xslfilename;
313	}
314
315
316	1;

Line 0 Link Here

(-)a/Koha/OAI/Client/Record.pm (+61 lines)
	1	package Koha::OAI::Client::Record;
	2
	3	# Copyright Prosentient Systems 2015
	4	#
	5	# This file is part of Koha.
	6	#
	7	# Koha is free software; you can redistribute it and/or modify it under the
	8	# terms of the GNU General Public License as published by the Free Software
	9	# Foundation; either version 3 of the License, or (at your option) any later
	10	# version.
	11	#
	12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
	13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	15	#
	16	# You should have received a copy of the GNU General Public License along
	17	# with Koha; if not, write to the Free Software Foundation, Inc.,
	18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
	19
	20	use Modern::Perl;
	21	use Carp;
	22
	23	use base qw(Koha::Object);
	24	use C4::Biblio;
	25
	26	=head1 NAME
	27
	28	Koha::OAI::Client::Record -
	29
	30	=head1 API
	31
	32	=head2 Class Methods
	33
	34	=cut
	35
	36	sub delete {
	37	my ( $self ) = @_;
	38	if ($self->record_type eq 'biblio' && $self->system_number){
	39	my $error = C4::Biblio::DelBiblio($self->system_number);
	40	if ($error){
	41	die $error;
	42	}
	43	}
	44	$self->SUPER::delete();
	45	}
	46
	47	=head3 type
	48
	49	=cut
	50
	51	sub type {
	52	return 'OaiHarvest';
	53	}
	54
	55	=head1 AUTHOR
	56
	57	David Cook <dcook@prosentient.com.au>
	58
	59	=cut
	60
	61	1;

Line 0 Link Here

(-)a/Koha/OAI/Client/Records.pm (+58 lines)
	1	package Koha::OAI::Client::Records;
	2
	3	# Copyright Prosentient Systems 2015
	4	#
	5	# This file is part of Koha.
	6	#
	7	# Koha is free software; you can redistribute it and/or modify it under the
	8	# terms of the GNU General Public License as published by the Free Software
	9	# Foundation; either version 3 of the License, or (at your option) any later
	10	# version.
	11	#
	12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
	13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	15	#
	16	# You should have received a copy of the GNU General Public License along
	17	# with Koha; if not, write to the Free Software Foundation, Inc.,
	18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
	19
	20	use Modern::Perl;
	21	use Carp;
	22
	23	use base qw(Koha::Objects);
	24	use Koha::OAI::Client::Record;
	25
	26	=head1 NAME
	27
	28	Koha::OAI::Client::Records -
	29
	30	=head1 API
	31
	32	=head2 Class Methods
	33
	34	=cut
	35
	36	=head3 type
	37
	38	=cut
	39
	40	sub type {
	41	return 'OaiHarvest';
	42	}
	43
	44	=head3 object_class
	45
	46	=cut
	47
	48	sub object_class {
	49	return 'Koha::OAI::Client::Record';
	50	}
	51
	52	=head1 AUTHOR
	53
	54	David Cook <dcook@prosentient.com.au>
	55
	56	=cut
	57
	58	1;

Line 0 Link Here

(-)a/Koha/OAI/Client/Repositories.pm (+62 lines)
	1	package Koha::OAI::Client::Repositories;
	2
	3	# Copyright Prosentient Systems 2015
	4	#
	5	# This file is part of Koha.
	6	#
	7	# Koha is free software; you can redistribute it and/or modify it under the
	8	# terms of the GNU General Public License as published by the Free Software
	9	# Foundation; either version 3 of the License, or (at your option) any later
	10	# version.
	11	#
	12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
	13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	15	#
	16	# You should have received a copy of the GNU General Public License along
	17	# with Koha; if not, write to the Free Software Foundation, Inc.,
	18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
	19
	20	use Modern::Perl;
	21
	22	use Carp;
	23
	24	use Koha::Database;
	25
	26	use Koha::OAI::Client::Repository;
	27
	28	use base qw(Koha::Objects);
	29
	30	=head1 NAME
	31
	32	Koha::OAI::Client::Repositories -
	33
	34	=head1 API
	35
	36	=head2 Class Methods
	37
	38	=cut
	39
	40	=head3 type
	41
	42	=cut
	43
	44	sub type {
	45	return 'OaiHarvestRepository';
	46	}
	47
	48	=head3 object_class
	49
	50	=cut
	51
	52	sub object_class {
	53	return 'Koha::OAI::Client::Repository';
	54	}
	55
	56	=head1 AUTHOR
	57
	58	David Cook <dcook@prosentient.com.au>
	59
	60	=cut
	61
	62	1;

Line 0 Link Here

(-)a/Koha/OAI/Client/Repository.pm (+418 lines)
		1	package Koha::OAI::Client::Repository;
		2
		3	# Copyright Prosentient Systems 2015
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, write to the Free Software Foundation, Inc.,
		18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		19
		20	use Modern::Perl;
		21	use Carp;
		22	use POSIX qw(strftime);
		23	use HTTP::OAI;
		24	use URI;
		25	use DateTime;
		26	use DateTime::Format::Strptime;
		27	use base qw(Koha::Object);
		28
		29	use Koha::OAI::Client::Records;
		30
		31	=head1 NAME
		32
		33	Koha::OAI::Client::Repository -
		34
		35	=head1 API
		36
		37	=head2 Class Methods
		38
		39	=cut
		40
		41	=head3 test_settings
		42
		43	=cut
		44
		45	sub test_settings {
		46	my ( $self ) = @_;
		47	my $errors = {};
		48	my $harvester = $self->harvester();
		49	if ($harvester){
		50	my $identify = $harvester->Identify;
		51	if ($identify->is_success){
		52
		53	#Test Granularity
		54	my $granularity_setting = $self->datetime_granularity;
		55	if ($granularity_setting eq "YYYY-MM-DDThh:mm:ssZ"){
		56	my $actual_granularity = $identify->granularity;
		57	if ($granularity_setting ne $actual_granularity){
		58	$errors->{second_granularity_not_supported} = 1;
		59	}
		60	}
		61
		62	#Test Set
		63	my $set_setting = $self->opt_set;
		64	if ($set_setting){
65	my $sets = $harvester->ListSets();
66	my $matched_set;
67	while ( my $set = $sets->next ){
68	if ($set_setting eq $set->setSpec){
69	$matched_set = 1;
70	last;
71	}
72	}
73	if ( ! $matched_set ){
74	$errors->{set_does_not_exist} = 1;
75	}
76	}
77
78	#Test Metadata prefix
79	my $metadata_prefix_setting = $self->metadata_prefix;
80	if ($metadata_prefix_setting){
81	my $metadata_formats = $harvester->ListMetadataFormats();
82	my $matched_format;
83	while ( my $metadata_format = $metadata_formats->next ){
84	if ($metadata_prefix_setting eq $metadata_format->metadataPrefix){
85	$matched_format = 1;
86	last;
87	}
88	}
89	if ( ! $matched_format ){
90	$errors->{metadata_prefix_does_not_exist} = 1;
91	}
92	}
93	} else {
94	if ($identify->is_error()){
95	foreach my $error ($identify->errors){
96	if ($error->code =~ /^404$/){
97	$errors->{url_not_found} = 1;
98	} elsif ($error->code =~ /^401$/){
99	$errors->{failed_authentication} = 1;
100	} else {
101	$errors->{generic_identify_error} = 1;
102	}
103	}
104	} else {
105	#This should never happen, but you never know...
106	$errors->{generic_identify_error} = 1;
107	}
108	}
109	} else {
110	$errors->{no_harvester} = 1;
111	}
112	return $errors;
113	}
114
115
116	=head3 validate
117
118	=cut
119
120	sub validate {
121	my ( $self ) = @_;
122	my $errors = {};
123	if ($self->base_url){
124	my $uri = URI->new($self->base_url);
125	if ($uri){
126	my $reference = ref $uri;
127	if ( ! grep { $reference eq $_ } ("URI::http","URI::https")){
128	$errors->{base_url}->{invalid} = 1;
129	}
130	}
131	} elsif (! defined $self->base_url){
132	#base_url is undefined
133	$errors->{base_url}->{required} = 1;
134	}
135	if ( ! defined $self->metadata_prefix ){
136	$errors->{metadata_prefix}->{required} = 1;
137	}
138	return $errors;
139	}
140
141	=head3 find_original_system_number
142
143	=cut
144
145	sub find_original_system_number {
146	my ( $self, $args ) = @_;
147
148	my $original_system_number;
149	my $metadata = $args->{metadata};
150	if ($metadata){
151	my $original_system_field = $self->original_system_field; #e.g. 001 or 999$c
152	if ($original_system_field){
153	#Select root element (e.g. <metadata>)
154	my $root = $metadata->documentElement();
155	#Select the first child of root (ie the original metadata record) (eg <record>)
156	my $original_metadata = $root->firstChild;
157
158	#Check to see if there is a MARC namespace
159	my $marcxml_prefix = $original_metadata->lookupNamespacePrefix("http://www.loc.gov/MARC21/slim");
160	if (defined $marcxml_prefix){
161	my ($tag,$code) = split(/\$/,$original_system_field); #e.g. split 999$c into 999 and c \|\| split 001 into 001
162	my $xpc = XML::LibXML::XPathContext->new;
163	$xpc->registerNs('marc', 'http://www.loc.gov/MARC21/slim');
164	my $fields = $xpc->find(qq(//marc:record/node()[\@tag="$tag"]),$original_metadata);
165	if ($fields){
166	#Use the first field found
167	my $field = $fields->get_node(1);
168	if ($field){
169	my $node_name = $field->nodeName;
170	if ($node_name eq 'controlfield'){
171	#If it's a controlfield, we can use the text without drilling down
172	$original_system_number = $field->textContent;
173	} elsif ($node_name eq 'datafield'){
174	#If it's a datafield, we need to drill down into the subfields
175	if ($code){
176	my $subfields = $field->find(qq(node()[\@code="$code"]));
177	if ($subfields){
178	#Use the first subfield found
179	my $subfield = $subfields->get_node(1);
180	if ($subfield){
181	$original_system_number = $subfield->textContent;
182	}
183
184	}
185	}
186	}
187	}
188	}
189	}
190	}
191	}
192	return $original_system_number;
193	}
194
195	=head3 queue_record
196
197	=cut
198
199	sub queue_record {
200	my ( $self, $args ) = @_;
201	my $record = $args->{record};
202	my $stylesheet = $args->{stylesheet};
203
204	my $action = "to_import";
205	if ( $record->is_deleted() ){
206	my $already_deleted_rows = Koha::OAI::Client::Records->search({ identifier => $record->identifier, action => "deleted" });
207	my $identified_rows = Koha::OAI::Client::Records->search({ identifier => $record->identifier });
208	if ($identified_rows->count() > 0 && $already_deleted_rows->count() == 0){
209	#Only queue records "to_delete", if the record has been previously harvested
210	$action = "to_delete";
211	} else {
212	return;
213	}
214	}
215
216	my $metadata = $record->metadata ? $record->metadata->dom : '';
217	my $original_system_number;
218	if ($metadata){
219	if ($stylesheet){
220	#Use a stylesheet to strip the OAI-PMH metadata XML wrapper
221	eval {
222	my $result = $stylesheet->transform($metadata);
223	if ($result){
224	$metadata = $result;
225	}
226	};
227	if ($@){
228	warn "Problem transforming harvested metadata with XSLT: $@";
229	}
230	}
231
232	$original_system_number = $self->find_original_system_number({metadata => $metadata});
233	}
234
235	my $entry = Koha::OAI::Client::Records->_resultset->find_or_new({
236	repository_id => $self->repository_id, #Internal ID for the OAI repository
237	action => $action,
238	identifier => $record->identifier, #External ID for the OAI record
239	datestamp => $record->datestamp, #External datestamp for the OAI record
240	metadata_prefix => $self->metadata_prefix, #External metadataPrefix for the OAI record
241	metadata => $metadata ? $metadata->toString(1) : undef,
242	record_type => $self->record_type,
243	original_system_number => $original_system_number,
244	},
245	{ key => "harvest_identifier_datestamp" }
246	);
247	if( !$entry->in_storage ) {
248	$entry->insert;
249	}
250	}
251
252	sub _format_datetime {
253	my ( $self, $args ) = @_;
254	my $datetime_string = $args->{datetime_string};
255	my $formatted_string;
256	if ($datetime_string && $datetime_string ne "0000-00-00 00:00:00"){
257	my $strp = DateTime::Format::Strptime->new(
258	pattern => '%F %T',
259	);
260	my $dt = $strp->parse_datetime($datetime_string);
261	if ($dt){
262	if ($self->datetime_granularity && $self->datetime_granularity eq 'YYYY-MM-DDThh:mm:ssZ'){
263	$formatted_string = $dt->strftime("%FT%TZ");
264	} else {
265	$formatted_string = $dt->strftime("%F");
266	}
267	}
268	}
269	return $formatted_string;
270	}
271
272	sub harvester {
273	my ( $self ) = @_;
274	my $harvester;
275	if ($self->base_url){
276	$harvester = new HTTP::OAI::Harvester( baseURL => $self->base_url );
277	#Use basic http authentication credentials if they're set
278	# if ($self->basic_realm && $self->basic_username && $self->basic_password){
279	#Decompose the URI in order to register the basic authentication credentials
280	my $uri = URI->new($self->base_url);
281	my $host = $uri->host;
282	my $port = $uri->port;
283	$harvester->credentials($host.":".$port, $self->basic_realm, $self->basic_username, $self->basic_password);
284	# }
285	}
286	return $harvester;
287	}
288
289	=head3 query_repository
290
291	=cut
292
293	sub query_repository {
294	my ( $self ) = @_;
295	my $response;
296	my $h = $self->harvester();
297	if ($h){
298	my $opt_from = $self->opt_from;
299	my $opt_until = $self->opt_until;
300	foreach my $datetime ($opt_from,$opt_until){
301	#Format the datetime strings from the database into the format and granularity
302	#expected by the target OAI-PMH server
303	$datetime = $self->_format_datetime({datetime_string => $datetime,});
304	}
305
306	#Issue the request to the OAI-PMH server
307	$response = $h->ListRecords(
308	metadataPrefix => $self->metadata_prefix,
309	from => $opt_from,
310	until => $opt_until,
311	set => $self->opt_set,
312	);
313	}
314	return $response;
315	}
316
317	=head3 harvest
318
319	=cut
320
321	sub harvest {
322	my ( $self, $args ) = @_;
323
324	my $verbose = $args->{verbose} // '';
325	my $stylesheet = $args->{stylesheet} // '';
326	$verbose && print "Harvesting records from ".$self->base_url."\n";
327
328	#Record the exact time before sending the first OAI-PMH request, and set "opt_from" to this after the harvest is finished
329	my $new_from_date = strftime "%Y-%m-%d %H:%M:%S", localtime;
330	#Send a request to the OAI-PMH repository
331	my $oai_response = $self->query_repository();
332	if ($oai_response){
333	if( $oai_response->is_error ){
334	warn "responseDate => " . $oai_response->responseDate . "\n";
335	warn "requestURL => " . $oai_response->requestURL . "\n";
336	warn "Error harvesting: " . $oai_response->message . "\n";
337	} else {
338	while( my $rec = $oai_response->next ) {
339	$verbose && print $rec->identifier."\n";
340	#Queue each record, so a different script can process them in Koha
341	$self->queue_record({record => $rec, stylesheet => $stylesheet, });
342	}
343	#Update opt_from after a successful harvest
344	$self->opt_from($new_from_date);
345	$verbose && print "Updating `opt_from` to $new_from_date \n";
346	$self->store();
347	}
348	}
349	}
350
351	#FIXME
352	=head3 reharvest
353
354	=cut
355
356	sub reharvest {
357	my ( $self, $args ) = @_;
358	my $verbose = $args->{verbose} // '';
359	eval {
360	$self->delete_harvest({ verbose => $verbose, });
361	};
362	if ($@){
363	warn $@;
364	} else {
365	$self->harvest({ verbose => $verbose, });
366	}
367	}
368
369	#FIXME
370	=head3 delete_harvest
371
372	=cut
373
374	sub delete_harvest {
375	my ( $self ) = @_;
376
377	my $count_of_records = 0;
378	my $count_of_deleted_records = 0;
379	my $problem_records = [];
380
381	if ($self->repository_id){
382	my $records = Koha::OAI::Client::Records->search({ "repository_id" => $self->repository_id });
383	while (my $record = $records->next){
384	#Increment overall count
385	$count_of_records++;
386	eval {
387	my $is_deleted = $record->delete();
388	if ($is_deleted){
389	$count_of_deleted_records++;
390	}
391	};
392	if ($@){
393	push(@$problem_records,$record);
394	}
395	}
396	}
397
398	if ($count_of_deleted_records < $count_of_records){
399	#Not all of this repository's records could be deleted. Manual intervention may be required to remove items
400	die $problem_records;
401	}
402	}
403
404	=head3 type
405
406	=cut
407
408	sub type {
409	return 'OaiHarvestRepository';
410	}
411
412	=head1 AUTHOR
413
414	David Cook <dcook@prosentient.com.au>
415
416	=cut
417
418	1;

Line 0 Link Here

(-)a/admin/oai_client.pl (+220 lines)
		1	#!/usr/bin/perl
		2
		3	# Copyright Prosentient Systems 2015
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19
		20	=head1 NAME
		21
		22	oai_client.pl
		23
		24	=head1 DESCRIPTION
		25
		26	Admin page to manage Koha's target OAI-PMH servers
		27
		28	=cut
		29
		30	use Modern::Perl;
		31	use CGI qw ( -utf8 );
		32	use C4::Auth;
		33	use C4::Output;
		34	use C4::Koha qw/GetFrameworksLoop/;
		35	use Koha::OAI::Client::Repository;
		36	use Koha::OAI::Client::Repositories;
		37	use Koha::OAI::Client::Records;
		38
		39	my $input = new CGI;
		40
		41	my ($template, $loggedinuser, $cookie, $flags) = get_template_and_user( {
		42	template_name => 'admin/oai_client.tt',
		43	query => $input,
		44	type => 'intranet',
		45	authnotrequired => 0,
		46	flagsrequired => { 'parameters' => 'parameters_remaining_permissions' },
		47	} );
		48
		49	my $op = $input->param('op');
		50	my $test_settings = $input->param('test_settings');
		51	my $repository_id = $input->param('repository_id');
		52
		53	my $base_url = $input->param('base_url') \|\| undef;
		54	my $metadata_prefix = $input->param('metadata_prefix') \|\| undef;
		55	my $opt_from = $input->param('opt_from') \|\| undef;
		56	my $opt_until = $input->param('opt_until') \|\| undef;
		57	my $opt_set = $input->param('opt_set') \|\| undef;
		58	my $active = $input->param('active');
		59	my $xslt_path = $input->param('xslt_path') \|\| undef;
		60	my $frameworkcode = $input->param('frameworkcode'); #This can be empty as the default framework is an empty value
		61	my $comments = $input->param('comments') \|\| undef;
		62	my $basic_username = $input->param('basic_username') \|\| undef;
		63	my $basic_password = $input->param('basic_password') \|\| undef;
		64	my $basic_realm = $input->param('basic_realm') \|\| undef;
65	my $datetime_granularity = $input->param('datetime_granularity');
66	my $record_type = $input->param('record_type');
67	my $original_system_field = $input->param('original_system_field') \|\| undef;
68
69	if ( $op ){
70	my $repository;
71	if ( $repository_id ){
72	#Fetch repository from database...
73	$repository = Koha::OAI::Client::Repositories->find($repository_id);
74	}
75
76	if ( $op eq "new" ){
77	$template->param(
78	frameworkloop => C4::Koha::GetFrameworksLoop(),
79	);
80	} elsif ( $op eq "create" ){
81	my $new_repository = Koha::OAI::Client::Repository->new({
82	base_url => $base_url,
83	metadata_prefix => $metadata_prefix,
84	opt_from => $opt_from,
85	opt_until => $opt_until,
86	opt_set => $opt_set,
87	active => $active,
88	xslt_path => $xslt_path,
89	frameworkcode => $frameworkcode,
90	comments => $comments,
91	basic_username => $basic_username,
92	basic_password => $basic_password,
93	basic_realm => $basic_realm,
94	datetime_granularity => $datetime_granularity,
95	record_type => $record_type,
96	original_system_field => $original_system_field,
97	});
98	my $errors = $new_repository->validate();
99	if (%$errors){
100	$op = "new";
101	$template->param(
102	repository => $new_repository,
103	frameworkloop => C4::Koha::GetFrameworksLoop($new_repository->frameworkcode),
104	errors => $errors,
105	);
106	} else {
107	$new_repository->store();
108	$op = "list";
109	}
110	} elsif ( $op eq "edit" ){
111	if ($repository){
112	$template->param(
113	repository => $repository,
114	frameworkloop => C4::Koha::GetFrameworksLoop($repository->frameworkcode),
115	);
116	}
117	} elsif ( ($op eq "view") \|\| ($op eq "reset") ){
118	if ($repository){
119
120	#If op eq reset, then delete the entire harvest for this repository
121	if ($op eq "reset"){
122	eval {
123	$repository->delete_harvest();
124	$template->param( reset_success => 1 );
125	};
126	if ($@){
127	my $problem_records = $@;
128	$template->param( problem_records => $problem_records );
129	}
130	}
131
132	#"add", "update"
133	my $imported_records = Koha::OAI::Client::Records->search({
134	repository_id => $repository->repository_id,
135	system_number => {'!=', undef},
136	}, {
137	group_by => [ qw/system_number/ ],
138	#Group results by system_number so you only get 1 result if you have many rows sharing the same system_number
139	#Also, group by system number because you might have imported record 1, deleted it, and then it gets re-imported later as a different Koha record
140	});
141	my $waiting_records = Koha::OAI::Client::Records->search({ repository_id => $repository->repository_id, action => 'to_import',});
142	my $to_delete_records = Koha::OAI::Client::Records->search({ repository_id => $repository->repository_id, action => 'to_delete',});
143	my $deleted_records = Koha::OAI::Client::Records->search({ repository_id => $repository->repository_id, action => 'deleted',});
144	my $error_records = Koha::OAI::Client::Records->search({ repository_id => $repository->repository_id, action => 'error',});
145
146	$template->param(
147	repository => $repository,
148	imported_records_count => $imported_records->count(),
149	waiting_records_count => $waiting_records->count(),
150	to_delete_records_count => $to_delete_records->count(),
151	deleted_records_count => $deleted_records->count(),
152	error_records_count => $error_records->count(),
153	);
154	}
155	if ($op eq "reset"){
156	$op = "view";
157	}
158	} elsif ( $op eq "update" ){
159	if ($repository){
160	$repository->set({
161	base_url => $base_url,
162	metadata_prefix => $metadata_prefix,
163	opt_from => $opt_from,
164	opt_until => $opt_until,
165	opt_set => $opt_set,
166	active => $active,
167	xslt_path => $xslt_path,
168	frameworkcode => $frameworkcode,
169	comments => $comments,
170	basic_username => $basic_username,
171	basic_password => $basic_password,
172	basic_realm => $basic_realm,
173	datetime_granularity => $datetime_granularity,
174	record_type => $record_type,
175	original_system_field => $original_system_field,
176	});
177	my $errors = $repository->validate();
178	if (%$errors \|\| $test_settings){
179	if (! %$errors && $test_settings){
180	my $test_errors = $repository->test_settings();
181	if (%$test_errors){
182	$template->param( test_errors => $test_errors );
183	} else {
184	$template->param( test_success => 1 );
185	}
186	}
187	$op = "edit";
188	$template->param(
189	repository => $repository,
190	frameworkloop => C4::Koha::GetFrameworksLoop($repository->frameworkcode),
191	errors => $errors,
192	);
193	} else {
194	$repository->store();
195	$op = "list";
196	}
197	}
198	} elsif ( $op eq "delete" ){
199	if ($repository){
200	$repository->delete;
201	}
202	$op = "list";
203	}
204	} else {
205	$op = "list";
206	}
207
208	if ( $op eq "list" ){
209	my @repositories = Koha::OAI::Client::Repositories->as_list();
210	$template->param(
211	repositories => \@repositories,
212	frameworkloop => C4::Koha::GetFrameworksLoop(),
213	);
214	}
215
216	$template->param(
217	op => $op,
218	);
219
220	output_html_with_http_headers $input, $cookie, $template->output;

Line 0 Link Here

(-)a/installer/data/mysql/atomicupdate/bug_10662-Build_OAI-PMH_Harvesting_Client_tables.sql (+51 lines)
	1	DROP TABLE IF EXISTS oai_harvest;
	2	DROP TABLE IF EXISTS oai_harvest_repositories;
	3
	4	--
	5	-- Table structure for table `oai_harvest_repositories`
	6	--
	7
	8	CREATE TABLE IF NOT EXISTS oai_harvest_repositories (
	9	repository_id int(11) NOT NULL AUTO_INCREMENT, -- primary key identifier
	10	base_url text NOT NULL, -- baseURL of the remote OAI-PMH repository (mandatory)
	11	basic_username text, -- username for basic http authentication
	12	basic_password text, -- password for basic http authentication
	13	basic_realm text, -- realm for basic http authentication
	14	datetime_granularity enum('YYYY-MM-DDThh:mm:ssZ','YYYY-MM-DD') NOT NULL DEFAULT 'YYYY-MM-DD', -- granularity to use in harvest request
	15	opt_from DATETIME DEFAULT NULL, -- "from" DATETIME for selective harvesting (optional - automatically updated by cronjob)
	16	opt_until DATETIME DEFAULT NULL, -- "until" DATETIME for selective harvesting (optional)
	17	opt_set varchar(45) DEFAULT NULL, -- the record set to harvest for selective harvesting (optional)
	18	metadata_prefix varchar(45) NOT NULL, -- metadata format (e.g. oai_dc, dc, marcxml)
	19	active int(1) NOT NULL DEFAULT '0', -- indicate whether this repo is actively harvested by Koha
	20	timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- date last modified. Maybe `datelastmodified` would be a better column name.
	21	xslt_path text, -- filepath to a local or external XSLT file to use for transforming incoming records into MARCXML
	22	frameworkcode varchar(4) NOT NULL DEFAULT '', -- framework to use when ingesting records
	23	comments text, -- limited number of characters (controlled by template) to describe the repository (optional - for librarian use rather than system use)
	24	record_type enum('biblio','auth','holdings') NOT NULL DEFAULT 'biblio',
	25	original_system_field varchar(45), -- Field where you can a find a record's original system number (e.g. 001 or 999$c)
	26	-- NOTE: In future, may need to add fields for holdings-biblio relationship (e.g. field to designate 004 in holdings record as link to a biblio's original system number)
	27	PRIMARY KEY (repository_id)
	28	) ENGINE=InnoDB DEFAULT CHARSET=utf8;
	29
	30	--
	31	-- Table structure for table `oai_harvest`
	32	--
	33
	34	CREATE TABLE IF NOT EXISTS oai_harvest (
	35	oai_harvest_id int(11) NOT NULL AUTO_INCREMENT, -- unique internal id for the OAI record
	36	repository_id int(11) NOT NULL, -- OAI repository from which the record was harvested
	37	system_number int(11) DEFAULT NULL, -- indicates the Koha record created from this record
	38	action varchar(45) DEFAULT NULL, -- indicates whether this record has been processed and how it's been processed (e.g. add, update, ignore)
	39	timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- last modified date in Koha. Maybe `datelastmodified` would be a better column name.
	40	identifier varchar(255) NOT NULL, -- the record's OAI ID from the OAI repository
	41	datestamp varchar(45) NOT NULL, -- the record's OAI datestamp from the OAI repository (e.g. YYYY-MM-DDTHH:MM:SSZ).
	42	metadata_prefix varchar(45) NOT NULL, -- the type of metadata (e.g. marcxml, oai_dc, etc.)
	43	metadata longtext, -- original metadata as transmitted via OAI-PMH
	44	record_type enum('biblio','auth','holdings') NOT NULL DEFAULT 'biblio',
	45	original_system_number varchar(45), -- original system number (e.g. the unique number in the 001 of the incoming record)
	46	-- NOTE: In future, may need to add fields for holdings-biblio relationship
	47	PRIMARY KEY (oai_harvest_id),
	48	UNIQUE KEY harvest_identifier_datestamp (identifier,datestamp),
	49	KEY FK_oai_harvest_1 (repository_id),
	50	CONSTRAINT FK_oai_harvest_1 FOREIGN KEY (repository_id) REFERENCES oai_harvest_repositories (repository_id) ON UPDATE NO ACTION
	51	) ENGINE=InnoDB DEFAULT CHARSET=utf8;

Line 0 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_client.tt (+326 lines)
		1	[% INCLUDE 'doc-head-open.inc' %]
		2	<title>Koha &rsaquo; Administration &rsaquo; OAI-PMH server targets</title>
		3	[% INCLUDE 'doc-head-close.inc' %]
		4	[% INCLUDE 'calendar.inc' %]
		5	<script type="text/javascript" src="[% interface %]/lib/jquery/plugins/jquery-ui-timepicker-addon.min.js"></script>
		6	[% INCLUDE 'timepicker.inc' %]
		7	[% IF ( op == "list" ) %]
		8	<link rel="stylesheet" type="text/css" href="[% themelang %]/css/datatables.css" />
		9	[% INCLUDE 'datatables.inc' %]
		10	<script type="text/javascript">
		11	//<![CDATA[
		12	$(document).ready(function() {
		13	$("#serverst").dataTable($.extend(true, {}, dataTablesDefaults, {
		14	"aoColumnDefs": [
		15	{ "aTargets": [8,9], "bSortable": false },
		16	],
		17	"sPaginationType": "four_button"
		18	}));
		19	});
		20	//]]>
		21	</script>
		22	[% ELSIF ( op == "edit" ) \|\| ( op == "new" ) %]
		23	<script type="text/javascript">
		24	//<![CDATA[
		25	$(document).ready(function() {
		26	$(".opt_datetime").datetimepicker({
		27	dateFormat: "yy-mm-dd",
		28	timeFormat: "HH:mm:ss",
		29	hour: 0,
		30	minute: 0,
		31	second: 0,
		32	showSecond: 1,
		33	});
		34	});
		35	//]]>
		36	</script>
		37	<style type="text/css">
		38	/* Override staff-global.css which hides second, millisecond, and microsecond sliders */
		39	.ui_tpicker_second {
		40	display: block;
		41	}
		42	.test-success {
		43	/* same color as .text-success in Bootstrap 2.2.2 */
		44	color:#468847;
		45	}
		46	</style>
		47	[% END %]
		48	</head>
		49
		50	<body id="admin_oai_server_targets" class="admin">
		51	[% INCLUDE 'header.inc' %]
		52	[% INCLUDE 'cat-search.inc' %]
		53	<div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/admin/admin-home.pl">Administration</a> &rsaquo; OAI-PMH server targets</div>
		54
		55	<div id="doc3" class="yui-t2">
		56
		57	<div id="bd">
		58	<div id="yui-main">
		59	<div class="yui-b">
		60	[% IF ( op ) %]
		61	[% IF ( op == "list" ) %]
		62	<div id="toolbar" class="btn-toolbar">
		63	<a id="newserver" class="btn btn-small" href="/cgi-bin/koha/admin/oai_client.pl?op=new"><i class="icon-plus"></i> New OAI-PMH server target</a>
		64	</div>
65	<h1>OAI-PMH server targets</h1>
66	<table id="serverst">
67	<thead>
68	<tr>
69	<th>Base URL</th>
70	<th>Metadata Prefix</th>
71	<th>From</th>
72	<th>Until</th>
73	<th>Set</th>
74	<th>Active</th>
75	<th>Path to XSLT</th>
76	<th>Record type</th>
77	<th>MARC framework</th>
78	<th>Original system field</th>
79	<th>Comments</th>
80	<th>Options</th>
81	</tr>
82	</thead>
83	<tbody>
84	[% FOREACH repository IN repositories %]
85	<tr>
86	<td>[% repository.base_url %]</td>
87	<td>[% repository.metadata_prefix %]</td>
88	<td>[% IF ( repository.opt_from ) != "0000-00-00 00:00:00"; repository.opt_from; END; %]</td>
89	<td>[% IF ( repository.opt_until ) != "0000-00-00 00:00:00"; repository.opt_until; END; %]</td>
90	<td>[% repository.opt_set %]</td>
91	<td>
92	[% IF ( repository.active == 1 ) %]
93	Active
94	[% ELSE %]
95	Inactive
96	[% END %]
97	</td>
98	<td>[% repository.xslt_path %]</td>
99	<td>[% repository.record_type %]</td>
100	<td>
101	[%
102	framework_name = '';
103	FOREACH framework IN frameworkloop;
104	IF ( framework.value == repository.frameworkcode );
105	framework_name = framework.description;
106	END;
107	END;
108	%]
109	[% IF ( framework_name ) %]
110	[% framework_name %]
111	[% ELSE %]
112	Default
113	[% END %]
114	</td>
115	<td>[% repository.original_system_field %]</td>
116	<td>[% repository.comments %]</td>
117	<td>
118	<a href="/cgi-bin/koha/admin/oai_client.pl?op=view&repository_id=[% repository.repository_id %]">View</a>
119	<a href="/cgi-bin/koha/admin/oai_client.pl?op=edit&repository_id=[% repository.repository_id %]">Edit</a>
120	<a href="/cgi-bin/koha/admin/oai_client.pl?op=delete&repository_id=[% repository.repository_id %]">Delete</a>
121	</td>
122	</tr>
123	[% END %]
124	</tbody>
125	</table>
126	[% ELSIF ( op == "view" ) %]
127	<h1>View OAI-PMH server target</h1>
128	<h2>[% repository.base_url %]</h2>
129	<fieldset class="rows">
130	<ol>
131	<li>Koha records created from harvested records: [% imported_records_count %]</li>
132	<li>Koha records deleted by harvested records: [% deleted_records_count %]</li>
133	<li>Harvested records waiting to be imported: [% waiting_records_count %]</li>
134	<li>Harvested records waiting to be deleted from Koha: [% to_delete_records_count %]</li>
135	<li>Harvested records in an error state: [% error_records_count %]</li>
136	</ol>
137	</fieldset>
138	<form action="/cgi-bin/koha/admin/oai_client.pl" name="reset-form" method="get" id="oai-server-reset" novalidate="novalidate">
139	<fieldset class="action">
140	<input type="hidden" name="op" value="reset" />
141	<input type="hidden" name="repository_id" value="[% repository.repository_id %]" />
142	<button type="submit">Reset repository harvest</button>
143	</fieldset>
144	</form>
145	[% IF ( problem_records ) %]
146	<p>Not all of this repository's records could be deleted. Manual intervention may be required to remove items. Review the following records:</p>
147	<ol>
148	[% FOREACH problem_record IN problem_records %]
149	[% IF ( problem_record.record_type == "biblio" ) %]
150	<li><a target="_blank" href="/cgi-bin/koha/catalogue/detail.pl?biblionumber=[% problem_record.system_number %]">[% problem_record.record_type %] - [% problem_record.system_number %]</a></li>
151	[% ELSE %]
152	<li>[% problem_record.record_type %] - [% problem_record.system_number %]</li>
153	[% END %]
154	[% END %]
155	</ol>
156	[% END %]
157	[% IF ( reset_success ) %]
158	<p>This repository no longer has any harvested records. Modify the "From" and "Until" dates to prepare the repository for the next harvest.</p>
159	[% END %]
160	[% ELSIF ( op == "edit" ) \|\| ( op == "new" ) %]
161	[% IF ( op == "new" ) %]
162	<h1>New OAI-PMH server target</h1>
163	[% ELSIF ( op == "edit" ) %]
164	<h1>Modify OAI-PMH server target</h1>
165	[% END %]
166	<form action="/cgi-bin/koha/admin/oai_client.pl" name="detail-form" method="post" id="oai-server-target-details" novalidate="novalidate">
167	[% IF ( op == "new" ) %]
168	<input type="hidden" name="op" value="create" />
169	[% ELSIF ( op == "edit" ) %]
170	<input type="hidden" name="repository_id" value="[% repository.repository_id %]" />
171	<input type="hidden" name="op" value="update" />
172	[% END %]
173	<fieldset class="rows">
174	<legend>HTTP parameters:</legend>
175	<ol>
176	<li>
177	<label for="base_url">Base URL: </label>
178	<input type="text" id="base_url" name="base_url" value="[% repository.base_url %]" size="60" />
179	[% IF ( errors.base_url.required ) %]<span class="required">Required</span>[% END %]
180	<span class="error">[% IF ( errors.base_url.invalid ) %](Invalid URL)[% END %]</span>
181	<span class="error">[% IF ( test_errors.url_not_found ) %](404 - URL Not Found)[% END %]</span>
182
183	</li>
184	<li>
185	<label for="basic_username">Basic auth username: </label>
186	<input type="text" id="basic_username" name="basic_username" value="[% repository.basic_username %]" size="60" />
187	<span class="error">[% IF ( test_errors.failed_authentication ) %](Authentication failed)[% END %]</span>
188	</li>
189	<li>
190	<label for="basic_password">Basic auth password: </label>
191	<input type="text" id="basic_password" name="basic_password" value="[% repository.basic_password %]" size="60" />
192	<span class="error">[% IF ( test_errors.failed_authentication ) %](Authentication failed)[% END %]</span>
193	</li>
194	<li>
195	<label for="basic_realm">Basic auth realm: </label>
196	<input type="text" id="basic_realm" name="basic_realm" value="[% repository.basic_realm %]" size="60" />
197	<span class="error">[% IF ( test_errors.failed_authentication ) %](Authentication failed)[% END %]</span>
198	</li>
199	</ol>
200	</fieldset>
201	<fieldset class="rows">
202	<legend>OAI-PMH parameters:</legend>
203	<ol>
204	<li>
205	<label for="opt_set">Set: </label>
206	<input type="text" name="opt_set" value="[% repository.opt_set %]" size="30" />
207	<span class="error">[% IF ( test_errors.set_does_not_exist ) %](Set unavailable)[% END %]</span>
208	</li>
209	<li>
210	<label for="metadata_prefix">Metadata prefix: </label>
211	<input type="text" name="metadata_prefix" value="[% repository.metadata_prefix %]" size="30" />
212	[% IF ( errors.metadata_prefix.required ) %]<span class="required">Required</span>[% END %]
213	[% IF ( test_errors.metadata_prefix_does_not_exist ) %]<span class="error">(Metadata prefix unavailable)</span>[% END %]
214
215	</li>
216	<li>
217	<label for="datetime_granularity">Granularity: </label>
218	<select name="datetime_granularity">
219	[% FOREACH granularity IN ['YYYY-MM-DD', 'YYYY-MM-DDThh:mm:ssZ'] %]
220	[% IF ( granularity == repository.datetime_granularity ) %]
221	<option value="[% granularity %]" selected="selected">[% granularity %]</option>
222	[% ELSE %]
223	<option value="[% granularity %]">[% granularity %]</option>
224	[% END %]
225	[% END %]
226	</select>
227	<!-- <span class="required">Required</span> -->
228	[% IF ( test_errors.second_granularity_not_supported ) %]<span class="error">(Second granularity is not supported)</span>[% END %]
229	</li>
230	<li>
231	<label for="opt_from">From: </label>
232	<input type="text" class="opt_datetime" name="opt_from" value="[% IF ( repository.opt_from ) != "0000-00-00 00:00:00"; repository.opt_from; END; %]" size="30" />
233	</li>
234	<li>
235	<label for="opt_until">Until: </label>
236	<input type="text" class="opt_datetime" name="opt_until" value="[% IF ( repository.opt_until ) != "0000-00-00 00:00:00"; repository.opt_until; END; %]" size="30" />
237	</li>
238	</ol>
239	</fieldset>
240	<fieldset class="action">
241	[% IF ( test_success ) %]
242	<p class="test-success">HTTP and OAI-PMH parameter tests completed successfully.</p>
243	[% END %]
244	[% IF (test_errors.no_harvester) %]
245	<p>Unable to create a OAI-PMH harvester with these settings</p>
246	[% END %]
247	[% IF (test_errors.generic_identify_error) %]
248	<p>An error was encountered using the "Identify" verb with the server target.</p>
249	[% END %]
250	<input type="submit" name="test_settings" value="Test HTTP and OAI-PMH parameters">
251	</fieldset>
252	<fieldset class="rows">
253	<legend>Import parameters:</legend>
254	<ol>
255	<li>
256	<label for="active">Active: </label>
257	[% IF ( repository.active == 1) %]
258	<input type="radio" name="active" value="1" checked="checked" />
259	[% ELSE %]
260	<input type="radio" name="active" value="1" />
261	[% END %]
262	Active
263	[% IF ( repository.active != 1) %]
264	<input type="radio" name="active" value="0" checked="checked" />
265	[% ELSE %]
266	<input type="radio" name="active" value="0" />
267	[% END %]
268	Inactive
269	</li>
270	<li>
271	<label for="xslt_path">Path to XSLT: </label>
272	<input type="text" name="xslt_path" value="[% repository.xslt_path %]" size="60" />
273	</li>
274	<li>
275	<label for="record_type">Record type:</label>
276	<select name="record_type">
277	[% record_types = [ 'biblio' ] #TODO: In future, add 'auth' and 'holdings' support here %]
278	[% FOREACH type IN record_types %]
279	[% IF type == repository.record_type %]
280	<option value="[% type %]" selected="selected">[% type %]</option>
281	[% ELSE %]
282	<option value="[% type %]">[% type %]</option>
283	[% END %]
284	[% END %]
285	</select>
286	<!-- <span class="required">Required</span> -->
287	</li>
288	[%# Only need framework for 'biblio' and 'holdings'. This will need to change for 'auth'. %]
289	<li>
290	<label for="frameworkcode">MARC framework: </label>
291	<select name="frameworkcode">
292	<option value="">Default</option>
293	[% FOREACH framework IN frameworkloop %]
294	[% IF ( framework.selected ) %]
295	<option value="[% framework.value %]" selected="selected">[% framework.description %]</option>
296	[% ELSE %]
297	<option value="[% framework.value %]">[% framework.description %]</option>
298	[% END %]
299	[% END %]
300	</select>
301	<!-- <span class="required">Required</span> -->
302	</li>
303	<li>
304	<label for="original_system_field">Original system field</label>
305	<input type="text" name="original_system_field" value="[% repository.original_system_field %]" size="60%" />
306	</li>
307	<li>
308	<label for="comments">Comments: </label>
309	<textarea name="comments">[% repository.comments %]</textarea>
310	</li>
311	</ol>
312	</fieldset>
313	<fieldset class="action">
314	<input type="submit" value="Save">
315	<a class="cancel" href="/cgi-bin/koha/admin/oai_client.pl">Cancel</a>
316	</fieldset>
317	</form>
318	[% END %]
319	[% END #/op %]
320	</div>
321	</div>
322	<div class="yui-b">
323	[% INCLUDE 'admin-menu.inc' %]
324	</div>
325	</div>
326	[% INCLUDE 'intranet-bottom.inc' %]

Line 0 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/xslt/MARC21slimFromOAI.xsl (+82 lines)
		1	<?xml version="1.0" encoding="UTF-8"?>
		2	<xsl:stylesheet version="1.0"
		3	xmlns:marc="http://www.loc.gov/MARC21/slim"
		4	xmlns:oai="http://www.openarchives.org/OAI/2.0/"
		5	xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
		6	<xsl:output method="xml" encoding="UTF-8" indent="yes"/>
		7
		8	<!-- pass in the OAI-PMH identifier for archival purposes -->
		9	<xsl:param name="identifier"/>
		10
		11	<!-- This is the first template which matches the incoming metadata -->
		12	<!-- We strip out the OAI metadata wrapper by doing nothing but applying the templates for marc:record -->
		13	<xsl:template match="oai:metadata">
		14	<xsl:apply-templates />
		15	</xsl:template>
		16
		17	<xsl:template match="marc:record">
		18	<xsl:copy>
		19	<!-- Apply all relevant templates for all attributes and elements -->
		20	<xsl:apply-templates select="@* \| *" mode="copy"/>
		21
		22	<!-- Add new node (or whatever else you want to do after copying the existing record) -->
		23	<xsl:if test="$identifier">
		24	<xsl:element name="datafield" xmlns="http://www.loc.gov/MARC21/slim">
		25	<xsl:attribute name="ind1"><xsl:text>8</xsl:text></xsl:attribute>
		26	<xsl:attribute name="ind2"><xsl:text> </xsl:text></xsl:attribute>
		27	<xsl:attribute name="tag">037</xsl:attribute>
		28
		29	<xsl:element name="subfield">
		30	<xsl:attribute name="code">a</xsl:attribute>
		31	<xsl:value-of select="$identifier"/>
		32	</xsl:element>
		33
		34	<xsl:element name="subfield">
		35	<xsl:attribute name="code">b</xsl:attribute>
		36	<xsl:text>OAI-PMH</xsl:text>
		37	</xsl:element>
		38	</xsl:element>
		39	</xsl:if>
		40	</xsl:copy>
		41	</xsl:template>
		42
		43
		44	<!-- Identity transformation: this template is the workhorse that copies attributes and nodes -->
		45	<!-- In terms of nodes, it'll apply to the leader, controlfield, and subfields. It won't apply to datafields, as we have a more specific template for those. -->
		46	<xsl:template match="@* \| node()" mode="copy">
		47	<!-- Create a copy of this attribute or node -->
		48	<xsl:copy>
		49	<!-- Recursively apply this template to the attributes and child nodes of this element -->
		50	<xsl:apply-templates select="@* \| node()" mode="copy"/>
		51	</xsl:copy>
		52	</xsl:template>
		53
		54	<xsl:template match="marc:datafield" mode="copy">
		55	<!-- Add subfields by changing the predicate in the select attribute (e.g. [@code='9' or @code='a']) -->
		56	<xsl:variable name="child_subfields_to_remove" select="child::*[@code='9']"/>
		57	<!-- Strip out all $9 subfields, as these provide links to authority records. These will nearly never be correct linkages, so strip them out. -->
		58	<xsl:choose>
		59	<xsl:when test="self::node()[@tag = '952' or @tag = '942' or @tag = '999']">
		60	<!-- STRIP DATAFIELDS -->
		61	<!-- Add datafields to strip by changing the predicate in the test attribute -->
		62	<!-- Strip out any 952 tags so that we don't have indexing problems in regards to unexpected items... -->
		63	<!-- Strip out any 942 tags. They'll contain local data (e.g. 942$c item type) which is local and thus won't correspond with the Koha that is importing these records -->
		64	<!-- Strip all 999 fields; this isn't strictly necessary though, as "C4::Biblio::_koha_add_biblio" and "C4::Biblio::_koha_add_biblioitem" will typically fix the 999$c and 999$d fields in MARC21 -->
65	<!-- NOTE: If you don't strip the 942 field, you'll need to make sure that the item type specified matches one that already exists in Koha or you'll have problems -->
66	</xsl:when>
67	<xsl:when test="count(child::*) = count($child_subfields_to_remove)">
68	<!-- STRIP DATAFIELDS WHICH WILL BECOME EMPTY -->
69	<!-- We don't want to output a datafield if we're going to remove all its children, as empty datafields cause fatal errors in MARC::Record -->
70	</xsl:when>
71	<xsl:otherwise>
72	<!-- Create a copy of the datafield element (without attributes or child nodes) -->
73	<xsl:copy>
74	<!-- Apply copy templates for datafield attributes -->
75	<xsl:apply-templates select="@*" mode="copy"/>
76	<!-- Apply copy templates for subfield nodes -->
77	<xsl:apply-templates select="marc:subfield[not(self::node() = $child_subfields_to_remove)]" mode="copy"/>
78	</xsl:copy>
79	</xsl:otherwise>
80	</xsl:choose>
81	</xsl:template>
82	</xsl:stylesheet>

Line 0 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/xslt/StripOAIWrapper.xsl (+22 lines)
	1	<?xml version="1.0" encoding="UTF-8" ?>
	2	<xsl:stylesheet version="1.0"
	3	xmlns:oai="http://www.openarchives.org/OAI/2.0/"
	4	xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
	5	<xsl:output method="xml" encoding="UTF-8" indent="yes"/>
	6
	7	<!-- This is the first template that matches the OAI-PMH root metadata wrapper -->
	8	<!-- We strip out the OAI-PMH metadata wrapper by doing nothing but applying the remaining templates to its child nodes -->
	9	<!-- NOTE: We need to strip the wrapper, so that we can access the actual record using modules like MARC::Record-->
	10	<xsl:template match="oai:metadata">
	11	<xsl:apply-templates />
	12	</xsl:template>
	13
	14	<!-- Identity transformation: this template copies attributes and nodes -->
	15	<xsl:template match="@* \| node()">
	16	<!-- Create a copy of this attribute or node -->
	17	<xsl:copy>
	18	<!-- Recursively apply this template to the attributes and child nodes of this element -->
	19	<xsl:apply-templates select="@* \| node()" />
	20	</xsl:copy>
	21	</xsl:template>
	22	</xsl:stylesheet>

Line 0 Link Here

(-)a/misc/cronjobs/oai/oai_harvester.pl (-1 / +180 lines)
0	-	1	#!/usr/bin/perl
		2
		3	# Copyright Prosentient Systems 2015
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, see <http://www.gnu.org/licenses>.
		18
		19	use Modern::Perl;
		20	use Getopt::Long;
		21	use Pod::Usage;
		22
		23
		24	=head1 NAME
		25
		26	oai_harvester.pl - cron script to download records via OAI-PMH, and import
		27	into Koha.
		28
		29	=head1 SYNOPSIS
		30
		31	perl oai_harvester.pl [ -r ][ -d ][ -i ]
		32
		33	=head1 OPTIONS
		34
		35	=over
		36
		37	=item B<--help>
		38
		39	Print a brief help message and exits.
		40
		41	=item B<--man>
		42
		43	Prints the manual page and exits.
		44
		45	=item B<-v>
		46
		47	Verbose. Without this flag set, only errors are reported.
		48
		49	=item B<-r>
		50
		51	Re-harvest the OAI server by resetting the "from" parameter to null.
		52
		53	=item B<-d>
		54
		55	Download records from the OAI-PMH server, and queue for importing.
		56
		57	=item B<-i>
		58
		59	Import queued records into Koha.
		60
		61	=back
		62
		63	=head1 DESCRIPTION
		64
		65	With the "-d" option, this script iterates through a list of
66	OAI servers (ie repositories), issues a ListRecords OAI-PMH request, and
67	downloads the results into the Koha database to await further
68	processing (ie adding new records and updating existing ones).
69
70	With the "-i" option, this script iterates through a list of
71	harvested OAI-PMH metadata records. Each record is checked
72	to see whether it needs to be added as a new record in Koha,
73	or if it needs to update a record that has previously been
74	harvested. After this checking, it is optionally transformed
75	using a user-defined XSLT. Finally, it is passed to
76	Koha's internal APIs to be added/updated in the database.
77
78
79	=cut
80
81	my $help = 0;
82	my $man = 0;
83	my $verbose = 0;
84	my $reset = 0;
85	my $download = 0;
86	my $import = 0;
87
88	GetOptions(
89	'h\|help\|?' => \$help,
90	'm\|man' => \$man,
91	'v' => \$verbose,
92	'd' => \$download,
93	'i' => \$import,
94	'r' => \$reset,
95	)or pod2usage(2);
96	pod2usage(1) if $help;
97	pod2usage( -verbose => 2 ) if $man;
98
99	if ($download){
100	require C4::Context;
101	require XML::LibXSLT;
102	require Koha::OAI::Client::Repositories;
103
104	my $stylesheet;
105	#Build path to XSLT filename
106	#FIXME: It would make more sense to have a utility XSLT directory outside of htdocs which is language independent...
107	my $htdocs = C4::Context->config('intrahtdocs');
108	my $theme = C4::Context->preference("template");
109	my $xslfilename = "$htdocs/$theme/en/xslt/StripOAIWrapper.xsl";
110	if ( -f $xslfilename ){
111	#FIXME: Ideally, it would be good to use Koha::XSLT_Handler here, but
112	#we want to output a XML::LibXML::Document instead of output_as_chars.
113	my $xslt = XML::LibXSLT->new();
114	my $style_doc = XML::LibXML->load_xml(location => $xslfilename);
115	$stylesheet = $xslt->parse_stylesheet($style_doc);
116	}
117
118	my $repositories = Koha::OAI::Client::Repositories->search({
119	active => 1,
120	base_url => {'!=', undef},
121	});
122	while (my $repository = $repositories->next){
123	if ($reset){
124	$verbose && print "Resetting opt_from to undefined...\n";
125	$repository->opt_from(undef);
126	}
127	$repository->harvest({ verbose => $verbose, stylesheet => $stylesheet, });
128	}
129	}
130
131	if ($import){
132	require Koha::OAI::Client::Importer;
133	require Koha::OAI::Client::Records;
134
135	#Import records, previously harvested via OAI-PMH, into Koha
136	my $oai_importer = Koha::OAI::Client::Importer->new();
137	if ($oai_importer){
138	my $records = Koha::OAI::Client::Records->search({
139	system_number => undef,
140	action => ["to_import","to_delete"],
141	});
142
143	$verbose && print $records->count." records to process\n";
144	if ($records){
145
146	my $number_imported = 0;
147	my $number_deleted = 0;
148
149	#Iterate through the records and import (i.e. add/update/re-add) or delete them in Koha
150	while( my $record = $records->next ) {
151	if ($record->action eq "to_import"){
152	eval {
153	$oai_importer->import_record($record);
154	};
155	if ($@){
156	warn "Error importing record (oai_harvest_id ".$record->oai_harvest_id.") into Koha:\n$@";
157	$record->action("error");
158	$record->store();
159	} else {
160	$verbose && print $record->action . " identifier (".$record->identifier.") (".$record->record_type.") as system number (".$record->system_number.")\n";
161	$number_imported++;
162	}
163	} elsif ($record->action eq "to_delete") {
164	eval {
165	$oai_importer->delete_record($record);
166	};
167	if ($@){
168	warn "Error deleting record (oai_harvest_id ".$record->oai_harvest_id.") from Koha:\n$@";
169	} else {
170	$number_deleted++;
171	}
172	}
173	}
174	$verbose && print "$number_imported records imported (i.e. added/updated)\n";
175	$verbose && print "$number_deleted records deleted \n";
176
177
178	}
179	}
180	}