Attachment #164619 for bug #35659

View | Details | Raw Unified | Return to bug 35659
Collapse All | Expand All




package Koha::Import::Oaipmh::Authorities;

# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License

use Modern::Perl;

use Koha::Import::Oaipmh::Authority;

use base qw(Koha::Objects);

=head1 NAME

Koha::Import::Oaipmh::Authorities

This object represents a collection of OAI-PMH records being imported as authorities

=head1 API

=head2 Methods


=head3 type

=cut

sub _type {
    return 'ImportOaipmhAuthority';
}

=head3 object_class

Koha::Object class

=cut

sub object_class {
    return 'Koha::Import::Oaipmh::Authority';
}
1;




package Koha::Import::Oaipmh::Authority;

# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License

use Modern::Perl;

use base qw(Koha::Object);

=head1 NAME

Koha::Import::Oaipmh::Authority

This object represents an OAI-PMH record being imported as an authority

=head1 API

=head2 Class Methods

=cut

=head3 _type

=cut

sub _type {
    return 'ImportOaipmhAuthority';
}

1;




package Koha::Import::Oaipmh::Biblio;

# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License

use Modern::Perl;

use base qw(Koha::Object);

=head1 NAME

Koha::Import::Oaipmh::Biblio

This object represents an OAI-PMH record being imported as a bibliographic record

=head1 API

=head2 Methods


=head3 _type

=cut

sub _type {
    return 'ImportOaipmhBiblio';
}

1;




package Koha::Import::Oaipmh::Biblios;

# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License

use Modern::Perl;

use Koha::Import::Oaipmh::Biblio;

use base qw(Koha::Objects);

=head1 NAME

Koha::Import::Oaipmh::Biblios

This object represents a collection of OAI-PMH records being imported as bibliographic records

=head1 API

=head2 Methods


=head3 type

=cut

sub _type {
    return 'ImportOaipmhBiblio';
}

=head3 object_class

Koha::Object class

=cut

sub object_class {
    return 'Koha::Import::Oaipmh::Biblio';
}
1;




# Copyright BibLibre 2023
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

package Koha::OAI::Client::Harvester;

=head1 NAME

Koha::OAI::Client::Harvester - OAI Harvester

=head1 DESCRIPTION

Koha::OAI::Client::Harvester contains OAI-PMH harvester main functions

=cut

use utf8;
use open qw( :std :utf8 );

use C4::Biblio qw( AddBiblio GetFrameworkCode ModBiblio DelBiblio );
use C4::AuthoritiesMarc qw (AddAuthority GuessAuthTypeCode ModAuthority DelAuthority );
use C4::Log qw( cronlogaction );
use HTTP::OAI;
use HTTP::OAI::Metadata::OAI_DC;
use Koha::DateUtils qw( dt_from_string );
use Koha::OaiServers;
use Koha::Import::Oaipmh::Biblio;
use Koha::Import::Oaipmh::Biblios;
use Koha::Import::Oaipmh::Authority;
use Koha::Import::Oaipmh::Authorities;
use Koha::XSLT::Base;
use MARC::File::XML;
use MARC::Record;
use Modern::Perl;
use DateTime;
use DateTime::Format::Strptime;
use Try::Tiny qw( catch try );
use Date::Calc qw(
    Add_Delta_Days
    Today
);

my $strp = DateTime::Format::Strptime->new(
    pattern   => '%FT%TZ',
    time_zone => 'floating',
);

our $verbose = 0;
our $server;
our $force       = 0;
our $days        = 0;
our $xslt_engine = Koha::XSLT::Base->new;

=head2 new

$harvester = Koha::OAI::Client::Harvester->new( { server => $server, verbose => $verbose, days => $days, force => $force } );

New instance of Koha::OAI::Client::Harvester

C<$server> An OAI repository (Koha::OaiServer)

C<$verbose> print log messages to stdout when enabled

C<$days> number of days to harvest from (optional)

C<$force> force harvesting (ignore records datestamps)

=cut

sub new {

    my ( $self, $args ) = @_;

    $server  = $args->{'server'};
    $verbose = $args->{'verbose'};
    $days    = $args->{'days'};
    $force   = $args->{'force'};
    return $self;
}

=head2 init

Starts harvesting

=cut

sub init {
    my ( $self, $args ) = @_;

    printlog("Starting OAI Harvest from repository");

    my $h = HTTP::OAI::Harvester->new(
        repository => HTTP::OAI::Identify->new(
            baseURL => $server->endpoint,
            version => '2.0',
        )
    );

    printlog( "Using HTTP::OAI::Harvester version " . $HTTP::OAI::Harvester::VERSION );

    printlog( "Using endpoint "
            . $server->endpoint
            . ", set "
            . $server->oai_set
            . ", dataformat "
            . $server->dataformat
            . ", recordtype "
            . $server->recordtype );

    try {
        my $lmdf    = $h->ListMetadataFormats();
        my $lmdflog = "";
        for ( $lmdf->metadataFormat ) {
            $lmdflog .= $_->metadataPrefix . " ";
        }
        printlog("Available metadata formats for this repository: $lmdflog");
    } catch {
        printlog("ListMetadataFormats failed");
    };

    if ( $server->add_xslt ) {
        printlog( "Using XSLT file " . $server->add_xslt );
    } else {
        printlog("Not using an XSLT file");
    }

    my $start_date = '';
    my $today_date = '';

    if ($days) {

        # Change this to yyyy-mm-dd
        my $dt_today = dt_from_string();
        my ( $nowyear, $nowmonth, $nowday ) = Today();
        my @date     = Add_Delta_Days( $nowyear, $nowmonth, $nowday, -$days );
        my $dt_start = DateTime->new( year => $date[0], month => $date[1], day => $date[2] );
        $start_date = $dt_start->ymd();
        printlog("Harvesting from $start_date");
    }

    printlog("Asking for records");
    my $response = $h->ListRecords(
        metadataPrefix => $server->dataformat,
        set            => $server->oai_set,
        from           => $start_date,
    );
    if ( $response->is_error ) {
        printlog( "Error requesting ListRecords: " . $response->code . " " . $response->message );
        exit;
    }
    printlog( "Request URL: " . $response->requestURL );

    my %stats;
    my @statuses = qw(added updated deleted in_error skipped total);
    foreach my $status (@statuses) {
        $stats{$status} = 0;
    }

    printlog("Starting processing results");
    while ( my $oai_record = $response->next ) {
        $stats{'total'}++;
        my $status = $self->processRecord($oai_record);
        $stats{$status}++;
    }

    my $results = '';
    foreach my $status (@statuses) {
        $results .= $stats{$status} . " $status\n";
    }
    printlog( "Harvest results:\n" . $results );

    if ( C4::Context->preference("OAI-PMH:HarvestEmailReport") ) {

        my $letter = C4::Letters::GetPreparedLetter(
            (
                module      => 'catalogue',
                letter_code => 'OAI_HARVEST_REPORT',
                substitute  => {
                    servername => $server->servername,
                    endpoint   => $server->endpoint,
                    set        => $server->oai_set,
                    dataformat => $server->dataformat,
                    recordtype => $server->recordtype,
                    added      => $stats{'added'},
                    updated    => $stats{'updated'},
                    deleted    => $stats{'deleted'},
                    skipped    => $stats{'skipped'},
                    in_error   => $stats{'in_error'},
                    total      => $stats{'total'}
                },
            )
        ) or printlog("OAI_HARVEST_REPORT letter not found");

        my $success = C4::Letters::EnqueueLetter(
            {
                letter     => $letter,
                to_address => C4::Context->preference("OAI-PMH:HarvestEmailReport"), message_transport_type => 'email'
            }
        );
        if ($success) {
            printlog("Email report enqueued");
        } else {
            printlog("Unable to enqueue report email");
        }
    }

    printlog("Ending OAI Harvest from repository");
}

=head2 processRecord

This method processes an incoming OAI record

=cut

sub processRecord {
    my $self       = shift;
    my $oai_record = shift;
    my $status     = '';
    unless ( $oai_record->identifier ) {
        printlog("No identifier found");
        $status = 'skipped';
        return $status;
    }

    my $to_delete;
    my $outputUnimarc;
    my $marcrecord;
    if ( $oai_record->status && $oai_record->status eq "deleted" ) {
        $to_delete = 1;
    } else {
        if ( $server->add_xslt ) {
            $outputUnimarc = $xslt_engine->transform( $oai_record->metadata->dom, $server->add_xslt );
            my $err = $xslt_engine->err;
            if ($err) {
                printlog("XSLT::Base error: $err");
                $status = 'in_error';
                return $status;
            }
        } else {
            $outputUnimarc = $oai_record->metadata->dom;
        }
        $marcrecord = MARC::Record->new_from_xml( $outputUnimarc, 'UTF-8' );
    }
    my $imported_record;
    if ( $server->recordtype eq "biblio" ) {
        $imported_record = Koha::Import::Oaipmh::Biblios->find(
            {
                repository => $server->endpoint,
                identifier => $oai_record->identifier,
                recordtype => $server->recordtype,
            }
        );
    } else {
        $imported_record = Koha::Import::Oaipmh::Authorities->find(
            {
                repository => $server->endpoint,
                identifier => $oai_record->identifier,
                recordtype => $server->recordtype,
            }
        );
    }

    if ($imported_record) {
        if ($to_delete) {
            if ( $server->recordtype eq "biblio" ) {
                my $biblionumber = $imported_record->biblionumber;
                my $error        = DelBiblio($biblionumber);
                if ($error) {
                    printlog(
                        "Record " . $oai_record->identifier . " not deleted, biblionumber: $biblionumber ($error)" );
                    $status = 'in_error';
                } else {
                    $imported_record->delete;
                    printlog( "Record " . $oai_record->identifier . " deleted, biblionumber: $biblionumber" );
                    $status = 'deleted';
                }
            } else {
                my $authid = $imported_record->authid;
                DelAuthority( { authid => $authid } );
                $imported_record->delete;
                printlog( "Record " . $oai_record->identifier . " deleted, authid: $authid" );
                $status = 'deleted';
            }
        } else {
            my $existing_dt = $strp->parse_datetime( $imported_record->datestamp );
            my $incoming_dt = $strp->parse_datetime( $oai_record->datestamp );

            if ( $force || !$incoming_dt || !$existing_dt || $incoming_dt > $existing_dt ) {
                if ( $server->recordtype eq "biblio" ) {
                    my $biblionumber = $imported_record->biblionumber;
                    my $result       = ModBiblio( $marcrecord, $biblionumber, GetFrameworkCode($biblionumber) );
                    printlog( "Record " . $oai_record->identifier . " updated, biblionumber: $biblionumber" );
                } else {
                    my $authid = $imported_record->authid;
                    my $result = ModAuthority( $authid, $marcrecord, GuessAuthTypeCode($marcrecord) );
                    printlog( "Record " . $oai_record->identifier . " updated, authid: $authid" );
                }
                $imported_record->update(
                    {
                        datestamp => $imported_record->datestamp,
                    }
                );
                $status = 'updated';
            } else {
                printlog( "Record "
                        . $oai_record->identifier
                        . " skipped (incoming record was not newer than existing record)" );
                $status = 'skipped';
            }
        }
    } elsif ( !$to_delete ) {
        if ( $server->recordtype eq "biblio" ) {
            my ( $biblionumber, $biblioitemnumber ) = AddBiblio($marcrecord);
            printlog( $oai_record->identifier . " added, biblionumber: $biblionumber" );
            Koha::Import::Oaipmh::Biblio->new(
                {
                    repository   => $server->endpoint,
                    identifier   => $oai_record->identifier,
                    biblionumber => $biblionumber,
                    recordtype   => $server->recordtype,
                    datestamp    => $oai_record->datestamp,
                }
            )->store;
        } else {
            my $authid = AddAuthority( $marcrecord, "", GuessAuthTypeCode($marcrecord) );
            printlog( $oai_record->identifier . " added, authid: $authid" );
            Koha::Import::Oaipmh::Authority->new(
                {
                    repository => $server->endpoint,
                    identifier => $oai_record->identifier,
                    authid     => $authid,
                    recordtype => $server->recordtype,
                    datestamp  => $oai_record->datestamp,
                }
            )->store;
        }
        $status = 'added';
    } else {
        printlog( "Record " . $oai_record->identifier . " skipped (record not present or already deleted)" );
        $status = 'skipped';
    }
    return $status;
}

=head2 printlog

This method adds a cronlog and prints to stdout if verbose is enabled

=cut

sub printlog {
    my $message = shift;
    $message = $server->servername . ": " . $message;
    print $message . "\n" if ($verbose);
    cronlogaction( { info => $message } );
}

1;




package Koha::OaiServer;

# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;


use Koha::Database;

use base qw(Koha::Object);

=head1 NAME

Koha::OaiServer - Koha OaiServer Object class

=head1 API

=head2 Class Methods

=cut

=head3 _type

Return type of Object relating to Schema ResultSet

=cut

sub _type {
    return 'Oaiserver';
}

1;




package Koha::OaiServers;

# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;


use Koha::Database;

use Koha::OaiServer;

use base qw(Koha::Objects);

=head1 NAME

Koha::OaiServers - Koha OaiServer Object set class

=head1 API

=head2 Class Methods

=cut

=head3 _type

Return type of object, relating to Schema ResultSet

=cut

sub _type {
    return 'Oaiserver';
}

=head3 object_class

Return object class

=cut

sub object_class {
    return 'Koha::OaiServer';
}

1;




#!/usr/bin/perl

# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

# This script is used to maintain the OAI servers table.
# Parameter $op is operation: list, new, edit, add_validated, delete_confirmed.
# add_validated saves a validated record and goes to list view.
# delete_confirmed deletes a record and goes to list view.

use Modern::Perl;
use CGI qw ( -utf8 );
use C4::Context;
use C4::Auth qw( get_template_and_user );
use C4::Output qw( output_html_with_http_headers );
use Koha::Database;
use Koha::OaiServers;

# Initialize CGI, template, database

my $input       = CGI->new;
my $op          = $input->param('op') || 'list';
my $id          = $input->param('id') || 0;
my $type        = $input->param('type') || '';
my $searchfield = '';

my ( $template, $loggedinuser, $cookie ) = get_template_and_user(
    {
        template_name => "admin/oaiservers.tt",
        query         => $input,
        type          => "intranet",
        flagsrequired => { parameters => 'manage_search_targets' },
    }
);
my $script_name = "/cgi-bin/koha/admin/oaiservers.pl";
my $path        = C4::Context->config('intrahtdocs') . "/prog/";

$template->param( script_name => $script_name, xslt_path => $path );

my $schema = Koha::Database->new()->schema();

# Main code
# First process a confirmed delete, or save a validated record

if ( $op eq 'delete_confirmed' && $id ) {
    my $server = Koha::OaiServers->find($id);
    if ($server) {
        $server->delete;
        $template->param( msg_deleted => 1, msg_add => $server->servername );
    } else {
        $template->param( msg_notfound => 1, msg_add => $id );
    }
    $id = 0;
} elsif ( $op eq 'add_validated' ) {
    my @fields = qw/endpoint oai_set dataformat
        recordtype servername
        add_xslt/;
    my $formdata = _form_data_hashref( $input, \@fields );
    if ($id) {
        my $server = Koha::OaiServers->find($id);
        if ($server) {
            $server->set($formdata)->store;
            $template->param( msg_updated => 1, msg_add => $formdata->{servername} );
        } else {
            $template->param( msg_notfound => 1, msg_add => $id );
        }
        $id = 0;
    } else {
        Koha::OaiServer->new($formdata)->store;
        $template->param( msg_added => 1, msg_add => $formdata->{servername} );
    }
} else {

    #use searchfield only in remaining operations
    $searchfield = $input->param('searchfield') || '';
}

# Now list multiple records, or edit one record

my $data = [];
if ( $op eq 'add' || $op eq 'edit' ) {
    $data = ServerSearch( $schema, $id, $searchfield ) if $searchfield || $id;
    delete $data->[0]->{id} if @$data && $op eq 'add';    #cloning record
    $template->param(
        add_form => 1, server => @$data ? $data->[0] : undef,
        op => $op, type => $op eq 'add' ? lc $type : ''
    );
} else {
    $data = ServerSearch( $schema, $id, $searchfield );
    $template->param(
        loop => \@$data, searchfield => $searchfield, id => $id,
        op   => 'list'
    );
}
output_html_with_http_headers $input, $cookie, $template->output;

# End of main code

sub ServerSearch {    #find server(s) by id or name
    my ( $schema, $id, $searchstring ) = @_;

    return Koha::OaiServers->search(
        $id ? { id => $id } : { servername => { like => $searchstring . '%' } },
    )->unblessed;
}

sub _form_data_hashref {
    my ( $input, $fieldref ) = @_;
    return { map { ( $_ => scalar $input->param($_) // '' ) } @$fieldref };
}




use Modern::Perl;

return {
    bug_number  => "35659",
    description => "OAI-PMH Harvester",
    up          => sub {
        my ($args) = @_;
        my ( $dbh, $out ) = @$args{qw(dbh out)};
        $dbh->do(q{DROP TABLE IF EXISTS `oaiservers`;});
        $dbh->do(
            q{
            CREATE TABLE `oaiservers` (
              `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
              `endpoint` varchar(255) NOT NULL COMMENT 'OAI endpoint (host + port + path)',
              `oai_set` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
              `servername` longtext NOT NULL COMMENT 'name given to the target by the library',
              `dataformat` enum('oai_dc','marc-xml', 'marcxml') NOT NULL DEFAULT 'oai_dc' COMMENT 'data format',
              `recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'server contains bibliographic or authority records',
              `add_xslt` longtext DEFAULT NULL COMMENT 'zero or more paths to XSLT files to be processed on the search results',
              PRIMARY KEY (`id`)
            ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
        }
        );
        say $out "Added new table 'oaiservers'";

        $dbh->do(q{DROP TABLE IF EXISTS `import_oaipmh_biblios`;});
        $dbh->do(
            q{
            CREATE TABLE `import_oaipmh_biblios` (
              `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
              `biblionumber` int(11) NOT NULL COMMENT 'unique identifier assigned to each koha record',
              `identifier` varchar(255) NOT NULL COMMENT 'OAI record identifier',
              `repository` varchar(255) NOT NULL COMMENT 'OAI repository',
              `recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'is the record bibliographic or authority',
              `datestamp` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
              `last_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
              PRIMARY KEY (id),
              KEY biblionumber (biblionumber),
              CONSTRAINT FK_import_oaipmh_biblios_1 FOREIGN KEY (biblionumber) REFERENCES biblio (biblionumber) ON DELETE CASCADE ON UPDATE NO ACTION
            ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
        }
        );

        $dbh->do(q{DROP TABLE IF EXISTS `import_oaipmh_authorities`;});
        $dbh->do(
            q{
            CREATE TABLE `import_oaipmh_authorities` (
              `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
              `authid` bigint(20) unsigned NOT NULL COMMENT 'unique identifier assigned to each koha record',
              `identifier` varchar(255) NOT NULL COMMENT 'OAI record identifier',
              `repository` varchar(255) NOT NULL COMMENT 'OAI repository',
              `recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'is the record bibliographic or authority',
              `datestamp` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
              `last_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
              PRIMARY KEY (id),
              KEY authid (authid),
              CONSTRAINT FK_import_oaipmh_authorities_1 FOREIGN KEY (authid) REFERENCES auth_header (authid) ON DELETE CASCADE ON UPDATE NO ACTION
            ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
        }
        );
        say $out "Added new tables 'import_oaipmh_biblios' and import_oaipmh_authorities";

        $dbh->do(
            q{
            UPDATE `permissions` SET description='Manage Z39.50 and SRU servers, OAI repositories configuration' WHERE code='manage_search_targets';
        }
        );
        say $out "Updated manage_search_targets permission description";

        $dbh->do(
            q{
            INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES
            ('OAI-PMH:HarvestEmailReport','','','If set, a report email will be sent to this address after an OAI harvest','Free');
        }
        );
        say $out "Added OAI-PMH:HarvestEmailReport system preference";

        $dbh->do(
            q{
            INSERT IGNORE INTO letter
            (module, code, branchcode, name, is_html, title, content, message_transport_type, lang)
            VALUES ('catalogue','OAI_HARVEST_REPORT','','OAI Harvest Report',0,'OAI Harvest report for [% servername %]','OAI Harvest report for [% servername %]:\n\nEndpoint: [% endpoint %]\nSet: [% set %]\nData format: [% dataformat %]\nRecord type: [% recordtype %]\n\n[% added %] records added\n[% updated %] records updated\n[% deleted %] records deleted\n[% skipped %] records skipped\n[% in_error %] records in error\n[% total %] total','email','default');
        }
        );
        say $out "Added OAI_HARVEST_REPORT letter";

    },
};




            - "Your hold on [% hold.biblio.title %] ([% hold.biblio.id %]) has been confirmed."
            - "You will be notified by the library when your item is available for pickup."
            - "Thank you!"

        - module: catalogue
          code: OAI_HARVEST_REPORT
          branchcode: ""
          name: "OAI Harvest Report"
          is_html: 0
          title: "OAI Harvest report for [% servername %]"
          message_transport_type: email
          lang: default
          content:
            - "OAI Harvest report for [% servername %]:"
            - ""
            - "Endpoint: [% endpoint %]"
            - "Set: [% set %]"
            - "Data format: [% dataformat %]"
            - "Record type: [% recordtype %]"
            - ""
            - "[% added %] records added"
            - "[% updated %] records updated"
            - "[% deleted %] records deleted"
            - "[% skipped %] records skipped"
            - "[% in_error %] records in error"
            - "[% total %] total"




) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Table structure for table `oaiservers`
--

DROP TABLE IF EXISTS `oaiservers`;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `oaiservers` (
  `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
  `endpoint` varchar(255) NOT NULL COMMENT 'OAI endpoint (host + port + path)',
  `oai_set` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
  `servername` longtext NOT NULL COMMENT 'name given to the target by the library',
  `dataformat` enum('oai_dc','marc-xml', 'marcxml') NOT NULL DEFAULT 'oai_dc' COMMENT 'data format',
  `recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'server contains bibliographic or authority records',
  `add_xslt` longtext DEFAULT NULL COMMENT 'zero or more paths to XSLT files to be processed on the search results',
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

--
-- Table structure for table `import_oaipmh_biblios`
--

DROP TABLE IF EXISTS `import_oaipmh_biblios`;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `import_oaipmh_biblios` (
  `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
  `biblionumber` int(11) NOT NULL COMMENT 'unique identifier assigned to each koha record',
  `identifier` varchar(255) NOT NULL COMMENT 'OAI record identifier',
  `repository` varchar(255) NOT NULL COMMENT 'OAI repository',
  `recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'is the record bibliographic or authority',
  `datestamp` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
  `last_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
  PRIMARY KEY (id),
  KEY biblionumber (biblionumber),
  CONSTRAINT FK_import_oaipmh_biblios_1 FOREIGN KEY (biblionumber) REFERENCES biblio (biblionumber) ON DELETE CASCADE ON UPDATE NO ACTION
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

--
-- Table structure for table `import_oaipmh_authorities`
--

DROP TABLE IF EXISTS `import_oaipmh_authorities`;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `import_oaipmh_authorities` (
  `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
  `authid` bigint(20) unsigned NOT NULL COMMENT 'unique identifier assigned to each koha record',
  `identifier` varchar(255) NOT NULL COMMENT 'OAI record identifier',
  `repository` varchar(255) NOT NULL COMMENT 'OAI repository',
  `recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'is the record bibliographic or authority',
  `datestamp` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
  `last_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
  PRIMARY KEY (id),
  KEY authid (authid),
  CONSTRAINT FK_import_oaipmh_authorities_1 FOREIGN KEY (authid) REFERENCES auth_header (authid) ON DELETE CASCADE ON UPDATE NO ACTION
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

--
-- Table structure for table `oauth_access_tokens`
--

Lines 439-444 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here

(-)a/installer/data/mysql/mandatory/sysprefs.sql (+1 lines)
439	('OAI-PMH:AutoUpdateSetsEmbedItemData', '0', '', 'Embed item information when automatically updating OAI sets. Requires OAI-PMH:AutoUpdateSets syspref to be enabled', 'YesNo'),	439	('OAI-PMH:AutoUpdateSetsEmbedItemData', '0', '', 'Embed item information when automatically updating OAI sets. Requires OAI-PMH:AutoUpdateSets syspref to be enabled', 'YesNo'),
440	('OAI-PMH:ConfFile','',NULL,'If empty, Koha OAI Server operates in normal mode, otherwise it operates in extended mode.','File'),	440	('OAI-PMH:ConfFile','',NULL,'If empty, Koha OAI Server operates in normal mode, otherwise it operates in extended mode.','File'),
441	('OAI-PMH:DeletedRecord','persistent','Koha\'s deletedbiblio table will never be deleted (persistent), might be deleted (transient), or will never have any data in it (no)','transient\|persistent\|no','Choice'),	441	('OAI-PMH:DeletedRecord','persistent','Koha\'s deletedbiblio table will never be deleted (persistent), might be deleted (transient), or will never have any data in it (no)','transient\|persistent\|no','Choice'),
		442	('OAI-PMH:HarvestEmailReport','','','If set, a report email will be sent to this address after an OAI harvest','Free'),
442	('OAI-PMH:MaxCount','50',NULL,'OAI-PMH maximum number of records by answer to ListRecords and ListIdentifiers queries','Integer'),	443	('OAI-PMH:MaxCount','50',NULL,'OAI-PMH maximum number of records by answer to ListRecords and ListIdentifiers queries','Integer'),
443	('OnSiteCheckoutAutoCheck','0','','Enable/Do not enable onsite checkout by default if last checkout was an onsite checkout','YesNo'),	444	('OnSiteCheckoutAutoCheck','0','','Enable/Do not enable onsite checkout by default if last checkout was an onsite checkout','YesNo'),
444	('OnSiteCheckouts','0','','Enable/Disable the on-site checkouts feature','YesNo'),	445	('OnSiteCheckouts','0','','Enable/Disable the on-site checkouts feature','YesNo'),

Lines 29-35 INSERT INTO permissions (module_bit, code, description) VALUES Link Here

(-)a/installer/data/mysql/mandatory/userpermissions.sql (-1 / +1 lines)
29	( 3, 'manage_item_search_fields', 'Manage item search fields'),	29	( 3, 'manage_item_search_fields', 'Manage item search fields'),
30	( 3, 'manage_search_engine_config', 'Manage search engine configuration'),	30	( 3, 'manage_search_engine_config', 'Manage search engine configuration'),
31	( 3, 'manage_marc_overlay_rules', 'Manage MARC overlay rules configuration'),	31	( 3, 'manage_marc_overlay_rules', 'Manage MARC overlay rules configuration'),
32	( 3, 'manage_search_targets', 'Manage Z39.50 and SRU server configuration'),	32	( 3, 'manage_search_targets', 'Manage Z39.50 and SRU servers, OAI repositories configuration'),
33	( 3, 'manage_didyoumean', 'Manage Did you mean? configuration'),	33	( 3, 'manage_didyoumean', 'Manage Did you mean? configuration'),
34	( 3, 'manage_column_config', 'Manage column configuration'),	34	( 3, 'manage_column_config', 'Manage column configuration'),
35	( 3, 'manage_sms_providers', 'Manage SMS cellular providers'),	35	( 3, 'manage_sms_providers', 'Manage SMS cellular providers'),

Lines 149-154 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc (+1 lines)
149	[% END %]	149	[% END %]
150	[% IF ( CAN_user_parameters_manage_search_targets ) %]	150	[% IF ( CAN_user_parameters_manage_search_targets ) %]
151	<li><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50/SRU servers</a></li>	151	<li><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50/SRU servers</a></li>
		152	<li><a href="/cgi-bin/koha/admin/oaiservers.pl">OAI repositories</a></li>
152	[% END %]	153	[% END %]
153	[% IF ( CAN_user_parameters_manage_smtp_servers ) %]	154	[% IF ( CAN_user_parameters_manage_smtp_servers ) %]
154	<li><a href="/cgi-bin/koha/admin/smtp_servers.pl">SMTP servers</a></li>	155	<li><a href="/cgi-bin/koha/admin/smtp_servers.pl">SMTP servers</a></li>




[% USE Koha %]
<!-- oai-admin-search.inc -->
<div id="header_search" role="tablist">
    <div class="tab-content">
        <div id="oai_search" role="tabpanel" class="tab-pane active">
            <form action="/cgi-bin/koha/admin/oaiservers.pl" method="post">
                <div class="form-title">
                    <label class="control-label">Search OAI repositories</label>
                </div>

                <div class="form-content">
                    <input class="head-searchbox form-control" type="text" name="searchfield" id="searchfield" value="[% searchfield | html %]" placeholder="OAI repository search" />
                </div>

                <button type="submit"><i class="fa fa-arrow-right"></i></button>
            </form>
        </div>

        [% IF ( CAN_user_circulate_circulate_remaining_permissions ) %]
            [% INCLUDE 'patron-search-box.inc' %]
        [% END %]
        [% IF ( CAN_user_catalogue ) %]
            [% INCLUDE 'catalogue-search-box.inc' %]
        [% END %]

    </div><!-- /.tab-content -->
    <ul class="nav nav-tabs" role="tablist">
        <li role="presentation" class="active">
            <a title="Search OAI repositories" href="#oai_search" aria-controls="z3950_search" role="tab" aria-expanded="true" data-toggle="tab" class="keep_text"><i class="fa fa-search"></i> <span class="tab-title">Search OAI repositories</span></a>
        </li>
        [% IF ( CAN_user_circulate_circulate_remaining_permissions ) %]
            <li role="presentation">
                <a title="Check out" href="#circ_search" aria-controls="circ_search" role="tab" data-toggle="tab" class="keep_text"><i class="fa fa-upload"></i> <span class="tab-title">Check out</span></a>
            </li>
        [% END %]
        [% IF ( CAN_user_catalogue ) %]
            <li role="presentation">
                <a title="Search catalog" href="#catalog_search" aria-controls="catalog_search" role="tab" data-toggle="tab" class="keep_text"><i class="fa fa-fw fa-search"></i> <span class="tab-title">Search catalog</span></a>
            </li>
        [% END %]
    </ul>
</div><!-- /#header_search -->
<!-- /oai-admin-search.inc -->

Lines 244-250 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/permissions.inc (-1 / +1 lines)
244	<span class="permissioncode">([% name \| html %])</span>	244	<span class="permissioncode">([% name \| html %])</span>
245	[%- CASE 'manage_search_targets' -%]	245	[%- CASE 'manage_search_targets' -%]
246	<span class="sub_permission manage_search_targets_subpermission">	246	<span class="sub_permission manage_search_targets_subpermission">
247	Manage Z39.50 and SRU server configuration	247	Manage Z39.50 and SRU servers, OAI repositories configuration
248	</span>	248	</span>
249	<span class="permissioncode">([% name \| html %])</span>	249	<span class="permissioncode">([% name \| html %])</span>
250	[%- CASE 'manage_didyoumean' -%]	250	[%- CASE 'manage_didyoumean' -%]

Lines 249-254 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt (+2 lines)
249	[% IF ( CAN_user_parameters_manage_search_targets ) %]	249	[% IF ( CAN_user_parameters_manage_search_targets ) %]
250	<dt><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50/SRU servers</a></dt>	250	<dt><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50/SRU servers</a></dt>
251	<dd>Define which external servers to query for MARC data</dd>	251	<dd>Define which external servers to query for MARC data</dd>
		252	<dt><a href="/cgi-bin/koha/admin/oaiservers.pl">OAI repositories</a></dt>
		253	<dd>Define which OAI repositories to harvest data from</dd>
252	[% END %]	254	[% END %]
253	[% IF ( CAN_user_parameters_manage_smtp_servers ) %]	255	[% IF ( CAN_user_parameters_manage_smtp_servers ) %]
254	<dt><a href="/cgi-bin/koha/admin/smtp_servers.pl">SMTP servers</a></dt>	256	<dt><a href="/cgi-bin/koha/admin/smtp_servers.pl">SMTP servers</a></dt>




[% USE raw %]
[% USE Asset %]
[% SET footerjs = 1 %]
[% INCLUDE 'doc-head-open.inc' %]

<title>
    [% IF op == 'edit' %]
        Modify OAI repository [% server.servername | html %] &rsaquo; [% END %]

    [% IF op ==  'add' %]
        New OAI repository &rsaquo; [% END %]

    OAI repositories &rsaquo; Administration &rsaquo; Koha
</title>
[% INCLUDE 'doc-head-close.inc' %]
</head>

<body id="admin_oaiservers" class="admin">
[% WRAPPER 'header.inc' %]
    [% INCLUDE 'oai-admin-search.inc' %]
[% END %]

[% WRAPPER 'sub-header.inc' %]
    [% WRAPPER breadcrumbs %]
        [% WRAPPER breadcrumb_item %]
            <a href="/cgi-bin/koha/admin/admin-home.pl">Administration</a>
        [% END %]

        [% IF op == 'edit' || op == 'add' %]
            [% WRAPPER breadcrumb_item %]
                <a href="/cgi-bin/koha/admin/oaiservers.pl">OAI repositories</a>
           [% END %]
        [% END %]

        [% IF op == 'edit' %]
            [% WRAPPER breadcrumb_item bc_active= 1 %]
                <span>Modify OAI repository [% server.servername | html %]</span>
           [% END %]

        [% ELSIF op == 'add' %]
            [% WRAPPER breadcrumb_item bc_active= 1 %]
                <span>New OAI repository</span>
           [% END %]

        [% ELSE %]
            [% WRAPPER breadcrumb_item bc_active= 1 %]
                <span>OAI repositories</span>
           [% END %]
        [% END %]
    [% END #/ WRAPPER breadcrumbs %]
[% END #/ WRAPPER sub-header.inc %]

<div class="main container-fluid">
    <div class="row">
        <div class="col-sm-10 col-sm-push-2">
            <main>

[% IF msg_deleted %]
    <div class="dialog alert">OAI repository deleted ([% msg_add | html %])</div>
[% ELSIF msg_updated %]
    <div class="dialog message">OAI repository updated ([% msg_add | html %])</div>
[% ELSIF msg_added %]
    <div class="dialog message">OAI repository added ([% msg_add | html %])</div>
[% ELSIF msg_notfound %]
    <div class="dialog alert">Error: Server with ID [% msg_add | html %] not found</div>
[% END %]

[% IF ( add_form ) %]
    <form action="[% script_name | html %]" name="Aform" method="post" id="serverentry">
        <input type="hidden" name="op" value="add_validated" />
        [% IF op == 'edit' %]
            <h1>Modify OAI repository</h1>
            <input type="hidden" name="id" value="[% server.id | html %]" />
        [% ELSE %]
            <h1>New OAI repository</h1>
        [% END %]
        <fieldset class="rows">
        <ol>
        <li><label for="name" class="required">Repository name: </label>
            [% IF server.servername %]
                <input type="text" name="servername" id="servername" size="65" maxlength="100" value="[% server.servername | html %]" required="required" />
            [% ELSE %]
                <input type="text" name="servername" id="servername" class="focus" size="65" maxlength="100" value="[% server.servername | html %]" required="required" />
            [% END %]
             <span class="required">Required</span>
        </li>

        <li><label for="endpoint" class="required">Endpoint: </label> <input type="text" name="endpoint" id="endpoint" size="30" value="[% server.endpoint | html %]" required="required" /> <span class="required">Required</span>
        </li>
        <li><label for="oai_set">Set: </label> <input type="text" name="oai_set" id="oai_set" value="[% server.oai_set | html %]" />
        </li>
        <li>
            <label for="dataformat">Data format: </label>
        <select name="dataformat" id="dataformat">
            <option value="oai_dc">Dublin Core (oai_dc)</option>
            <option value="marc-xml">Marcxml (marc-xml)</option>
            <option value="marcxml">Marcxml (marcxml)</option>
        </select>
            <div class="hint">OAI can send records in various formats. Choose one.</div>
        </li>

        <li><label for="recordtype">Record type: </label>
        <select name="recordtype" id="recordtype">
            <option value="biblio">Bibliographic</option>
            <option value="authority">Authority</option>
        </select>
        </li>
        <li>
        <label for="add_xslt">XSLT file for transforming results: </label>
        <input type="text" name="add_xslt" id="add_xslt" size="100" placeholder="[% xslt_path | html %]" value="[% server.add_xslt | html %]"/>
        </li>

        </ol>
        </fieldset>

        <fieldset class="action"><input type="submit" class="btn btn-primary" value="Save" /> <a class="cancel" href="/cgi-bin/koha/admin/oaiservers.pl">Cancel</a></fieldset>
    </form>
[% END %]

[% IF op == 'list' %]
    <div id="toolbar" class="btn-toolbar">
        <a id="newserver" class="btn btn-default" href="/cgi-bin/koha/admin/oaiservers.pl?op=add"><i class="fa fa-plus"></i> New OAI repository</a>
    </div>
    <h1>OAI repositories administration</h1>
    [% IF id %]
        You searched for record [% id | html %]
    [% ELSIF searchfield %]
        You searched for [% searchfield | html %]
    [% END %]

    <div class="page-section">
    <table id="serverst">

        <thead><tr><th>Id</th><th>Repository name</th><th>Endpoint</th><th>Set</th><th>Data format</th><th>Record type</th><th class="noExport">Actions</th>
        </tr></thead>
        <tbody>
        [% FOREACH loo IN loop %]
            <tr>
            <td>[% loo.id | html %]</td>
            <td><a href="/cgi-bin/koha/admin/oaiservers.pl?op=edit&amp;id=[% loo.id | uri %]">[% loo.servername | html %]</a></td><td>[% loo.endpoint | html %]</td><td>[% loo.oai_set | html %]</td>
            <td>[% loo.dataformat | html %]</td>
            <td>[% IF ( loo.recordtype == 'biblio' )  %]
                <span>Bibliographic</span>
                [% ELSIF ( loo.recordtype == 'authority' ) %]
                <span>Authority</span>
                [% END %]
            </td>
            <td>
                <div class="btn-group dropup">
                    <a class="btn btn-default btn-xs dropdown-toggle" id="reportactions[% savedreport.id | html %]" role="button" data-toggle="dropdown" href="#">
                        Actions <b class="caret"></b>
                    </a>
                    <ul class="dropdown-menu pull-right" role="menu" aria-labelledby="reportactions[% savedreport.id | html %]">
                        <li><a href="/cgi-bin/koha/admin/oaiservers.pl?op=edit&amp;id=[% loo.id | uri %]"><i class="fa-solid fa-pencil" aria-hidden="true"></i> Edit</a></li>
                        <li><a href="/cgi-bin/koha/admin/oaiservers.pl?op=add&amp;id=[% loo.id | uri %]"><i class="fa fa-copy"></i> Copy</a></li>
                        <li><a href="/cgi-bin/koha/admin/oaiservers.pl?op=delete_confirmed&amp;id=[% loo.id | uri %]" class="delete" data-servername="[% loo.servername | html %]"><i class="fa fa-trash-can"></i> Delete</a></li>
                    </ul>
                </div>
            </td>
            </tr>
        [% END %]
        </tbody>
    </table>
    </div> <!-- /.page-section -->
[% END %]

            </main>
        </div> <!-- /.col-sm-10.col-sm-push-2 -->

        <div class="col-sm-2 col-sm-pull-10">
            <aside>
                [% INCLUDE 'admin-menu.inc' %]
            </aside>
        </div> <!-- /.col-sm-2.col-sm-pull-10 -->
     </div> <!-- /.row -->

[% MACRO jsinclude BLOCK %]
    [% Asset.js("js/admin-menu.js") | $raw %]
    [% IF op == 'list' %]
        [% INCLUDE 'datatables.inc' %]
    [% END %]

    <script>
        [% IF ( add_form ) %]
            $(document).ready(function(){
                // Update selects for dataformat and recordtype
                [% IF server %]
                    $("#dataformat").val('[% server.dataformat | html %]');
                    $("#recordtype").val('[% server.recordtype | html %]');
                [% END %]
                $( "#serverentry" ).validate({
                    rules: {
                        servername: { required: true },
                        host: { required: true },
                        port: {
                            number: true
                        },
                        rank: { number: true },
                    }
                });
                $("#serverentry").submit(function( event ) {
                    // enable recordtype to include field in post
                    $('#recordtype').prop('disabled',false);
                });
                $("#servername").on("blur",function(){
                    toUC(this);
                });
            });
        [% ELSE %]
            $(document).ready(function() {
                $("#serverst").dataTable($.extend(true, {}, dataTablesDefaults, {
                    "aoColumnDefs": [
                        { "aTargets": [-1], "bSortable": false, "bSearchable": false },
                    ],
                    "sPaginationType": "full"
                }));
                $(".delete").on("click",function(e){
                    var servername = $(this).data("servername");
                    if( confirm( _("Are you sure you want to delete repository %s?").format(servername) ) ) {
                        return true;
                    } else {
                        e.preventDefault();
                    }
                });
            });
        [% END %]
    </script>
[% END %]
[% INCLUDE 'intranet-bottom.inc' %]

Lines 91-96 Web services: Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref (+5 lines)
91	transient: might be emptied or truncated at some point (transient)	91	transient: might be emptied or truncated at some point (transient)
92	no: will never have any data in it (no)	92	no: will never have any data in it (no)
93	- "."	93	- "."
		94	-
		95	- If set, a report email will be sent to this address after an OAI harvest
		96	- pref: "OAI-PMH:HarvestEmailReport"
		97	class: Text
		98
94	ILS-DI:	99	ILS-DI:
95	-	100	-
96	- pref: ILS-DI	101	- pref: ILS-DI




<?xml version="1.0" encoding="UTF-8"?>
<xsl:stylesheet version="1.0"
    xmlns:dc="http://purl.org/dc/elements/1.1/"
    xmlns:dcterms="http://purl.org/dc/terms/1.1"
    xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/
        http://www.openarchives.org/OAI/2.0/oai_dc.xsd"
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
    xmlns="http://www.loc.gov/MARC21/slim"  exclude-result-prefixes="dc dcterms oai_dc">

    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>

    <xsl:template match="/">
        <collection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" >
            <xsl:apply-templates />
        </collection>
    </xsl:template>

    <xsl:template match="oai_dc:dc">
        <record>

            <xsl:variable name="FamilyName" select="dc:FamilyName"/>
            <xsl:variable name="GivenName" select="dc:GivenName"/>

            <datafield tag="100" ind1="0" ind2=" ">
                <subfield code="a">
                    <xsl:value-of select="concat($FamilyName,', ',$GivenName)"/>
                </subfield>
            </datafield>

        </record>
   </xsl:template>

</xsl:stylesheet>




<?xml version="1.0" encoding="UTF-8"?>
<xsl:stylesheet version="1.0"
    xmlns:dc="http://purl.org/dc/elements/1.1/"
    xmlns:dcterms="http://purl.org/dc/terms/1.1"
    xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/
        http://www.openarchives.org/OAI/2.0/oai_dc.xsd"
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
    xmlns="http://www.loc.gov/MARC21/slim"  exclude-result-prefixes="dc dcterms oai_dc">

    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>

    <xsl:template match="/">
        <collection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" >
            <xsl:apply-templates />
        </collection>
    </xsl:template>

    <xsl:template match="oai_dc:dc">
        <record>

            <xsl:for-each select="dc:FamilyName">
                <datafield tag="200" ind1="0" ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>

            <xsl:for-each select="dc:GivenName">
                <datafield tag="200" ind1="0" ind2=" ">
                    <subfield code="b">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>

        </record>
   </xsl:template>

</xsl:stylesheet>




<?xml version="1.0" encoding="UTF-8"?>
<xsl:stylesheet version="1.0"
    xmlns:dc="http://purl.org/dc/elements/1.1/"
    xmlns:dcterms="http://purl.org/dc/terms/1.1"
    xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/
        http://www.openarchives.org/OAI/2.0/oai_dc.xsd"
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
    xmlns="http://www.loc.gov/MARC21/slim"  exclude-result-prefixes="dc dcterms oai_dc">

    <xsl:import href="MARC21slimUtils.xsl"/>
    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>


    <xsl:template match="/">
        <collection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" >
            <xsl:apply-templates />
        </collection>
    </xsl:template>

    <xsl:template name="OAI-PMH">
             <xsl:for-each select = "ListRecords/record/metadata/oai_dc:dc">
                <xsl:apply-templates  />
             </xsl:for-each>
             <xsl:for-each select = "GetRecord/record/metadata/oai_dc:dc">
                <xsl:apply-templates  />
             </xsl:for-each>
    </xsl:template>

    <xsl:template match="text()" />
    <xsl:template match="oai_dc:dc">
        <record>
            <xsl:element name="leader">
                <xsl:variable name="type" select="dc:type"/>
                <xsl:variable name="leader06">
                    <xsl:choose>
                        <xsl:when test="$type='collection'">p</xsl:when>
                        <xsl:when test="$type='dataset'">m</xsl:when>
                        <xsl:when test="$type='event'">r</xsl:when>
                        <xsl:when test="$type='image'">k</xsl:when>
                        <xsl:when test="$type='interactive resource'">m</xsl:when>
                        <xsl:when test="$type='service'">m</xsl:when>
                        <xsl:when test="$type='software'">m</xsl:when>
                        <xsl:when test="$type='sound'">i</xsl:when>
                        <xsl:when test="$type='text'">a</xsl:when>
                        <xsl:otherwise>a</xsl:otherwise>
                    </xsl:choose>
                </xsl:variable>
                <xsl:variable name="leader07">
                    <xsl:choose>
                        <xsl:when test="$type='collection'">c</xsl:when>
                        <xsl:otherwise>m</xsl:otherwise>
                    </xsl:choose>
                </xsl:variable>
                <xsl:value-of select="concat('      ',$leader06,$leader07,'         3u     ')"/>
            </xsl:element>

            <datafield tag="042" ind1=" " ind2=" ">
                <subfield code="a">dc</subfield>
            </datafield>



            <xsl:for-each select="dc:creator">
                <xsl:choose>
                    <xsl:when test="(.!='') and (position()=1)">
                        <xsl:call-template name="persname_template">
                            <xsl:with-param name="string" select="." />
                            <xsl:with-param name="field" select="'100'" />
                            <xsl:with-param name="ind1" select = "'1'" />
                            <xsl:with-param name="ind2" select = "'0'" />
                            <xsl:with-param name="type" select="'author'" />
                        </xsl:call-template>
                    </xsl:when>
                    <xsl:otherwise>
                        <xsl:if test=".!=''">
                            <xsl:call-template name="persname_template">
                                <xsl:with-param name="string" select="." />
                                <xsl:with-param name="field" select="'700'" />
                                <xsl:with-param name="ind1" select = "'1'" />
                                <xsl:with-param name="ind2" select = "'0'" />
                                <xsl:with-param name="type" select="'author'" />
                            </xsl:call-template>
                        </xsl:if>
                    </xsl:otherwise>
                </xsl:choose>
            </xsl:for-each>


            <xsl:for-each select="dc:title[1]">
                <datafield tag="245" ind1="0" ind2="0">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>

            <xsl:for-each select="dc:title[position()>1]">
                <xsl:if test=".!=''">
                    <datafield tag="246" ind1="3" ind2="3">
                        <subfield code="a">
                            <xsl:value-of select="."/>
                        </subfield>
                    </datafield>
                </xsl:if>
            </xsl:for-each>

            <xsl:choose>
                <xsl:when test="dc:publisher">
                    <xsl:if test="translate(dc:publisher/.,'.,:;','')!=''">
                        <datafield tag="260" ind1=" " ind2=" ">
                        <xsl:choose>
                            <xsl:when test="dc:date">
                                <subfield code="b"><xsl:value-of select="dc:publisher[1]"/>, </subfield>
                                <xsl:if test="translate(dc:date[1]/., '.,:;','')!=''">
                                    <subfield code="c"><xsl:value-of select="dc:date[1]" />.</subfield>
                                </xsl:if>
                            </xsl:when>
                            <xsl:otherwise>
                                <subfield code="b"><xsl:value-of select="dc:publisher[1]"/>.</subfield>
                            </xsl:otherwise>
                        </xsl:choose>
                        </datafield>
                    </xsl:if>
                </xsl:when>
                <xsl:otherwise>
                    <xsl:if test="translate(dc:date[1],'.,:;','')!=''">
                        <datafield tag="260" ind1=" " ind2=" ">
                            <subfield code="c"><xsl:value-of select="dc:date[1]" />.</subfield>
                        </datafield>
                    </xsl:if>
                </xsl:otherwise>
            </xsl:choose>

            <xsl:for-each select="dc:coverage">
                <xsl:choose>
                    <xsl:when test="translate(., '0123456789-.?','')=''">
                        <!--Likely;this is a date-->
                        <datafield tag="500" ind1=" " ind2=" ">
                            <subfield code="a"><xsl:value-of select="."/></subfield>
                        </datafield>
                    </xsl:when>
                    <xsl:otherwise>
                        <!--likely a geographic subject, we will print this later-->
                    </xsl:otherwise>
                </xsl:choose>
            </xsl:for-each>

            <xsl:for-each select="dc:identifier">
                <xsl:if test="position()!=last()">
                    <datafield tag="500" ind1=" " ind2=" ">
                        <subfield code="a"><xsl:value-of select="." /></subfield>
                    </datafield>
                </xsl:if>
            </xsl:for-each>

            <xsl:for-each select="dc:description">
                <datafield tag="520" ind1=" " ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="normalize-space(.)"/>
                    </subfield>
                </datafield>
            </xsl:for-each>


            <xsl:for-each select="dc:rights">
                <datafield tag="540" ind1=" " ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>


            <xsl:for-each select="dc:language">
                <datafield tag="546" ind1=" " ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>



            <xsl:for-each select="dc:subject">
                <datafield tag="690" ind1=" " ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>

            <xsl:for-each select="dc:coverage">
                <xsl:choose>
                    <xsl:when test="translate(., '0123456789-.?','')=''">
                        <!--Likely; this is a date-->
                    </xsl:when>
                    <xsl:otherwise>
                        <!--likely a geographic subject-->
                        <datafield tag="691" ind1=" " ind2=" ">
                            <subfield code="a"><xsl:value-of select="." /></subfield>
                        </datafield>
                    </xsl:otherwise>
                </xsl:choose>
            </xsl:for-each>


            <xsl:for-each select="dc:type">
                <datafield tag="655" ind1="7" ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                    <subfield code="2">local</subfield>
                </datafield>
            </xsl:for-each>



            <xsl:for-each select="dc:contributer">
                    <xsl:call-template name="persname_template">
                        <xsl:with-param name="string" select="." />
                        <xsl:with-param name="field" select="'100'" />
                        <xsl:with-param name="ind1" select = "'1'" />
                        <xsl:with-param name="ind2" select = "'0'" />
                        <xsl:with-param name="type" select="'contributor'" />
                    </xsl:call-template>
            </xsl:for-each>

            <xsl:for-each select="dc:source">
                <datafield tag="786" ind1="0" ind2=" ">
                    <subfield code="n">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>

            <xsl:for-each select="dc:relation">
                <datafield tag="787" ind1="0" ind2=" ">
                    <subfield code="n">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>

            <xsl:if test="dc:identifier">
                <datafield tag="856" ind1="4" ind2="1">
                    <subfield code="u"><xsl:value-of select="dc:identifier[last()]" /></subfield>
                    <subfield code="z">Connect to this object online.</subfield>
                </datafield>
            </xsl:if>
        </record>

    </xsl:template>
<xsl:template name="persname_template">
      <xsl:param name="string" />
      <xsl:param name="field" />
      <xsl:param name="ind1" />
      <xsl:param name="ind2" />
      <xsl:param name="type" />
      <datafield>
         <xsl:attribute name="tag">
            <xsl:value-of select="$field" />
         </xsl:attribute>
         <xsl:attribute name="ind1">
            <xsl:value-of select="$ind1" />
         </xsl:attribute>
         <xsl:attribute name="ind2">
            <xsl:value-of select="$ind2" />
         </xsl:attribute>

         <!-- Sample input: Brightman, Samuel C. (Samuel Charles), 1911-1992 -->
         <!-- Sample output: $aBrightman, Samuel C. $q(Samuel Charles), $d1911-. -->
         <!-- will handle names with dashes e.g. Bourke-White, Margaret -->

         <!-- CAPTURE PRIMARY NAME BY LOOKING FOR A PAREN OR A DASH OR NEITHER -->
         <xsl:choose>
            <!-- IF A PAREN, STOP AT AN OPENING PAREN -->
            <xsl:when test="contains($string, '(')!=0">
               <subfield code="a">
                  <xsl:value-of select="substring-before($string, '(')" />
               </subfield>
            </xsl:when>
            <!-- IF A DASH, CHECK IF IT'S A DATE OR PART OF THE NAME -->
            <xsl:when test="contains($string, '-')!=0">
               <xsl:variable name="name_1" select="substring-before($string, '-')" />
               <xsl:choose>
                  <!-- IF IT'S A DATE REMOVE IT -->
                  <xsl:when test="translate(substring($name_1, (string-length($name_1)), 1), '0123456789', '9999999999') = '9'">
                     <xsl:variable name="name" select="substring($name_1, 1, (string-length($name_1)-6))" />
                     <subfield code="a">
                        <xsl:value-of select="$name" />
                     </subfield>
                  </xsl:when>
                  <!-- IF IT'S NOT A DATE, CHECK WHETHER THERE IS A DATE LATER -->
                  <xsl:otherwise>
                     <xsl:variable name="remainder" select="substring-after($string, '-')" />
                     <xsl:choose>
                        <!-- IF THERE'S A DASH, ASSUME IT'S A DATE AND REMOVE IT -->
                        <xsl:when test="contains($remainder, '-')!=0">
                           <xsl:variable name="tmp" select="substring-before($remainder, '-')" />
                           <xsl:variable name="name_2" select="substring($tmp, 1, (string-length($tmp)-6))" />
                           <subfield code="a">
                              <xsl:value-of select="$name_1" />-<xsl:value-of select="$name_2" />
                           </subfield>
                        </xsl:when>
                        <!-- IF THERE'S NO DASH IN THE REMAINDER, OUTPUT IT -->
                        <xsl:otherwise>
                           <subfield code="a">
                              <xsl:value-of select="$string" />
                           </subfield>
                        </xsl:otherwise>
                     </xsl:choose>
                  </xsl:otherwise>
               </xsl:choose>
            </xsl:when>
            <!-- NO DASHES, NO PARENS, JUST OUTPUT THE NAME -->
            <xsl:otherwise>
               <subfield code="a">
                  <xsl:value-of select="$string" />
               </subfield>
            </xsl:otherwise>
         </xsl:choose>

         <!-- CAPTURE SECONDARY NAME IN PARENS FOR SUBFIELD Q -->
         <xsl:if test="contains($string, '(')!=0">
            <xsl:variable name="subq_tmp" select="substring-after($string, '(')" />
            <xsl:variable name="subq" select="substring-before($subq_tmp, ')')" />
            <subfield code="q">
               (<xsl:value-of select="$subq" />)
            </subfield>
         </xsl:if>

         <!-- CAPTURE DATE FOR SUBFIELD D, ASSUME DATE IS LAST ITEM IN FIELD -->
         <!-- Note: does not work if name has a dash in it -->
         <xsl:if test="contains($string, '-')!=0">
            <xsl:variable name="date_tmp" select="substring-before($string, '-')" />
            <xsl:variable name="remainder" select="substring-after($string, '-')" />
            <xsl:choose>
               <!-- CHECK SECOND HALF FOR ANOTHER DASH; IF PRESENT, ASSUME THAT IS DATE -->
               <xsl:when test="contains($remainder, '-')!=0">
                  <xsl:variable name="tmp" select="substring-before($remainder, '-')" />
                  <xsl:variable name="date_1" select="substring($remainder, (string-length($tmp)-3))" />
                  <!-- CHECK WHETHER IT HAS A NUMBER BEFORE IT AND IF SO, OUTPUT IT AS DATE -->
                  <xsl:if test="translate(substring($date_1, 1, 1), '0123456789', '9999999999') = '9'">
                     <subfield code="d">
                        <xsl:value-of select="$date_1" />.
                     </subfield>
                  </xsl:if>
               </xsl:when>
               <!-- OTHERWISE THIS IS THE ONLY DASH SO TAKE IT -->
               <xsl:otherwise>
                  <xsl:variable name="date_2" select="substring($string, (string-length($date_tmp)-3))" />
                  <!-- CHECK WHETHER IT HAS A NUMBER BEFORE IT AND IF SO, OUTPUT IT AS DATE -->
                  <xsl:if test="translate(substring($date_2, 1, 1), '0123456789', '9999999999') = '9'">
                     <subfield code="d">
                        <xsl:value-of select="$date_2" />.
                     </subfield>
                  </xsl:if>
               </xsl:otherwise>
            </xsl:choose>
         </xsl:if>
         <subfield code="e"><xsl:value-of select="$type" /></subfield>
      </datafield>
   </xsl:template>

</xsl:stylesheet>




<?xml version="1.0" encoding="UTF-8"?>
<xsl:stylesheet version="1.0"
    xmlns:dc="http://purl.org/dc/elements/1.1/"
    xmlns:dcterms="http://purl.org/dc/terms/1.1"
    xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/
        http://www.openarchives.org/OAI/2.0/oai_dc.xsd"
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
    xmlns="http://www.loc.gov/MARC21/slim"  exclude-result-prefixes="dc dcterms oai_dc">

    <xsl:import href="MARC21slimUtils.xsl"/>
    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>


    <xsl:template match="/">
        <collection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" >
            <xsl:apply-templates />
        </collection>
    </xsl:template>

    <xsl:template name="OAI-PMH">
             <xsl:for-each select = "ListRecords/record/metadata/oai_dc:dc">
                <xsl:apply-templates  />
             </xsl:for-each>
             <xsl:for-each select = "GetRecord/record/metadata/oai_dc:dc">
                <xsl:apply-templates  />
             </xsl:for-each>
    </xsl:template>

    <xsl:template match="text()" />
    <xsl:template match="oai_dc:dc">
        <record>

            <xsl:for-each select="dc:creator">
                <xsl:call-template name="persname_template">
                    <xsl:with-param name="string" select="." />
                    <xsl:with-param name="field" select="'700'" />
                    <xsl:with-param name="ind1" select = "'1'" />
                    <xsl:with-param name="ind2" select = "'0'" />
                    <xsl:with-param name="type" select="'author'" />
                </xsl:call-template>
            </xsl:for-each>


            <xsl:for-each select="dc:title[1]">
                <datafield tag="200" ind1="0" ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>

            <xsl:for-each select="dc:title[position()>1]">
                <xsl:if test=".!=''">
                    <datafield tag="532" ind1="3" ind2="0">
                        <subfield code="a">
                            <xsl:value-of select="."/>
                        </subfield>
                    </datafield>
                </xsl:if>
            </xsl:for-each>

            <xsl:choose>
                <xsl:when test="dc:publisher">
                    <xsl:if test="translate(dc:publisher/.,'.,:;','')!=''">
                        <datafield tag="214" ind1=" " ind2=" ">
                        <xsl:choose>
                            <xsl:when test="dc:date">
                                <subfield code="c"><xsl:value-of select="dc:publisher[1]"/>, </subfield>
                                <xsl:if test="translate(dc:date[1]/., '.,:;','')!=''">
                                    <subfield code="d"><xsl:value-of select="dc:date[1]" />.</subfield>
                                </xsl:if>
                            </xsl:when>
                            <xsl:otherwise>
                                <subfield code="c"><xsl:value-of select="dc:publisher[1]"/>.</subfield>
                            </xsl:otherwise>
                        </xsl:choose>
                        </datafield>
                    </xsl:if>
                </xsl:when>
                <xsl:otherwise>
                    <xsl:if test="translate(dc:date[1],'.,:;','')!=''">
                        <datafield tag="214" ind1=" " ind2=" ">
                            <subfield code="d"><xsl:value-of select="dc:date[1]" />.</subfield>
                        </datafield>
                    </xsl:if>
                </xsl:otherwise>
            </xsl:choose>


            <xsl:for-each select="dc:contributor">
                <datafield tag="701" ind1=" " ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>


            <xsl:for-each select="dc:coverage">
                <xsl:choose>
                    <xsl:when test="translate(., '0123456789-.?','')=''">
                        <!--Likely;this is a date-->
                        <datafield tag="215" ind1=" " ind2=" ">
                            <subfield code="a"><xsl:value-of select="."/></subfield>
                        </datafield>
                    </xsl:when>
                    <xsl:otherwise>
                        <!--likely a geographic subject, we will print this later-->
                    </xsl:otherwise>
                </xsl:choose>
            </xsl:for-each>


            <xsl:for-each select="dc:description">
                <datafield tag="330" ind1=" " ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="normalize-space(.)"/>
                    </subfield>
                </datafield>
            </xsl:for-each>


            <!-- no match -->
            <xsl:for-each select="dc:rights">
                <datafield tag="371" ind1=" " ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>


            <xsl:for-each select="dc:language">
                <datafield tag="101" ind1=" " ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>


            <xsl:for-each select="dc:subject">
                <datafield tag="610" ind1=" " ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>


            <!-- Resource type will most likely need a specific instance-based mapping -->
            <!--
            <xsl:for-each select="dc:type">
                <datafield tag="099" ind1=" " ind2=" ">
                    <subfield code="t">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>
            -->


            <!-- no match -->
            <xsl:for-each select="dc:coverage">
                <xsl:choose>
                    <xsl:when test="translate(., '0123456789-.?','')=''">
                        <!--Likely; this is a date-->
                    </xsl:when>
                    <xsl:otherwise>
                        <!--likely a geographic subject-->
                        <datafield tag="691" ind1=" " ind2=" ">
                            <subfield code="a"><xsl:value-of select="." /></subfield>
                        </datafield>
                    </xsl:otherwise>
                </xsl:choose>
            </xsl:for-each>


            <!-- no match -->
            <xsl:for-each select="dc:type">
                <datafield tag="655" ind1="7" ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                    <subfield code="2">local</subfield>
                </datafield>
            </xsl:for-each>


            <xsl:for-each select="dc:contributer">
                    <xsl:call-template name="persname_template">
                        <xsl:with-param name="string" select="." />
                        <xsl:with-param name="field" select="'702'" />
                        <xsl:with-param name="ind1" select = "'1'" />
                        <xsl:with-param name="ind2" select = "'0'" />
                        <xsl:with-param name="type" select="'contributor'" />
                    </xsl:call-template>
            </xsl:for-each>

            <!-- no match -->
            <xsl:for-each select="dc:source">
                <datafield tag="786" ind1="0" ind2=" ">
                    <subfield code="n">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>

            <!-- TODO: 455 456 470 488-->
            <xsl:for-each select="dc:relation">
                <datafield tag="488" ind1="0" ind2=" ">
                    <subfield code="e">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>

            <xsl:if test="dc:identifier">
                <datafield tag="856" ind1="4" ind2="1">
                    <subfield code="u"><xsl:value-of select="dc:identifier[last()]" /></subfield>
                    <subfield code="z">Connect to this object online.</subfield>
                </datafield>
            </xsl:if>
        </record>

    </xsl:template>
<xsl:template name="persname_template">
      <xsl:param name="string" />
      <xsl:param name="field" />
      <xsl:param name="ind1" />
      <xsl:param name="ind2" />
      <xsl:param name="type" />
      <datafield>
         <xsl:attribute name="tag">
            <xsl:value-of select="$field" />
         </xsl:attribute>
         <xsl:attribute name="ind1">
            <xsl:value-of select="$ind1" />
         </xsl:attribute>
         <xsl:attribute name="ind2">
            <xsl:value-of select="$ind2" />
         </xsl:attribute>

         <!-- Sample input: Brightman, Samuel C. (Samuel Charles), 1911-1992 -->
         <!-- Sample output: $aBrightman, Samuel C. $q(Samuel Charles), $d1911-. -->
         <!-- will handle names with dashes e.g. Bourke-White, Margaret -->

         <!-- CAPTURE PRIMARY NAME BY LOOKING FOR A PAREN OR A DASH OR NEITHER -->
         <xsl:choose>
            <!-- IF A PAREN, STOP AT AN OPENING PAREN -->
            <xsl:when test="contains($string, '(')!=0">
               <subfield code="a">
                  <xsl:value-of select="substring-before($string, '(')" />
               </subfield>
            </xsl:when>
            <!-- IF A DASH, CHECK IF IT'S A DATE OR PART OF THE NAME -->
            <xsl:when test="contains($string, '-')!=0">
               <xsl:variable name="name_1" select="substring-before($string, '-')" />
               <xsl:choose>
                  <!-- IF IT'S A DATE REMOVE IT -->
                  <xsl:when test="translate(substring($name_1, (string-length($name_1)), 1), '0123456789', '9999999999') = '9'">
                     <xsl:variable name="name" select="substring($name_1, 1, (string-length($name_1)-6))" />
                     <subfield code="a">
                        <xsl:value-of select="$name" />
                     </subfield>
                  </xsl:when>
                  <!-- IF IT'S NOT A DATE, CHECK WHETHER THERE IS A DATE LATER -->
                  <xsl:otherwise>
                     <xsl:variable name="remainder" select="substring-after($string, '-')" />
                     <xsl:choose>
                        <!-- IF THERE'S A DASH, ASSUME IT'S A DATE AND REMOVE IT -->
                        <xsl:when test="contains($remainder, '-')!=0">
                           <xsl:variable name="tmp" select="substring-before($remainder, '-')" />
                           <xsl:variable name="name_2" select="substring($tmp, 1, (string-length($tmp)-6))" />
                           <subfield code="a">
                              <xsl:value-of select="$name_1" />-<xsl:value-of select="$name_2" />
                           </subfield>
                        </xsl:when>
                        <!-- IF THERE'S NO DASH IN THE REMAINDER, OUTPUT IT -->
                        <xsl:otherwise>
                           <subfield code="a">
                              <xsl:value-of select="$string" />
                           </subfield>
                        </xsl:otherwise>
                     </xsl:choose>
                  </xsl:otherwise>
               </xsl:choose>
            </xsl:when>
            <!-- NO DASHES, NO PARENS, JUST OUTPUT THE NAME -->
            <xsl:otherwise>
               <subfield code="a">
                  <xsl:value-of select="$string" />
               </subfield>
            </xsl:otherwise>
         </xsl:choose>

         <!-- CAPTURE SECONDARY NAME IN PARENS FOR SUBFIELD Q -->
         <xsl:if test="contains($string, '(')!=0">
            <xsl:variable name="subq_tmp" select="substring-after($string, '(')" />
            <xsl:variable name="subq" select="substring-before($subq_tmp, ')')" />
            <subfield code="q">
               (<xsl:value-of select="$subq" />)
            </subfield>
         </xsl:if>

         <!-- CAPTURE DATE FOR SUBFIELD D, ASSUME DATE IS LAST ITEM IN FIELD -->
         <!-- Note: does not work if name has a dash in it -->
         <xsl:if test="contains($string, '-')!=0">
            <xsl:variable name="date_tmp" select="substring-before($string, '-')" />
            <xsl:variable name="remainder" select="substring-after($string, '-')" />
            <xsl:choose>
               <!-- CHECK SECOND HALF FOR ANOTHER DASH; IF PRESENT, ASSUME THAT IS DATE -->
               <xsl:when test="contains($remainder, '-')!=0">
                  <xsl:variable name="tmp" select="substring-before($remainder, '-')" />
                  <xsl:variable name="date_1" select="substring($remainder, (string-length($tmp)-3))" />
                  <!-- CHECK WHETHER IT HAS A NUMBER BEFORE IT AND IF SO, OUTPUT IT AS DATE -->
                  <xsl:if test="translate(substring($date_1, 1, 1), '0123456789', '9999999999') = '9'">
                     <subfield code="d">
                        <xsl:value-of select="$date_1" />.
                     </subfield>
                  </xsl:if>
               </xsl:when>
               <!-- OTHERWISE THIS IS THE ONLY DASH SO TAKE IT -->
               <xsl:otherwise>
                  <xsl:variable name="date_2" select="substring($string, (string-length($date_tmp)-3))" />
                  <!-- CHECK WHETHER IT HAS A NUMBER BEFORE IT AND IF SO, OUTPUT IT AS DATE -->
                  <xsl:if test="translate(substring($date_2, 1, 1), '0123456789', '9999999999') = '9'">
                     <subfield code="d">
                        <xsl:value-of select="$date_2" />.
                     </subfield>
                  </xsl:if>
               </xsl:otherwise>
            </xsl:choose>
         </xsl:if>
         <subfield code="e"><xsl:value-of select="$type" /></subfield>
      </datafield>
   </xsl:template>

</xsl:stylesheet>




<?xml version="1.0" encoding="UTF-8"?>
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:marc="http://www.loc.gov/MARC21/slim">
  <xsl:output method="xml" version="1.0" encoding="UTF-8" indent="yes"/>

 <!-- keep comments -->
  <xsl:template match="*[name()='metadata']">
    <xsl:apply-templates />
  </xsl:template>

  <xsl:template match="*">
    <!-- remove element prefix -->
    <xsl:element name="{local-name()}">
      <!-- process attributes -->
      <xsl:for-each select="@*">
        <!-- remove attribute prefix -->
        <xsl:attribute name="{local-name()}">
          <xsl:value-of select="."/>
        </xsl:attribute>
      </xsl:for-each>
      <xsl:apply-templates/>
    </xsl:element>
  </xsl:template>
</xsl:stylesheet>




#!/usr/bin/perl
#
# Copyright 2023 BibLibre
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

use warnings;
use strict;
use utf8;

use Getopt::Long qw( GetOptions );
use Koha::Script -cron;
use Koha::OAI::Client::Harvester;
use Koha::OaiServers;
use C4::Log qw( cronlogaction );
use Try::Tiny qw( catch try );

my $command_line_options = join( " ", @ARGV );

my ( $help, $verbose, $id, $days, $list, $force );

GetOptions(
    'h|help'         => \$help,
    'v|verbose'      => \$verbose,
    'r|repository:i' => \$id,
    'd|days:i'       => \$days,
    'l|list'         => \$list,
    'f|force'        => \$force,
);
my $usage = << 'ENDUSAGE';

This script starts an OAI Harvest

This script has the following parameters :
    -h --help: this message
    -v --verbose
    -r --repository: id of the OAI repository
    -d --days: number of days to harvest from (optional)
    -l --list: list the OAI repositories
    -f --force: force harvesting (ignore records datestamps)

ENDUSAGE

if ($help) {
    print $usage;
    exit;
}

if ($list) {
    my $servers = Koha::OaiServers->search( {}, { order_by => { -asc => 'id' } } )->unblessed;
    print "The following repositories are available: \n\n";
    foreach my $server (@$servers) {
        print $server->{'id'} . ": "
            . $server->{'servername'}
            . ", endpoint: "
            . $server->{'endpoint'}
            . ", set: "
            . $server->{'oai_set'}
            . ", recordtype: "
            . $server->{'recordtype'} . "\n";
    }
    print "\n";
    exit;
}

if ( !$id ) {
    print "The repository parameter is mandatory.\n";
    print $usage . "\n";
}

my $server = Koha::OaiServers->find($id);

unless ($server) {
    print "OAI Server $id unknown\n";
    exit;
}

my $script_handler = Koha::Script->new( { script => $0 } );

try {
    $script_handler->lock_exec;
} catch {
    my $message = "Skipping execution of $0";
    print "$message\n" if $verbose;
    cronlogaction( { info => $message } );
    exit;
};

cronlogaction( { action => 'Start', info => "Starting OAI Harvest" } );
cronlogaction( { info   => "Command line: $command_line_options" } );

my $harvester =
    Koha::OAI::Client::Harvester->new( { server => $server, verbose => $verbose, days => $days, force => $force } );
$harvester->init();

cronlogaction( { action => 'End', info => "Ending OAI Harvest" } );
exit(0);




#!/usr/bin/perl

# Copyright 2024 BibLibre
#
# This file is part of Koha
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;
use Test::More tests => 5;
use Test::Exception;

use t::lib::TestBuilder;
use HTTP::OAI;
use HTTP::OAI::Metadata::OAI_DC;
use HTTP::OAI::Record;
use HTTP::OAI::Encapsulation;
use Koha::Database;
use Koha::OaiServer;
use Koha::OaiServers;
use Koha::OAI::Client::Harvester;

my $schema  = Koha::Database->new->schema;
my $builder = t::lib::TestBuilder->new;

$schema->storage->txn_begin;

my $new_oai_1 = Koha::OaiServer->new(
    {
        endpoint   => 'my_host1.org',
        oai_set    => 'set1',
        servername => 'my_test_1',
        dataformat => 'oai_dc',
        recordtype => 'biblio',
        add_xslt   => '',
    }
)->store;

my $harvester = Koha::OAI::Client::Harvester->new( { server => $new_oai_1, verbose => 1, days => 1, force => 1 } );

my $record =
    '<metadata xmlns="http://www.openarchives.org/OAI/2.0/"><oai_dc:dc xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd"><dc:title>Pièces diverses </dc:title><dc:identifier>ARCH/0320 [cote]</dc:identifier><dc:relation>FR-920509801 [RCR établissement]</dc:relation><dc:relation>Central obrera boliviana [Fonds ou collection]</dc:relation><dc:format>1 carton</dc:format><dc:date>1971/2000</dc:date><dc:type>Archives et manuscrits</dc:type></oai_dc:dc></metadata>';

my $r = HTTP::OAI::Record->new();
$r->header->datestamp('2017-05-10T09:18:13Z');
$r->metadata( HTTP::OAI::Metadata->new( dom => $record ) );

my $status = $harvester->processRecord($r);
is( $status, 'skipped', 'Record with no identifier is skipped' );

$r->header->identifier('oai:myarchive.org:oid-233');

$status = $harvester->processRecord($r);
is( $status, 'added', 'Record is added' );

$status = $harvester->processRecord($r);
is( $status, 'updated', 'When force is used, record is updated' );

$harvester = Koha::OAI::Client::Harvester->new( { server => $new_oai_1, verbose => 1, days => 1, force => 0 } );
$status    = $harvester->processRecord($r);
is( $status, 'skipped', 'When force is not used, record is skipped (already up to date)' );

$r->header->datestamp('2018-05-10T09:18:13Z');
$status = $harvester->processRecord($r);
is( $status, 'updated', 'When force is not used, record is updated if newer' );

$schema->storage->txn_rollback;




#!/usr/bin/perl

# Copyright 2015 Koha Development team
#
# This file is part of Koha
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;
use Test::More tests => 2;
use Test::Exception;

use t::lib::TestBuilder;
use Koha::Database;
use Koha::OaiServer;
use Koha::OaiServers;

my $schema  = Koha::Database->new->schema;
my $builder = t::lib::TestBuilder->new;

subtest 'new, find and delete tests' => sub {
    plan tests => 4;
    $schema->storage->txn_begin;
    my $nb_of_oais = Koha::OaiServers->search->count;
    my $new_oai_1  = Koha::OaiServer->new(
        {
            endpoint   => 'my_host1.org',
            oai_set    => 'set1',
            servername => 'my_test_1',
            dataformat => 'oai_dc',
            recordtype => 'biblio',
        }
    )->store;
    my $new_oai_2 = Koha::OaiServer->new(
        {
            endpoint   => 'my_host2.org',
            oai_set    => 'set2',
            servername => 'my_test_2',
            dataformat => 'marcxml',
            recordtype => 'authority',
        }
    )->store;

    like( $new_oai_1->id, qr|^\d+$|, 'Adding a new oai server should have set the id' );
    is( Koha::OaiServers->search->count, $nb_of_oais + 2, 'The 2 oai servers should have been added' );

    my $retrieved_oai_1 = Koha::OaiServers->find( $new_oai_1->id );
    is(
        $retrieved_oai_1->servername, $new_oai_1->servername,
        'Find a oai server by id should return the correct oai server'
    );

    $retrieved_oai_1->delete;
    is( Koha::OaiServers->search->count, $nb_of_oais + 1, 'Delete should have deleted the oai server' );

    $schema->storage->txn_rollback;
};

subtest 'Check NOT NULL without default values' => sub {
    plan tests => 5;
    $schema->storage->txn_begin;
    local $SIG{__WARN__} = sub { };    # TODO Needed it for suppressing DBIx warns

    my $server = Koha::OaiServer->new(
        {
            oai_set    => 'set3',
            dataformat => 'marcxml',
            recordtype => 'biblio',
        }
    );

    throws_ok { $server->store } 'DBIx::Class::Exception', 'Exception on empty endpoint';
    like( $@->{msg}, qr/'endpoint' doesn't have a default value/, 'Verified that DBIx blamed endpoint' );

    $server->endpoint('endpoint_added');
    throws_ok { $server->store } 'DBIx::Class::Exception', 'Exception on empty servername';
    like( $@->{msg}, qr/'servername' doesn't have a default value/, 'Verified that DBIx blamed servername' );

    $server->servername('servername_added');

    lives_ok { $server->store } 'No exceptions anymore';

    $schema->storage->txn_rollback;
};

Return to bug 35659

Lines 2467-2469 tables: Link Here

(-)a/installer/data/mysql/en/mandatory/sample_notices.yml (+23 lines)
2467	- "Your hold on [% hold.biblio.title %] ([% hold.biblio.id %]) has been confirmed."	2467	- "Your hold on [% hold.biblio.title %] ([% hold.biblio.id %]) has been confirmed."
2468	- "You will be notified by the library when your item is available for pickup."	2468	- "You will be notified by the library when your item is available for pickup."
2469	- "Thank you!"	2469	- "Thank you!"
		2470
		2471	- module: catalogue
		2472	code: OAI_HARVEST_REPORT
		2473	branchcode: ""
		2474	name: "OAI Harvest Report"
		2475	is_html: 0
		2476	title: "OAI Harvest report for [% servername %]"
		2477	message_transport_type: email
		2478	lang: default
		2479	content:
		2480	- "OAI Harvest report for [% servername %]:"
		2481	- ""
		2482	- "Endpoint: [% endpoint %]"
		2483	- "Set: [% set %]"
		2484	- "Data format: [% dataformat %]"
		2485	- "Record type: [% recordtype %]"
		2486	- ""
		2487	- "[% added %] records added"
		2488	- "[% updated %] records updated"
		2489	- "[% deleted %] records deleted"
		2490	- "[% skipped %] records skipped"
		2491	- "[% in_error %] records in error"
		2492	- "[% total %] total"

Lines 4819-4824 CREATE TABLE `oai_sets_mappings` ( Link Here

(-)a/installer/data/mysql/kohastructure.sql (+58 lines)
4819	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;	4819	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
4820	/!40101 SET character_set_client = @saved_cs_client /;	4820	/!40101 SET character_set_client = @saved_cs_client /;
4821		4821
		4822	--
		4823	-- Table structure for table `oaiservers`
		4824	--
		4825
		4826	DROP TABLE IF EXISTS `oaiservers`;
		4827	/!40101 SET @saved_cs_client = @@character_set_client /;
		4828	/!40101 SET character_set_client = utf8 /;
		4829	CREATE TABLE `oaiservers` (
		4830	`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
		4831	`endpoint` varchar(255) NOT NULL COMMENT 'OAI endpoint (host + port + path)',
		4832	`oai_set` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
		4833	`servername` longtext NOT NULL COMMENT 'name given to the target by the library',
		4834	`dataformat` enum('oai_dc','marc-xml', 'marcxml') NOT NULL DEFAULT 'oai_dc' COMMENT 'data format',
		4835	`recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'server contains bibliographic or authority records',
		4836	`add_xslt` longtext DEFAULT NULL COMMENT 'zero or more paths to XSLT files to be processed on the search results',
		4837	PRIMARY KEY (`id`)
		4838	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
		4839
		4840	--
		4841	-- Table structure for table `import_oaipmh_biblios`
		4842	--
		4843
		4844	DROP TABLE IF EXISTS `import_oaipmh_biblios`;
		4845	/!40101 SET @saved_cs_client = @@character_set_client /;
		4846	/!40101 SET character_set_client = utf8 /;
		4847	CREATE TABLE `import_oaipmh_biblios` (
		4848	`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
		4849	`biblionumber` int(11) NOT NULL COMMENT 'unique identifier assigned to each koha record',
		4850	`identifier` varchar(255) NOT NULL COMMENT 'OAI record identifier',
		4851	`repository` varchar(255) NOT NULL COMMENT 'OAI repository',
		4852	`recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'is the record bibliographic or authority',
		4853	`datestamp` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
		4854	`last_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
		4855	PRIMARY KEY (id),
		4856	KEY biblionumber (biblionumber),
		4857	CONSTRAINT FK_import_oaipmh_biblios_1 FOREIGN KEY (biblionumber) REFERENCES biblio (biblionumber) ON DELETE CASCADE ON UPDATE NO ACTION
		4858	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
		4859
		4860	--
		4861	-- Table structure for table `import_oaipmh_authorities`
		4862	--
		4863
		4864	DROP TABLE IF EXISTS `import_oaipmh_authorities`;
		4865	/!40101 SET @saved_cs_client = @@character_set_client /;
		4866	/!40101 SET character_set_client = utf8 /;
		4867	CREATE TABLE `import_oaipmh_authorities` (
		4868	`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
		4869	`authid` bigint(20) unsigned NOT NULL COMMENT 'unique identifier assigned to each koha record',
		4870	`identifier` varchar(255) NOT NULL COMMENT 'OAI record identifier',
		4871	`repository` varchar(255) NOT NULL COMMENT 'OAI repository',
		4872	`recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'is the record bibliographic or authority',
		4873	`datestamp` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
		4874	`last_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
		4875	PRIMARY KEY (id),
		4876	KEY authid (authid),
		4877	CONSTRAINT FK_import_oaipmh_authorities_1 FOREIGN KEY (authid) REFERENCES auth_header (authid) ON DELETE CASCADE ON UPDATE NO ACTION
		4878	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
		4879
4822	--	4880	--
4823	-- Table structure for table `oauth_access_tokens`	4881	-- Table structure for table `oauth_access_tokens`
4824	--	4882	--

Line 0 Link Here

(-)a/Koha/Import/Oaipmh/Authorities.pm (+51 lines)
	1	package Koha::Import::Oaipmh::Authorities;
	2
	3	# This file is part of Koha.
	4	#
	5	# Koha is free software; you can redistribute it and/or modify it
	6	# under the terms of the GNU General Public License as published by
	7	# the Free Software Foundation; either version 3 of the License, or
	8	# (at your option) any later version.
	9	#
	10	# Koha is distributed in the hope that it will be useful, but
	11	# WITHOUT ANY WARRANTY; without even the implied warranty of
	12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	# GNU General Public License for more details.
	14	#
	15	# You should have received a copy of the GNU General Public License
	16
	17	use Modern::Perl;
	18
	19	use Koha::Import::Oaipmh::Authority;
	20
	21	use base qw(Koha::Objects);
	22
	23	=head1 NAME
	24
	25	Koha::Import::Oaipmh::Authorities
	26
	27	This object represents a collection of OAI-PMH records being imported as authorities
	28
	29	=head1 API
	30
	31	=head2 Methods
	32
	33
	34	=head3 type
	35
	36	=cut
	37
	38	sub _type {
	39	return 'ImportOaipmhAuthority';
	40	}
	41
	42	=head3 object_class
	43
	44	Koha::Object class
	45
	46	=cut
	47
	48	sub object_class {
	49	return 'Koha::Import::Oaipmh::Authority';
	50	}
	51	1;

Line 0 Link Here

(-)a/Koha/Import/Oaipmh/Authority.pm (+41 lines)
	1	package Koha::Import::Oaipmh::Authority;
	2
	3	# This file is part of Koha.
	4	#
	5	# Koha is free software; you can redistribute it and/or modify it
	6	# under the terms of the GNU General Public License as published by
	7	# the Free Software Foundation; either version 3 of the License, or
	8	# (at your option) any later version.
	9	#
	10	# Koha is distributed in the hope that it will be useful, but
	11	# WITHOUT ANY WARRANTY; without even the implied warranty of
	12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	# GNU General Public License for more details.
	14	#
	15	# You should have received a copy of the GNU General Public License
	16
	17	use Modern::Perl;
	18
	19	use base qw(Koha::Object);
	20
	21	=head1 NAME
	22
	23	Koha::Import::Oaipmh::Authority
	24
	25	This object represents an OAI-PMH record being imported as an authority
	26
	27	=head1 API
	28
	29	=head2 Class Methods
	30
	31	=cut
	32
	33	=head3 _type
	34
	35	=cut
	36
	37	sub _type {
	38	return 'ImportOaipmhAuthority';
	39	}
	40
	41	1;

Line 0 Link Here

(-)a/Koha/Import/Oaipmh/Biblio.pm (+40 lines)
	1	package Koha::Import::Oaipmh::Biblio;
	2
	3	# This file is part of Koha.
	4	#
	5	# Koha is free software; you can redistribute it and/or modify it
	6	# under the terms of the GNU General Public License as published by
	7	# the Free Software Foundation; either version 3 of the License, or
	8	# (at your option) any later version.
	9	#
	10	# Koha is distributed in the hope that it will be useful, but
	11	# WITHOUT ANY WARRANTY; without even the implied warranty of
	12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	# GNU General Public License for more details.
	14	#
	15	# You should have received a copy of the GNU General Public License
	16
	17	use Modern::Perl;
	18
	19	use base qw(Koha::Object);
	20
	21	=head1 NAME
	22
	23	Koha::Import::Oaipmh::Biblio
	24
	25	This object represents an OAI-PMH record being imported as a bibliographic record
	26
	27	=head1 API
	28
	29	=head2 Methods
	30
	31
	32	=head3 _type
	33
	34	=cut
	35
	36	sub _type {
	37	return 'ImportOaipmhBiblio';
	38	}
	39
	40	1;

Line 0 Link Here

(-)a/Koha/Import/Oaipmh/Biblios.pm (+51 lines)
	1	package Koha::Import::Oaipmh::Biblios;
	2
	3	# This file is part of Koha.
	4	#
	5	# Koha is free software; you can redistribute it and/or modify it
	6	# under the terms of the GNU General Public License as published by
	7	# the Free Software Foundation; either version 3 of the License, or
	8	# (at your option) any later version.
	9	#
	10	# Koha is distributed in the hope that it will be useful, but
	11	# WITHOUT ANY WARRANTY; without even the implied warranty of
	12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	# GNU General Public License for more details.
	14	#
	15	# You should have received a copy of the GNU General Public License
	16
	17	use Modern::Perl;
	18
	19	use Koha::Import::Oaipmh::Biblio;
	20
	21	use base qw(Koha::Objects);
	22
	23	=head1 NAME
	24
	25	Koha::Import::Oaipmh::Biblios
	26
	27	This object represents a collection of OAI-PMH records being imported as bibliographic records
	28
	29	=head1 API
	30
	31	=head2 Methods
	32
	33
	34	=head3 type
	35
	36	=cut
	37
	38	sub _type {
	39	return 'ImportOaipmhBiblio';
	40	}
	41
	42	=head3 object_class
	43
	44	Koha::Object class
	45
	46	=cut
	47
	48	sub object_class {
	49	return 'Koha::Import::Oaipmh::Biblio';
	50	}
	51	1;

Line 0 Link Here

(-)a/Koha/OAI/Client/Harvester.pm (+372 lines)
		1	# Copyright BibLibre 2023
		2	#
		3	# This file is part of Koha.
		4	#
		5	# Koha is free software; you can redistribute it and/or modify it
		6	# under the terms of the GNU General Public License as published by
		7	# the Free Software Foundation; either version 3 of the License, or
		8	# (at your option) any later version.
		9	#
		10	# Koha is distributed in the hope that it will be useful, but
		11	# WITHOUT ANY WARRANTY; without even the implied warranty of
		12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		13	# GNU General Public License for more details.
		14	#
		15	# You should have received a copy of the GNU General Public License
		16	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		17
		18	package Koha::OAI::Client::Harvester;
		19
		20	=head1 NAME
		21
		22	Koha::OAI::Client::Harvester - OAI Harvester
		23
		24	=head1 DESCRIPTION
		25
		26	Koha::OAI::Client::Harvester contains OAI-PMH harvester main functions
		27
		28	=cut
		29
		30	use utf8;
		31	use open qw( :std :utf8 );
		32
		33	use C4::Biblio qw( AddBiblio GetFrameworkCode ModBiblio DelBiblio );
		34	use C4::AuthoritiesMarc qw (AddAuthority GuessAuthTypeCode ModAuthority DelAuthority );
		35	use C4::Log qw( cronlogaction );
		36	use HTTP::OAI;
		37	use HTTP::OAI::Metadata::OAI_DC;
		38	use Koha::DateUtils qw( dt_from_string );
		39	use Koha::OaiServers;
		40	use Koha::Import::Oaipmh::Biblio;
		41	use Koha::Import::Oaipmh::Biblios;
		42	use Koha::Import::Oaipmh::Authority;
		43	use Koha::Import::Oaipmh::Authorities;
		44	use Koha::XSLT::Base;
		45	use MARC::File::XML;
		46	use MARC::Record;
		47	use Modern::Perl;
		48	use DateTime;
		49	use DateTime::Format::Strptime;
		50	use Try::Tiny qw( catch try );
		51	use Date::Calc qw(
		52	Add_Delta_Days
		53	Today
		54	);
		55
		56	my $strp = DateTime::Format::Strptime->new(
		57	pattern => '%FT%TZ',
		58	time_zone => 'floating',
		59	);
		60
		61	our $verbose = 0;
		62	our $server;
		63	our $force = 0;
		64	our $days = 0;
65	our $xslt_engine = Koha::XSLT::Base->new;
66
67	=head2 new
68
69	$harvester = Koha::OAI::Client::Harvester->new( { server => $server, verbose => $verbose, days => $days, force => $force } );
70
71	New instance of Koha::OAI::Client::Harvester
72
73	C<$server> An OAI repository (Koha::OaiServer)
74
75	C<$verbose> print log messages to stdout when enabled
76
77	C<$days> number of days to harvest from (optional)
78
79	C<$force> force harvesting (ignore records datestamps)
80
81	=cut
82
83	sub new {
84
85	my ( $self, $args ) = @_;
86
87	$server = $args->{'server'};
88	$verbose = $args->{'verbose'};
89	$days = $args->{'days'};
90	$force = $args->{'force'};
91	return $self;
92	}
93
94	=head2 init
95
96	Starts harvesting
97
98	=cut
99
100	sub init {
101	my ( $self, $args ) = @_;
102
103	printlog("Starting OAI Harvest from repository");
104
105	my $h = HTTP::OAI::Harvester->new(
106	repository => HTTP::OAI::Identify->new(
107	baseURL => $server->endpoint,
108	version => '2.0',
109	)
110	);
111
112	printlog( "Using HTTP::OAI::Harvester version " . $HTTP::OAI::Harvester::VERSION );
113
114	printlog( "Using endpoint "
115	. $server->endpoint
116	. ", set "
117	. $server->oai_set
118	. ", dataformat "
119	. $server->dataformat
120	. ", recordtype "
121	. $server->recordtype );
122
123	try {
124	my $lmdf = $h->ListMetadataFormats();
125	my $lmdflog = "";
126	for ( $lmdf->metadataFormat ) {
127	$lmdflog .= $_->metadataPrefix . " ";
128	}
129	printlog("Available metadata formats for this repository: $lmdflog");
130	} catch {
131	printlog("ListMetadataFormats failed");
132	};
133
134	if ( $server->add_xslt ) {
135	printlog( "Using XSLT file " . $server->add_xslt );
136	} else {
137	printlog("Not using an XSLT file");
138	}
139
140	my $start_date = '';
141	my $today_date = '';
142
143	if ($days) {
144
145	# Change this to yyyy-mm-dd
146	my $dt_today = dt_from_string();
147	my ( $nowyear, $nowmonth, $nowday ) = Today();
148	my @date = Add_Delta_Days( $nowyear, $nowmonth, $nowday, -$days );
149	my $dt_start = DateTime->new( year => $date[0], month => $date[1], day => $date[2] );
150	$start_date = $dt_start->ymd();
151	printlog("Harvesting from $start_date");
152	}
153
154	printlog("Asking for records");
155	my $response = $h->ListRecords(
156	metadataPrefix => $server->dataformat,
157	set => $server->oai_set,
158	from => $start_date,
159	);
160	if ( $response->is_error ) {
161	printlog( "Error requesting ListRecords: " . $response->code . " " . $response->message );
162	exit;
163	}
164	printlog( "Request URL: " . $response->requestURL );
165
166	my %stats;
167	my @statuses = qw(added updated deleted in_error skipped total);
168	foreach my $status (@statuses) {
169	$stats{$status} = 0;
170	}
171
172	printlog("Starting processing results");
173	while ( my $oai_record = $response->next ) {
174	$stats{'total'}++;
175	my $status = $self->processRecord($oai_record);
176	$stats{$status}++;
177	}
178
179	my $results = '';
180	foreach my $status (@statuses) {
181	$results .= $stats{$status} . " $status\n";
182	}
183	printlog( "Harvest results:\n" . $results );
184
185	if ( C4::Context->preference("OAI-PMH:HarvestEmailReport") ) {
186
187	my $letter = C4::Letters::GetPreparedLetter(
188	(
189	module => 'catalogue',
190	letter_code => 'OAI_HARVEST_REPORT',
191	substitute => {
192	servername => $server->servername,
193	endpoint => $server->endpoint,
194	set => $server->oai_set,
195	dataformat => $server->dataformat,
196	recordtype => $server->recordtype,
197	added => $stats{'added'},
198	updated => $stats{'updated'},
199	deleted => $stats{'deleted'},
200	skipped => $stats{'skipped'},
201	in_error => $stats{'in_error'},
202	total => $stats{'total'}
203	},
204	)
205	) or printlog("OAI_HARVEST_REPORT letter not found");
206
207	my $success = C4::Letters::EnqueueLetter(
208	{
209	letter => $letter,
210	to_address => C4::Context->preference("OAI-PMH:HarvestEmailReport"), message_transport_type => 'email'
211	}
212	);
213	if ($success) {
214	printlog("Email report enqueued");
215	} else {
216	printlog("Unable to enqueue report email");
217	}
218	}
219
220	printlog("Ending OAI Harvest from repository");
221	}
222
223	=head2 processRecord
224
225	This method processes an incoming OAI record
226
227	=cut
228
229	sub processRecord {
230	my $self = shift;
231	my $oai_record = shift;
232	my $status = '';
233	unless ( $oai_record->identifier ) {
234	printlog("No identifier found");
235	$status = 'skipped';
236	return $status;
237	}
238
239	my $to_delete;
240	my $outputUnimarc;
241	my $marcrecord;
242	if ( $oai_record->status && $oai_record->status eq "deleted" ) {
243	$to_delete = 1;
244	} else {
245	if ( $server->add_xslt ) {
246	$outputUnimarc = $xslt_engine->transform( $oai_record->metadata->dom, $server->add_xslt );
247	my $err = $xslt_engine->err;
248	if ($err) {
249	printlog("XSLT::Base error: $err");
250	$status = 'in_error';
251	return $status;
252	}
253	} else {
254	$outputUnimarc = $oai_record->metadata->dom;
255	}
256	$marcrecord = MARC::Record->new_from_xml( $outputUnimarc, 'UTF-8' );
257	}
258	my $imported_record;
259	if ( $server->recordtype eq "biblio" ) {
260	$imported_record = Koha::Import::Oaipmh::Biblios->find(
261	{
262	repository => $server->endpoint,
263	identifier => $oai_record->identifier,
264	recordtype => $server->recordtype,
265	}
266	);
267	} else {
268	$imported_record = Koha::Import::Oaipmh::Authorities->find(
269	{
270	repository => $server->endpoint,
271	identifier => $oai_record->identifier,
272	recordtype => $server->recordtype,
273	}
274	);
275	}
276
277	if ($imported_record) {
278	if ($to_delete) {
279	if ( $server->recordtype eq "biblio" ) {
280	my $biblionumber = $imported_record->biblionumber;
281	my $error = DelBiblio($biblionumber);
282	if ($error) {
283	printlog(
284	"Record " . $oai_record->identifier . " not deleted, biblionumber: $biblionumber ($error)" );
285	$status = 'in_error';
286	} else {
287	$imported_record->delete;
288	printlog( "Record " . $oai_record->identifier . " deleted, biblionumber: $biblionumber" );
289	$status = 'deleted';
290	}
291	} else {
292	my $authid = $imported_record->authid;
293	DelAuthority( { authid => $authid } );
294	$imported_record->delete;
295	printlog( "Record " . $oai_record->identifier . " deleted, authid: $authid" );
296	$status = 'deleted';
297	}
298	} else {
299	my $existing_dt = $strp->parse_datetime( $imported_record->datestamp );
300	my $incoming_dt = $strp->parse_datetime( $oai_record->datestamp );
301
302	if ( $force \|\| !$incoming_dt \|\| !$existing_dt \|\| $incoming_dt > $existing_dt ) {
303	if ( $server->recordtype eq "biblio" ) {
304	my $biblionumber = $imported_record->biblionumber;
305	my $result = ModBiblio( $marcrecord, $biblionumber, GetFrameworkCode($biblionumber) );
306	printlog( "Record " . $oai_record->identifier . " updated, biblionumber: $biblionumber" );
307	} else {
308	my $authid = $imported_record->authid;
309	my $result = ModAuthority( $authid, $marcrecord, GuessAuthTypeCode($marcrecord) );
310	printlog( "Record " . $oai_record->identifier . " updated, authid: $authid" );
311	}
312	$imported_record->update(
313	{
314	datestamp => $imported_record->datestamp,
315	}
316	);
317	$status = 'updated';
318	} else {
319	printlog( "Record "
320	. $oai_record->identifier
321	. " skipped (incoming record was not newer than existing record)" );
322	$status = 'skipped';
323	}
324	}
325	} elsif ( !$to_delete ) {
326	if ( $server->recordtype eq "biblio" ) {
327	my ( $biblionumber, $biblioitemnumber ) = AddBiblio($marcrecord);
328	printlog( $oai_record->identifier . " added, biblionumber: $biblionumber" );
329	Koha::Import::Oaipmh::Biblio->new(
330	{
331	repository => $server->endpoint,
332	identifier => $oai_record->identifier,
333	biblionumber => $biblionumber,
334	recordtype => $server->recordtype,
335	datestamp => $oai_record->datestamp,
336	}
337	)->store;
338	} else {
339	my $authid = AddAuthority( $marcrecord, "", GuessAuthTypeCode($marcrecord) );
340	printlog( $oai_record->identifier . " added, authid: $authid" );
341	Koha::Import::Oaipmh::Authority->new(
342	{
343	repository => $server->endpoint,
344	identifier => $oai_record->identifier,
345	authid => $authid,
346	recordtype => $server->recordtype,
347	datestamp => $oai_record->datestamp,
348	}
349	)->store;
350	}
351	$status = 'added';
352	} else {
353	printlog( "Record " . $oai_record->identifier . " skipped (record not present or already deleted)" );
354	$status = 'skipped';
355	}
356	return $status;
357	}
358
359	=head2 printlog
360
361	This method adds a cronlog and prints to stdout if verbose is enabled
362
363	=cut
364
365	sub printlog {
366	my $message = shift;
367	$message = $server->servername . ": " . $message;
368	print $message . "\n" if ($verbose);
369	cronlogaction( { info => $message } );
370	}
371
372	1;

Line 0 Link Here

(-)a/Koha/OaiServer.pm (+45 lines)
	1	package Koha::OaiServer;
	2
	3	# This file is part of Koha.
	4	#
	5	# Koha is free software; you can redistribute it and/or modify it
	6	# under the terms of the GNU General Public License as published by
	7	# the Free Software Foundation; either version 3 of the License, or
	8	# (at your option) any later version.
	9	#
	10	# Koha is distributed in the hope that it will be useful, but
	11	# WITHOUT ANY WARRANTY; without even the implied warranty of
	12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	# GNU General Public License for more details.
	14	#
	15	# You should have received a copy of the GNU General Public License
	16	# along with Koha; if not, see <http://www.gnu.org/licenses>.
	17
	18	use Modern::Perl;
	19
	20
	21	use Koha::Database;
	22
	23	use base qw(Koha::Object);
	24
	25	=head1 NAME
	26
	27	Koha::OaiServer - Koha OaiServer Object class
	28
	29	=head1 API
	30
	31	=head2 Class Methods
	32
	33	=cut
	34
	35	=head3 _type
	36
	37	Return type of Object relating to Schema ResultSet
	38
	39	=cut
	40
	41	sub _type {
	42	return 'Oaiserver';
	43	}
	44
	45	1;

Line 0 Link Here

(-)a/Koha/OaiServers.pm (+57 lines)
	1	package Koha::OaiServers;
	2
	3	# This file is part of Koha.
	4	#
	5	# Koha is free software; you can redistribute it and/or modify it
	6	# under the terms of the GNU General Public License as published by
	7	# the Free Software Foundation; either version 3 of the License, or
	8	# (at your option) any later version.
	9	#
	10	# Koha is distributed in the hope that it will be useful, but
	11	# WITHOUT ANY WARRANTY; without even the implied warranty of
	12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	# GNU General Public License for more details.
	14	#
	15	# You should have received a copy of the GNU General Public License
	16	# along with Koha; if not, see <http://www.gnu.org/licenses>.
	17
	18	use Modern::Perl;
	19
	20
	21	use Koha::Database;
	22
	23	use Koha::OaiServer;
	24
	25	use base qw(Koha::Objects);
	26
	27	=head1 NAME
	28
	29	Koha::OaiServers - Koha OaiServer Object set class
	30
	31	=head1 API
	32
	33	=head2 Class Methods
	34
	35	=cut
	36
	37	=head3 _type
	38
	39	Return type of object, relating to Schema ResultSet
	40
	41	=cut
	42
	43	sub _type {
	44	return 'Oaiserver';
	45	}
	46
	47	=head3 object_class
	48
	49	Return object class
	50
	51	=cut
	52
	53	sub object_class {
	54	return 'Koha::OaiServer';
	55	}
	56
	57	1;

Line 0 Link Here

(-)a/admin/oaiservers.pl (+122 lines)
		1	#!/usr/bin/perl
		2
		3	# This file is part of Koha.
		4	#
		5	# Koha is free software; you can redistribute it and/or modify it
		6	# under the terms of the GNU General Public License as published by
		7	# the Free Software Foundation; either version 3 of the License, or
		8	# (at your option) any later version.
		9	#
		10	# Koha is distributed in the hope that it will be useful, but
		11	# WITHOUT ANY WARRANTY; without even the implied warranty of
		12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		13	# GNU General Public License for more details.
		14	#
		15	# You should have received a copy of the GNU General Public License
		16	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		17
		18	# This script is used to maintain the OAI servers table.
		19	# Parameter $op is operation: list, new, edit, add_validated, delete_confirmed.
		20	# add_validated saves a validated record and goes to list view.
		21	# delete_confirmed deletes a record and goes to list view.
		22
		23	use Modern::Perl;
		24	use CGI qw ( -utf8 );
		25	use C4::Context;
		26	use C4::Auth qw( get_template_and_user );
		27	use C4::Output qw( output_html_with_http_headers );
		28	use Koha::Database;
		29	use Koha::OaiServers;
		30
		31	# Initialize CGI, template, database
		32
		33	my $input = CGI->new;
		34	my $op = $input->param('op') \|\| 'list';
		35	my $id = $input->param('id') \|\| 0;
		36	my $type = $input->param('type') \|\| '';
		37	my $searchfield = '';
		38
		39	my ( $template, $loggedinuser, $cookie ) = get_template_and_user(
		40	{
		41	template_name => "admin/oaiservers.tt",
		42	query => $input,
		43	type => "intranet",
		44	flagsrequired => { parameters => 'manage_search_targets' },
		45	}
		46	);
		47	my $script_name = "/cgi-bin/koha/admin/oaiservers.pl";
		48	my $path = C4::Context->config('intrahtdocs') . "/prog/";
		49
		50	$template->param( script_name => $script_name, xslt_path => $path );
		51
		52	my $schema = Koha::Database->new()->schema();
		53
		54	# Main code
		55	# First process a confirmed delete, or save a validated record
		56
		57	if ( $op eq 'delete_confirmed' && $id ) {
		58	my $server = Koha::OaiServers->find($id);
		59	if ($server) {
		60	$server->delete;
		61	$template->param( msg_deleted => 1, msg_add => $server->servername );
		62	} else {
		63	$template->param( msg_notfound => 1, msg_add => $id );
		64	}
65	$id = 0;
66	} elsif ( $op eq 'add_validated' ) {
67	my @fields = qw/endpoint oai_set dataformat
68	recordtype servername
69	add_xslt/;
70	my $formdata = _form_data_hashref( $input, \@fields );
71	if ($id) {
72	my $server = Koha::OaiServers->find($id);
73	if ($server) {
74	$server->set($formdata)->store;
75	$template->param( msg_updated => 1, msg_add => $formdata->{servername} );
76	} else {
77	$template->param( msg_notfound => 1, msg_add => $id );
78	}
79	$id = 0;
80	} else {
81	Koha::OaiServer->new($formdata)->store;
82	$template->param( msg_added => 1, msg_add => $formdata->{servername} );
83	}
84	} else {
85
86	#use searchfield only in remaining operations
87	$searchfield = $input->param('searchfield') \|\| '';
88	}
89
90	# Now list multiple records, or edit one record
91
92	my $data = [];
93	if ( $op eq 'add' \|\| $op eq 'edit' ) {
94	$data = ServerSearch( $schema, $id, $searchfield ) if $searchfield \|\| $id;
95	delete $data->[0]->{id} if @$data && $op eq 'add'; #cloning record
96	$template->param(
97	add_form => 1, server => @$data ? $data->[0] : undef,
98	op => $op, type => $op eq 'add' ? lc $type : ''
99	);
100	} else {
101	$data = ServerSearch( $schema, $id, $searchfield );
102	$template->param(
103	loop => \@$data, searchfield => $searchfield, id => $id,
104	op => 'list'
105	);
106	}
107	output_html_with_http_headers $input, $cookie, $template->output;
108
109	# End of main code
110
111	sub ServerSearch { #find server(s) by id or name
112	my ( $schema, $id, $searchstring ) = @_;
113
114	return Koha::OaiServers->search(
115	$id ? { id => $id } : { servername => { like => $searchstring . '%' } },
116	)->unblessed;
117	}
118
119	sub _form_data_hashref {
120	my ( $input, $fieldref ) = @_;
121	return { map { ( $_ => scalar $input->param($_) // '' ) } @$fieldref };
122	}

Line 0 Link Here

(-)a/installer/data/mysql/atomicupdate/Bug-35659.pl (+88 lines)
		1	use Modern::Perl;
		2
		3	return {
		4	bug_number => "35659",
		5	description => "OAI-PMH Harvester",
		6	up => sub {
		7	my ($args) = @_;
		8	my ( $dbh, $out ) = @$args{qw(dbh out)};
		9	$dbh->do(q{DROP TABLE IF EXISTS `oaiservers`;});
		10	$dbh->do(
		11	q{
		12	CREATE TABLE `oaiservers` (
		13	`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
		14	`endpoint` varchar(255) NOT NULL COMMENT 'OAI endpoint (host + port + path)',
		15	`oai_set` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
		16	`servername` longtext NOT NULL COMMENT 'name given to the target by the library',
		17	`dataformat` enum('oai_dc','marc-xml', 'marcxml') NOT NULL DEFAULT 'oai_dc' COMMENT 'data format',
		18	`recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'server contains bibliographic or authority records',
		19	`add_xslt` longtext DEFAULT NULL COMMENT 'zero or more paths to XSLT files to be processed on the search results',
		20	PRIMARY KEY (`id`)
		21	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
		22	}
		23	);
		24	say $out "Added new table 'oaiservers'";
		25
		26	$dbh->do(q{DROP TABLE IF EXISTS `import_oaipmh_biblios`;});
		27	$dbh->do(
		28	q{
		29	CREATE TABLE `import_oaipmh_biblios` (
		30	`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
		31	`biblionumber` int(11) NOT NULL COMMENT 'unique identifier assigned to each koha record',
		32	`identifier` varchar(255) NOT NULL COMMENT 'OAI record identifier',
		33	`repository` varchar(255) NOT NULL COMMENT 'OAI repository',
		34	`recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'is the record bibliographic or authority',
		35	`datestamp` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
		36	`last_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
		37	PRIMARY KEY (id),
		38	KEY biblionumber (biblionumber),
		39	CONSTRAINT FK_import_oaipmh_biblios_1 FOREIGN KEY (biblionumber) REFERENCES biblio (biblionumber) ON DELETE CASCADE ON UPDATE NO ACTION
		40	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
		41	}
		42	);
		43
		44	$dbh->do(q{DROP TABLE IF EXISTS `import_oaipmh_authorities`;});
		45	$dbh->do(
		46	q{
		47	CREATE TABLE `import_oaipmh_authorities` (
		48	`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
		49	`authid` bigint(20) unsigned NOT NULL COMMENT 'unique identifier assigned to each koha record',
		50	`identifier` varchar(255) NOT NULL COMMENT 'OAI record identifier',
		51	`repository` varchar(255) NOT NULL COMMENT 'OAI repository',
		52	`recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'is the record bibliographic or authority',
		53	`datestamp` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
		54	`last_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
		55	PRIMARY KEY (id),
		56	KEY authid (authid),
		57	CONSTRAINT FK_import_oaipmh_authorities_1 FOREIGN KEY (authid) REFERENCES auth_header (authid) ON DELETE CASCADE ON UPDATE NO ACTION
		58	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
		59	}
		60	);
		61	say $out "Added new tables 'import_oaipmh_biblios' and import_oaipmh_authorities";
		62
		63	$dbh->do(
		64	q{
65	UPDATE `permissions` SET description='Manage Z39.50 and SRU servers, OAI repositories configuration' WHERE code='manage_search_targets';
66	}
67	);
68	say $out "Updated manage_search_targets permission description";
69
70	$dbh->do(
71	q{
72	INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES
73	('OAI-PMH:HarvestEmailReport','','','If set, a report email will be sent to this address after an OAI harvest','Free');
74	}
75	);
76	say $out "Added OAI-PMH:HarvestEmailReport system preference";
77
78	$dbh->do(
79	q{
80	INSERT IGNORE INTO letter
81	(module, code, branchcode, name, is_html, title, content, message_transport_type, lang)
82	VALUES ('catalogue','OAI_HARVEST_REPORT','','OAI Harvest Report',0,'OAI Harvest report for [% servername %]','OAI Harvest report for [% servername %]:\n\nEndpoint: [% endpoint %]\nSet: [% set %]\nData format: [% dataformat %]\nRecord type: [% recordtype %]\n\n[% added %] records added\n[% updated %] records updated\n[% deleted %] records deleted\n[% skipped %] records skipped\n[% in_error %] records in error\n[% total %] total','email','default');
83	}
84	);
85	say $out "Added OAI_HARVEST_REPORT letter";
86
87	},
88	};

Line 0 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/oai-admin-search.inc (+43 lines)
	1	[% USE Koha %]
	2	<!-- oai-admin-search.inc -->
	3	<div id="header_search" role="tablist">
	4	<div class="tab-content">
	5	<div id="oai_search" role="tabpanel" class="tab-pane active">
	6	<form action="/cgi-bin/koha/admin/oaiservers.pl" method="post">
	7	<div class="form-title">
	8	<label class="control-label">Search OAI repositories</label>
	9	</div>
	10
	11	<div class="form-content">
	12	<input class="head-searchbox form-control" type="text" name="searchfield" id="searchfield" value="[% searchfield \| html %]" placeholder="OAI repository search" />
	13	</div>
	14
	15	<button type="submit"><i class="fa fa-arrow-right"></i></button>
	16	</form>
	17	</div>
	18
	19	[% IF ( CAN_user_circulate_circulate_remaining_permissions ) %]
	20	[% INCLUDE 'patron-search-box.inc' %]
	21	[% END %]
	22	[% IF ( CAN_user_catalogue ) %]
	23	[% INCLUDE 'catalogue-search-box.inc' %]
	24	[% END %]
	25
	26	</div><!-- /.tab-content -->
	27	<ul class="nav nav-tabs" role="tablist">
	28	<li role="presentation" class="active">
	29	<a title="Search OAI repositories" href="#oai_search" aria-controls="z3950_search" role="tab" aria-expanded="true" data-toggle="tab" class="keep_text"><i class="fa fa-search"></i> <span class="tab-title">Search OAI repositories</span></a>
	30	</li>
	31	[% IF ( CAN_user_circulate_circulate_remaining_permissions ) %]
	32	<li role="presentation">
	33	<a title="Check out" href="#circ_search" aria-controls="circ_search" role="tab" data-toggle="tab" class="keep_text"><i class="fa fa-upload"></i> <span class="tab-title">Check out</span></a>
	34	</li>
	35	[% END %]
	36	[% IF ( CAN_user_catalogue ) %]
	37	<li role="presentation">
	38	<a title="Search catalog" href="#catalog_search" aria-controls="catalog_search" role="tab" data-toggle="tab" class="keep_text"><i class="fa fa-fw fa-search"></i> <span class="tab-title">Search catalog</span></a>
	39	</li>
	40	[% END %]
	41	</ul>
	42	</div><!-- /#header_search -->
	43	<!-- /oai-admin-search.inc -->

Line 0 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oaiservers.tt (+229 lines)
		1	[% USE raw %]
		2	[% USE Asset %]
		3	[% SET footerjs = 1 %]
		4	[% INCLUDE 'doc-head-open.inc' %]
		5
		6	<title>
		7	[% IF op == 'edit' %]
		8	Modify OAI repository [% server.servername \| html %] &rsaquo; [% END %]
		9
		10	[% IF op == 'add' %]
		11	New OAI repository &rsaquo; [% END %]
		12
		13	OAI repositories &rsaquo; Administration &rsaquo; Koha
		14	</title>
		15	[% INCLUDE 'doc-head-close.inc' %]
		16	</head>
		17
		18	<body id="admin_oaiservers" class="admin">
		19	[% WRAPPER 'header.inc' %]
		20	[% INCLUDE 'oai-admin-search.inc' %]
		21	[% END %]
		22
		23	[% WRAPPER 'sub-header.inc' %]
		24	[% WRAPPER breadcrumbs %]
		25	[% WRAPPER breadcrumb_item %]
		26	<a href="/cgi-bin/koha/admin/admin-home.pl">Administration</a>
		27	[% END %]
		28
		29	[% IF op == 'edit' \|\| op == 'add' %]
		30	[% WRAPPER breadcrumb_item %]
		31	<a href="/cgi-bin/koha/admin/oaiservers.pl">OAI repositories</a>
		32	[% END %]
		33	[% END %]
		34
		35	[% IF op == 'edit' %]
		36	[% WRAPPER breadcrumb_item bc_active= 1 %]
		37	<span>Modify OAI repository [% server.servername \| html %]</span>
		38	[% END %]
		39
		40	[% ELSIF op == 'add' %]
		41	[% WRAPPER breadcrumb_item bc_active= 1 %]
		42	<span>New OAI repository</span>
		43	[% END %]
		44
		45	[% ELSE %]
		46	[% WRAPPER breadcrumb_item bc_active= 1 %]
		47	<span>OAI repositories</span>
		48	[% END %]
		49	[% END %]
		50	[% END #/ WRAPPER breadcrumbs %]
		51	[% END #/ WRAPPER sub-header.inc %]
		52
		53	<div class="main container-fluid">
		54	<div class="row">
		55	<div class="col-sm-10 col-sm-push-2">
		56	<main>
		57
		58	[% IF msg_deleted %]
		59	<div class="dialog alert">OAI repository deleted ([% msg_add \| html %])</div>
		60	[% ELSIF msg_updated %]
		61	<div class="dialog message">OAI repository updated ([% msg_add \| html %])</div>
		62	[% ELSIF msg_added %]
		63	<div class="dialog message">OAI repository added ([% msg_add \| html %])</div>
		64	[% ELSIF msg_notfound %]
65	<div class="dialog alert">Error: Server with ID [% msg_add \| html %] not found</div>
66	[% END %]
67
68	[% IF ( add_form ) %]
69	<form action="[% script_name \| html %]" name="Aform" method="post" id="serverentry">
70	<input type="hidden" name="op" value="add_validated" />
71	[% IF op == 'edit' %]
72	<h1>Modify OAI repository</h1>
73	<input type="hidden" name="id" value="[% server.id \| html %]" />
74	[% ELSE %]
75	<h1>New OAI repository</h1>
76	[% END %]
77	<fieldset class="rows">
78	<ol>
79	<li><label for="name" class="required">Repository name: </label>
80	[% IF server.servername %]
81	<input type="text" name="servername" id="servername" size="65" maxlength="100" value="[% server.servername \| html %]" required="required" />
82	[% ELSE %]
83	<input type="text" name="servername" id="servername" class="focus" size="65" maxlength="100" value="[% server.servername \| html %]" required="required" />
84	[% END %]
85	<span class="required">Required</span>
86	</li>
87
88	<li><label for="endpoint" class="required">Endpoint: </label> <input type="text" name="endpoint" id="endpoint" size="30" value="[% server.endpoint \| html %]" required="required" /> <span class="required">Required</span>
89	</li>
90	<li><label for="oai_set">Set: </label> <input type="text" name="oai_set" id="oai_set" value="[% server.oai_set \| html %]" />
91	</li>
92	<li>
93	<label for="dataformat">Data format: </label>
94	<select name="dataformat" id="dataformat">
95	<option value="oai_dc">Dublin Core (oai_dc)</option>
96	<option value="marc-xml">Marcxml (marc-xml)</option>
97	<option value="marcxml">Marcxml (marcxml)</option>
98	</select>
99	<div class="hint">OAI can send records in various formats. Choose one.</div>
100	</li>
101
102	<li><label for="recordtype">Record type: </label>
103	<select name="recordtype" id="recordtype">
104	<option value="biblio">Bibliographic</option>
105	<option value="authority">Authority</option>
106	</select>
107	</li>
108	<li>
109	<label for="add_xslt">XSLT file for transforming results: </label>
110	<input type="text" name="add_xslt" id="add_xslt" size="100" placeholder="[% xslt_path \| html %]" value="[% server.add_xslt \| html %]"/>
111	</li>
112
113	</ol>
114	</fieldset>
115
116	<fieldset class="action"><input type="submit" class="btn btn-primary" value="Save" /> <a class="cancel" href="/cgi-bin/koha/admin/oaiservers.pl">Cancel</a></fieldset>
117	</form>
118	[% END %]
119
120	[% IF op == 'list' %]
121	<div id="toolbar" class="btn-toolbar">
122	<a id="newserver" class="btn btn-default" href="/cgi-bin/koha/admin/oaiservers.pl?op=add"><i class="fa fa-plus"></i> New OAI repository</a>
123	</div>
124	<h1>OAI repositories administration</h1>
125	[% IF id %]
126	You searched for record [% id \| html %]
127	[% ELSIF searchfield %]
128	You searched for [% searchfield \| html %]
129	[% END %]
130
131	<div class="page-section">
132	<table id="serverst">
133
134	<thead><tr><th>Id</th><th>Repository name</th><th>Endpoint</th><th>Set</th><th>Data format</th><th>Record type</th><th class="noExport">Actions</th>
135	</tr></thead>
136	<tbody>
137	[% FOREACH loo IN loop %]
138	<tr>
139	<td>[% loo.id \| html %]</td>
140	<td><a href="/cgi-bin/koha/admin/oaiservers.pl?op=edit&id=[% loo.id \| uri %]">[% loo.servername \| html %]</a></td><td>[% loo.endpoint \| html %]</td><td>[% loo.oai_set \| html %]</td>
141	<td>[% loo.dataformat \| html %]</td>
142	<td>[% IF ( loo.recordtype == 'biblio' ) %]
143	<span>Bibliographic</span>
144	[% ELSIF ( loo.recordtype == 'authority' ) %]
145	<span>Authority</span>
146	[% END %]
147	</td>
148	<td>
149	<div class="btn-group dropup">
150	<a class="btn btn-default btn-xs dropdown-toggle" id="reportactions[% savedreport.id \| html %]" role="button" data-toggle="dropdown" href="#">
151	Actions <b class="caret"></b>
152	</a>
153	<ul class="dropdown-menu pull-right" role="menu" aria-labelledby="reportactions[% savedreport.id \| html %]">
154	<li><a href="/cgi-bin/koha/admin/oaiservers.pl?op=edit&id=[% loo.id \| uri %]"><i class="fa-solid fa-pencil" aria-hidden="true"></i> Edit</a></li>
155	<li><a href="/cgi-bin/koha/admin/oaiservers.pl?op=add&id=[% loo.id \| uri %]"><i class="fa fa-copy"></i> Copy</a></li>
156	<li><a href="/cgi-bin/koha/admin/oaiservers.pl?op=delete_confirmed&id=[% loo.id \| uri %]" class="delete" data-servername="[% loo.servername \| html %]"><i class="fa fa-trash-can"></i> Delete</a></li>
157	</ul>
158	</div>
159	</td>
160	</tr>
161	[% END %]
162	</tbody>
163	</table>
164	</div> <!-- /.page-section -->
165	[% END %]
166
167	</main>
168	</div> <!-- /.col-sm-10.col-sm-push-2 -->
169
170	<div class="col-sm-2 col-sm-pull-10">
171	<aside>
172	[% INCLUDE 'admin-menu.inc' %]
173	</aside>
174	</div> <!-- /.col-sm-2.col-sm-pull-10 -->
175	</div> <!-- /.row -->
176
177	[% MACRO jsinclude BLOCK %]
178	[% Asset.js("js/admin-menu.js") \| $raw %]
179	[% IF op == 'list' %]
180	[% INCLUDE 'datatables.inc' %]
181	[% END %]
182
183	<script>
184	[% IF ( add_form ) %]
185	$(document).ready(function(){
186	// Update selects for dataformat and recordtype
187	[% IF server %]
188	$("#dataformat").val('[% server.dataformat \| html %]');
189	$("#recordtype").val('[% server.recordtype \| html %]');
190	[% END %]
191	$( "#serverentry" ).validate({
192	rules: {
193	servername: { required: true },
194	host: { required: true },
195	port: {
196	number: true
197	},
198	rank: { number: true },
199	}
200	});
201	$("#serverentry").submit(function( event ) {
202	// enable recordtype to include field in post
203	$('#recordtype').prop('disabled',false);
204	});
205	$("#servername").on("blur",function(){
206	toUC(this);
207	});
208	});
209	[% ELSE %]
210	$(document).ready(function() {
211	$("#serverst").dataTable($.extend(true, {}, dataTablesDefaults, {
212	"aoColumnDefs": [
213	{ "aTargets": [-1], "bSortable": false, "bSearchable": false },
214	],
215	"sPaginationType": "full"
216	}));
217	$(".delete").on("click",function(e){
218	var servername = $(this).data("servername");
219	if( confirm( _("Are you sure you want to delete repository %s?").format(servername) ) ) {
220	return true;
221	} else {
222	e.preventDefault();
223	}
224	});
225	});
226	[% END %]
227	</script>
228	[% END %]
229	[% INCLUDE 'intranet-bottom.inc' %]

Line 0 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/xslt/AuthOAIDCtoMARC21XML.xsl (+35 lines)
	1	<?xml version="1.0" encoding="UTF-8"?>
	2	<xsl:stylesheet version="1.0"
	3	xmlns:dc="http://purl.org/dc/elements/1.1/"
	4	xmlns:dcterms="http://purl.org/dc/terms/1.1"
	5	xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
	6	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	7	xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/
	8	http://www.openarchives.org/OAI/2.0/oai_dc.xsd"
	9	xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
	10	xmlns="http://www.loc.gov/MARC21/slim" exclude-result-prefixes="dc dcterms oai_dc">
	11
	12	<xsl:output method="xml" encoding="UTF-8" indent="yes"/>
	13
	14	<xsl:template match="/">
	15	<collection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" >
	16	<xsl:apply-templates />
	17	</collection>
	18	</xsl:template>
	19
	20	<xsl:template match="oai_dc:dc">
	21	<record>
	22
	23	<xsl:variable name="FamilyName" select="dc:FamilyName"/>
	24	<xsl:variable name="GivenName" select="dc:GivenName"/>
	25
	26	<datafield tag="100" ind1="0" ind2=" ">
	27	<subfield code="a">
	28	<xsl:value-of select="concat($FamilyName,', ',$GivenName)"/>
	29	</subfield>
	30	</datafield>
	31
	32	</record>
	33	</xsl:template>
	34
	35	</xsl:stylesheet>

Line 0 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/xslt/OAIMarcxml2KohaMarcxml.xsl (+23 lines)
	1	<?xml version="1.0" encoding="UTF-8"?>
	2	<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:marc="http://www.loc.gov/MARC21/slim">
	3	<xsl:output method="xml" version="1.0" encoding="UTF-8" indent="yes"/>
	4
	5	<!-- keep comments -->
	6	<xsl:template match="*[name()='metadata']">
	7	<xsl:apply-templates />
	8	</xsl:template>
	9
	10	<xsl:template match="*">
	11	<!-- remove element prefix -->
	12	<xsl:element name="{local-name()}">
	13	<!-- process attributes -->
	14	<xsl:for-each select="@*">
	15	<!-- remove attribute prefix -->
	16	<xsl:attribute name="{local-name()}">
	17	<xsl:value-of select="."/>
	18	</xsl:attribute>
	19	</xsl:for-each>
	20	<xsl:apply-templates/>
	21	</xsl:element>
	22	</xsl:template>
	23	</xsl:stylesheet>

Line 0 Link Here

(-)a/misc/cronjobs/harvest_oai.pl (+110 lines)
		1	#!/usr/bin/perl
		2	#
		3	# Copyright 2023 BibLibre
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19
		20	use warnings;
		21	use strict;
		22	use utf8;
		23
		24	use Getopt::Long qw( GetOptions );
		25	use Koha::Script -cron;
		26	use Koha::OAI::Client::Harvester;
		27	use Koha::OaiServers;
		28	use C4::Log qw( cronlogaction );
		29	use Try::Tiny qw( catch try );
		30
		31	my $command_line_options = join( " ", @ARGV );
		32
		33	my ( $help, $verbose, $id, $days, $list, $force );
		34
		35	GetOptions(
		36	'h\|help' => \$help,
		37	'v\|verbose' => \$verbose,
		38	'r\|repository:i' => \$id,
		39	'd\|days:i' => \$days,
		40	'l\|list' => \$list,
		41	'f\|force' => \$force,
		42	);
		43	my $usage = << 'ENDUSAGE';
		44
		45	This script starts an OAI Harvest
		46
		47	This script has the following parameters :
		48	-h --help: this message
		49	-v --verbose
		50	-r --repository: id of the OAI repository
		51	-d --days: number of days to harvest from (optional)
		52	-l --list: list the OAI repositories
		53	-f --force: force harvesting (ignore records datestamps)
		54
		55	ENDUSAGE
		56
		57	if ($help) {
		58	print $usage;
		59	exit;
		60	}
		61
		62	if ($list) {
		63	my $servers = Koha::OaiServers->search( {}, { order_by => { -asc => 'id' } } )->unblessed;
		64	print "The following repositories are available: \n\n";
65	foreach my $server (@$servers) {
66	print $server->{'id'} . ": "
67	. $server->{'servername'}
68	. ", endpoint: "
69	. $server->{'endpoint'}
70	. ", set: "
71	. $server->{'oai_set'}
72	. ", recordtype: "
73	. $server->{'recordtype'} . "\n";
74	}
75	print "\n";
76	exit;
77	}
78
79	if ( !$id ) {
80	print "The repository parameter is mandatory.\n";
81	print $usage . "\n";
82	}
83
84	my $server = Koha::OaiServers->find($id);
85
86	unless ($server) {
87	print "OAI Server $id unknown\n";
88	exit;
89	}
90
91	my $script_handler = Koha::Script->new( { script => $0 } );
92
93	try {
94	$script_handler->lock_exec;
95	} catch {
96	my $message = "Skipping execution of $0";
97	print "$message\n" if $verbose;
98	cronlogaction( { info => $message } );
99	exit;
100	};
101
102	cronlogaction( { action => 'Start', info => "Starting OAI Harvest" } );
103	cronlogaction( { info => "Command line: $command_line_options" } );
104
105	my $harvester =
106	Koha::OAI::Client::Harvester->new( { server => $server, verbose => $verbose, days => $days, force => $force } );
107	$harvester->init();
108
109	cronlogaction( { action => 'End', info => "Ending OAI Harvest" } );
110	exit(0);

Line 0 Link Here

(-)a/t/db_dependent/Koha/OAIHarvester.t (+78 lines)
		1	#!/usr/bin/perl
		2
		3	# Copyright 2024 BibLibre
		4	#
		5	# This file is part of Koha
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19
		20	use Modern::Perl;
		21	use Test::More tests => 5;
		22	use Test::Exception;
		23
		24	use t::lib::TestBuilder;
		25	use HTTP::OAI;
		26	use HTTP::OAI::Metadata::OAI_DC;
		27	use HTTP::OAI::Record;
		28	use HTTP::OAI::Encapsulation;
		29	use Koha::Database;
		30	use Koha::OaiServer;
		31	use Koha::OaiServers;
		32	use Koha::OAI::Client::Harvester;
		33
		34	my $schema = Koha::Database->new->schema;
		35	my $builder = t::lib::TestBuilder->new;
		36
		37	$schema->storage->txn_begin;
		38
		39	my $new_oai_1 = Koha::OaiServer->new(
		40	{
		41	endpoint => 'my_host1.org',
		42	oai_set => 'set1',
		43	servername => 'my_test_1',
		44	dataformat => 'oai_dc',
		45	recordtype => 'biblio',
		46	add_xslt => '',
		47	}
		48	)->store;
		49
		50	my $harvester = Koha::OAI::Client::Harvester->new( { server => $new_oai_1, verbose => 1, days => 1, force => 1 } );
		51
		52	my $record =
		53	'<metadata xmlns="http://www.openarchives.org/OAI/2.0/"><oai_dc:dc xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd"><dc:title>Pièces diverses </dc:title><dc:identifier>ARCH/0320 [cote]</dc:identifier><dc:relation>FR-920509801 [RCR établissement]</dc:relation><dc:relation>Central obrera boliviana [Fonds ou collection]</dc:relation><dc:format>1 carton</dc:format><dc:date>1971/2000</dc:date><dc:type>Archives et manuscrits</dc:type></oai_dc:dc></metadata>';
		54
		55	my $r = HTTP::OAI::Record->new();
		56	$r->header->datestamp('2017-05-10T09:18:13Z');
		57	$r->metadata( HTTP::OAI::Metadata->new( dom => $record ) );
		58
		59	my $status = $harvester->processRecord($r);
		60	is( $status, 'skipped', 'Record with no identifier is skipped' );
		61
		62	$r->header->identifier('oai:myarchive.org:oid-233');
		63
		64	$status = $harvester->processRecord($r);
65	is( $status, 'added', 'Record is added' );
66
67	$status = $harvester->processRecord($r);
68	is( $status, 'updated', 'When force is used, record is updated' );
69
70	$harvester = Koha::OAI::Client::Harvester->new( { server => $new_oai_1, verbose => 1, days => 1, force => 0 } );
71	$status = $harvester->processRecord($r);
72	is( $status, 'skipped', 'When force is not used, record is skipped (already up to date)' );
73
74	$r->header->datestamp('2018-05-10T09:18:13Z');
75	$status = $harvester->processRecord($r);
76	is( $status, 'updated', 'When force is not used, record is updated if newer' );
77
78	$schema->storage->txn_rollback;

Line 0 Link Here

(-)a/t/db_dependent/Koha/OAIServers.t (-1 / +95 lines)
0	-	1	#!/usr/bin/perl
		2
		3	# Copyright 2015 Koha Development team
		4	#
		5	# This file is part of Koha
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19
		20	use Modern::Perl;
		21	use Test::More tests => 2;
		22	use Test::Exception;
		23
		24	use t::lib::TestBuilder;
		25	use Koha::Database;
		26	use Koha::OaiServer;
		27	use Koha::OaiServers;
		28
		29	my $schema = Koha::Database->new->schema;
		30	my $builder = t::lib::TestBuilder->new;
		31
		32	subtest 'new, find and delete tests' => sub {
		33	plan tests => 4;
		34	$schema->storage->txn_begin;
		35	my $nb_of_oais = Koha::OaiServers->search->count;
		36	my $new_oai_1 = Koha::OaiServer->new(
		37	{
		38	endpoint => 'my_host1.org',
		39	oai_set => 'set1',
		40	servername => 'my_test_1',
		41	dataformat => 'oai_dc',
		42	recordtype => 'biblio',
		43	}
		44	)->store;
		45	my $new_oai_2 = Koha::OaiServer->new(
		46	{
		47	endpoint => 'my_host2.org',
		48	oai_set => 'set2',
		49	servername => 'my_test_2',
		50	dataformat => 'marcxml',
		51	recordtype => 'authority',
		52	}
		53	)->store;
		54
		55	like( $new_oai_1->id, qr\|^\d+$\|, 'Adding a new oai server should have set the id' );
		56	is( Koha::OaiServers->search->count, $nb_of_oais + 2, 'The 2 oai servers should have been added' );
		57
		58	my $retrieved_oai_1 = Koha::OaiServers->find( $new_oai_1->id );
		59	is(
		60	$retrieved_oai_1->servername, $new_oai_1->servername,
		61	'Find a oai server by id should return the correct oai server'
		62	);
		63
		64	$retrieved_oai_1->delete;
		65	is( Koha::OaiServers->search->count, $nb_of_oais + 1, 'Delete should have deleted the oai server' );
66
67	$schema->storage->txn_rollback;
68	};
69
70	subtest 'Check NOT NULL without default values' => sub {
71	plan tests => 5;
72	$schema->storage->txn_begin;
73	local $SIG{__WARN__} = sub { }; # TODO Needed it for suppressing DBIx warns
74
75	my $server = Koha::OaiServer->new(
76	{
77	oai_set => 'set3',
78	dataformat => 'marcxml',
79	recordtype => 'biblio',
80	}
81	);
82
83	throws_ok { $server->store } 'DBIx::Class::Exception', 'Exception on empty endpoint';
84	like( $@->{msg}, qr/'endpoint' doesn't have a default value/, 'Verified that DBIx blamed endpoint' );
85
86	$server->endpoint('endpoint_added');
87	throws_ok { $server->store } 'DBIx::Class::Exception', 'Exception on empty servername';
88	like( $@->{msg}, qr/'servername' doesn't have a default value/, 'Verified that DBIx blamed servername' );
89
90	$server->servername('servername_added');
91
92	lives_ok { $server->store } 'No exceptions anymore';
93
94	$schema->storage->txn_rollback;
95	};