Attachment #160356 for bug #35659

View | Details | Raw Unified | Return to bug 35659
Collapse All | Expand All




package Koha::Import::Oaipmh::Authorities;

# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License

use Modern::Perl;

use Koha::Import::Oaipmh::Authority;

use base qw(Koha::Objects);

=head1 NAME

Koha::Import::Oaipmh::Authorities

This object represents a collection of OAI-PMH records being imported as authorities

=head1 API

=head2 Methods


=head3 type

=cut

sub _type {
    return 'ImportOaipmhAuthority';
}

=head3 object_class

Koha::Object class

=cut

sub object_class {
    return 'Koha::Import::Oaipmh::Authority';
}
1;




package Koha::Import::Oaipmh::Authority;

# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License

use Modern::Perl;

use base qw(Koha::Object);

=head1 NAME

Koha::Import::Oaipmh::Authority

This object represents an OAI-PMH record being imported as an authority

=head1 API

=head2 Class Methods

=cut

=head3 type

=cut

sub _type {
    return 'ImportOaipmhAuthority';
}

1;




package Koha::Import::Oaipmh::Biblio;

# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License

use Modern::Perl;

use base qw(Koha::Object);

=head1 NAME

Koha::Import::Oaipmh::Biblio

This object represents an OAI-PMH record being imported as a bibliographic record

=head1 API

=head2 Methods


=head3 type

=cut

sub _type {
    return 'ImportOaipmhBiblio';
}

1;




package Koha::Import::Oaipmh::Biblios;

# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License

use Modern::Perl;

use Koha::Import::Oaipmh::Biblio;

use base qw(Koha::Objects);

=head1 NAME

Koha::Import::Oaipmh::Biblios

This object represents a collection of OAI-PMH records being imported as bibliographic records

=head1 API

=head2 Methods


=head3 type

=cut

sub _type {
    return 'ImportOaipmhBiblio';
}

=head3 object_class

Koha::Object class

=cut

sub object_class {
    return 'Koha::Import::Oaipmh::Biblio';
}
1;




# Copyright BibLibre 2023
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

package Koha::OAI::Client::Harvester;

use utf8;
use open qw( :std :utf8 );

use C4::Biblio qw( AddBiblio GetFrameworkCode ModBiblio DelBiblio );
use C4::AuthoritiesMarc qw (AddAuthority GuessAuthTypeCode ModAuthority DelAuthority );
use C4::Log qw( cronlogaction );
use HTTP::OAI;
use HTTP::OAI::Metadata::OAI_DC;
use Koha::DateUtils qw( dt_from_string );
use Koha::OaiServers;
use Koha::Import::Oaipmh::Biblio;
use Koha::Import::Oaipmh::Biblios;
use Koha::Import::Oaipmh::Authority;
use Koha::Import::Oaipmh::Authorities;
use Koha::XSLT::Base;
use MARC::File::XML;
use MARC::Record;
use Modern::Perl;
use DateTime;
use DateTime::Format::Strptime;
use Try::Tiny qw( catch try );
use Date::Calc qw(
    Add_Delta_Days
    Today
);

my $strp = DateTime::Format::Strptime->new(
    pattern   => '%FT%TZ',
    time_zone => 'floating',
);

our $verbose    = 0;
our $servername = '';

sub new {

    my ( $self, $args ) = @_;

    my $id = $args->{'server'};
    $verbose = $args->{'verbose'};
    my $days  = $args->{'days'};
    my $force = $args->{'force'};

    return unless $id;

    my $server = Koha::OaiServers->find($id);
    $servername = $server->servername;

    unless ($server) {
        printlog("OAI Server $id unknown\n");
        return;
    }

    printlog("Starting OAI Harvest from repository");

    my $h = HTTP::OAI::Harvester->new(
        repository => HTTP::OAI::Identify->new(
            baseURL => $server->endpoint,
            version => '2.0',
        )
    );

    printlog( "Using HTTP::OAI::Harvester version " . $HTTP::OAI::Harvester::VERSION );

    printlog( "Using endpoint "
            . $server->endpoint
            . ", set "
            . $server->oai_set
            . ", dataformat "
            . $server->dataformat
            . ", recordtype "
            . $server->recordtype );

    try {
        my $lmdf    = $h->ListMetadataFormats();
        my $lmdflog = "";
        for ( $lmdf->metadataFormat ) {
            $lmdflog .= $_->metadataPrefix . " ";
        }
        printlog("Available metadata formats for this repository: $lmdflog");
    } catch {
        printlog("ListMetadataFormats failed");
    };

    #my $response = $h->Identify;
    #print "Repository supports protocol version ", $response->version, "\n";
    if ( $server->add_xslt ) {
        printlog( "Using XSLT file " . $server->add_xslt );
    } else {
        printlog("Not using an XSLT file");
    }

    my $start_date = '';
    my $today_date = '';

    if ($days) {

        # Change this to yyyy-mm-dd
        my $dt_today = dt_from_string();
        my ( $nowyear, $nowmonth, $nowday ) = Today();
        my @date     = Add_Delta_Days( $nowyear, $nowmonth, $nowday, -$days );
        my $dt_start = DateTime->new( year => $date[0], month => $date[1], day => $date[2] );
        $start_date = $dt_start->ymd();
        printlog("Harvesting from $start_date");
    }

    printlog("Asking for records");
    my $response = $h->ListRecords(
        metadataPrefix => $server->dataformat,
        set            => $server->oai_set,
        from           => $start_date,
    );
    if ( $response->is_error ) {
        printlog( "Error requesting ListRecords: " . $response->code . " " . $response->message );
        exit;
    }
    printlog( "Request URL: " . $response->requestURL );

    my $xslt_engine = Koha::XSLT::Base->new;
    my $added       = 0;
    my $updated     = 0;
    my $deleted     = 0;
    my $in_error    = 0;
    my $skipped     = 0;
    my $total       = 0;

    printlog("Starting processing results");
    while ( my $oai_record = $response->next ) {
        $total++;
        unless ( $oai_record->identifier ) {
            printlog("No identifier found");
            $skipped++;
            next;
        }

        my $to_delete;
        my $outputUnimarc;
        my $marcrecord;
        if ( $oai_record->status && $oai_record->status eq "deleted" ) {
            $to_delete = 1;
        } else {
            if ( $server->add_xslt ) {
                $outputUnimarc = $xslt_engine->transform( $oai_record->metadata->dom, $server->add_xslt );
                my $err = $xslt_engine->err;
                if ($err) {
                    printlog("XSLT::Base error: $err");
                    $in_error++;
                    next;
                }
            } else {
                $outputUnimarc = $oai_record->metadata->dom;
            }
            $marcrecord = MARC::Record->new_from_xml( $outputUnimarc, 'UTF-8' );
        }
        my $imported_record;
        if ( $server->recordtype eq "biblio" ) {
            $imported_record = Koha::Import::Oaipmh::Biblios->find(
                {
                    repository => $server->endpoint,
                    identifier => $oai_record->identifier,
                    recordtype => $server->recordtype,
                }
            );
        } else {
            $imported_record = Koha::Import::Oaipmh::Authorities->find(
                {
                    repository => $server->endpoint,
                    identifier => $oai_record->identifier,
                    recordtype => $server->recordtype,
                }
            );
        }

        if ($imported_record) {
            if ($to_delete) {
                if ( $server->recordtype eq "biblio" ) {
                    my $biblionumber = $imported_record->biblionumber;
                    my $error        = DelBiblio($biblionumber);
                    if ($error) {
                        printlog( "Record "
                                . $oai_record->identifier
                                . " not deleted, biblionumber: $biblionumber ($error)" );
                        $in_error++;
                    } else {
                        $imported_record->delete;
                        printlog( "Record " . $oai_record->identifier . " deleted, biblionumber: $biblionumber" );
                        $deleted++;
                    }
                } else {
                    my $authid = $imported_record->authid;
                    DelAuthority( { authid => $authid } );
                    $imported_record->delete;
                    printlog( "Record " . $oai_record->identifier . " deleted, authid: $authid" );
                    $deleted++;
                }
            } else {
                my $incoming_dt = $strp->parse_datetime( $imported_record->datestamp );
                my $existing_dt = $strp->parse_datetime( $oai_record->datestamp );
                if ( $force || !$incoming_dt || !$existing_dt || $incoming_dt > $existing_dt ) {
                    if ( $server->recordtype eq "biblio" ) {
                        my $biblionumber = $imported_record->biblionumber;
                        my $result       = ModBiblio( $marcrecord, $biblionumber, GetFrameworkCode($biblionumber) );
                        printlog( "Record " . $oai_record->identifier . " updated, biblionumber: $biblionumber" );
                    } else {
                        my $authid = $imported_record->authid;
                        my $result = ModAuthority( $authid, $marcrecord, GuessAuthTypeCode($marcrecord) );
                        printlog( "Record " . $oai_record->identifier . " updated, authid: $authid" );
                    }
                    $imported_record->update(
                        {
                            datestamp => $imported_record->datestamp,
                        }
                    );
                    $updated++;
                } else {
                    printlog( "Record "
                            . $oai_record->identifier
                            . " skipped (incoming record was not newer than existing record)" );
                    $skipped++;
                }
            }
        } elsif ( !$to_delete ) {
            if ( $server->recordtype eq "biblio" ) {
                my ( $biblionumber, $biblioitemnumber ) = AddBiblio($marcrecord);
                printlog( $oai_record->identifier . " added, biblionumber: $biblionumber" );
                Koha::Import::Oaipmh::Biblio->new(
                    {
                        repository   => $server->endpoint,
                        identifier   => $oai_record->identifier,
                        biblionumber => $biblionumber,
                        recordtype   => $server->recordtype,
                        datestamp    => $oai_record->datestamp,
                    }
                )->store;
            } else {
                my $authid = AddAuthority( $marcrecord, "", GuessAuthTypeCode($marcrecord) );
                printlog( $oai_record->identifier . " added, authid: $authid" );
                Koha::Import::Oaipmh::Authority->new(
                    {
                        repository => $server->endpoint,
                        identifier => $oai_record->identifier,
                        authid     => $authid,
                        recordtype => $server->recordtype,
                        datestamp  => $oai_record->datestamp,
                    }
                )->store;
            }
            $added++;
        } else {
            printlog( "Record " . $oai_record->identifier . " skipped (record not present or already deleted)" );
            $skipped++;
        }
    }
    printlog(
        "Harvest results:\n$added added\n$updated updated\n$deleted deleted\n$skipped skipped\n$in_error errors\n$total total"
    );

    if ( C4::Context->preference("OAI-PMH:HarvestEmailReport") ) {

        my $letter = C4::Letters::GetPreparedLetter(
            (
                module      => 'catalogue',
                letter_code => 'OAI_HARVEST_REPORT',
                substitute  => {
                    servername => $servername,
                    endpoint   => $server->endpoint,
                    set        => $server->oai_set,
                    dataformat => $server->dataformat,
                    recordtype => $server->recordtype,
                    added      => $added,
                    updated    => $updated,
                    deleted    => $deleted,
                    skipped    => $skipped,
                    in_error   => $in_error,
                    total      => $total
                },
            )
        ) or printlog("OAI_HARVEST_REPORT letter not found");

        my $success = C4::Letters::EnqueueLetter(
            {
                letter     => $letter,
                to_address => C4::Context->preference("OAI-PMH:HarvestEmailReport"), message_transport_type => 'email'
            }
        );
        if ($success) {
            printlog("Email report enqueued");
        } else {
            printlog("Unable to enqueue report email");
        }
    }

    printlog("Ending OAI Harvest from repository");
}

sub printlog {
    my $message = shift;
    $message = $servername . ": " . $message;
    print $message . "\n" if ($verbose);
    cronlogaction( { info => $message } );
}

1;




package Koha::OaiServer;

# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;


use Koha::Database;

use base qw(Koha::Object);

=head1 NAME

Koha::OaiServer - Koha OaiServer Object class

=head1 API

=head2 Class Methods

=cut

=head3 _type

Return type of Object relating to Schema ResultSet

=cut

sub _type {
    return 'Oaiserver';
}

1;




package Koha::OaiServers;

# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;


use Koha::Database;

use Koha::OaiServer;

use base qw(Koha::Objects);

=head1 NAME

Koha::OaiServers - Koha OaiServer Object set class

=head1 API

=head2 Class Methods

=cut

=head3 type

Return type of object, relating to Schema ResultSet

=cut

sub _type {
    return 'Oaiserver';
}

=head3 object_class

Return object class

=cut

sub object_class {
    return 'Koha::OaiServer';
}

1;




#!/usr/bin/perl

# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

# This script is used to maintain the OAI servers table.
# Parameter $op is operation: list, new, edit, add_validated, delete_confirmed.
# add_validated saves a validated record and goes to list view.
# delete_confirmed deletes a record and goes to list view.

use Modern::Perl;
use CGI qw ( -utf8 );
use C4::Context;
use C4::Auth qw( get_template_and_user );
use C4::Output qw( output_html_with_http_headers );
use Koha::Database;
use Koha::OaiServers;

# Initialize CGI, template, database

my $input       = CGI->new;
my $op          = $input->param('op') || 'list';
my $id          = $input->param('id') || 0;
my $type        = $input->param('type') || '';
my $searchfield = '';

my ( $template, $loggedinuser, $cookie ) = get_template_and_user(
    {
        template_name => "admin/oaiservers.tt",
        query         => $input,
        type          => "intranet",
        flagsrequired => { parameters => 'manage_search_targets' },
    }
);
my $script_name = "/cgi-bin/koha/admin/oaiservers.pl";
my $path        = C4::Context->config('intrahtdocs') . "/prog/";

$template->param( script_name => $script_name, xslt_path => $path );

my $schema = Koha::Database->new()->schema();

# Main code
# First process a confirmed delete, or save a validated record

if ( $op eq 'delete_confirmed' && $id ) {
    my $server = Koha::OaiServers->find($id);
    if ($server) {
        $server->delete;
        $template->param( msg_deleted => 1, msg_add => $server->servername );
    } else {
        $template->param( msg_notfound => 1, msg_add => $id );
    }
    $id = 0;
} elsif ( $op eq 'add_validated' ) {
    my @fields = qw/endpoint oai_set dataformat
        recordtype servername
        add_xslt/;
    my $formdata = _form_data_hashref( $input, \@fields );
    if ($id) {
        my $server = Koha::OaiServers->find($id);
        if ($server) {
            $server->set($formdata)->store;
            $template->param( msg_updated => 1, msg_add => $formdata->{servername} );
        } else {
            $template->param( msg_notfound => 1, msg_add => $id );
        }
        $id = 0;
    } else {
        Koha::OaiServer->new($formdata)->store;
        $template->param( msg_added => 1, msg_add => $formdata->{servername} );
    }
} else {

    #use searchfield only in remaining operations
    $searchfield = $input->param('searchfield') || '';
}

# Now list multiple records, or edit one record

my $data = [];
if ( $op eq 'add' || $op eq 'edit' ) {
    $data = ServerSearch( $schema, $id, $searchfield ) if $searchfield || $id;
    delete $data->[0]->{id} if @$data && $op eq 'add';    #cloning record
    $template->param(
        add_form => 1, server => @$data ? $data->[0] : undef,
        op => $op, type => $op eq 'add' ? lc $type : ''
    );
} else {
    $data = ServerSearch( $schema, $id, $searchfield );
    $template->param(
        loop => \@$data, searchfield => $searchfield, id => $id,
        op   => 'list'
    );
}
output_html_with_http_headers $input, $cookie, $template->output;

# End of main code

sub ServerSearch {    #find server(s) by id or name
    my ( $schema, $id, $searchstring ) = @_;

    return Koha::OaiServers->search(
        $id ? { id => $id } : { servername => { like => $searchstring . '%' } },
    )->unblessed;
}

sub _form_data_hashref {
    my ( $input, $fieldref ) = @_;
    return { map { ( $_ => scalar $input->param($_) // '' ) } @$fieldref };
}




use Modern::Perl;

return {
    bug_number  => "35659",
    description => "OAI-PMH Harvester",
    up          => sub {
        my ($args) = @_;
        my ( $dbh, $out ) = @$args{qw(dbh out)};
        $dbh->do(q{DROP TABLE IF EXISTS `oaiservers`;});
        $dbh->do(
            q{
            CREATE TABLE `oaiservers` (
              `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
              `endpoint` varchar(255) NOT NULL COMMENT 'OAI endpoint (host + port + path)',
              `oai_set` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
              `servername` longtext NOT NULL COMMENT 'name given to the target by the library',
              `dataformat` enum('oai_dc','marc-xml', 'marcxml') NOT NULL DEFAULT 'oai_dc' COMMENT 'data format',
              `recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'server contains bibliographic or authority records',
              `add_xslt` longtext DEFAULT NULL COMMENT 'zero or more paths to XSLT files to be processed on the search results',
              PRIMARY KEY (`id`)
            ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
        }
        );
        say $out "Added new table 'oaiservers'";

        $dbh->do(q{DROP TABLE IF EXISTS `import_oaipmh_biblios`;});
        $dbh->do(
            q{
            CREATE TABLE `import_oaipmh_biblios` (
              `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
              `biblionumber` int(11) NOT NULL COMMENT 'unique identifier assigned to each koha record',
              `identifier` varchar(255) NOT NULL COMMENT 'OAI record identifier',
              `repository` varchar(255) NOT NULL COMMENT 'OAI repository',
              `recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'is the record bibliographic or authority',
              `datestamp` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
              `last_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
              PRIMARY KEY (id),
              KEY biblionumber (biblionumber),
              CONSTRAINT FK_import_oaipmh_biblios_1 FOREIGN KEY (biblionumber) REFERENCES biblio (biblionumber) ON DELETE CASCADE ON UPDATE NO ACTION
            ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
        }
        );

        $dbh->do(q{DROP TABLE IF EXISTS `import_oaipmh_authorities`;});
        $dbh->do(
            q{
            CREATE TABLE `import_oaipmh_authorities` (
              `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
              `authid` bigint(20) unsigned NOT NULL COMMENT 'unique identifier assigned to each koha record',
              `identifier` varchar(255) NOT NULL COMMENT 'OAI record identifier',
              `repository` varchar(255) NOT NULL COMMENT 'OAI repository',
              `recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'is the record bibliographic or authority',
              `datestamp` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
              `last_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
              PRIMARY KEY (id),
              KEY authid (authid),
              CONSTRAINT FK_import_oaipmh_authorities_1 FOREIGN KEY (authid) REFERENCES auth_header (authid) ON DELETE CASCADE ON UPDATE NO ACTION
            ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
        }
        );
        say $out "Added new tables 'import_oaipmh_biblios' and import_oaipmh_authorities";

        $dbh->do(
            q{
            UPDATE `permissions` SET description='Manage Z39.50 and SRU servers, OAI repositories configuration' WHERE code='manage_search_targets';
        }
        );
        say $out "Updated manage_search_targets permission description";

        $dbh->do(
            q{
            INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES
            ('OAI-PMH:HarvestEmailReport','','','If set, a report email will be sent to this address after an OAI harvest','Free');
        }
        );
        say $out "Added OAI-PMH:HarvestEmailReport system preference";

        $dbh->do(
            q{
            INSERT IGNORE INTO letter
            (module, code, branchcode, name, is_html, title, content, message_transport_type, lang)
            VALUES ('catalogue','OAI_HARVEST_REPORT','','OAI Harvest Report',0,'OAI Harvest report for [% servername %]','OAI Harvest report for [% servername %]:\n\nEndpoint: [% endpoint %]\nSet: [% set %]\nData format: [% dataformat %]\nRecord type: [% recordtype %]\n\n[% added %] records added\n[% updated %] records updated\n[% deleted %] records deleted\n[% skipped %] records skipped\n[% in_error %] records in error\n[% total %] total','email','default');
        }
        );
        say $out "Added OAI_HARVEST_REPORT letter";

    },
};




            - "Your hold on [% hold.biblio.title %] ([% hold.biblio.id %]) has been confirmed."
            - "You will be notified by the library when your item is available for pickup."
            - "Thank you!"

        - module: catalogue
          code: OAI_HARVEST_REPORT
          branchcode: ""
          name: "OAI Harvest Report"
          is_html: 0
          title: "OAI Harvest report for [% servername %]"
          message_transport_type: email
          lang: default
          content:
            - "OAI Harvest report for [% servername %]:"
            - ""
            - "Endpoint: [% endpoint %]"
            - "Set: [% set %]"
            - "Data format: [% dataformat %]"
            - "Record type: [% recordtype %]"
            - ""
            - "[% added %] records added"
            - "[% updated %] records updated"
            - "[% deleted %] records deleted"
            - "[% skipped %] records skipped"
            - "[% in_error %] records in error"
            - "[% total %] total"




) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Table structure for table `oaiservers`
--

DROP TABLE IF EXISTS `oaiservers`;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `oaiservers` (
  `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
  `endpoint` varchar(255) NOT NULL COMMENT 'OAI endpoint (host + port + path)',
  `oai_set` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
  `servername` longtext NOT NULL COMMENT 'name given to the target by the library',
  `dataformat` enum('oai_dc','marc-xml', 'marcxml') NOT NULL DEFAULT 'oai_dc' COMMENT 'data format',
  `recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'server contains bibliographic or authority records',
  `add_xslt` longtext DEFAULT NULL COMMENT 'zero or more paths to XSLT files to be processed on the search results',
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

--
-- Table structure for table `import_oaipmh_biblios`
--

DROP TABLE IF EXISTS `import_oaipmh_biblios`;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `import_oaipmh_biblios` (
  `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
  `biblionumber` int(11) NOT NULL COMMENT 'unique identifier assigned to each koha record',
  `identifier` varchar(255) NOT NULL COMMENT 'OAI record identifier',
  `repository` varchar(255) NOT NULL COMMENT 'OAI repository',
  `recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'is the record bibliographic or authority',
  `datestamp` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
  `last_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
  PRIMARY KEY (id),
  KEY biblionumber (biblionumber),
  CONSTRAINT FK_import_oaipmh_biblios_1 FOREIGN KEY (biblionumber) REFERENCES biblio (biblionumber) ON DELETE CASCADE ON UPDATE NO ACTION
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

--
-- Table structure for table `import_oaipmh_authorities`
--

DROP TABLE IF EXISTS `import_oaipmh_authorities`;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `import_oaipmh_authorities` (
  `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
  `authid` bigint(20) unsigned NOT NULL COMMENT 'unique identifier assigned to each koha record',
  `identifier` varchar(255) NOT NULL COMMENT 'OAI record identifier',
  `repository` varchar(255) NOT NULL COMMENT 'OAI repository',
  `recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'is the record bibliographic or authority',
  `datestamp` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
  `last_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
  PRIMARY KEY (id),
  KEY authid (authid),
  CONSTRAINT FK_import_oaipmh_authorities_1 FOREIGN KEY (authid) REFERENCES auth_header (authid) ON DELETE CASCADE ON UPDATE NO ACTION
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

--
-- Table structure for table `oauth_access_tokens`
--

Lines 436-441 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here

(-)a/installer/data/mysql/mandatory/sysprefs.sql (+1 lines)
436	('OAI-PMH:ConfFile','',NULL,'If empty, Koha OAI Server operates in normal mode, otherwise it operates in extended mode.','File'),	436	('OAI-PMH:ConfFile','',NULL,'If empty, Koha OAI Server operates in normal mode, otherwise it operates in extended mode.','File'),
437	('OAI-PMH:DeletedRecord','persistent','Koha\'s deletedbiblio table will never be deleted (persistent), might be deleted (transient), or will never have any data in it (no)','transient\|persistent\|no','Choice'),	437	('OAI-PMH:DeletedRecord','persistent','Koha\'s deletedbiblio table will never be deleted (persistent), might be deleted (transient), or will never have any data in it (no)','transient\|persistent\|no','Choice'),
438	('OAI-PMH:MaxCount','50',NULL,'OAI-PMH maximum number of records by answer to ListRecords and ListIdentifiers queries','Integer'),	438	('OAI-PMH:MaxCount','50',NULL,'OAI-PMH maximum number of records by answer to ListRecords and ListIdentifiers queries','Integer'),
		439	('OAI-PMH:HarvestEmailReport','','','If set, a report email will be sent to this address after an OAI harvest','Free'),
439	('OPACAcquisitionDetails','0','','Show the acquisition details at the OPAC','YesNo'),	440	('OPACAcquisitionDetails','0','','Show the acquisition details at the OPAC','YesNo'),
440	('OpacAddMastheadLibraryPulldown','0','','Adds a pulldown menu to select the library to search on the opac masthead.','YesNo'),	441	('OpacAddMastheadLibraryPulldown','0','','Adds a pulldown menu to select the library to search on the opac masthead.','YesNo'),
441	('OpacAdvSearchMoreOptions','pubdate,itemtype,language,subtype,sorting,location','Show search options for the expanded view (More options)','pubdate\|itemtype\|language\|subtype\|sorting\|location','multiple'),	442	('OpacAdvSearchMoreOptions','pubdate,itemtype,language,subtype,sorting,location','Show search options for the expanded view (More options)','pubdate\|itemtype\|language\|subtype\|sorting\|location','multiple'),

Lines 29-35 INSERT INTO permissions (module_bit, code, description) VALUES Link Here

(-)a/installer/data/mysql/mandatory/userpermissions.sql (-1 / +1 lines)
29	( 3, 'manage_item_search_fields', 'Manage item search fields'),	29	( 3, 'manage_item_search_fields', 'Manage item search fields'),
30	( 3, 'manage_search_engine_config', 'Manage search engine configuration'),	30	( 3, 'manage_search_engine_config', 'Manage search engine configuration'),
31	( 3, 'manage_marc_overlay_rules', 'Manage MARC overlay rules configuration'),	31	( 3, 'manage_marc_overlay_rules', 'Manage MARC overlay rules configuration'),
32	( 3, 'manage_search_targets', 'Manage Z39.50 and SRU server configuration'),	32	( 3, 'manage_search_targets', 'Manage Z39.50 and SRU servers, OAI repositories configuration'),
33	( 3, 'manage_didyoumean', 'Manage Did you mean? configuration'),	33	( 3, 'manage_didyoumean', 'Manage Did you mean? configuration'),
34	( 3, 'manage_column_config', 'Manage column configuration'),	34	( 3, 'manage_column_config', 'Manage column configuration'),
35	( 3, 'manage_sms_providers', 'Manage SMS cellular providers'),	35	( 3, 'manage_sms_providers', 'Manage SMS cellular providers'),

Lines 149-154 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc (+1 lines)
149	[% END %]	149	[% END %]
150	[% IF ( CAN_user_parameters_manage_search_targets ) %]	150	[% IF ( CAN_user_parameters_manage_search_targets ) %]
151	<li><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50/SRU servers</a></li>	151	<li><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50/SRU servers</a></li>
		152	<li><a href="/cgi-bin/koha/admin/oaiservers.pl">OAI repositories</a></li>
152	[% END %]	153	[% END %]
153	[% IF ( CAN_user_parameters_manage_smtp_servers ) %]	154	[% IF ( CAN_user_parameters_manage_smtp_servers ) %]
154	<li><a href="/cgi-bin/koha/admin/smtp_servers.pl">SMTP servers</a></li>	155	<li><a href="/cgi-bin/koha/admin/smtp_servers.pl">SMTP servers</a></li>




[% USE Koha %]
<!-- oai-admin-search.inc -->
<div id="header_search" role="tablist">
    <div class="tab-content">
        <div id="oai_search" role="tabpanel" class="tab-pane active">
            <form action="/cgi-bin/koha/admin/oaiservers.pl" method="post">
                <div class="form-title">
                    <label class="control-label">Search OAI repositories</label>
                </div>

                <div class="form-content">
                    <input class="head-searchbox form-control" type="text" name="searchfield" id="searchfield" value="[% searchfield | html %]" placeholder="OAI repository search" />
                </div>

                <button type="submit"><i class="fa fa-arrow-right"></i></button>
            </form>
        </div>

        [% IF ( CAN_user_circulate_circulate_remaining_permissions ) %]
            [% INCLUDE 'patron-search-box.inc' %]
        [% END %]
        [% IF ( CAN_user_catalogue ) %]
            [% INCLUDE 'catalogue-search-box.inc' %]
        [% END %]

    </div><!-- /.tab-content -->
    <ul class="nav nav-tabs" role="tablist">
        <li role="presentation" class="active">
            <a title="Search OAI repositories" href="#oai_search" aria-controls="z3950_search" role="tab" aria-expanded="true" data-toggle="tab" class="keep_text"><i class="fa fa-search"></i> <span class="tab-title">Search OAI repositories</span></a>
        </li>
        [% IF ( CAN_user_circulate_circulate_remaining_permissions ) %]
            <li role="presentation">
                <a title="Check out" href="#circ_search" aria-controls="circ_search" role="tab" data-toggle="tab" class="keep_text"><i class="fa fa-upload"></i> <span class="tab-title">Check out</span></a>
            </li>
        [% END %]
        [% IF ( CAN_user_catalogue ) %]
            <li role="presentation">
                <a title="Search catalog" href="#catalog_search" aria-controls="catalog_search" role="tab" data-toggle="tab" class="keep_text"><i class="fa fa-fw fa-search"></i> <span class="tab-title">Search catalog</span></a>
            </li>
        [% END %]
    </ul>
</div><!-- /#header_search -->
<!-- /oai-admin-search.inc -->

Lines 244-250 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/permissions.inc (-1 / +1 lines)
244	<span class="permissioncode">([% name \| html %])</span>	244	<span class="permissioncode">([% name \| html %])</span>
245	[%- CASE 'manage_search_targets' -%]	245	[%- CASE 'manage_search_targets' -%]
246	<span class="sub_permission manage_search_targets_subpermission">	246	<span class="sub_permission manage_search_targets_subpermission">
247	Manage Z39.50 and SRU server configuration	247	Manage Z39.50 and SRU servers, OAI repositories configuration
248	</span>	248	</span>
249	<span class="permissioncode">([% name \| html %])</span>	249	<span class="permissioncode">([% name \| html %])</span>
250	[%- CASE 'manage_didyoumean' -%]	250	[%- CASE 'manage_didyoumean' -%]

Lines 248-253 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt (+2 lines)
248	[% IF ( CAN_user_parameters_manage_search_targets ) %]	248	[% IF ( CAN_user_parameters_manage_search_targets ) %]
249	<dt><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50/SRU servers</a></dt>	249	<dt><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50/SRU servers</a></dt>
250	<dd>Define which external servers to query for MARC data</dd>	250	<dd>Define which external servers to query for MARC data</dd>
		251	<dt><a href="/cgi-bin/koha/admin/oaiservers.pl">OAI repositories</a></dt>
		252	<dd>Define which OAI repositories to harvest data from</dd>
251	[% END %]	253	[% END %]
252	[% IF ( CAN_user_parameters_manage_smtp_servers ) %]	254	[% IF ( CAN_user_parameters_manage_smtp_servers ) %]
253	<dt><a href="/cgi-bin/koha/admin/smtp_servers.pl">SMTP servers</a></dt>	255	<dt><a href="/cgi-bin/koha/admin/smtp_servers.pl">SMTP servers</a></dt>




[% USE raw %]
[% USE Asset %]
[% SET footerjs = 1 %]
[% INCLUDE 'doc-head-open.inc' %]

<title>
    [% IF op == 'edit' %]
        Modify OAI repository [% server.servername | html %] &rsaquo; [% END %]

    [% IF op ==  'add' %]
        New OAI repository &rsaquo; [% END %]

    OAI repositories &rsaquo; Administration &rsaquo; Koha
</title>
[% INCLUDE 'doc-head-close.inc' %]
</head>

<body id="admin_oaiservers" class="admin">
[% WRAPPER 'header.inc' %]
    [% INCLUDE 'oai-admin-search.inc' %]
[% END %]

[% WRAPPER 'sub-header.inc' %]
    [% WRAPPER breadcrumbs %]
        [% WRAPPER breadcrumb_item %]
            <a href="/cgi-bin/koha/admin/admin-home.pl">Administration</a>
        [% END %]

        [% IF op == 'edit' || op == 'add' %]
            [% WRAPPER breadcrumb_item %]
                <a href="/cgi-bin/koha/admin/oaiservers.pl">OAI repositories</a>
           [% END %]
        [% END %]

        [% IF op == 'edit' %]
            [% WRAPPER breadcrumb_item bc_active= 1 %]
                <span>Modify OAI repository [% server.servername | html %]</span>
           [% END %]

        [% ELSIF op == 'add' %]
            [% WRAPPER breadcrumb_item bc_active= 1 %]
                <span>New OAI repository</span>
           [% END %]

        [% ELSE %]
            [% WRAPPER breadcrumb_item bc_active= 1 %]
                <span>OAI repositories</span>
           [% END %]
        [% END %]
    [% END #/ WRAPPER breadcrumbs %]
[% END #/ WRAPPER sub-header.inc %]

<div class="main container-fluid">
    <div class="row">
        <div class="col-sm-10 col-sm-push-2">
            <main>

[% IF msg_deleted %]
    <div class="dialog alert">OAI repository deleted ([% msg_add | html %])</div>
[% ELSIF msg_updated %]
    <div class="dialog message">OAI repository updated ([% msg_add | html %])</div>
[% ELSIF msg_added %]
    <div class="dialog message">OAI repository added ([% msg_add | html %])</div>
[% ELSIF msg_notfound %]
    <div class="dialog alert">Error: Server with ID [% msg_add | html %] not found</div>
[% END %]

[% IF ( add_form ) %]
    <form action="[% script_name | html %]" name="Aform" method="post" id="serverentry">
        <input type="hidden" name="op" value="add_validated" />
        [% IF op == 'edit' %]
            <h1>Modify OAI repository</h1>
            <input type="hidden" name="id" value="[% server.id | html %]" />
        [% ELSE %]
            <h1>New OAI repository</h1>
        [% END %]
        <fieldset class="rows">
        <ol>
        <li><label for="name" class="required">Repository name: </label>
            [% IF server.servername %]
                <input type="text" name="servername" id="servername" size="65" maxlength="100" value="[% server.servername | html %]" required="required" />
            [% ELSE %]
                <input type="text" name="servername" id="servername" class="focus" size="65" maxlength="100" value="[% server.servername | html %]" required="required" />
            [% END %]
             <span class="required">Required</span>
        </li>

        <li><label for="endpoint" class="required">Endpoint: </label> <input type="text" name="endpoint" id="endpoint" size="30" value="[% server.endpoint | html %]" required="required" /> <span class="required">Required</span>
        </li>
        <li><label for="oai_set">Set: </label> <input type="text" name="oai_set" id="oai_set" value="[% server.oai_set | html %]" />
        </li>
        <li>
            <label for="dataformat">Data format: </label>
        <select name="dataformat" id="dataformat">
            <option value="oai_dc">Dublin Core (oai_dc)</option>
            <option value="marc-xml">Marcxml (marc-xml)</option>
            <option value="marcxml">Marcxml (marcxml)</option>
        </select>
            <div class="hint">OAI can send records in various formats. Choose one.</div>
        </li>

        <li><label for="recordtype">Record type: </label>
        <select name="recordtype" id="recordtype">
            <option value="biblio">Bibliographic</option>
            <option value="authority">Authority</option>
        </select>
        </li>
        <li>
        <label for="add_xslt">XSLT file for transforming results: </label>
        <input type="text" name="add_xslt" id="add_xslt" size="100" placeholder="[% xslt_path | html %]" value="[% server.add_xslt | html %]"/>
        </li>

        </ol>
        </fieldset>

        <fieldset class="action"><input type="submit" class="btn btn-primary" value="Save" /> <a class="cancel" href="/cgi-bin/koha/admin/oaiservers.pl">Cancel</a></fieldset>
    </form>
[% END %]

[% IF op == 'list' %]
    <div id="toolbar" class="btn-toolbar">
        <a id="newserver" class="btn btn-default" href="/cgi-bin/koha/admin/oaiservers.pl?op=add"><i class="fa fa-plus"></i> New OAI repository</a>
    </div>
    <h1>OAI repositories administration</h1>
    [% IF id %]
        You searched for record [% id | html %]
    [% ELSIF searchfield %]
        You searched for [% searchfield | html %]
    [% END %]

    <div class="page-section">
    <table id="serverst">

        <thead><tr><th>Id</th><th>Repository name</th><th>Endpoint</th><th>Set</th><th>Data format</th><th>Record type</th><th class="noExport">Actions</th>
        </tr></thead>
        <tbody>
        [% FOREACH loo IN loop %]
            <tr>
            <td>[% loo.id | html %]</td>
            <td><a href="/cgi-bin/koha/admin/oaiservers.pl?op=edit&amp;id=[% loo.id | uri %]">[% loo.servername | html %]</a></td><td>[% loo.endpoint | html %]</td><td>[% loo.oai_set | html %]</td>
            <td>[% loo.dataformat | html %]</td>
            <td>[% IF ( loo.recordtype == 'biblio' )  %]
                <span>Bibliographic</span>
                [% ELSIF ( loo.recordtype == 'authority' ) %]
                <span>Authority</span>
                [% END %]
            </td>
            <td>
                <div class="btn-group dropup">
                    <a class="btn btn-default btn-xs dropdown-toggle" id="reportactions[% savedreport.id | html %]" role="button" data-toggle="dropdown" href="#">
                        Actions <b class="caret"></b>
                    </a>
                    <ul class="dropdown-menu pull-right" role="menu" aria-labelledby="reportactions[% savedreport.id | html %]">
                        <li><a href="/cgi-bin/koha/admin/oaiservers.pl?op=edit&amp;id=[% loo.id | uri %]"><i class="fa-solid fa-pencil" aria-hidden="true"></i> Edit</a></li>
                        <li><a href="/cgi-bin/koha/admin/oaiservers.pl?op=add&amp;id=[% loo.id | uri %]"><i class="fa fa-copy"></i> Copy</a></li>
                        <li><a href="/cgi-bin/koha/admin/oaiservers.pl?op=delete_confirmed&amp;id=[% loo.id | uri %]" class="delete" data-servername="[% loo.servername | html %]"><i class="fa fa-trash-can"></i> Delete</a></li>
                    </ul>
                </div>
            </td>
            </tr>
        [% END %]
        </tbody>
    </table>
    </div> <!-- /.page-section -->
[% END %]

            </main>
        </div> <!-- /.col-sm-10.col-sm-push-2 -->

        <div class="col-sm-2 col-sm-pull-10">
            <aside>
                [% INCLUDE 'admin-menu.inc' %]
            </aside>
        </div> <!-- /.col-sm-2.col-sm-pull-10 -->
     </div> <!-- /.row -->

[% MACRO jsinclude BLOCK %]
    [% Asset.js("js/admin-menu.js") | $raw %]
    [% IF op == 'list' %]
        [% INCLUDE 'datatables.inc' %]
    [% END %]

    <script>
        [% IF ( add_form ) %]
            $(document).ready(function(){
                // Update selects for dataformat and recordtype
                [% IF server %]
                    $("#dataformat").val('[% server.dataformat | html %]');
                    $("#recordtype").val('[% server.recordtype | html %]');
                [% END %]
                $( "#serverentry" ).validate({
                    rules: {
                        servername: { required: true },
                        host: { required: true },
                        port: {
                            number: true
                        },
                        rank: { number: true },
                    }
                });
                $("#serverentry").submit(function( event ) {
                    // enable recordtype to include field in post
                    $('#recordtype').prop('disabled',false);
                });
                $("#servername").on("blur",function(){
                    toUC(this);
                });
            });
        [% ELSE %]
            $(document).ready(function() {
                $("#serverst").dataTable($.extend(true, {}, dataTablesDefaults, {
                    "aoColumnDefs": [
                        { "aTargets": [-1], "bSortable": false, "bSearchable": false },
                    ],
                    "sPaginationType": "full"
                }));
                $(".delete").on("click",function(e){
                    var servername = $(this).data("servername");
                    if( confirm( _("Are you sure you want to delete repository %s?").format(servername) ) ) {
                        return true;
                    } else {
                        e.preventDefault();
                    }
                });
            });
        [% END %]
    </script>
[% END %]
[% INCLUDE 'intranet-bottom.inc' %]

Lines 80-85 Web services: Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref (+5 lines)
80	transient: might be emptied or truncated at some point (transient)	80	transient: might be emptied or truncated at some point (transient)
81	no: will never have any data in it (no)	81	no: will never have any data in it (no)
82	- "."	82	- "."
		83	-
		84	- If set, a report email will be sent to this address after an OAI harvest
		85	- pref: "OAI-PMH:HarvestEmailReport"
		86	class: Text
		87
83	ILS-DI:	88	ILS-DI:
84	-	89	-
85	- pref: ILS-DI	90	- pref: ILS-DI




<?xml version="1.0" encoding="UTF-8"?>
<xsl:stylesheet version="1.0"
    xmlns:dc="http://purl.org/dc/elements/1.1/"
    xmlns:dcterms="http://purl.org/dc/terms/1.1"
    xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/
        http://www.openarchives.org/OAI/2.0/oai_dc.xsd"
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
    xmlns="http://www.loc.gov/MARC21/slim"  exclude-result-prefixes="dc dcterms oai_dc">

    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>

    <xsl:template match="/">
        <collection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" >
            <xsl:apply-templates />
        </collection>
    </xsl:template>

    <xsl:template match="oai_dc:dc">
        <record>

            <xsl:variable name="FamilyName" select="dc:FamilyName"/>
            <xsl:variable name="GivenName" select="dc:GivenName"/>

            <datafield tag="100" ind1="0" ind2=" ">
                <subfield code="a">
                    <xsl:value-of select="concat($FamilyName,', ',$GivenName)"/>
                </subfield>
            </datafield>

        </record>
   </xsl:template>

</xsl:stylesheet>





<?xml version="1.0" encoding="UTF-8"?>
<xsl:stylesheet version="1.0"
    xmlns:dc="http://purl.org/dc/elements/1.1/"
    xmlns:dcterms="http://purl.org/dc/terms/1.1"
    xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/
        http://www.openarchives.org/OAI/2.0/oai_dc.xsd"
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
    xmlns="http://www.loc.gov/MARC21/slim"  exclude-result-prefixes="dc dcterms oai_dc">

    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>

    <xsl:template match="/">
        <collection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" >
            <xsl:apply-templates />
        </collection>
    </xsl:template>

    <xsl:template match="oai_dc:dc">
        <record>

            <xsl:for-each select="dc:FamilyName">
                <datafield tag="200" ind1="0" ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>

            <xsl:for-each select="dc:GivenName">
                <datafield tag="200" ind1="0" ind2=" ">
                    <subfield code="b">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>

        </record>
   </xsl:template>

</xsl:stylesheet>




<?xml version="1.0" encoding="UTF-8"?>
<xsl:stylesheet version="1.0" 
    xmlns:dc="http://purl.org/dc/elements/1.1/" 
    xmlns:dcterms="http://purl.org/dc/terms/1.1" 
    xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" 
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/
        http://www.openarchives.org/OAI/2.0/oai_dc.xsd"
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
    xmlns="http://www.loc.gov/MARC21/slim"  exclude-result-prefixes="dc dcterms oai_dc">
 
    <xsl:import href="MARC21slimUtils.xsl"/>
    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>
    
  
    <xsl:template match="/">
        <collection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" >
            <xsl:apply-templates />
        </collection>
    </xsl:template>
    
    <xsl:template name="OAI-PMH">
             <xsl:for-each select = "ListRecords/record/metadata/oai_dc:dc">
                <xsl:apply-templates  />
             </xsl:for-each>
             <xsl:for-each select = "GetRecord/record/metadata/oai_dc:dc">
                <xsl:apply-templates  />
             </xsl:for-each>
    </xsl:template>

    <xsl:template match="text()" />
    <xsl:template match="oai_dc:dc">
        <record>
            <xsl:element name="leader">
                <xsl:variable name="type" select="dc:type"/>
                <xsl:variable name="leader06">
                    <xsl:choose>
                        <xsl:when test="$type='collection'">p</xsl:when>
                        <xsl:when test="$type='dataset'">m</xsl:when>
                        <xsl:when test="$type='event'">r</xsl:when>
                        <xsl:when test="$type='image'">k</xsl:when>
                        <xsl:when test="$type='interactive resource'">m</xsl:when>
                        <xsl:when test="$type='service'">m</xsl:when>
                        <xsl:when test="$type='software'">m</xsl:when>
                        <xsl:when test="$type='sound'">i</xsl:when>
                        <xsl:when test="$type='text'">a</xsl:when>
                        <xsl:otherwise>a</xsl:otherwise>
                    </xsl:choose>
                </xsl:variable>
                <xsl:variable name="leader07">
                    <xsl:choose>
                        <xsl:when test="$type='collection'">c</xsl:when>
                        <xsl:otherwise>m</xsl:otherwise>
                    </xsl:choose>
                </xsl:variable>
                <xsl:value-of select="concat('      ',$leader06,$leader07,'         3u     ')"/>
            </xsl:element>

            <datafield tag="042" ind1=" " ind2=" ">
                <subfield code="a">dc</subfield>
            </datafield>


            
            <xsl:for-each select="dc:creator">
                <xsl:choose>
                    <xsl:when test="(.!='') and (position()=1)">
                        <xsl:call-template name="persname_template">
                            <xsl:with-param name="string" select="." />
                            <xsl:with-param name="field" select="'100'" />
                            <xsl:with-param name="ind1" select = "'1'" />
                            <xsl:with-param name="ind2" select = "'0'" />
                            <xsl:with-param name="type" select="'author'" />
                        </xsl:call-template>
                    </xsl:when>
                    <xsl:otherwise>
                        <xsl:if test=".!=''">
                            <xsl:call-template name="persname_template">
                                <xsl:with-param name="string" select="." />
                                <xsl:with-param name="field" select="'700'" />
                                <xsl:with-param name="ind1" select = "'1'" />
                                <xsl:with-param name="ind2" select = "'0'" />
                                <xsl:with-param name="type" select="'author'" />
                            </xsl:call-template>
                        </xsl:if>
                    </xsl:otherwise>
                </xsl:choose>
            </xsl:for-each>
            
        
            <xsl:for-each select="dc:title[1]">
                <datafield tag="245" ind1="0" ind2="0">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>

            <xsl:for-each select="dc:title[position()>1]">
                <xsl:if test=".!=''">
                    <datafield tag="246" ind1="3" ind2="3">
                        <subfield code="a">
                            <xsl:value-of select="."/>
                        </subfield>
                    </datafield>
                </xsl:if>
            </xsl:for-each>

            <xsl:choose>
                <xsl:when test="dc:publisher">
                    <xsl:if test="translate(dc:publisher/.,'.,:;','')!=''">
                        <datafield tag="260" ind1=" " ind2=" ">
                        <xsl:choose>
                            <xsl:when test="dc:date">   
                                <subfield code="b"><xsl:value-of select="dc:publisher[1]"/>, </subfield>    
                                <xsl:if test="translate(dc:date[1]/., '.,:;','')!=''">
                                    <subfield code="c"><xsl:value-of select="dc:date[1]" />.</subfield>
                                </xsl:if>
                            </xsl:when>
                            <xsl:otherwise>
                                <subfield code="b"><xsl:value-of select="dc:publisher[1]"/>.</subfield> 
                            </xsl:otherwise>
                        </xsl:choose>
                        </datafield>
                    </xsl:if>
                </xsl:when>
                <xsl:otherwise>
                    <xsl:if test="translate(dc:date[1],'.,:;','')!=''">
                        <datafield tag="260" ind1=" " ind2=" ">
                            <subfield code="c"><xsl:value-of select="dc:date[1]" />.</subfield>
                        </datafield>
                    </xsl:if>
                </xsl:otherwise>
            </xsl:choose>

            <xsl:for-each select="dc:coverage">
                <xsl:choose>
                    <xsl:when test="translate(., '0123456789-.?','')=''">
                        <!--Likely;this is a date-->
                        <datafield tag="500" ind1=" " ind2=" ">
                            <subfield code="a"><xsl:value-of select="."/></subfield>
                        </datafield>
                    </xsl:when>
                    <xsl:otherwise>
                        <!--likely a geographic subject, we will print this later-->
                    </xsl:otherwise>
                </xsl:choose>
            </xsl:for-each>

            <xsl:for-each select="dc:identifier">
                <xsl:if test="position()!=last()">
                    <datafield tag="500" ind1=" " ind2=" ">
                        <subfield code="a"><xsl:value-of select="." /></subfield>
                    </datafield>
                </xsl:if>
            </xsl:for-each>
            
            <xsl:for-each select="dc:description">
                <datafield tag="520" ind1=" " ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="normalize-space(.)"/>
                    </subfield>
                </datafield>
            </xsl:for-each>


            <xsl:for-each select="dc:rights">
                <datafield tag="540" ind1=" " ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>


            <xsl:for-each select="dc:language">
                <datafield tag="546" ind1=" " ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>



            <xsl:for-each select="dc:subject">
                <datafield tag="690" ind1=" " ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>
            
            <xsl:for-each select="dc:coverage">
                <xsl:choose>
                    <xsl:when test="translate(., '0123456789-.?','')=''">
                        <!--Likely; this is a date-->
                    </xsl:when>
                    <xsl:otherwise>
                        <!--likely a geographic subject-->
                        <datafield tag="691" ind1=" " ind2=" ">
                            <subfield code="a"><xsl:value-of select="." /></subfield>
                        </datafield>
                    </xsl:otherwise>
                </xsl:choose>
            </xsl:for-each>
                                                                                            

            <xsl:for-each select="dc:type">
                <datafield tag="655" ind1="7" ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                    <subfield code="2">local</subfield>
                </datafield>
            </xsl:for-each>

            

            <xsl:for-each select="dc:contributer">
                    <xsl:call-template name="persname_template">
                        <xsl:with-param name="string" select="." />
                        <xsl:with-param name="field" select="'100'" />
                        <xsl:with-param name="ind1" select = "'1'" />
                        <xsl:with-param name="ind2" select = "'0'" />
                        <xsl:with-param name="type" select="'contributor'" />
                    </xsl:call-template>
            </xsl:for-each>
            
            <xsl:for-each select="dc:source">
                <datafield tag="786" ind1="0" ind2=" ">
                    <subfield code="n">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>

            <xsl:for-each select="dc:relation">
                <datafield tag="787" ind1="0" ind2=" ">
                    <subfield code="n">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>

            <xsl:if test="dc:identifier">
                <datafield tag="856" ind1="4" ind2="1">
                    <subfield code="u"><xsl:value-of select="dc:identifier[last()]" /></subfield>
                    <subfield code="z">Connect to this object online.</subfield>
                </datafield>
            </xsl:if>
        </record>       

    </xsl:template>
<xsl:template name="persname_template">
      <xsl:param name="string" />
      <xsl:param name="field" />
      <xsl:param name="ind1" />
      <xsl:param name="ind2" />
      <xsl:param name="type" />
      <datafield>
         <xsl:attribute name="tag">
            <xsl:value-of select="$field" />
         </xsl:attribute>
         <xsl:attribute name="ind1">
            <xsl:value-of select="$ind1" />
         </xsl:attribute>
         <xsl:attribute name="ind2">
            <xsl:value-of select="$ind2" />
         </xsl:attribute>

         <!-- Sample input: Brightman, Samuel C. (Samuel Charles), 1911-1992 -->
         <!-- Sample output: $aBrightman, Samuel C. $q(Samuel Charles), $d1911-. -->
         <!-- will handle names with dashes e.g. Bourke-White, Margaret -->

         <!-- CAPTURE PRIMARY NAME BY LOOKING FOR A PAREN OR A DASH OR NEITHER -->
         <xsl:choose>
            <!-- IF A PAREN, STOP AT AN OPENING PAREN -->
            <xsl:when test="contains($string, '(')!=0">
               <subfield code="a">
                  <xsl:value-of select="substring-before($string, '(')" />
               </subfield>
            </xsl:when>
            <!-- IF A DASH, CHECK IF IT'S A DATE OR PART OF THE NAME -->
            <xsl:when test="contains($string, '-')!=0">
               <xsl:variable name="name_1" select="substring-before($string, '-')" />
               <xsl:choose>
                  <!-- IF IT'S A DATE REMOVE IT -->
                  <xsl:when test="translate(substring($name_1, (string-length($name_1)), 1), '0123456789', '9999999999') = '9'">
                     <xsl:variable name="name" select="substring($name_1, 1, (string-length($name_1)-6))" />
                     <subfield code="a">
                        <xsl:value-of select="$name" />
                     </subfield>
                  </xsl:when>
                  <!-- IF IT'S NOT A DATE, CHECK WHETHER THERE IS A DATE LATER -->
                  <xsl:otherwise>
                     <xsl:variable name="remainder" select="substring-after($string, '-')" />
                     <xsl:choose>
                        <!-- IF THERE'S A DASH, ASSUME IT'S A DATE AND REMOVE IT -->
                        <xsl:when test="contains($remainder, '-')!=0">
                           <xsl:variable name="tmp" select="substring-before($remainder, '-')" />
                           <xsl:variable name="name_2" select="substring($tmp, 1, (string-length($tmp)-6))" />
                           <subfield code="a">
                              <xsl:value-of select="$name_1" />-<xsl:value-of select="$name_2" />
                           </subfield>
                        </xsl:when>
                        <!-- IF THERE'S NO DASH IN THE REMAINDER, OUTPUT IT -->
                        <xsl:otherwise>
                           <subfield code="a">
                              <xsl:value-of select="$string" />
                           </subfield>
                        </xsl:otherwise>
                     </xsl:choose>
                  </xsl:otherwise>
               </xsl:choose>
            </xsl:when>
            <!-- NO DASHES, NO PARENS, JUST OUTPUT THE NAME -->
            <xsl:otherwise>
               <subfield code="a">
                  <xsl:value-of select="$string" />
               </subfield>
            </xsl:otherwise>
         </xsl:choose>

         <!-- CAPTURE SECONDARY NAME IN PARENS FOR SUBFIELD Q -->
         <xsl:if test="contains($string, '(')!=0">
            <xsl:variable name="subq_tmp" select="substring-after($string, '(')" />
            <xsl:variable name="subq" select="substring-before($subq_tmp, ')')" />
            <subfield code="q">
               (<xsl:value-of select="$subq" />)
            </subfield>
         </xsl:if>

         <!-- CAPTURE DATE FOR SUBFIELD D, ASSUME DATE IS LAST ITEM IN FIELD -->
         <!-- Note: does not work if name has a dash in it -->
         <xsl:if test="contains($string, '-')!=0">
            <xsl:variable name="date_tmp" select="substring-before($string, '-')" />
            <xsl:variable name="remainder" select="substring-after($string, '-')" />
            <xsl:choose>
               <!-- CHECK SECOND HALF FOR ANOTHER DASH; IF PRESENT, ASSUME THAT IS DATE -->
               <xsl:when test="contains($remainder, '-')!=0">
                  <xsl:variable name="tmp" select="substring-before($remainder, '-')" />
                  <xsl:variable name="date_1" select="substring($remainder, (string-length($tmp)-3))" />
                  <!-- CHECK WHETHER IT HAS A NUMBER BEFORE IT AND IF SO, OUTPUT IT AS DATE -->
                  <xsl:if test="translate(substring($date_1, 1, 1), '0123456789', '9999999999') = '9'">
                     <subfield code="d">
                        <xsl:value-of select="$date_1" />.
                     </subfield>
                  </xsl:if>
               </xsl:when>
               <!-- OTHERWISE THIS IS THE ONLY DASH SO TAKE IT -->
               <xsl:otherwise>
                  <xsl:variable name="date_2" select="substring($string, (string-length($date_tmp)-3))" />
                  <!-- CHECK WHETHER IT HAS A NUMBER BEFORE IT AND IF SO, OUTPUT IT AS DATE -->
                  <xsl:if test="translate(substring($date_2, 1, 1), '0123456789', '9999999999') = '9'">
                     <subfield code="d">
                        <xsl:value-of select="$date_2" />.
                     </subfield>
                  </xsl:if>
               </xsl:otherwise>
            </xsl:choose>
         </xsl:if>
         <subfield code="e"><xsl:value-of select="$type" /></subfield>
      </datafield>
   </xsl:template>
    
</xsl:stylesheet>




<?xml version="1.0" encoding="UTF-8"?>
<xsl:stylesheet version="1.0"
    xmlns:dc="http://purl.org/dc/elements/1.1/"
    xmlns:dcterms="http://purl.org/dc/terms/1.1"
    xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/
        http://www.openarchives.org/OAI/2.0/oai_dc.xsd"
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
    xmlns="http://www.loc.gov/MARC21/slim"  exclude-result-prefixes="dc dcterms oai_dc">

    <xsl:import href="MARC21slimUtils.xsl"/>
    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>


    <xsl:template match="/">
        <collection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" >
            <xsl:apply-templates />
        </collection>
    </xsl:template>

    <xsl:template name="OAI-PMH">
             <xsl:for-each select = "ListRecords/record/metadata/oai_dc:dc">
                <xsl:apply-templates  />
             </xsl:for-each>
             <xsl:for-each select = "GetRecord/record/metadata/oai_dc:dc">
                <xsl:apply-templates  />
             </xsl:for-each>
    </xsl:template>

    <xsl:template match="text()" />
    <xsl:template match="oai_dc:dc">
        <record>

            <xsl:for-each select="dc:creator">
                <xsl:call-template name="persname_template">
                    <xsl:with-param name="string" select="." />
                    <xsl:with-param name="field" select="'700'" />
                    <xsl:with-param name="ind1" select = "'1'" />
                    <xsl:with-param name="ind2" select = "'0'" />
                    <xsl:with-param name="type" select="'author'" />
                </xsl:call-template>
            </xsl:for-each>


            <xsl:for-each select="dc:title[1]">
                <datafield tag="200" ind1="0" ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>

            <xsl:for-each select="dc:title[position()>1]">
                <xsl:if test=".!=''">
                    <datafield tag="532" ind1="3" ind2="0">
                        <subfield code="a">
                            <xsl:value-of select="."/>
                        </subfield>
                    </datafield>
                </xsl:if>
            </xsl:for-each>

            <xsl:choose>
                <xsl:when test="dc:publisher">
                    <xsl:if test="translate(dc:publisher/.,'.,:;','')!=''">
                        <datafield tag="214" ind1=" " ind2=" ">
                        <xsl:choose>
                            <xsl:when test="dc:date">
                                <subfield code="c"><xsl:value-of select="dc:publisher[1]"/>, </subfield>
                                <xsl:if test="translate(dc:date[1]/., '.,:;','')!=''">
                                    <subfield code="d"><xsl:value-of select="dc:date[1]" />.</subfield>
                                </xsl:if>
                            </xsl:when>
                            <xsl:otherwise>
                                <subfield code="c"><xsl:value-of select="dc:publisher[1]"/>.</subfield>
                            </xsl:otherwise>
                        </xsl:choose>
                        </datafield>
                    </xsl:if>
                </xsl:when>
                <xsl:otherwise>
                    <xsl:if test="translate(dc:date[1],'.,:;','')!=''">
                        <datafield tag="214" ind1=" " ind2=" ">
                            <subfield code="d"><xsl:value-of select="dc:date[1]" />.</subfield>
                        </datafield>
                    </xsl:if>
                </xsl:otherwise>
            </xsl:choose>


            <xsl:for-each select="dc:contributor">
                <datafield tag="701" ind1=" " ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>


            <xsl:for-each select="dc:coverage">
                <xsl:choose>
                    <xsl:when test="translate(., '0123456789-.?','')=''">
                        <!--Likely;this is a date-->
                        <datafield tag="215" ind1=" " ind2=" ">
                            <subfield code="a"><xsl:value-of select="."/></subfield>
                        </datafield>
                    </xsl:when>
                    <xsl:otherwise>
                        <!--likely a geographic subject, we will print this later-->
                    </xsl:otherwise>
                </xsl:choose>
            </xsl:for-each>


            <xsl:for-each select="dc:description">
                <datafield tag="330" ind1=" " ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="normalize-space(.)"/>
                    </subfield>
                </datafield>
            </xsl:for-each>


            <!-- no match -->
            <xsl:for-each select="dc:rights">
                <datafield tag="371" ind1=" " ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>


            <xsl:for-each select="dc:language">
                <datafield tag="101" ind1=" " ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>


            <xsl:for-each select="dc:subject">
                <datafield tag="610" ind1=" " ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>


            <!-- Resource type will most likely need a specific instance-based mapping -->
            <!--
            <xsl:for-each select="dc:type">
                <datafield tag="099" ind1=" " ind2=" ">
                    <subfield code="t">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>
            -->


            <!-- no match -->
            <xsl:for-each select="dc:coverage">
                <xsl:choose>
                    <xsl:when test="translate(., '0123456789-.?','')=''">
                        <!--Likely; this is a date-->
                    </xsl:when>
                    <xsl:otherwise>
                        <!--likely a geographic subject-->
                        <datafield tag="691" ind1=" " ind2=" ">
                            <subfield code="a"><xsl:value-of select="." /></subfield>
                        </datafield>
                    </xsl:otherwise>
                </xsl:choose>
            </xsl:for-each>


            <!-- no match -->
            <xsl:for-each select="dc:type">
                <datafield tag="655" ind1="7" ind2=" ">
                    <subfield code="a">
                        <xsl:value-of select="."/>
                    </subfield>
                    <subfield code="2">local</subfield>
                </datafield>
            </xsl:for-each>


            <xsl:for-each select="dc:contributer">
                    <xsl:call-template name="persname_template">
                        <xsl:with-param name="string" select="." />
                        <xsl:with-param name="field" select="'702'" />
                        <xsl:with-param name="ind1" select = "'1'" />
                        <xsl:with-param name="ind2" select = "'0'" />
                        <xsl:with-param name="type" select="'contributor'" />
                    </xsl:call-template>
            </xsl:for-each>

            <!-- no match -->
            <xsl:for-each select="dc:source">
                <datafield tag="786" ind1="0" ind2=" ">
                    <subfield code="n">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>

            <!-- TODO: 455 456 470 488-->
            <xsl:for-each select="dc:relation">
                <datafield tag="488" ind1="0" ind2=" ">
                    <subfield code="e">
                        <xsl:value-of select="."/>
                    </subfield>
                </datafield>
            </xsl:for-each>

            <xsl:if test="dc:identifier">
                <datafield tag="856" ind1="4" ind2="1">
                    <subfield code="u"><xsl:value-of select="dc:identifier[last()]" /></subfield>
                    <subfield code="z">Connect to this object online.</subfield>
                </datafield>
            </xsl:if>
        </record>

    </xsl:template>
<xsl:template name="persname_template">
      <xsl:param name="string" />
      <xsl:param name="field" />
      <xsl:param name="ind1" />
      <xsl:param name="ind2" />
      <xsl:param name="type" />
      <datafield>
         <xsl:attribute name="tag">
            <xsl:value-of select="$field" />
         </xsl:attribute>
         <xsl:attribute name="ind1">
            <xsl:value-of select="$ind1" />
         </xsl:attribute>
         <xsl:attribute name="ind2">
            <xsl:value-of select="$ind2" />
         </xsl:attribute>

         <!-- Sample input: Brightman, Samuel C. (Samuel Charles), 1911-1992 -->
         <!-- Sample output: $aBrightman, Samuel C. $q(Samuel Charles), $d1911-. -->
         <!-- will handle names with dashes e.g. Bourke-White, Margaret -->

         <!-- CAPTURE PRIMARY NAME BY LOOKING FOR A PAREN OR A DASH OR NEITHER -->
         <xsl:choose>
            <!-- IF A PAREN, STOP AT AN OPENING PAREN -->
            <xsl:when test="contains($string, '(')!=0">
               <subfield code="a">
                  <xsl:value-of select="substring-before($string, '(')" />
               </subfield>
            </xsl:when>
            <!-- IF A DASH, CHECK IF IT'S A DATE OR PART OF THE NAME -->
            <xsl:when test="contains($string, '-')!=0">
               <xsl:variable name="name_1" select="substring-before($string, '-')" />
               <xsl:choose>
                  <!-- IF IT'S A DATE REMOVE IT -->
                  <xsl:when test="translate(substring($name_1, (string-length($name_1)), 1), '0123456789', '9999999999') = '9'">
                     <xsl:variable name="name" select="substring($name_1, 1, (string-length($name_1)-6))" />
                     <subfield code="a">
                        <xsl:value-of select="$name" />
                     </subfield>
                  </xsl:when>
                  <!-- IF IT'S NOT A DATE, CHECK WHETHER THERE IS A DATE LATER -->
                  <xsl:otherwise>
                     <xsl:variable name="remainder" select="substring-after($string, '-')" />
                     <xsl:choose>
                        <!-- IF THERE'S A DASH, ASSUME IT'S A DATE AND REMOVE IT -->
                        <xsl:when test="contains($remainder, '-')!=0">
                           <xsl:variable name="tmp" select="substring-before($remainder, '-')" />
                           <xsl:variable name="name_2" select="substring($tmp, 1, (string-length($tmp)-6))" />
                           <subfield code="a">
                              <xsl:value-of select="$name_1" />-<xsl:value-of select="$name_2" />
                           </subfield>
                        </xsl:when>
                        <!-- IF THERE'S NO DASH IN THE REMAINDER, OUTPUT IT -->
                        <xsl:otherwise>
                           <subfield code="a">
                              <xsl:value-of select="$string" />
                           </subfield>
                        </xsl:otherwise>
                     </xsl:choose>
                  </xsl:otherwise>
               </xsl:choose>
            </xsl:when>
            <!-- NO DASHES, NO PARENS, JUST OUTPUT THE NAME -->
            <xsl:otherwise>
               <subfield code="a">
                  <xsl:value-of select="$string" />
               </subfield>
            </xsl:otherwise>
         </xsl:choose>

         <!-- CAPTURE SECONDARY NAME IN PARENS FOR SUBFIELD Q -->
         <xsl:if test="contains($string, '(')!=0">
            <xsl:variable name="subq_tmp" select="substring-after($string, '(')" />
            <xsl:variable name="subq" select="substring-before($subq_tmp, ')')" />
            <subfield code="q">
               (<xsl:value-of select="$subq" />)
            </subfield>
         </xsl:if>

         <!-- CAPTURE DATE FOR SUBFIELD D, ASSUME DATE IS LAST ITEM IN FIELD -->
         <!-- Note: does not work if name has a dash in it -->
         <xsl:if test="contains($string, '-')!=0">
            <xsl:variable name="date_tmp" select="substring-before($string, '-')" />
            <xsl:variable name="remainder" select="substring-after($string, '-')" />
            <xsl:choose>
               <!-- CHECK SECOND HALF FOR ANOTHER DASH; IF PRESENT, ASSUME THAT IS DATE -->
               <xsl:when test="contains($remainder, '-')!=0">
                  <xsl:variable name="tmp" select="substring-before($remainder, '-')" />
                  <xsl:variable name="date_1" select="substring($remainder, (string-length($tmp)-3))" />
                  <!-- CHECK WHETHER IT HAS A NUMBER BEFORE IT AND IF SO, OUTPUT IT AS DATE -->
                  <xsl:if test="translate(substring($date_1, 1, 1), '0123456789', '9999999999') = '9'">
                     <subfield code="d">
                        <xsl:value-of select="$date_1" />.
                     </subfield>
                  </xsl:if>
               </xsl:when>
               <!-- OTHERWISE THIS IS THE ONLY DASH SO TAKE IT -->
               <xsl:otherwise>
                  <xsl:variable name="date_2" select="substring($string, (string-length($date_tmp)-3))" />
                  <!-- CHECK WHETHER IT HAS A NUMBER BEFORE IT AND IF SO, OUTPUT IT AS DATE -->
                  <xsl:if test="translate(substring($date_2, 1, 1), '0123456789', '9999999999') = '9'">
                     <subfield code="d">
                        <xsl:value-of select="$date_2" />.
                     </subfield>
                  </xsl:if>
               </xsl:otherwise>
            </xsl:choose>
         </xsl:if>
         <subfield code="e"><xsl:value-of select="$type" /></subfield>
      </datafield>
   </xsl:template>

</xsl:stylesheet>




<?xml version="1.0" encoding="UTF-8"?>
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:marc="http://www.loc.gov/MARC21/slim">
  <xsl:output method="xml" version="1.0" encoding="UTF-8" indent="yes"/>

 <!-- keep comments -->
  <xsl:template match="*[name()='metadata']">
    <xsl:apply-templates />
  </xsl:template>

  <xsl:template match="*">
    <!-- remove element prefix -->
    <xsl:element name="{local-name()}">
      <!-- process attributes -->
      <xsl:for-each select="@*">
        <!-- remove attribute prefix -->
        <xsl:attribute name="{local-name()}">
          <xsl:value-of select="."/>
        </xsl:attribute>
      </xsl:for-each>
      <xsl:apply-templates/>
    </xsl:element>
  </xsl:template>
</xsl:stylesheet>




#!/usr/bin/perl
#
# Copyright 2023 BibLibre
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

use warnings;
use strict;
use utf8;

use Getopt::Long qw( GetOptions );
use Koha::Script -cron;
use Koha::OAI::Client::Harvester;
use Koha::OaiServers;
use C4::Log qw( cronlogaction );
use Try::Tiny qw( catch try );

my $command_line_options = join( " ", @ARGV );

my ( $help, $verbose, $id, $days, $list, $force );

GetOptions(
    'h|help'         => \$help,
    'v|verbose'      => \$verbose,
    'r|repository:i' => \$id,
    'd|days:i'       => \$days,
    'l|list'         => \$list,
    'f|force'        => \$force,
);
my $usage = << 'ENDUSAGE';

This script starts an OAI Harvest

This script has the following parameters :
    -h --help: this message
    -v --verbose
    -r --repository: id of the OAI repository
    -d --days: number of days to harvest from (optional)
    -l --list: list the OAI repositories
    -f --force: force harvesting (ignore records datestamps)

ENDUSAGE

if ($help) {
    print $usage;
    exit;
}

if ($list) {
    my $servers = Koha::OaiServers->search( {}, { order_by => { -asc => 'id' } } )->unblessed;
    print "The following repositories are available: \n\n";
    foreach my $server (@$servers) {
        print $server->{'id'} . ": "
            . $server->{'servername'}
            . ", endpoint: "
            . $server->{'endpoint'}
            . ", set: "
            . $server->{'oai_set'}
            . ", recordtype: "
            . $server->{'recordtype'} . "\n";
    }
    print "\n";
    exit;
}

if ( !$id ) {
    print "The repository parameter is mandatory.\n";
    print $usage . "\n";
}

my $script_handler = Koha::Script->new( { script => $0 } );

try {
    $script_handler->lock_exec;
} catch {
    my $message = "Skipping execution of $0";
    print "$message\n" if $verbose;
    cronlogaction( { info => $message } );
    exit;
};

cronlogaction( { action => 'Start', info => "Starting OAI Harvest" } );
cronlogaction( { info   => "Command line: $command_line_options" } );

my $harvester =
    Koha::OAI::Client::Harvester->new( { server => $id, verbose => $verbose, days => $days, force => $force } );

cronlogaction( { action => 'End', info => "Ending OAI Harvest" } );
exit(0);




#!/usr/bin/perl

# Copyright 2015 Koha Development team
#
# This file is part of Koha
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;
use Test::More tests => 2;
use Test::Exception;

use t::lib::TestBuilder;
use Koha::Database;
use Koha::OaiServer;
use Koha::OaiServers;

my $schema  = Koha::Database->new->schema;
my $builder = t::lib::TestBuilder->new;

subtest 'new, find and delete tests' => sub {
    plan tests => 4;
    $schema->storage->txn_begin;
    my $nb_of_oais = Koha::OaiServers->search->count;
    my $new_oai_1  = Koha::OaiServer->new(
        {
            endpoint   => 'my_host1.org',
            oai_set    => 'set1',
            servername => 'my_test_1',
            dataformat => 'oai_dc',
            recordtype => 'biblio',
        }
    )->store;
    my $new_oai_2 = Koha::OaiServer->new(
        {
            endpoint   => 'my_host2.org',
            oai_set    => 'set2',
            servername => 'my_test_2',
            dataformat => 'marcxml',
            recordtype => 'authority',
        }
    )->store;

    like( $new_oai_1->id, qr|^\d+$|, 'Adding a new oai server should have set the id' );
    is( Koha::OaiServers->search->count, $nb_of_oais + 2, 'The 2 oai servers should have been added' );

    my $retrieved_oai_1 = Koha::OaiServers->find( $new_oai_1->id );
    is(
        $retrieved_oai_1->servername, $new_oai_1->servername,
        'Find a oai server by id should return the correct oai server'
    );

    $retrieved_oai_1->delete;
    is( Koha::OaiServers->search->count, $nb_of_oais + 1, 'Delete should have deleted the oai server' );

    $schema->storage->txn_rollback;
};

subtest 'Check NOT NULL without default values' => sub {
    plan tests => 5;
    $schema->storage->txn_begin;
    local $SIG{__WARN__} = sub { };    # TODO Needed it for suppressing DBIx warns

    my $server = Koha::OaiServer->new(
        {
            oai_set    => 'set3',
            dataformat => 'marcxml',
            recordtype => 'biblio',
        }
    );

    throws_ok { $server->store } 'DBIx::Class::Exception', 'Exception on empty endpoint';
    like( $@->{msg}, qr/'endpoint' doesn't have a default value/, 'Verified that DBIx blamed endpoint' );

    $server->endpoint('endpoint_added');
    throws_ok { $server->store } 'DBIx::Class::Exception', 'Exception on empty servername';
    like( $@->{msg}, qr/'servername' doesn't have a default value/, 'Verified that DBIx blamed servername' );

    $server->servername('servername_added');

    lives_ok { $server->store } 'No exceptions anymore';

    $schema->storage->txn_rollback;
};

Return to bug 35659

Lines 2467-2469 tables: Link Here

(-)a/installer/data/mysql/en/mandatory/sample_notices.yml (+23 lines)
2467	- "Your hold on [% hold.biblio.title %] ([% hold.biblio.id %]) has been confirmed."	2467	- "Your hold on [% hold.biblio.title %] ([% hold.biblio.id %]) has been confirmed."
2468	- "You will be notified by the library when your item is available for pickup."	2468	- "You will be notified by the library when your item is available for pickup."
2469	- "Thank you!"	2469	- "Thank you!"
		2470
		2471	- module: catalogue
		2472	code: OAI_HARVEST_REPORT
		2473	branchcode: ""
		2474	name: "OAI Harvest Report"
		2475	is_html: 0
		2476	title: "OAI Harvest report for [% servername %]"
		2477	message_transport_type: email
		2478	lang: default
		2479	content:
		2480	- "OAI Harvest report for [% servername %]:"
		2481	- ""
		2482	- "Endpoint: [% endpoint %]"
		2483	- "Set: [% set %]"
		2484	- "Data format: [% dataformat %]"
		2485	- "Record type: [% recordtype %]"
		2486	- ""
		2487	- "[% added %] records added"
		2488	- "[% updated %] records updated"
		2489	- "[% deleted %] records deleted"
		2490	- "[% skipped %] records skipped"
		2491	- "[% in_error %] records in error"
		2492	- "[% total %] total"

Lines 4819-4824 CREATE TABLE `oai_sets_mappings` ( Link Here

(-)a/installer/data/mysql/kohastructure.sql (+58 lines)
4819	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;	4819	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
4820	/!40101 SET character_set_client = @saved_cs_client /;	4820	/!40101 SET character_set_client = @saved_cs_client /;
4821		4821
		4822	--
		4823	-- Table structure for table `oaiservers`
		4824	--
		4825
		4826	DROP TABLE IF EXISTS `oaiservers`;
		4827	/!40101 SET @saved_cs_client = @@character_set_client /;
		4828	/!40101 SET character_set_client = utf8 /;
		4829	CREATE TABLE `oaiservers` (
		4830	`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
		4831	`endpoint` varchar(255) NOT NULL COMMENT 'OAI endpoint (host + port + path)',
		4832	`oai_set` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
		4833	`servername` longtext NOT NULL COMMENT 'name given to the target by the library',
		4834	`dataformat` enum('oai_dc','marc-xml', 'marcxml') NOT NULL DEFAULT 'oai_dc' COMMENT 'data format',
		4835	`recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'server contains bibliographic or authority records',
		4836	`add_xslt` longtext DEFAULT NULL COMMENT 'zero or more paths to XSLT files to be processed on the search results',
		4837	PRIMARY KEY (`id`)
		4838	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
		4839
		4840	--
		4841	-- Table structure for table `import_oaipmh_biblios`
		4842	--
		4843
		4844	DROP TABLE IF EXISTS `import_oaipmh_biblios`;
		4845	/!40101 SET @saved_cs_client = @@character_set_client /;
		4846	/!40101 SET character_set_client = utf8 /;
		4847	CREATE TABLE `import_oaipmh_biblios` (
		4848	`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
		4849	`biblionumber` int(11) NOT NULL COMMENT 'unique identifier assigned to each koha record',
		4850	`identifier` varchar(255) NOT NULL COMMENT 'OAI record identifier',
		4851	`repository` varchar(255) NOT NULL COMMENT 'OAI repository',
		4852	`recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'is the record bibliographic or authority',
		4853	`datestamp` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
		4854	`last_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
		4855	PRIMARY KEY (id),
		4856	KEY biblionumber (biblionumber),
		4857	CONSTRAINT FK_import_oaipmh_biblios_1 FOREIGN KEY (biblionumber) REFERENCES biblio (biblionumber) ON DELETE CASCADE ON UPDATE NO ACTION
		4858	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
		4859
		4860	--
		4861	-- Table structure for table `import_oaipmh_authorities`
		4862	--
		4863
		4864	DROP TABLE IF EXISTS `import_oaipmh_authorities`;
		4865	/!40101 SET @saved_cs_client = @@character_set_client /;
		4866	/!40101 SET character_set_client = utf8 /;
		4867	CREATE TABLE `import_oaipmh_authorities` (
		4868	`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
		4869	`authid` bigint(20) unsigned NOT NULL COMMENT 'unique identifier assigned to each koha record',
		4870	`identifier` varchar(255) NOT NULL COMMENT 'OAI record identifier',
		4871	`repository` varchar(255) NOT NULL COMMENT 'OAI repository',
		4872	`recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'is the record bibliographic or authority',
		4873	`datestamp` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
		4874	`last_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
		4875	PRIMARY KEY (id),
		4876	KEY authid (authid),
		4877	CONSTRAINT FK_import_oaipmh_authorities_1 FOREIGN KEY (authid) REFERENCES auth_header (authid) ON DELETE CASCADE ON UPDATE NO ACTION
		4878	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
		4879
4822	--	4880	--
4823	-- Table structure for table `oauth_access_tokens`	4881	-- Table structure for table `oauth_access_tokens`
4824	--	4882	--

Line 0 Link Here

(-)a/Koha/Import/Oaipmh/Authorities.pm (+51 lines)
	1	package Koha::Import::Oaipmh::Authorities;
	2
	3	# This file is part of Koha.
	4	#
	5	# Koha is free software; you can redistribute it and/or modify it
	6	# under the terms of the GNU General Public License as published by
	7	# the Free Software Foundation; either version 3 of the License, or
	8	# (at your option) any later version.
	9	#
	10	# Koha is distributed in the hope that it will be useful, but
	11	# WITHOUT ANY WARRANTY; without even the implied warranty of
	12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	# GNU General Public License for more details.
	14	#
	15	# You should have received a copy of the GNU General Public License
	16
	17	use Modern::Perl;
	18
	19	use Koha::Import::Oaipmh::Authority;
	20
	21	use base qw(Koha::Objects);
	22
	23	=head1 NAME
	24
	25	Koha::Import::Oaipmh::Authorities
	26
	27	This object represents a collection of OAI-PMH records being imported as authorities
	28
	29	=head1 API
	30
	31	=head2 Methods
	32
	33
	34	=head3 type
	35
	36	=cut
	37
	38	sub _type {
	39	return 'ImportOaipmhAuthority';
	40	}
	41
	42	=head3 object_class
	43
	44	Koha::Object class
	45
	46	=cut
	47
	48	sub object_class {
	49	return 'Koha::Import::Oaipmh::Authority';
	50	}
	51	1;

Line 0 Link Here

(-)a/Koha/Import/Oaipmh/Authority.pm (+41 lines)
	1	package Koha::Import::Oaipmh::Authority;
	2
	3	# This file is part of Koha.
	4	#
	5	# Koha is free software; you can redistribute it and/or modify it
	6	# under the terms of the GNU General Public License as published by
	7	# the Free Software Foundation; either version 3 of the License, or
	8	# (at your option) any later version.
	9	#
	10	# Koha is distributed in the hope that it will be useful, but
	11	# WITHOUT ANY WARRANTY; without even the implied warranty of
	12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	# GNU General Public License for more details.
	14	#
	15	# You should have received a copy of the GNU General Public License
	16
	17	use Modern::Perl;
	18
	19	use base qw(Koha::Object);
	20
	21	=head1 NAME
	22
	23	Koha::Import::Oaipmh::Authority
	24
	25	This object represents an OAI-PMH record being imported as an authority
	26
	27	=head1 API
	28
	29	=head2 Class Methods
	30
	31	=cut
	32
	33	=head3 type
	34
	35	=cut
	36
	37	sub _type {
	38	return 'ImportOaipmhAuthority';
	39	}
	40
	41	1;

Line 0 Link Here

(-)a/Koha/Import/Oaipmh/Biblio.pm (+40 lines)
	1	package Koha::Import::Oaipmh::Biblio;
	2
	3	# This file is part of Koha.
	4	#
	5	# Koha is free software; you can redistribute it and/or modify it
	6	# under the terms of the GNU General Public License as published by
	7	# the Free Software Foundation; either version 3 of the License, or
	8	# (at your option) any later version.
	9	#
	10	# Koha is distributed in the hope that it will be useful, but
	11	# WITHOUT ANY WARRANTY; without even the implied warranty of
	12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	# GNU General Public License for more details.
	14	#
	15	# You should have received a copy of the GNU General Public License
	16
	17	use Modern::Perl;
	18
	19	use base qw(Koha::Object);
	20
	21	=head1 NAME
	22
	23	Koha::Import::Oaipmh::Biblio
	24
	25	This object represents an OAI-PMH record being imported as a bibliographic record
	26
	27	=head1 API
	28
	29	=head2 Methods
	30
	31
	32	=head3 type
	33
	34	=cut
	35
	36	sub _type {
	37	return 'ImportOaipmhBiblio';
	38	}
	39
	40	1;

Line 0 Link Here

(-)a/Koha/Import/Oaipmh/Biblios.pm (+51 lines)
	1	package Koha::Import::Oaipmh::Biblios;
	2
	3	# This file is part of Koha.
	4	#
	5	# Koha is free software; you can redistribute it and/or modify it
	6	# under the terms of the GNU General Public License as published by
	7	# the Free Software Foundation; either version 3 of the License, or
	8	# (at your option) any later version.
	9	#
	10	# Koha is distributed in the hope that it will be useful, but
	11	# WITHOUT ANY WARRANTY; without even the implied warranty of
	12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	# GNU General Public License for more details.
	14	#
	15	# You should have received a copy of the GNU General Public License
	16
	17	use Modern::Perl;
	18
	19	use Koha::Import::Oaipmh::Biblio;
	20
	21	use base qw(Koha::Objects);
	22
	23	=head1 NAME
	24
	25	Koha::Import::Oaipmh::Biblios
	26
	27	This object represents a collection of OAI-PMH records being imported as bibliographic records
	28
	29	=head1 API
	30
	31	=head2 Methods
	32
	33
	34	=head3 type
	35
	36	=cut
	37
	38	sub _type {
	39	return 'ImportOaipmhBiblio';
	40	}
	41
	42	=head3 object_class
	43
	44	Koha::Object class
	45
	46	=cut
	47
	48	sub object_class {
	49	return 'Koha::Import::Oaipmh::Biblio';
	50	}
	51	1;

Line 0 Link Here

(-)a/Koha/OAI/Client/Harvester.pm (+322 lines)
		1	# Copyright BibLibre 2023
		2	#
		3	# This file is part of Koha.
		4	#
		5	# Koha is free software; you can redistribute it and/or modify it
		6	# under the terms of the GNU General Public License as published by
		7	# the Free Software Foundation; either version 3 of the License, or
		8	# (at your option) any later version.
		9	#
		10	# Koha is distributed in the hope that it will be useful, but
		11	# WITHOUT ANY WARRANTY; without even the implied warranty of
		12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		13	# GNU General Public License for more details.
		14	#
		15	# You should have received a copy of the GNU General Public License
		16	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		17
		18	package Koha::OAI::Client::Harvester;
		19
		20	use utf8;
		21	use open qw( :std :utf8 );
		22
		23	use C4::Biblio qw( AddBiblio GetFrameworkCode ModBiblio DelBiblio );
		24	use C4::AuthoritiesMarc qw (AddAuthority GuessAuthTypeCode ModAuthority DelAuthority );
		25	use C4::Log qw( cronlogaction );
		26	use HTTP::OAI;
		27	use HTTP::OAI::Metadata::OAI_DC;
		28	use Koha::DateUtils qw( dt_from_string );
		29	use Koha::OaiServers;
		30	use Koha::Import::Oaipmh::Biblio;
		31	use Koha::Import::Oaipmh::Biblios;
		32	use Koha::Import::Oaipmh::Authority;
		33	use Koha::Import::Oaipmh::Authorities;
		34	use Koha::XSLT::Base;
		35	use MARC::File::XML;
		36	use MARC::Record;
		37	use Modern::Perl;
		38	use DateTime;
		39	use DateTime::Format::Strptime;
		40	use Try::Tiny qw( catch try );
		41	use Date::Calc qw(
		42	Add_Delta_Days
		43	Today
		44	);
		45
		46	my $strp = DateTime::Format::Strptime->new(
		47	pattern => '%FT%TZ',
		48	time_zone => 'floating',
		49	);
		50
		51	our $verbose = 0;
		52	our $servername = '';
		53
		54	sub new {
		55
		56	my ( $self, $args ) = @_;
		57
		58	my $id = $args->{'server'};
		59	$verbose = $args->{'verbose'};
		60	my $days = $args->{'days'};
		61	my $force = $args->{'force'};
		62
		63	return unless $id;
		64
65	my $server = Koha::OaiServers->find($id);
66	$servername = $server->servername;
67
68	unless ($server) {
69	printlog("OAI Server $id unknown\n");
70	return;
71	}
72
73	printlog("Starting OAI Harvest from repository");
74
75	my $h = HTTP::OAI::Harvester->new(
76	repository => HTTP::OAI::Identify->new(
77	baseURL => $server->endpoint,
78	version => '2.0',
79	)
80	);
81
82	printlog( "Using HTTP::OAI::Harvester version " . $HTTP::OAI::Harvester::VERSION );
83
84	printlog( "Using endpoint "
85	. $server->endpoint
86	. ", set "
87	. $server->oai_set
88	. ", dataformat "
89	. $server->dataformat
90	. ", recordtype "
91	. $server->recordtype );
92
93	try {
94	my $lmdf = $h->ListMetadataFormats();
95	my $lmdflog = "";
96	for ( $lmdf->metadataFormat ) {
97	$lmdflog .= $_->metadataPrefix . " ";
98	}
99	printlog("Available metadata formats for this repository: $lmdflog");
100	} catch {
101	printlog("ListMetadataFormats failed");
102	};
103
104	#my $response = $h->Identify;
105	#print "Repository supports protocol version ", $response->version, "\n";
106	if ( $server->add_xslt ) {
107	printlog( "Using XSLT file " . $server->add_xslt );
108	} else {
109	printlog("Not using an XSLT file");
110	}
111
112	my $start_date = '';
113	my $today_date = '';
114
115	if ($days) {
116
117	# Change this to yyyy-mm-dd
118	my $dt_today = dt_from_string();
119	my ( $nowyear, $nowmonth, $nowday ) = Today();
120	my @date = Add_Delta_Days( $nowyear, $nowmonth, $nowday, -$days );
121	my $dt_start = DateTime->new( year => $date[0], month => $date[1], day => $date[2] );
122	$start_date = $dt_start->ymd();
123	printlog("Harvesting from $start_date");
124	}
125
126	printlog("Asking for records");
127	my $response = $h->ListRecords(
128	metadataPrefix => $server->dataformat,
129	set => $server->oai_set,
130	from => $start_date,
131	);
132	if ( $response->is_error ) {
133	printlog( "Error requesting ListRecords: " . $response->code . " " . $response->message );
134	exit;
135	}
136	printlog( "Request URL: " . $response->requestURL );
137
138	my $xslt_engine = Koha::XSLT::Base->new;
139	my $added = 0;
140	my $updated = 0;
141	my $deleted = 0;
142	my $in_error = 0;
143	my $skipped = 0;
144	my $total = 0;
145
146	printlog("Starting processing results");
147	while ( my $oai_record = $response->next ) {
148	$total++;
149	unless ( $oai_record->identifier ) {
150	printlog("No identifier found");
151	$skipped++;
152	next;
153	}
154
155	my $to_delete;
156	my $outputUnimarc;
157	my $marcrecord;
158	if ( $oai_record->status && $oai_record->status eq "deleted" ) {
159	$to_delete = 1;
160	} else {
161	if ( $server->add_xslt ) {
162	$outputUnimarc = $xslt_engine->transform( $oai_record->metadata->dom, $server->add_xslt );
163	my $err = $xslt_engine->err;
164	if ($err) {
165	printlog("XSLT::Base error: $err");
166	$in_error++;
167	next;
168	}
169	} else {
170	$outputUnimarc = $oai_record->metadata->dom;
171	}
172	$marcrecord = MARC::Record->new_from_xml( $outputUnimarc, 'UTF-8' );
173	}
174	my $imported_record;
175	if ( $server->recordtype eq "biblio" ) {
176	$imported_record = Koha::Import::Oaipmh::Biblios->find(
177	{
178	repository => $server->endpoint,
179	identifier => $oai_record->identifier,
180	recordtype => $server->recordtype,
181	}
182	);
183	} else {
184	$imported_record = Koha::Import::Oaipmh::Authorities->find(
185	{
186	repository => $server->endpoint,
187	identifier => $oai_record->identifier,
188	recordtype => $server->recordtype,
189	}
190	);
191	}
192
193	if ($imported_record) {
194	if ($to_delete) {
195	if ( $server->recordtype eq "biblio" ) {
196	my $biblionumber = $imported_record->biblionumber;
197	my $error = DelBiblio($biblionumber);
198	if ($error) {
199	printlog( "Record "
200	. $oai_record->identifier
201	. " not deleted, biblionumber: $biblionumber ($error)" );
202	$in_error++;
203	} else {
204	$imported_record->delete;
205	printlog( "Record " . $oai_record->identifier . " deleted, biblionumber: $biblionumber" );
206	$deleted++;
207	}
208	} else {
209	my $authid = $imported_record->authid;
210	DelAuthority( { authid => $authid } );
211	$imported_record->delete;
212	printlog( "Record " . $oai_record->identifier . " deleted, authid: $authid" );
213	$deleted++;
214	}
215	} else {
216	my $incoming_dt = $strp->parse_datetime( $imported_record->datestamp );
217	my $existing_dt = $strp->parse_datetime( $oai_record->datestamp );
218	if ( $force \|\| !$incoming_dt \|\| !$existing_dt \|\| $incoming_dt > $existing_dt ) {
219	if ( $server->recordtype eq "biblio" ) {
220	my $biblionumber = $imported_record->biblionumber;
221	my $result = ModBiblio( $marcrecord, $biblionumber, GetFrameworkCode($biblionumber) );
222	printlog( "Record " . $oai_record->identifier . " updated, biblionumber: $biblionumber" );
223	} else {
224	my $authid = $imported_record->authid;
225	my $result = ModAuthority( $authid, $marcrecord, GuessAuthTypeCode($marcrecord) );
226	printlog( "Record " . $oai_record->identifier . " updated, authid: $authid" );
227	}
228	$imported_record->update(
229	{
230	datestamp => $imported_record->datestamp,
231	}
232	);
233	$updated++;
234	} else {
235	printlog( "Record "
236	. $oai_record->identifier
237	. " skipped (incoming record was not newer than existing record)" );
238	$skipped++;
239	}
240	}
241	} elsif ( !$to_delete ) {
242	if ( $server->recordtype eq "biblio" ) {
243	my ( $biblionumber, $biblioitemnumber ) = AddBiblio($marcrecord);
244	printlog( $oai_record->identifier . " added, biblionumber: $biblionumber" );
245	Koha::Import::Oaipmh::Biblio->new(
246	{
247	repository => $server->endpoint,
248	identifier => $oai_record->identifier,
249	biblionumber => $biblionumber,
250	recordtype => $server->recordtype,
251	datestamp => $oai_record->datestamp,
252	}
253	)->store;
254	} else {
255	my $authid = AddAuthority( $marcrecord, "", GuessAuthTypeCode($marcrecord) );
256	printlog( $oai_record->identifier . " added, authid: $authid" );
257	Koha::Import::Oaipmh::Authority->new(
258	{
259	repository => $server->endpoint,
260	identifier => $oai_record->identifier,
261	authid => $authid,
262	recordtype => $server->recordtype,
263	datestamp => $oai_record->datestamp,
264	}
265	)->store;
266	}
267	$added++;
268	} else {
269	printlog( "Record " . $oai_record->identifier . " skipped (record not present or already deleted)" );
270	$skipped++;
271	}
272	}
273	printlog(
274	"Harvest results:\n$added added\n$updated updated\n$deleted deleted\n$skipped skipped\n$in_error errors\n$total total"
275	);
276
277	if ( C4::Context->preference("OAI-PMH:HarvestEmailReport") ) {
278
279	my $letter = C4::Letters::GetPreparedLetter(
280	(
281	module => 'catalogue',
282	letter_code => 'OAI_HARVEST_REPORT',
283	substitute => {
284	servername => $servername,
285	endpoint => $server->endpoint,
286	set => $server->oai_set,
287	dataformat => $server->dataformat,
288	recordtype => $server->recordtype,
289	added => $added,
290	updated => $updated,
291	deleted => $deleted,
292	skipped => $skipped,
293	in_error => $in_error,
294	total => $total
295	},
296	)
297	) or printlog("OAI_HARVEST_REPORT letter not found");
298
299	my $success = C4::Letters::EnqueueLetter(
300	{
301	letter => $letter,
302	to_address => C4::Context->preference("OAI-PMH:HarvestEmailReport"), message_transport_type => 'email'
303	}
304	);
305	if ($success) {
306	printlog("Email report enqueued");
307	} else {
308	printlog("Unable to enqueue report email");
309	}
310	}
311
312	printlog("Ending OAI Harvest from repository");
313	}
314
315	sub printlog {
316	my $message = shift;
317	$message = $servername . ": " . $message;
318	print $message . "\n" if ($verbose);
319	cronlogaction( { info => $message } );
320	}
321
322	1;

Line 0 Link Here

(-)a/Koha/OaiServer.pm (+45 lines)
	1	package Koha::OaiServer;
	2
	3	# This file is part of Koha.
	4	#
	5	# Koha is free software; you can redistribute it and/or modify it
	6	# under the terms of the GNU General Public License as published by
	7	# the Free Software Foundation; either version 3 of the License, or
	8	# (at your option) any later version.
	9	#
	10	# Koha is distributed in the hope that it will be useful, but
	11	# WITHOUT ANY WARRANTY; without even the implied warranty of
	12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	# GNU General Public License for more details.
	14	#
	15	# You should have received a copy of the GNU General Public License
	16	# along with Koha; if not, see <http://www.gnu.org/licenses>.
	17
	18	use Modern::Perl;
	19
	20
	21	use Koha::Database;
	22
	23	use base qw(Koha::Object);
	24
	25	=head1 NAME
	26
	27	Koha::OaiServer - Koha OaiServer Object class
	28
	29	=head1 API
	30
	31	=head2 Class Methods
	32
	33	=cut
	34
	35	=head3 _type
	36
	37	Return type of Object relating to Schema ResultSet
	38
	39	=cut
	40
	41	sub _type {
	42	return 'Oaiserver';
	43	}
	44
	45	1;

Line 0 Link Here

(-)a/Koha/OaiServers.pm (+57 lines)
	1	package Koha::OaiServers;
	2
	3	# This file is part of Koha.
	4	#
	5	# Koha is free software; you can redistribute it and/or modify it
	6	# under the terms of the GNU General Public License as published by
	7	# the Free Software Foundation; either version 3 of the License, or
	8	# (at your option) any later version.
	9	#
	10	# Koha is distributed in the hope that it will be useful, but
	11	# WITHOUT ANY WARRANTY; without even the implied warranty of
	12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	# GNU General Public License for more details.
	14	#
	15	# You should have received a copy of the GNU General Public License
	16	# along with Koha; if not, see <http://www.gnu.org/licenses>.
	17
	18	use Modern::Perl;
	19
	20
	21	use Koha::Database;
	22
	23	use Koha::OaiServer;
	24
	25	use base qw(Koha::Objects);
	26
	27	=head1 NAME
	28
	29	Koha::OaiServers - Koha OaiServer Object set class
	30
	31	=head1 API
	32
	33	=head2 Class Methods
	34
	35	=cut
	36
	37	=head3 type
	38
	39	Return type of object, relating to Schema ResultSet
	40
	41	=cut
	42
	43	sub _type {
	44	return 'Oaiserver';
	45	}
	46
	47	=head3 object_class
	48
	49	Return object class
	50
	51	=cut
	52
	53	sub object_class {
	54	return 'Koha::OaiServer';
	55	}
	56
	57	1;

Line 0 Link Here

(-)a/admin/oaiservers.pl (+122 lines)
		1	#!/usr/bin/perl
		2
		3	# This file is part of Koha.
		4	#
		5	# Koha is free software; you can redistribute it and/or modify it
		6	# under the terms of the GNU General Public License as published by
		7	# the Free Software Foundation; either version 3 of the License, or
		8	# (at your option) any later version.
		9	#
		10	# Koha is distributed in the hope that it will be useful, but
		11	# WITHOUT ANY WARRANTY; without even the implied warranty of
		12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		13	# GNU General Public License for more details.
		14	#
		15	# You should have received a copy of the GNU General Public License
		16	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		17
		18	# This script is used to maintain the OAI servers table.
		19	# Parameter $op is operation: list, new, edit, add_validated, delete_confirmed.
		20	# add_validated saves a validated record and goes to list view.
		21	# delete_confirmed deletes a record and goes to list view.
		22
		23	use Modern::Perl;
		24	use CGI qw ( -utf8 );
		25	use C4::Context;
		26	use C4::Auth qw( get_template_and_user );
		27	use C4::Output qw( output_html_with_http_headers );
		28	use Koha::Database;
		29	use Koha::OaiServers;
		30
		31	# Initialize CGI, template, database
		32
		33	my $input = CGI->new;
		34	my $op = $input->param('op') \|\| 'list';
		35	my $id = $input->param('id') \|\| 0;
		36	my $type = $input->param('type') \|\| '';
		37	my $searchfield = '';
		38
		39	my ( $template, $loggedinuser, $cookie ) = get_template_and_user(
		40	{
		41	template_name => "admin/oaiservers.tt",
		42	query => $input,
		43	type => "intranet",
		44	flagsrequired => { parameters => 'manage_search_targets' },
		45	}
		46	);
		47	my $script_name = "/cgi-bin/koha/admin/oaiservers.pl";
		48	my $path = C4::Context->config('intrahtdocs') . "/prog/";
		49
		50	$template->param( script_name => $script_name, xslt_path => $path );
		51
		52	my $schema = Koha::Database->new()->schema();
		53
		54	# Main code
		55	# First process a confirmed delete, or save a validated record
		56
		57	if ( $op eq 'delete_confirmed' && $id ) {
		58	my $server = Koha::OaiServers->find($id);
		59	if ($server) {
		60	$server->delete;
		61	$template->param( msg_deleted => 1, msg_add => $server->servername );
		62	} else {
		63	$template->param( msg_notfound => 1, msg_add => $id );
		64	}
65	$id = 0;
66	} elsif ( $op eq 'add_validated' ) {
67	my @fields = qw/endpoint oai_set dataformat
68	recordtype servername
69	add_xslt/;
70	my $formdata = _form_data_hashref( $input, \@fields );
71	if ($id) {
72	my $server = Koha::OaiServers->find($id);
73	if ($server) {
74	$server->set($formdata)->store;
75	$template->param( msg_updated => 1, msg_add => $formdata->{servername} );
76	} else {
77	$template->param( msg_notfound => 1, msg_add => $id );
78	}
79	$id = 0;
80	} else {
81	Koha::OaiServer->new($formdata)->store;
82	$template->param( msg_added => 1, msg_add => $formdata->{servername} );
83	}
84	} else {
85
86	#use searchfield only in remaining operations
87	$searchfield = $input->param('searchfield') \|\| '';
88	}
89
90	# Now list multiple records, or edit one record
91
92	my $data = [];
93	if ( $op eq 'add' \|\| $op eq 'edit' ) {
94	$data = ServerSearch( $schema, $id, $searchfield ) if $searchfield \|\| $id;
95	delete $data->[0]->{id} if @$data && $op eq 'add'; #cloning record
96	$template->param(
97	add_form => 1, server => @$data ? $data->[0] : undef,
98	op => $op, type => $op eq 'add' ? lc $type : ''
99	);
100	} else {
101	$data = ServerSearch( $schema, $id, $searchfield );
102	$template->param(
103	loop => \@$data, searchfield => $searchfield, id => $id,
104	op => 'list'
105	);
106	}
107	output_html_with_http_headers $input, $cookie, $template->output;
108
109	# End of main code
110
111	sub ServerSearch { #find server(s) by id or name
112	my ( $schema, $id, $searchstring ) = @_;
113
114	return Koha::OaiServers->search(
115	$id ? { id => $id } : { servername => { like => $searchstring . '%' } },
116	)->unblessed;
117	}
118
119	sub _form_data_hashref {
120	my ( $input, $fieldref ) = @_;
121	return { map { ( $_ => scalar $input->param($_) // '' ) } @$fieldref };
122	}

Line 0 Link Here

(-)a/installer/data/mysql/atomicupdate/Bug-35659.pl (+88 lines)
		1	use Modern::Perl;
		2
		3	return {
		4	bug_number => "35659",
		5	description => "OAI-PMH Harvester",
		6	up => sub {
		7	my ($args) = @_;
		8	my ( $dbh, $out ) = @$args{qw(dbh out)};
		9	$dbh->do(q{DROP TABLE IF EXISTS `oaiservers`;});
		10	$dbh->do(
		11	q{
		12	CREATE TABLE `oaiservers` (
		13	`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
		14	`endpoint` varchar(255) NOT NULL COMMENT 'OAI endpoint (host + port + path)',
		15	`oai_set` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
		16	`servername` longtext NOT NULL COMMENT 'name given to the target by the library',
		17	`dataformat` enum('oai_dc','marc-xml', 'marcxml') NOT NULL DEFAULT 'oai_dc' COMMENT 'data format',
		18	`recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'server contains bibliographic or authority records',
		19	`add_xslt` longtext DEFAULT NULL COMMENT 'zero or more paths to XSLT files to be processed on the search results',
		20	PRIMARY KEY (`id`)
		21	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
		22	}
		23	);
		24	say $out "Added new table 'oaiservers'";
		25
		26	$dbh->do(q{DROP TABLE IF EXISTS `import_oaipmh_biblios`;});
		27	$dbh->do(
		28	q{
		29	CREATE TABLE `import_oaipmh_biblios` (
		30	`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
		31	`biblionumber` int(11) NOT NULL COMMENT 'unique identifier assigned to each koha record',
		32	`identifier` varchar(255) NOT NULL COMMENT 'OAI record identifier',
		33	`repository` varchar(255) NOT NULL COMMENT 'OAI repository',
		34	`recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'is the record bibliographic or authority',
		35	`datestamp` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
		36	`last_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
		37	PRIMARY KEY (id),
		38	KEY biblionumber (biblionumber),
		39	CONSTRAINT FK_import_oaipmh_biblios_1 FOREIGN KEY (biblionumber) REFERENCES biblio (biblionumber) ON DELETE CASCADE ON UPDATE NO ACTION
		40	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
		41	}
		42	);
		43
		44	$dbh->do(q{DROP TABLE IF EXISTS `import_oaipmh_authorities`;});
		45	$dbh->do(
		46	q{
		47	CREATE TABLE `import_oaipmh_authorities` (
		48	`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'unique identifier assigned by Koha',
		49	`authid` bigint(20) unsigned NOT NULL COMMENT 'unique identifier assigned to each koha record',
		50	`identifier` varchar(255) NOT NULL COMMENT 'OAI record identifier',
		51	`repository` varchar(255) NOT NULL COMMENT 'OAI repository',
		52	`recordtype` enum('authority','biblio') NOT NULL DEFAULT 'biblio' COMMENT 'is the record bibliographic or authority',
		53	`datestamp` varchar(255) DEFAULT NULL COMMENT 'OAI set to harvest',
		54	`last_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
		55	PRIMARY KEY (id),
		56	KEY authid (authid),
		57	CONSTRAINT FK_import_oaipmh_authorities_1 FOREIGN KEY (authid) REFERENCES auth_header (authid) ON DELETE CASCADE ON UPDATE NO ACTION
		58	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
		59	}
		60	);
		61	say $out "Added new tables 'import_oaipmh_biblios' and import_oaipmh_authorities";
		62
		63	$dbh->do(
		64	q{
65	UPDATE `permissions` SET description='Manage Z39.50 and SRU servers, OAI repositories configuration' WHERE code='manage_search_targets';
66	}
67	);
68	say $out "Updated manage_search_targets permission description";
69
70	$dbh->do(
71	q{
72	INSERT IGNORE INTO systempreferences (variable, value, options, explanation, type) VALUES
73	('OAI-PMH:HarvestEmailReport','','','If set, a report email will be sent to this address after an OAI harvest','Free');
74	}
75	);
76	say $out "Added OAI-PMH:HarvestEmailReport system preference";
77
78	$dbh->do(
79	q{
80	INSERT IGNORE INTO letter
81	(module, code, branchcode, name, is_html, title, content, message_transport_type, lang)
82	VALUES ('catalogue','OAI_HARVEST_REPORT','','OAI Harvest Report',0,'OAI Harvest report for [% servername %]','OAI Harvest report for [% servername %]:\n\nEndpoint: [% endpoint %]\nSet: [% set %]\nData format: [% dataformat %]\nRecord type: [% recordtype %]\n\n[% added %] records added\n[% updated %] records updated\n[% deleted %] records deleted\n[% skipped %] records skipped\n[% in_error %] records in error\n[% total %] total','email','default');
83	}
84	);
85	say $out "Added OAI_HARVEST_REPORT letter";
86
87	},
88	};

Line 0 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/oai-admin-search.inc (+43 lines)
	1	[% USE Koha %]
	2	<!-- oai-admin-search.inc -->
	3	<div id="header_search" role="tablist">
	4	<div class="tab-content">
	5	<div id="oai_search" role="tabpanel" class="tab-pane active">
	6	<form action="/cgi-bin/koha/admin/oaiservers.pl" method="post">
	7	<div class="form-title">
	8	<label class="control-label">Search OAI repositories</label>
	9	</div>
	10
	11	<div class="form-content">
	12	<input class="head-searchbox form-control" type="text" name="searchfield" id="searchfield" value="[% searchfield \| html %]" placeholder="OAI repository search" />
	13	</div>
	14
	15	<button type="submit"><i class="fa fa-arrow-right"></i></button>
	16	</form>
	17	</div>
	18
	19	[% IF ( CAN_user_circulate_circulate_remaining_permissions ) %]
	20	[% INCLUDE 'patron-search-box.inc' %]
	21	[% END %]
	22	[% IF ( CAN_user_catalogue ) %]
	23	[% INCLUDE 'catalogue-search-box.inc' %]
	24	[% END %]
	25
	26	</div><!-- /.tab-content -->
	27	<ul class="nav nav-tabs" role="tablist">
	28	<li role="presentation" class="active">
	29	<a title="Search OAI repositories" href="#oai_search" aria-controls="z3950_search" role="tab" aria-expanded="true" data-toggle="tab" class="keep_text"><i class="fa fa-search"></i> <span class="tab-title">Search OAI repositories</span></a>
	30	</li>
	31	[% IF ( CAN_user_circulate_circulate_remaining_permissions ) %]
	32	<li role="presentation">
	33	<a title="Check out" href="#circ_search" aria-controls="circ_search" role="tab" data-toggle="tab" class="keep_text"><i class="fa fa-upload"></i> <span class="tab-title">Check out</span></a>
	34	</li>
	35	[% END %]
	36	[% IF ( CAN_user_catalogue ) %]
	37	<li role="presentation">
	38	<a title="Search catalog" href="#catalog_search" aria-controls="catalog_search" role="tab" data-toggle="tab" class="keep_text"><i class="fa fa-fw fa-search"></i> <span class="tab-title">Search catalog</span></a>
	39	</li>
	40	[% END %]
	41	</ul>
	42	</div><!-- /#header_search -->
	43	<!-- /oai-admin-search.inc -->

Line 0 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oaiservers.tt (+229 lines)
		1	[% USE raw %]
		2	[% USE Asset %]
		3	[% SET footerjs = 1 %]
		4	[% INCLUDE 'doc-head-open.inc' %]
		5
		6	<title>
		7	[% IF op == 'edit' %]
		8	Modify OAI repository [% server.servername \| html %] &rsaquo; [% END %]
		9
		10	[% IF op == 'add' %]
		11	New OAI repository &rsaquo; [% END %]
		12
		13	OAI repositories &rsaquo; Administration &rsaquo; Koha
		14	</title>
		15	[% INCLUDE 'doc-head-close.inc' %]
		16	</head>
		17
		18	<body id="admin_oaiservers" class="admin">
		19	[% WRAPPER 'header.inc' %]
		20	[% INCLUDE 'oai-admin-search.inc' %]
		21	[% END %]
		22
		23	[% WRAPPER 'sub-header.inc' %]
		24	[% WRAPPER breadcrumbs %]
		25	[% WRAPPER breadcrumb_item %]
		26	<a href="/cgi-bin/koha/admin/admin-home.pl">Administration</a>
		27	[% END %]
		28
		29	[% IF op == 'edit' \|\| op == 'add' %]
		30	[% WRAPPER breadcrumb_item %]
		31	<a href="/cgi-bin/koha/admin/oaiservers.pl">OAI repositories</a>
		32	[% END %]
		33	[% END %]
		34
		35	[% IF op == 'edit' %]
		36	[% WRAPPER breadcrumb_item bc_active= 1 %]
		37	<span>Modify OAI repository [% server.servername \| html %]</span>
		38	[% END %]
		39
		40	[% ELSIF op == 'add' %]
		41	[% WRAPPER breadcrumb_item bc_active= 1 %]
		42	<span>New OAI repository</span>
		43	[% END %]
		44
		45	[% ELSE %]
		46	[% WRAPPER breadcrumb_item bc_active= 1 %]
		47	<span>OAI repositories</span>
		48	[% END %]
		49	[% END %]
		50	[% END #/ WRAPPER breadcrumbs %]
		51	[% END #/ WRAPPER sub-header.inc %]
		52
		53	<div class="main container-fluid">
		54	<div class="row">
		55	<div class="col-sm-10 col-sm-push-2">
		56	<main>
		57
		58	[% IF msg_deleted %]
		59	<div class="dialog alert">OAI repository deleted ([% msg_add \| html %])</div>
		60	[% ELSIF msg_updated %]
		61	<div class="dialog message">OAI repository updated ([% msg_add \| html %])</div>
		62	[% ELSIF msg_added %]
		63	<div class="dialog message">OAI repository added ([% msg_add \| html %])</div>
		64	[% ELSIF msg_notfound %]
65	<div class="dialog alert">Error: Server with ID [% msg_add \| html %] not found</div>
66	[% END %]
67
68	[% IF ( add_form ) %]
69	<form action="[% script_name \| html %]" name="Aform" method="post" id="serverentry">
70	<input type="hidden" name="op" value="add_validated" />
71	[% IF op == 'edit' %]
72	<h1>Modify OAI repository</h1>
73	<input type="hidden" name="id" value="[% server.id \| html %]" />
74	[% ELSE %]
75	<h1>New OAI repository</h1>
76	[% END %]
77	<fieldset class="rows">
78	<ol>
79	<li><label for="name" class="required">Repository name: </label>
80	[% IF server.servername %]
81	<input type="text" name="servername" id="servername" size="65" maxlength="100" value="[% server.servername \| html %]" required="required" />
82	[% ELSE %]
83	<input type="text" name="servername" id="servername" class="focus" size="65" maxlength="100" value="[% server.servername \| html %]" required="required" />
84	[% END %]
85	<span class="required">Required</span>
86	</li>
87
88	<li><label for="endpoint" class="required">Endpoint: </label> <input type="text" name="endpoint" id="endpoint" size="30" value="[% server.endpoint \| html %]" required="required" /> <span class="required">Required</span>
89	</li>
90	<li><label for="oai_set">Set: </label> <input type="text" name="oai_set" id="oai_set" value="[% server.oai_set \| html %]" />
91	</li>
92	<li>
93	<label for="dataformat">Data format: </label>
94	<select name="dataformat" id="dataformat">
95	<option value="oai_dc">Dublin Core (oai_dc)</option>
96	<option value="marc-xml">Marcxml (marc-xml)</option>
97	<option value="marcxml">Marcxml (marcxml)</option>
98	</select>
99	<div class="hint">OAI can send records in various formats. Choose one.</div>
100	</li>
101
102	<li><label for="recordtype">Record type: </label>
103	<select name="recordtype" id="recordtype">
104	<option value="biblio">Bibliographic</option>
105	<option value="authority">Authority</option>
106	</select>
107	</li>
108	<li>
109	<label for="add_xslt">XSLT file for transforming results: </label>
110	<input type="text" name="add_xslt" id="add_xslt" size="100" placeholder="[% xslt_path \| html %]" value="[% server.add_xslt \| html %]"/>
111	</li>
112
113	</ol>
114	</fieldset>
115
116	<fieldset class="action"><input type="submit" class="btn btn-primary" value="Save" /> <a class="cancel" href="/cgi-bin/koha/admin/oaiservers.pl">Cancel</a></fieldset>
117	</form>
118	[% END %]
119
120	[% IF op == 'list' %]
121	<div id="toolbar" class="btn-toolbar">
122	<a id="newserver" class="btn btn-default" href="/cgi-bin/koha/admin/oaiservers.pl?op=add"><i class="fa fa-plus"></i> New OAI repository</a>
123	</div>
124	<h1>OAI repositories administration</h1>
125	[% IF id %]
126	You searched for record [% id \| html %]
127	[% ELSIF searchfield %]
128	You searched for [% searchfield \| html %]
129	[% END %]
130
131	<div class="page-section">
132	<table id="serverst">
133
134	<thead><tr><th>Id</th><th>Repository name</th><th>Endpoint</th><th>Set</th><th>Data format</th><th>Record type</th><th class="noExport">Actions</th>
135	</tr></thead>
136	<tbody>
137	[% FOREACH loo IN loop %]
138	<tr>
139	<td>[% loo.id \| html %]</td>
140	<td><a href="/cgi-bin/koha/admin/oaiservers.pl?op=edit&id=[% loo.id \| uri %]">[% loo.servername \| html %]</a></td><td>[% loo.endpoint \| html %]</td><td>[% loo.oai_set \| html %]</td>
141	<td>[% loo.dataformat \| html %]</td>
142	<td>[% IF ( loo.recordtype == 'biblio' ) %]
143	<span>Bibliographic</span>
144	[% ELSIF ( loo.recordtype == 'authority' ) %]
145	<span>Authority</span>
146	[% END %]
147	</td>
148	<td>
149	<div class="btn-group dropup">
150	<a class="btn btn-default btn-xs dropdown-toggle" id="reportactions[% savedreport.id \| html %]" role="button" data-toggle="dropdown" href="#">
151	Actions <b class="caret"></b>
152	</a>
153	<ul class="dropdown-menu pull-right" role="menu" aria-labelledby="reportactions[% savedreport.id \| html %]">
154	<li><a href="/cgi-bin/koha/admin/oaiservers.pl?op=edit&id=[% loo.id \| uri %]"><i class="fa-solid fa-pencil" aria-hidden="true"></i> Edit</a></li>
155	<li><a href="/cgi-bin/koha/admin/oaiservers.pl?op=add&id=[% loo.id \| uri %]"><i class="fa fa-copy"></i> Copy</a></li>
156	<li><a href="/cgi-bin/koha/admin/oaiservers.pl?op=delete_confirmed&id=[% loo.id \| uri %]" class="delete" data-servername="[% loo.servername \| html %]"><i class="fa fa-trash-can"></i> Delete</a></li>
157	</ul>
158	</div>
159	</td>
160	</tr>
161	[% END %]
162	</tbody>
163	</table>
164	</div> <!-- /.page-section -->
165	[% END %]
166
167	</main>
168	</div> <!-- /.col-sm-10.col-sm-push-2 -->
169
170	<div class="col-sm-2 col-sm-pull-10">
171	<aside>
172	[% INCLUDE 'admin-menu.inc' %]
173	</aside>
174	</div> <!-- /.col-sm-2.col-sm-pull-10 -->
175	</div> <!-- /.row -->
176
177	[% MACRO jsinclude BLOCK %]
178	[% Asset.js("js/admin-menu.js") \| $raw %]
179	[% IF op == 'list' %]
180	[% INCLUDE 'datatables.inc' %]
181	[% END %]
182
183	<script>
184	[% IF ( add_form ) %]
185	$(document).ready(function(){
186	// Update selects for dataformat and recordtype
187	[% IF server %]
188	$("#dataformat").val('[% server.dataformat \| html %]');
189	$("#recordtype").val('[% server.recordtype \| html %]');
190	[% END %]
191	$( "#serverentry" ).validate({
192	rules: {
193	servername: { required: true },
194	host: { required: true },
195	port: {
196	number: true
197	},
198	rank: { number: true },
199	}
200	});
201	$("#serverentry").submit(function( event ) {
202	// enable recordtype to include field in post
203	$('#recordtype').prop('disabled',false);
204	});
205	$("#servername").on("blur",function(){
206	toUC(this);
207	});
208	});
209	[% ELSE %]
210	$(document).ready(function() {
211	$("#serverst").dataTable($.extend(true, {}, dataTablesDefaults, {
212	"aoColumnDefs": [
213	{ "aTargets": [-1], "bSortable": false, "bSearchable": false },
214	],
215	"sPaginationType": "full"
216	}));
217	$(".delete").on("click",function(e){
218	var servername = $(this).data("servername");
219	if( confirm( _("Are you sure you want to delete repository %s?").format(servername) ) ) {
220	return true;
221	} else {
222	e.preventDefault();
223	}
224	});
225	});
226	[% END %]
227	</script>
228	[% END %]
229	[% INCLUDE 'intranet-bottom.inc' %]

Line 0 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/xslt/AuthOAIDCtoMARC21XML.xsl (+36 lines)
	1	<?xml version="1.0" encoding="UTF-8"?>
	2	<xsl:stylesheet version="1.0"
	3	xmlns:dc="http://purl.org/dc/elements/1.1/"
	4	xmlns:dcterms="http://purl.org/dc/terms/1.1"
	5	xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
	6	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	7	xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/
	8	http://www.openarchives.org/OAI/2.0/oai_dc.xsd"
	9	xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
	10	xmlns="http://www.loc.gov/MARC21/slim" exclude-result-prefixes="dc dcterms oai_dc">
	11
	12	<xsl:output method="xml" encoding="UTF-8" indent="yes"/>
	13
	14	<xsl:template match="/">
	15	<collection xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" >
	16	<xsl:apply-templates />
	17	</collection>
	18	</xsl:template>
	19
	20	<xsl:template match="oai_dc:dc">
	21	<record>
	22
	23	<xsl:variable name="FamilyName" select="dc:FamilyName"/>
	24	<xsl:variable name="GivenName" select="dc:GivenName"/>
	25
	26	<datafield tag="100" ind1="0" ind2=" ">
	27	<subfield code="a">
	28	<xsl:value-of select="concat($FamilyName,', ',$GivenName)"/>
	29	</subfield>
	30	</datafield>
	31
	32	</record>
	33	</xsl:template>
	34
	35	</xsl:stylesheet>
	36

Line 0 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/xslt/OAIMarcxml2KohaMarcxml.xsl (+23 lines)
	1	<?xml version="1.0" encoding="UTF-8"?>
	2	<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:marc="http://www.loc.gov/MARC21/slim">
	3	<xsl:output method="xml" version="1.0" encoding="UTF-8" indent="yes"/>
	4
	5	<!-- keep comments -->
	6	<xsl:template match="*[name()='metadata']">
	7	<xsl:apply-templates />
	8	</xsl:template>
	9
	10	<xsl:template match="*">
	11	<!-- remove element prefix -->
	12	<xsl:element name="{local-name()}">
	13	<!-- process attributes -->
	14	<xsl:for-each select="@*">
	15	<!-- remove attribute prefix -->
	16	<xsl:attribute name="{local-name()}">
	17	<xsl:value-of select="."/>
	18	</xsl:attribute>
	19	</xsl:for-each>
	20	<xsl:apply-templates/>
	21	</xsl:element>
	22	</xsl:template>
	23	</xsl:stylesheet>

Line 0 Link Here

(-)a/misc/cronjobs/harvest_oai.pl (+102 lines)
		1	#!/usr/bin/perl
		2	#
		3	# Copyright 2023 BibLibre
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19
		20	use warnings;
		21	use strict;
		22	use utf8;
		23
		24	use Getopt::Long qw( GetOptions );
		25	use Koha::Script -cron;
		26	use Koha::OAI::Client::Harvester;
		27	use Koha::OaiServers;
		28	use C4::Log qw( cronlogaction );
		29	use Try::Tiny qw( catch try );
		30
		31	my $command_line_options = join( " ", @ARGV );
		32
		33	my ( $help, $verbose, $id, $days, $list, $force );
		34
		35	GetOptions(
		36	'h\|help' => \$help,
		37	'v\|verbose' => \$verbose,
		38	'r\|repository:i' => \$id,
		39	'd\|days:i' => \$days,
		40	'l\|list' => \$list,
		41	'f\|force' => \$force,
		42	);
		43	my $usage = << 'ENDUSAGE';
		44
		45	This script starts an OAI Harvest
		46
		47	This script has the following parameters :
		48	-h --help: this message
		49	-v --verbose
		50	-r --repository: id of the OAI repository
		51	-d --days: number of days to harvest from (optional)
		52	-l --list: list the OAI repositories
		53	-f --force: force harvesting (ignore records datestamps)
		54
		55	ENDUSAGE
		56
		57	if ($help) {
		58	print $usage;
		59	exit;
		60	}
		61
		62	if ($list) {
		63	my $servers = Koha::OaiServers->search( {}, { order_by => { -asc => 'id' } } )->unblessed;
		64	print "The following repositories are available: \n\n";
65	foreach my $server (@$servers) {
66	print $server->{'id'} . ": "
67	. $server->{'servername'}
68	. ", endpoint: "
69	. $server->{'endpoint'}
70	. ", set: "
71	. $server->{'oai_set'}
72	. ", recordtype: "
73	. $server->{'recordtype'} . "\n";
74	}
75	print "\n";
76	exit;
77	}
78
79	if ( !$id ) {
80	print "The repository parameter is mandatory.\n";
81	print $usage . "\n";
82	}
83
84	my $script_handler = Koha::Script->new( { script => $0 } );
85
86	try {
87	$script_handler->lock_exec;
88	} catch {
89	my $message = "Skipping execution of $0";
90	print "$message\n" if $verbose;
91	cronlogaction( { info => $message } );
92	exit;
93	};
94
95	cronlogaction( { action => 'Start', info => "Starting OAI Harvest" } );
96	cronlogaction( { info => "Command line: $command_line_options" } );
97
98	my $harvester =
99	Koha::OAI::Client::Harvester->new( { server => $id, verbose => $verbose, days => $days, force => $force } );
100
101	cronlogaction( { action => 'End', info => "Ending OAI Harvest" } );
102	exit(0);

Line 0 Link Here

(-)a/t/db_dependent/Koha/OAIServers.t (-1 / +95 lines)
0	-	1	#!/usr/bin/perl
		2
		3	# Copyright 2015 Koha Development team
		4	#
		5	# This file is part of Koha
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19
		20	use Modern::Perl;
		21	use Test::More tests => 2;
		22	use Test::Exception;
		23
		24	use t::lib::TestBuilder;
		25	use Koha::Database;
		26	use Koha::OaiServer;
		27	use Koha::OaiServers;
		28
		29	my $schema = Koha::Database->new->schema;
		30	my $builder = t::lib::TestBuilder->new;
		31
		32	subtest 'new, find and delete tests' => sub {
		33	plan tests => 4;
		34	$schema->storage->txn_begin;
		35	my $nb_of_oais = Koha::OaiServers->search->count;
		36	my $new_oai_1 = Koha::OaiServer->new(
		37	{
		38	endpoint => 'my_host1.org',
		39	oai_set => 'set1',
		40	servername => 'my_test_1',
		41	dataformat => 'oai_dc',
		42	recordtype => 'biblio',
		43	}
		44	)->store;
		45	my $new_oai_2 = Koha::OaiServer->new(
		46	{
		47	endpoint => 'my_host2.org',
		48	oai_set => 'set2',
		49	servername => 'my_test_2',
		50	dataformat => 'marcxml',
		51	recordtype => 'authority',
		52	}
		53	)->store;
		54
		55	like( $new_oai_1->id, qr\|^\d+$\|, 'Adding a new oai server should have set the id' );
		56	is( Koha::OaiServers->search->count, $nb_of_oais + 2, 'The 2 oai servers should have been added' );
		57
		58	my $retrieved_oai_1 = Koha::OaiServers->find( $new_oai_1->id );
		59	is(
		60	$retrieved_oai_1->servername, $new_oai_1->servername,
		61	'Find a oai server by id should return the correct oai server'
		62	);
		63
		64	$retrieved_oai_1->delete;
		65	is( Koha::OaiServers->search->count, $nb_of_oais + 1, 'Delete should have deleted the oai server' );
66
67	$schema->storage->txn_rollback;
68	};
69
70	subtest 'Check NOT NULL without default values' => sub {
71	plan tests => 5;
72	$schema->storage->txn_begin;
73	local $SIG{__WARN__} = sub { }; # TODO Needed it for suppressing DBIx warns
74
75	my $server = Koha::OaiServer->new(
76	{
77	oai_set => 'set3',
78	dataformat => 'marcxml',
79	recordtype => 'biblio',
80	}
81	);
82
83	throws_ok { $server->store } 'DBIx::Class::Exception', 'Exception on empty endpoint';
84	like( $@->{msg}, qr/'endpoint' doesn't have a default value/, 'Verified that DBIx blamed endpoint' );
85
86	$server->endpoint('endpoint_added');
87	throws_ok { $server->store } 'DBIx::Class::Exception', 'Exception on empty servername';
88	like( $@->{msg}, qr/'servername' doesn't have a default value/, 'Verified that DBIx blamed servername' );
89
90	$server->servername('servername_added');
91
92	lives_ok { $server->store } 'No exceptions anymore';
93
94	$schema->storage->txn_rollback;
95	};