Attachment #99739 for bug #10662

View | Details | Raw Unified | Return to bug 10662
Collapse All | Expand All




package Koha::Daemon;

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.
#

use Modern::Perl;
use POSIX; #For daemonizing
use Fcntl qw(:flock); #For pidfile

=head1 NAME

Koha::Daemon

=head1 SYNOPSIS

  use Koha::Daemon;
  my $daemon = Koha::Daemon->new({
    pidfile => "/path/to/file.pid",
    logfile => "/path/to/file.log",
    daemonize => 1,
  });
  $daemon->run();

=head1 METHODS

=head2 new

Create object

=head2 run

Run the daemon.

This method calls all the internal methods which
are do everything necessary to run the daemon.

=head1 INTERNAL METHODS

=head2 daemonize

Internal function for setting object up as a proper daemon
(e.g. forking, setting permissions, changing directories,
 closing file handles, etc.)

=head2 get_pidfile

Internal function to get a filehandle for the pidfile

=head2 lock_pidfile

Internal function to lock the pidfile, so that only one daemon
can run against this pidfile

=head2 log_to_file

Internal function to log to a file

=head2 make_pidfilehandle

Internal function to make a filehandle for pidfile

=head2 write_pidfile

Internal function to write pid to pidfile

=cut

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;
    return bless ($args, $class);
}

#######################################################################
#NOTE: On Debian, you can use the daemon binary to make a process into a daemon,
# but the following can be used if you don't want to use that program.

sub daemonize {
    my ($self) = @_;

    my $pid = fork;

    die "Couldn't fork: $!" unless defined($pid);
    if ($pid){
        exit; #Parent exit
    }

    #Become a session leader (ie detach program from controlling terminal)
    POSIX::setsid() or die "Can't start a new session: $!";

    #Change to known system directory
    chdir('/');

    #Reset the file creation mask so only the daemon owner can read/write files it creates
    umask(066);

    #Close inherited file handles, so that you can truly run in the background.
    open STDIN,  '<', '/dev/null';
    open STDOUT, '>', '/dev/null';
    open STDERR, '>&STDOUT';
}

sub log_to_file {
    my ($self,$logfile) = @_;

    #Open a filehandle to append to a log file
    my $opened = open(my $fh, '>>', $logfile);
    if ($opened){
        $fh->autoflush(1); #Make filehandle hot (ie don't buffer)
        *STDOUT = *$fh; #Re-assign STDOUT to LOG | --stdout
        *STDERR = *STDOUT; #Re-assign STDERR to STDOUT | --stderr
    }
    else {
        die "Unable to open a filehandle for $logfile: $!\n"; # --output
    }
}

sub make_pidfilehandle {
    my ($self,$pidfile) = @_;
    if ( ! -f $pidfile ){
        open(my $fh, '>', $pidfile) or die "Unable to write to $pidfile: $!\n";
        close($fh);
    }
    open(my $pidfilehandle, '+<', $pidfile) or die "Unable to open a filehandle for $pidfile: $!\n";
    return $pidfilehandle;
}

sub get_pidfile {
    my ($self,$pidfile) = @_;
    #NOTE: We need to save the filehandle in the object, so any locks persist for the life of the object
    my $pidfilehandle = $self->{pidfilehandle} ||= $self->make_pidfilehandle($pidfile);
    return $pidfilehandle;
}

sub lock_pidfile {
    my ($self,$pidfilehandle) = @_;
    my $locked;
    if (flock($pidfilehandle, LOCK_EX|LOCK_NB)){
        $locked = 1;

    }
    return $locked;
}

sub write_pidfile {
    my ($self,$pidfilehandle) = @_;
    if ($pidfilehandle){
        truncate($pidfilehandle, 0);
        print $pidfilehandle $$."\n" or die $!;
        #Flush the filehandle so you're not suffering from buffering
        $pidfilehandle->flush();
        return 1;
    }
}

sub run {
    my ($self) = @_;
    my $pidfile = $self->{pidfile};
    my $logfile = $self->{logfile};

    if ($pidfile){
        my $pidfilehandle = $self->get_pidfile($pidfile);
        if ($pidfilehandle){
            my $locked = $self->lock_pidfile($pidfilehandle);
            if ( ! $locked ) {
                die "$0 is unable to lock pidfile...\n";
            }
        }
    }

    if (my $configure = $self->{configure}){
        $configure->($self);
    }

    if ($self->{daemonize}){
        $self->daemonize();
    }

    if ($pidfile){
        my $pidfilehandle = $self->get_pidfile($pidfile);
        if ($pidfilehandle){
            $self->write_pidfile($pidfilehandle);
        }
    }

    if ($logfile){
        $self->log_to_file($logfile);
    }

    if (my $loop = $self->{loop}){
        $loop->($self);
    }
}

1;




package Koha::OAI::Harvester;

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.
#

use Modern::Perl;
use POE qw(Component::JobQueue);
use JSON;
use Sereal::Encoder;
use Sereal::Decoder;
use IO::Handle;
use File::Copy;
use File::Path qw(make_path remove_tree);
use DateTime;
use DateTime::Format::Strptime;

use C4::Context;
use Koha::OAI::Harvester::ImportQueues;

=head1 NAME

Koha::OAI::Harvester

=head1 SYNOPSIS

my $harvester = Koha::OAI::Harvester->spawn({
    Downloader => $downloader,
    DownloaderWorkers => $download_workers,
    Importer => $importer,
    ImporterWorkers => $import_workers,
    ImportQueuePoll => $import_poll,
    logger => $logger,
    state_file => $statefile,
    spooldir => $spooldir,
});

=head2 METHODS

=cut

my $day_granularity = DateTime::Format::Strptime->new(
    pattern   => '%F',
);
my $seconds_granularity = DateTime::Format::Strptime->new(
    pattern   => '%FT%TZ',
);

=head2 new

Create object

=cut

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;
    return bless ($args, $class);
}

=head2 spawn

    Creates POE sessions for handling tasks, downloading, and importing

=cut

sub spawn {
    my ($class, $args) = @_;
    my $self = $class->new($args);
    my $downloader = $self->{Downloader};
    my $importer = $self->{Importer};
    my $download_worker_limit = ( $self->{DownloaderWorkers} && int($self->{DownloaderWorkers}) ) ? $self->{DownloaderWorkers} : 1;
    my $import_worker_limit = ( $self->{ImporterWorkers} && int($self->{ImporterWorkers}) ) ? $self->{ImporterWorkers} : 1;
    my $import_queue_poll = ( $self->{ImportQueuePoll} && int($self->{ImportQueuePoll}) ) ? $self->{ImportQueuePoll} : 5;

    #NOTE: This job queue should always be created before the
    #harvester so that you can start download jobs immediately
    #upon spawning the harvester.
    POE::Component::JobQueue->spawn(
        Alias         => 'oai-downloader',
        WorkerLimit   => $download_worker_limit,
        Worker        => sub {
            my ($postback, $task) = @_;
            if ($downloader){
                if ($task->{status} eq "active"){
                    $downloader->run({
                        postback => $postback,
                        task => $task,
                   });
                }
            }
        },
        Passive => {},
    );

    POE::Session->create(
        object_states => [
            $self => {
                _start => "on_start",
                get_task => "get_task",
                list_tasks => "list_tasks",
                create_task => "create_task",
                start_task => "start_task",
                stop_task => "stop_task",
                delete_task => "delete_task",
                repeat_task => "repeat_task",
                register => "register",
                deregister => "deregister",
                restore_state => "restore_state",
                save_state => "save_state",
                is_task_finished => "is_task_finished",
                does_task_repeat => "does_task_repeat",
                download_postback => "download_postback",
                reset_imports_status => "reset_imports_status",
            },
        ],
    );

    POE::Component::JobQueue->spawn(
        Alias         => 'oai-importer',
        WorkerLimit   => $import_worker_limit,
        Worker        => sub {
            my $meta_postback = shift;

            #NOTE: We need to only retrieve queue items for active tasks. Otherwise,
            #the importer will just spin its wheels on inactive tasks and do nothing.
            my $active_tasks = $poe_kernel->call("harvester","list_tasks","active");
            my @active_uuids = map { $_->{uuid} } @$active_tasks;

            my $rs = Koha::OAI::Harvester::ImportQueues->new->search({
                uuid => \@active_uuids,
                status => "new"
            },{
                order_by => { -asc => 'id' },
                rows => 1,
            });
            my $result = $rs->single;
            if ($result){
                $result->status("wip");
                $result->update;
                my $task = {
                    id => $result->id,
                    uuid => $result->uuid,
                    result => $result->result,
                };

                my $postback = $meta_postback->($task);
                $importer->run({
                    postback => $postback,
                    task => $task,
                });
            }
        },
        Active => {
            PollInterval => $import_queue_poll,
            AckAlias => undef,
            AckState => undef,
        },
    );

    return;
}

=head2 on_start

    Internal method that starts the harvester

=cut

sub on_start {
    my ($self, $kernel, $heap) = @_[OBJECT, KERNEL,HEAP];
    $kernel->alias_set('harvester');
    $heap->{scoreboard} = {};

    #Reset any 'wip' imports to 'new' so they can be re-tried.
    $kernel->call("harvester","reset_imports_status");

    #Restore state from state file
    $kernel->call("harvester","restore_state");
}

=head2 download_postback

    Internal method

=cut

#NOTE: This isn't really implemented at the moment, as it's not really necessary.
sub download_postback {
    my ($kernel, $request_packet, $response_packet) = @_[KERNEL, ARG0, ARG1];
    my $message = $response_packet->[0];
}

=head2 deregister

    Remove the worker session from the harvester's in-memory scoreboard,
    unset the downloading flag if downloading is completed.

=cut

sub deregister {
    my ($self, $kernel, $heap, $session, $sender, $type) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0];

    my $scoreboard = $heap->{scoreboard};

    my $logger = $self->{logger};
    $logger->debug("Start deregistering $sender as $type task");

    my $task_uuid = delete $scoreboard->{session}->{$sender};
    #NOTE: If you don't check each step of the hashref, autovivication can lead to surprises.
    if ($task_uuid){
        if ($scoreboard->{task}->{$task_uuid}){
            if ($scoreboard->{task}->{$task_uuid}->{$type}){
                delete $scoreboard->{task}->{$task_uuid}->{$type}->{$sender};
            }
        }
    }

    my $task = $heap->{tasks}->{$task_uuid};
    if ($task && $task->{status} && ($task->{status} eq "active") ){
        #NOTE: Each task only has 1 download session, so we can now set/unset flags for the task.
        #NOTE: We should unset the downloading flag, if we're not going to repeat the task.
        if ($type eq "download"){
            my $task_repeats = $kernel->call("harvester","does_task_repeat",$task_uuid);
            if ($task_repeats){
                my $interval = $task->{interval};

                $task->{effective_from} = delete $task->{effective_until};
                $task->{download_timer} = $kernel->delay_set("repeat_task", $interval, $task_uuid);
            }
            else {
                $task->{downloading} = 0;
                $kernel->call("harvester","save_state");
                $kernel->call("harvester","is_task_finished",$task_uuid);
            }
        }
        elsif ($type eq 'import'){
            $kernel->call("harvester","is_task_finished",$task_uuid);
        }
    }
    $logger->debug("End deregistering $sender as $type task");
}


=head2 is_task_finished

    This event handler checks if the task has finished downloading and importing record.
    If it is finished downloading and importing, the task is deleted from the harvester.

    This only applies to non-repeating tasks.

=cut

sub is_task_finished {
    my ($self, $kernel, $heap, $session, $uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
    my $task = $kernel->call("harvester","get_task",$uuid);
    if ($task && (! $task->{downloading}) ){
        my $count = $self->get_import_count_for_task($uuid);
        if ( ! $count ) {
            #Clear this task out of the harvester as it's finished.
            $kernel->call("harvester","delete_task",$uuid);
            return 1;
        }
    }
    return 0;
}

=head2 register

    Internal method for registering a task on the harvester's scoreboard

=cut

sub register {
    my ($self, $kernel, $heap, $session, $sender, $type, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0,ARG1];
    my $logger = $self->{logger};

    my $scoreboard = $heap->{scoreboard};


    if ($type && $task_uuid){
        $logger->debug("Registering $sender as $type for $task_uuid");

        my $task = $heap->{tasks}->{$task_uuid};
        if ($task){

            if ($type){
                #Register the task uuid with the session id as a key for later recall
                $scoreboard->{session}->{$sender} = $task_uuid;

                #Register the session id as a certain type of session for a task
                $scoreboard->{task}->{$task_uuid}->{$type}->{$sender} = 1;

                if ($type eq "download"){
                    $task->{downloading} = 1;

                    my $task_repeats = $kernel->call("harvester","does_task_repeat",$task_uuid);
                    if ($task_repeats){

                        #NOTE: Set an effective until, so we know we're not getting records any newer than
                        #this moment.
                        my $dt = DateTime->now();
                        if ($dt){
                            #NOTE: Ideally, I'd like to make sure that we can use 'seconds' granularity, but
                            #it's valid for 'from' to be null, so it's impossible to know from the data whether
                            #or not the repository will support the seconds granularity.
                            #NOTE: Ideally, it would be good to use either 'day' granularity or 'seconds' granularity,
                            #but at the moment the interval is expressed as seconds only.
                            $dt->set_formatter($seconds_granularity);
                            $task->{effective_until} = "$dt";
                        }
                    }

                    $kernel->call("harvester","save_state");
                }
            }
        }
    }
}

=head2 does_task_repeat

    Internal method for checking if a task is supposed to repeat after it finishes

=cut

sub does_task_repeat {
    my ($self, $kernel, $heap, $session, $uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
    my $task = $kernel->call("harvester","get_task",$uuid);
    if ($task){
        my $interval = $task->{interval};
        my $parameters = $task->{parameters};
        if ($parameters){
            my $oai_pmh = $parameters->{oai_pmh};
            if ($oai_pmh){
                if ( $interval && ($oai_pmh->{verb} eq "ListRecords") && (! $oai_pmh->{until}) ){
                    return 1;
                }
            }
        }
    }
    return 0;
}

=head2 reset_imports_status

    Internal method that resets the import queue from wip to new

=cut

sub reset_imports_status {
    my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION];

    my $rs = Koha::OAI::Harvester::ImportQueues->new->search({
                status => "wip",
    });
    $rs->update({
        status => "new",
    });
}

=head2 restore_state

    Method to restore the harvester to a pre-existing state.

=cut

sub restore_state {
    my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION];

    my $state_file = $self->{state_file};
    if ($state_file){
        my $state_backup = "$state_file~";

        #NOTE: If there is a state backup, it means we crashed while saving the state. Otherwise,
        #let's try the regular state file if it exists.
        my $file_to_restore = ( -f $state_backup ) ? $state_backup : ( ( -f $state_file ) ? $state_file : undef );
        if ( $file_to_restore ){
            my $opened = open( my $fh, '<', $file_to_restore ) or die "Couldn't open state: $!";
            if ($opened){
                local $/;
                my $in = <$fh>;
                my $decoder = Sereal::Decoder->new;
                my $state = $decoder->decode($in);
                if ($state){
                    if ($state->{tasks}){
                        #Restore tasks from our saved state
                        $heap->{tasks} = $state->{tasks};
                        foreach my $uuid ( keys %{$heap->{tasks}} ){
                            my $task = $heap->{tasks}->{$uuid};

                            #If tasks were still downloading, restart the task
                            if ( ($task->{status} && $task->{status} eq "active") && $task->{downloading} ){
                                $task->{status} = "new";
                                $kernel->call("harvester","start_task",$task->{uuid});
                            }
                        }
                    }
                }
            }
        }
    }
}

=head2 save_state

    Method to save the existing state of the harvester

=cut

sub save_state {
    my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION];
    my $state_file = $self->{state_file};
    my $state_backup = "$state_file~";

    #Make a backup of existing state record
    my $moved = move($state_file,$state_backup);

    my $opened = open(my $fh, ">", $state_file) or die "Couldn't save state: $!";
    if ($opened){
        $fh->autoflush(1);
        my $tasks = $heap->{tasks};
        my $harvester_state = {
            tasks => $tasks,
        };
        my $encoder = Sereal::Encoder->new;
        my $out = $encoder->encode($harvester_state);
        local $\;
        my $printed = print $fh $out;
        if ($printed){
            close $fh;
            unlink($state_backup);
            return 1;
        }
    }
    return 0;
}

=head2 get_task

    This event handler returns a task from a harvester using the task's
    uuid as an argument.

=cut

sub get_task {
    my ($self, $kernel, $heap, $session, $uuid, $sender) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0, SENDER];

    if ( ! $uuid && $sender ){
        my $scoreboard = $heap->{scoreboard};
        my $uuid_by_session = $scoreboard->{session}->{$sender};
        if ($uuid_by_session){
            $uuid = $uuid_by_session;
        }
    }

    my $tasks = $heap->{tasks};
    if ($tasks && $uuid){
        my $task = $tasks->{$uuid};
        if ($task){
            return $task;
        }
    }
    return 0;
}

=head2 get_import_count_for_task

    This gets the count of the number of imports exist for a certain task

=cut

sub get_import_count_for_task {
    my ($self,$uuid) = @_;
    my $count = undef;
    if ($uuid){
        my $items = Koha::OAI::Harvester::ImportQueues->new->search({
            uuid => $uuid,
        });
        $count = $items->count;
    }
    return $count;
}

=head2 list_tasks

    This event handler returns a list of tasks that have been submitted
    to the harvester. It returns data like uuid, status, parameters,
    number of pending imports, etc.

=cut

sub list_tasks {
    my ($self, $kernel, $heap, $session, $status) = @_[OBJECT, KERNEL,HEAP,SESSION, ARG0];
    my @tasks = ();
    foreach my $uuid (sort keys %{$heap->{tasks}}){
        my $task = $heap->{tasks}->{$uuid};
        my $items = Koha::OAI::Harvester::ImportQueues->new->search({
            uuid => $uuid,
        });
        my $count = $items->count // 0;
        $task->{pending_imports} = $count;
        if ( ( ! $status ) || ( $status && $status eq $task->{status} ) ){
            push(@tasks, $task);
        }

    }
    return \@tasks;
}

=head2 create_task

    This event handler creates a spool directory for the task's imports.
    It also adds it to the harvester's memory and then saves memory to
    a persistent datastore.

    Newly created tasks have a status of "new".

=cut

sub create_task {
    my ($self, $kernel, $heap, $session, $incoming_task) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
    my $logger = $self->{logger};
    if ($incoming_task){
        my $uuid = $incoming_task->{uuid};
        if ( ! $heap->{tasks}->{$uuid} ){

            #Step One: assign a spool directory to this task
            my $spooldir = $self->{spooldir} // "/tmp";
            my $task_spooldir = "$spooldir/$uuid";
            if ( ! -d $task_spooldir ){
                my $made_spool_directory = make_path($task_spooldir);
                if ( ! $made_spool_directory ){
                    if ($logger){
                        $logger->warn("Unable to make task-specific spool directory at '$task_spooldir'");
                    }
                    return 0;
                }
            }
            $incoming_task->{spooldir} = $task_spooldir;

            #Step Two: assign new status
            $incoming_task->{status} = "new";

            #Step Three: add task to harvester's memory
            $heap->{tasks}->{ $uuid } = $incoming_task;

            #Step Four: save state
            $kernel->call($session,"save_state");
            return 1;
        }
    }
    return 0;
}

=head2 start_task

    This event handler marks a task as active in the harvester's memory,
    save the memory to a persistent datastore, then enqueues the task,
    so that it can be directed to the next available download worker.

    Newly started tasks have a status of "active".

=cut

sub start_task {
    my ($self, $session,$kernel,$heap,$uuid) = @_[OBJECT, SESSION,KERNEL,HEAP,ARG0];
    my $task = $heap->{tasks}->{$uuid};
    if ($task){
        if ( $task->{status} ne "active" ){

            #Clear any pre-existing error states
            delete $task->{error} if $task->{error};

            #Step One: mark task as active
            $task->{status} = "active";

            #Step Two: save state
            $kernel->call("harvester","save_state");

            #Step Three: enqueue task
            $kernel->post("oai-downloader",'enqueue','download_postback', $task);

            return 1;
        }
    }
    return 0;
}

=head2 repeat_task

    This method re-queues a task for downloading

=cut

sub repeat_task {
    my ($self, $session,$kernel,$heap,$uuid) = @_[OBJECT, SESSION,KERNEL,HEAP,ARG0];
    my $task = $heap->{tasks}->{$uuid};
    if ($task){
        my $interval = $task->{interval};
        if ($task->{downloading} && $interval){
            $kernel->post("oai-downloader",'enqueue','download_postback', $task);
        }
    }
}

=head2 stop_task

    This event handler prevents new workers from spawning, kills
    existing workers, and stops pending imports from being imported.

    Newly stopped tasks have a status of "stopped".

=cut

sub stop_task {
    my ($self, $kernel, $heap, $session, $sender, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0];

    my $task = $heap->{tasks}->{$task_uuid};

    if ($task && $task->{status} && $task->{status} ne "stopped" ){

        #Step One: deactivate task, so no new workers can be started
        $task->{status} = "stopped";
        #NOTE: You could also clear timers for new downloads, but that's probably unnecessary because of this step.

        #Step Two: kill workers
        my $scoreboard = $heap->{scoreboard};
        my $session_types = $scoreboard->{task}->{$task_uuid};
        if ($session_types){
            foreach my $type ( keys %$session_types ){
                my $sessions = $session_types->{$type};
                if ($sessions){
                    foreach my $session (keys %$sessions){
                        if ($session){
                            $kernel->signal($session, "cancel");
                        }
                    }
                }
            }
            #Remove the task uuid from the task key of the scoreboard
            delete $scoreboard->{task}->{$task_uuid};
            #NOTE: The task uuid will still exist under the session key,
            #but the sessions will deregister themselves and clean that up for you.
        }

        #Step Three: stop pending imports for this task
        my $items = Koha::OAI::Harvester::ImportQueues->new->search({
            uuid => $task_uuid,
        });
        my $rows_updated = $items->update({
            status => "stopped",
        });

        #Step Four: save state
        $kernel->call("harvester","save_state");
        return 1;
    }
    return 0;
}

=head2 delete_task

    Deleted tasks are stopped, pending imports are deleted from the
    database and file system, and then the task is removed from the harvester.

=cut

sub delete_task {
    my ($self, $kernel, $heap, $session, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];

    my $task = $heap->{tasks}->{$task_uuid};
    if ($task){
        #Step One: stop task
        $kernel->call($session,"stop_task",$task_uuid);

        #Step Two: delete pending imports in database
        my $items = Koha::OAI::Harvester::ImportQueues->new->search({
            uuid => $task_uuid,
        });
        if ($items){
            my $rows_deleted = $items->delete;
            #NOTE: shows 0E0 instead of 0
        }

        #Step Three: remove task specific spool directory and files within it
        my $spooldir = $task->{spooldir};
        if ($spooldir){
            my $files_deleted = remove_tree($spooldir, { safe => 1 });
        }

        delete $heap->{tasks}->{$task_uuid};

        #Step Four: save state
        $kernel->call("harvester","save_state");
        return 1;
    }
    return 0;
}

1;




package Koha::OAI::Harvester::Biblio;

# Copyright Prosentient Systems 2018
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;

use Carp;

use base qw(Koha::Object);

=head1 NAME

Koha::OAI::Harvester::Biblio

=head1 SYNOPSIS

    use Koha::OAI::Harvester::Biblio;
    my $biblio = Koha::OAI::Harvester::Biblios->find($id);

=head1 METHODS

=cut

sub _type {
    return 'OaiHarvesterBiblio';
}

=head1 AUTHOR

David Cook <dcook@prosentient.com.au>

=cut

1;




package Koha::OAI::Harvester::Biblios;

# Copyright Prosentient Systems 2018
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;

use Carp;

use Koha::Database;

use Koha::OAI::Harvester::Biblio;

use base qw(Koha::Objects);

=head1 NAME

Koha::OAI::Harvester::Biblios

=head2 METHODS

=cut

=head3 _type

=cut

sub _type {
    return 'OaiHarvesterBiblio';
}

=head3 object_class

=cut

sub object_class {
    return 'Koha::OAI::Harvester::Biblio';
}

=head1 AUTHOR

David Cook <dcook@prosentient.com.au>

=cut

1;




package Koha::OAI::Harvester::Client;

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;
use URI;
use IO::Socket::UNIX;
use IO::Select;
use JSON;
use Encode;

=head1 NAME

Koha::OAI::Harvester::Client

=head1 SYNOPSIS

  use Koha::OAI::Harvester::Client;
  my $client = Koha::OAI::Harvester::Client->new($client_config);

=head1 METHODS

=head2 new

Create object

=cut

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;
    return bless ($args, $class);
}

=head2 connect

Connect to Koha::OAI::Harvester::Listener via a Unix socket

=cut

sub connect {
    my ($self) =  @_;
    my $socket_uri = $self->{socket_uri};
    if ($socket_uri){
        my $uri = URI->new($socket_uri);
        if ($uri && $uri->scheme eq 'unix'){
            my $socket_path = $uri->path;
            my $socket = IO::Socket::UNIX->new(
                Type => IO::Socket::UNIX::SOCK_STREAM(),
                Peer => $socket_path,
            );
            if ($socket){
                my $select = new IO::Select();
                $select->add($socket);

                $self->{_select} = $select;
                $self->{_socket} = $socket;
                my $message = $self->_read();
                if ($message){
                    if ($message eq 'HELLO'){
                        $self->{_connected} = 1;
                        return 1;
                    }
                }
            }
            else {
                warn "Failed to create socket."
            }
        }
    }
    return 0;
}

=head2 create

    Create a task on the harvester

=cut

sub create {
    my ($self,$task) = @_;
    my $message = {
        command => "create",
        body => {
            task => $task,
        }
    };
    my ($status) = $self->_exchange($message);
    return $status;
}

=head2 start

    Start a task on the harvester

=cut

sub start {
    my ($self,$uuid) = @_;
    my $message = {
        command => "start",
        body => {
            task => {
                uuid => $uuid,
            },
        }
    };
    my ($status) = $self->_exchange($message);
    return $status;
}

=head2 stop

    Stop a task on the harvester

=cut

sub stop {
    my ($self,$uuid) = @_;
    my $message = {
        command => "stop",
        body => {
            task => {
                uuid => $uuid,
            },
        }
    };
    my ($status) = $self->_exchange($message);
    return $status;
}

=head2 delete

    Delete a task on the harvester

=cut

sub delete {
    my ($self,$uuid) = @_;
    my $message = {
        command => "delete",
        body => {
            task => {
                uuid => $uuid,
            },
        }
    };
    my ($status) = $self->_exchange($message);
    return $status;
}

=head2 list

    List all tasks on the harvester

=cut

sub list {
    my ($self) = @_;
    my $message = {
        command => "list",
    };
    my ($status,$tasks) = $self->_exchange($message);
    return $tasks;
}

sub _exchange {
    my ($self,$message) = @_;
    my $status = 0;
    my $data;
    if ($message){
        my $output = to_json($message);
        if ($output){
            $self->_write($output);
            my $json_response = $self->_read();
            if ($json_response){
                my $response = from_json($json_response);
                $data = $response->{data} if $response->{data};
                $status = 1 if $response->{msg} && $response->{msg} eq "success";
            }
        }
    }
    return ($status,$data);
}

sub _write {
    my ($self, $output) = @_;
    if ($output){
        if (my $select = $self->{_select}){
            if (my @filehandles = $select->can_write(5)){
                foreach my $filehandle (@filehandles){
                    #Localize output record separator as null
                    local $\ = "\x00";
                    $output = Encode::encode("UTF-8",$output);
                    print $filehandle $output;
                }
            }
        }
    }
}

sub _read {
    my ($self) = @_;
    if (my $select = $self->{_select}){
        if (my @filehandles = $select->can_read(5)){
            foreach my $filehandle (@filehandles){
                #Localize input record separator as null
                local $/ = "\x00";
                my $message = <$filehandle>;
                chomp($message) if $message;
                $message = Encode::decode("UTF-8",$message);
                return $message;
            }
        }
    }
}

1;




package Koha::OAI::Harvester::Downloader;

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;
use URI;
use XML::LibXML::Reader;
use IO::Handle;
use JSON;

=head1 NAME

Koha::OAI::Harvester::Downloader

=head1 SYNOPSIS

    use Koha::OAI::Harvester::Downloader;
    my $oai_downloader = Koha::OAI::Harvester::Downloader->new();

    This class is used within a Koha::OAI::Harvester::Work::Download:: module.

=head1 METHODS

=head2 new

    Create object

=cut

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;
    return bless ($args, $class);
}

=head2 BuildURL

    Takes a baseURL and a mix of required and optional OAI-PMH arguments,
    and makes them into a suitable URL for an OAI-PMH request.

=cut

sub BuildURL {
    my ($self, $args) = @_;
    my $baseURL = $args->{baseURL};
    my $url = URI->new($baseURL);
    if ($url && $url->isa("URI")){
        my $verb = $args->{verb};
        if ($verb){
            my %parameters = (
                verb => $verb,
            );
            if ($verb eq "ListRecords"){
                my $resumptionToken = $args->{resumptionToken};
                my $metadataPrefix = $args->{metadataPrefix};
                if ($resumptionToken){
                    $parameters{resumptionToken} = $resumptionToken;
                }
                elsif ($metadataPrefix){
                    $parameters{metadataPrefix} = $metadataPrefix;
                    #Only add optional parameters if they're provided
                    foreach my $param ( qw( from until set ) ){
                        $parameters{$param} = $args->{$param} if $args->{$param};
                    }
                }
                else {
                    warn "BuildURL() requires an argument of either resumptionToken or metadataPrefix";
                    return;
                }
            }
            elsif ($verb eq "GetRecord"){
                my $metadataPrefix = $args->{metadataPrefix};
                my $identifier = $args->{identifier};
                if ($metadataPrefix && $identifier){
                    $parameters{metadataPrefix} = $metadataPrefix;
                    $parameters{identifier} = $identifier;
                }
                else {
                    warn "BuildURL() requires an argument of metadataPrefix and an argument of identifier";
                    return;
                }
            }
            $url->query_form(%parameters);
            return $url;
        }
        else {
            warn "BuildURL() requires a verb of GetRecord or ListRecords";
            return;
        }
    }
    else {
        warn "BuildURL() requires a base URL of type URI.";
        return;
    }
}

=head2 GetXMLStream

    Fork a child process to send the HTTP request, which sends chunks
    of XML via a pipe to the parent process.

    The parent process creates and returns a XML::LibXML::Reader object,
    which reads the XML stream coming through the pipe.

    Normally, using a DOM reader, you must wait to read the entire XML document
    into memory. However, using a stream reader, chunks are read into memory,
    processed, then discarded. It's faster and more efficient.

=cut

sub GetXMLStream {
    my ($self, $args) = @_;
    my $url = $args->{url};
    my $user_agent = $args->{user_agent};
    if ($url && $user_agent){
        pipe( CHILD, PARENT ) or die "Cannot created connected pipes: $!";
        CHILD->autoflush(1);
        PARENT->autoflush(1);
        if ( my $pid = fork ){
            #Parent process
            close PARENT;
            return \*CHILD;
        }
        else {
            #Child process
            close CHILD;
            my $response = $self->_request({
                url => $url,
                user_agent => $user_agent,
                file_handle => \*PARENT,
            });
            if ($response && $response->is_success){
                #HTTP request has successfully finished, so we close the file handle and exit the process
                close PARENT;
                CORE::exit(); #some modules like mod_perl redefine exit
            }
            else {
                warn "[child $$] OAI-PMH unsuccessful. Response status: ".$response->status_line."\n" if $response;
                CORE::exit();
            }
        }
    }
    else {
        warn "GetXMLStream() requires a 'url' argument and a 'user_agent' argument";
        return;
    }
}

sub _request {
    my ($self, $args) = @_;
    my $url = $args->{url};
    my $user_agent = $args->{user_agent};
    my $fh = $args->{file_handle};

    if ($url && $user_agent && $fh){
        my $request = HTTP::Request->new( GET => $url );
        my $response = $user_agent->request( $request, sub {
                my ($chunk_of_data, $ref_to_response, $ref_to_protocol) = @_;
                print $fh $chunk_of_data;
        });
        return $response;
    }
    else {
        warn "_request() requires a 'url' argument, 'user_agent' argument, and 'file_handle' argument.";
        return;
    }
}

=head2 ParseXMLStream

    Parse XML using XML::LibXML::Reader from a file handle (e.g. a stream)

=cut


sub ParseXMLStream {
    my ($self, $args) = @_;

    my $each_callback = $args->{each_callback};
    my $fh = $args->{file_handle};
    if ($fh){
        my $reader = XML::LibXML::Reader->new( FD => $fh, no_blanks => 1 );
        my $pattern = XML::LibXML::Pattern->new('oai:OAI-PMH|/oai:OAI-PMH/*', { 'oai' => "http://www.openarchives.org/OAI/2.0/" });

        my $repository;

        warn "Start parsing...";
        while (my $rv = $reader->nextPatternMatch($pattern)){
            #$rv == 1; successful
            #$rv == 0; end of document reached
            #$rv == -1; error
            if ($rv == -1){
                die "Parser error!";
            }
            #NOTE: We do this so we only get the opening tag of the element.
            next unless $reader->nodeType == XML_READER_TYPE_ELEMENT;

            my $localname = $reader->localName;
            if ( $localname eq "request" ){
                my $node = $reader->copyCurrentNode(1);
                $repository = $node->textContent;
            }
            elsif ( $localname eq "error" ){
                #See https://www.openarchives.org/OAI/openarchivesprotocol.html#ErrorConditions
                #We should probably die under all circumstances except "noRecordsMatch"
                my $node = $reader->copyCurrentNode(1);
                if ($node){
                    my $code = $node->getAttribute("code");
                    if ($code){
                        if ($code ne "noRecordsMatch"){
                            warn "Error code: $code";
                            die;
                        }
                    }
                }
            }
            elsif ( ($localname eq "ListRecords") || ($localname eq "GetRecord") ){
                my $each_pattern = XML::LibXML::Pattern->new('//oai:record|oai:resumptionToken', { 'oai' => "http://www.openarchives.org/OAI/2.0/" });
                while (my $each_rv =  $reader->nextPatternMatch($each_pattern)){
                    if ($rv == "-1"){
                        #NOTE: -1 denotes an error state
                        warn "Error getting pattern match";
                    }
                    next unless $reader->nodeType == XML_READER_TYPE_ELEMENT;
                    if ($reader->localName eq "record"){
                        my $node = $reader->copyCurrentNode(1);
                        #NOTE: Without the UTF-8 flag, UTF-8 data will be corrupted.
                        my $document = XML::LibXML::Document->new('1.0', 'UTF-8');
                        $document->setDocumentElement($node);

                       #Per record callback
                        if ($each_callback){
                            $each_callback->({
                                repository => $repository,
                                document => $document,
                            });
                        }
                    }
                    elsif ($reader->localName eq "resumptionToken"){
                        my $resumptionToken = $reader->readInnerXml;
                        return ($resumptionToken,$repository);

                    }
                }
            }
        } #/OAI-PMH document match
    }
    else {
        warn "ParseXMLStream() requires a 'file_handle' argument.";
    }
}

=head2 harvest

    Given a URL and a user-agent, start downloading OAI-PMH records from
    that URL.

=cut

sub harvest {
    my ($self,$args) = @_;
    my $url = $args->{url};
    my $ua = $args->{user_agent};
    my $callback = $args->{callback};
    my $complete_callback = $args->{complete_callback};

    if ($url && $ua){

        #NOTE: http://search.cpan.org/~shlomif/XML-LibXML-2.0128/lib/XML/LibXML/Parser.pod#ERROR_REPORTING
        while($url){
            warn "URL = $url";
            warn "Creating child process to download and feed parent process parser.";
            my $stream = $self->GetXMLStream({
                url => $url,
                user_agent => $ua,
            });

            warn "Creating parent process parser.";
            my ($resumptionToken) = $self->ParseXMLStream({
                file_handle => $stream,
                each_callback => $callback,
            });
            warn "Finished parsing current XML document.";

            if ($resumptionToken){
                #If there's a resumptionToken at the end of the stream,
                #we build a new URL and repeat this process again.
                $url->query_form({
                    verb => "ListRecords",
                    resumptionToken => $resumptionToken,
                });
            }
            else {
                warn "Finished harvest.";
                last;
            }

            warn "Reap child process downloader.";
            #Reap the dead child requester process before performing another request,
            #so we don't fill up the process table with zombie children.
            while ((my $child = waitpid(-1, 0)) > 0) {
                warn "Parent $$ reaped child process $child" . ($? ? " with exit code $?" : '') . ".\n";
            }
        }

        if ($complete_callback){
            warn "Run complete callback.";

            #Clear query string
            $url->query_form({});

            #Run complete callback using the actual URL from the request.
            $complete_callback->({
                repository => $url,
            });
        }
    }
}

1;




package Koha::OAI::Harvester::Histories;

# Copyright Prosentient Systems 2018
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;

use Carp;

use Koha::Database;

use Koha::OAI::Harvester::History;

use base qw(Koha::Objects);

=head1 NAME

Koha::OAI::Harvester::Histories

=head2 METHODS

=cut

=head3 _type

=cut

sub _type {
    return 'OaiHarvesterHistory';
}

=head3 object_class

=cut

sub object_class {
    return 'Koha::OAI::Harvester::History';
}

=head1 AUTHOR

David Cook <dcook@prosentient.com.au>

=cut

1;




package Koha::OAI::Harvester::History;

# Copyright Prosentient Systems 2018
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;

use Carp;

use base qw(Koha::Object);

=head1 NAME

Koha::OAI::Harvester::History

=head1 SYNOPSIS

    use Koha::OAI::Harvester::History;
    my $request = Koha::OAI::Harvester::History->find($id);

=head1 METHODS

=cut

sub _type {
    return 'OaiHarvesterHistory';
}

=head1 AUTHOR

David Cook <dcook@prosentient.com.au>

=cut

1;




package Koha::OAI::Harvester::Import::MARCXML;

# Copyright 2016 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.
#

use Modern::Perl;
use MARC::Record;

use C4::Context;
use C4::Biblio;

use constant MAX_MATCHES => 99999; #NOTE: This is an arbitrary value. We want to get all matches.

=head1 NAME

Koha::OAI::Harvester::Import::MARCXML

=head1 SYNOPSIS

    use Koha::OAI::Harvester::Import::MARCXML;
    my $marcxml = eval { Koha::OAI::Harvester::Import::MARCXML->new({ dom => $results, }) };

=head1 METHODS

=head2 new

    Create object

=cut

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;
    if ( (! $args->{dom}) && (! $args->{marc_record}) ){
        die "You must provide either a dom or marc_record argument to this constructor";
    }
    if ( $args->{dom} && ( ! $args->{marc_record} ) ){
        my $dom = $args->{dom};
        my $xml = $dom->toString(2);
        my $marcflavour = C4::Context->preference('marcflavour') || 'MARC21';
        my $marc_record = eval {MARC::Record::new_from_xml( $xml, "utf8", $marcflavour)};
        if ($@){
            die "Unable to create MARC::Record object";
        }
        if ($marc_record){
            $args->{marc_record} = $marc_record;
        }
    }
    return bless ($args, $class);
}

=head2 import_record

    Import a record into Koha

=cut

sub import_record {
    my ($self,$args) = @_;
    my $framework = $args->{framework};
    my $record_type = $args->{record_type};
    my $matcher = $args->{matcher};
    my $koha_id = $args->{koha_id};

    my $action = "error";

    #Try to find a matching Koha MARCXML record via Zebra
    if (! $koha_id && $matcher){
        my $matched_id = $self->_try_matcher({
            matcher => $matcher,
        });
        if ($matched_id){
            $koha_id = $matched_id;
        }
    }

    if ($koha_id){
        #Update
        ($action) = $self->_mod_koha_record({
            record_type => $record_type,
            framework => $framework,
            koha_id => $koha_id,
        });
    }
    else {
        #Add
        ($action,$koha_id) = $self->_add_koha_record({
            record_type => $record_type,
            framework => $framework,
        });
    }

    return ($action,$koha_id);
}

sub _try_matcher {
    my ($self, $args) = @_;
    my $marc_record = $self->{marc_record};
    my $matcher = $args->{matcher};
    my $matched_id;
    my @matches = $matcher->get_matches($marc_record, MAX_MATCHES);
    if (@matches){
        my $bestrecordmatch = shift @matches;
        if ($bestrecordmatch && $bestrecordmatch->{record_id}){
            $matched_id = $bestrecordmatch->{record_id};
        }
    }
    return $matched_id;
}

sub _add_koha_record {
    my ($self, $args) = @_;
    my $marc_record = $self->{marc_record};
    my $record_type = $args->{record_type} // "biblio";
    my $framework = $args->{framework};
    my $koha_id;
    my $action = "error";
    if ($record_type eq "biblio"){
        #NOTE: Strip item fields to prevent any accidentally getting through.
        C4::Biblio::_strip_item_fields($marc_record,$framework);
        my ($biblionumber,$biblioitemnumber) = C4::Biblio::AddBiblio($marc_record,$framework);
        if ($biblionumber){
            $action = "added";
            $koha_id = $biblionumber;
        }
    }
    return ($action,$koha_id);
}

sub _mod_koha_record {
    my ($self, $args) = @_;
    my $marc_record = $self->{marc_record};
    my $record_type = $args->{record_type} // "biblio";
    my $framework = $args->{framework};
    my $koha_id = $args->{koha_id};
    my $action = "error";
    if ($record_type eq "biblio"){
        #NOTE: Strip item fields to prevent any accidentally getting through.
        C4::Biblio::_strip_item_fields($marc_record,$framework);
        my $updated = C4::Biblio::ModBiblio($marc_record, $koha_id, $framework);
        if ($updated){
            $action = "updated";
        }
    }
    return ($action);
}

1;




package Koha::OAI::Harvester::Import::Record;

# Copyright 2016 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.
#

use Modern::Perl;
use XML::LibXML;
use XML::LibXSLT;
use URI;
use File::Basename;

use C4::Context;
use C4::Biblio;

use Koha::OAI::Harvester::Import::MARCXML;
use Koha::OAI::Harvester::Biblios;
use Koha::OAI::Harvester::History;

=head1 NAME

Koha::OAI::Harvester::Import::Record

=head1 SYNOPSIS

    use Koha::OAI::Harvester::Import::Record;
    my $oai_record = Koha::OAI::Harvester::Import::Record->new({
        doc => $dom,
        repository => $repository,
    });

=head1 METHODS

=head2 new

    Create object

=cut

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;

    die "You must provide a 'doc' argument to the constructor" unless $args->{doc};
    die "You must provide a 'repository' argument to the constructor" unless $args->{repository};

    if (my $doc = $args->{doc}){

        #Get the root element
        my $root = $doc->documentElement;

        #Register namespaces for searching purposes
        my $xpc = XML::LibXML::XPathContext->new();
        $xpc->registerNs('oai','http://www.openarchives.org/OAI/2.0/');

        my $xpath_identifier = XML::LibXML::XPathExpression->new("oai:header/oai:identifier");
        my $identifier = $xpc->findnodes($xpath_identifier,$root)->shift;
        $args->{header_identifier} = $identifier->textContent;

        my $xpath_datestamp = XML::LibXML::XPathExpression->new("oai:header/oai:datestamp");
        my $datestamp = $xpc->findnodes($xpath_datestamp,$root)->shift;
        $args->{header_datestamp} = $datestamp->textContent;

        my $xpath_status = XML::LibXML::XPathExpression->new(q{oai:header/@status});
        my $status_node = $xpc->findnodes($xpath_status,$root)->shift;
        $args->{header_status} = $status_node ? $status_node->textContent : "";
    }

    return bless ($args, $class);
}

=head2 is_deleted_upstream

    Returns true if OAI-PMH record is deleted upstream

=cut

sub is_deleted_upstream {
    my ($self, $args) = @_;
    if ($self->{header_status}){
        if ($self->{header_status} eq "deleted"){
            return 1;
        }
    }
    return 0;
}

=head2 set_filter

    $self->set_filter("/path/to/filter.xsl");

    Set a XSLT to use to filter records on import. This
    takes a full filepath as an argument.

=cut

sub set_filter {
    my ($self, $filter_definition) = @_;

    #Source a default XSLT to use for filtering
    my $htdocs  = C4::Context->config('intrahtdocs');
    my $theme   = C4::Context->preference("template");
    $self->{filter} = "$htdocs/$theme/en/xslt/StripOAIPMH.xsl";
    $self->{filter_type} = "xslt";

    if ($filter_definition && $filter_definition ne "default"){
        my ($filter_type, $filter) = $self->_parse_filter($filter_definition);
        if ($filter_type eq "xslt"){
            if (  -f $filter ){
                $self->{filter} = $filter;
                $self->{filter_type} = "xslt";
            }
        }
    }
}

sub _parse_filter {
    my ($self,$filter_definition) = @_;
    my ($type,$filter);
    my $filter_uri = URI->new($filter_definition);
    if ($filter_uri){
        my $scheme = $filter_uri->scheme;
        if ( ($scheme && $scheme eq "file") || ! $scheme ){
            my $path = $filter_uri->path;
            #Filters may theoretically be .xsl or .pm files
            my($filename, $dirs, $suffix) = fileparse($path,(".xsl",".pm"));
            if ($suffix){
                if ( $suffix eq ".xsl"){
                    $type = "xslt";
                    $filter = $path;
                }
            }
        }
    }
    return ($type,$filter);
}

=head2 filter

    Filters the OAI-PMH record using a filter

=cut

sub filter {
    my ($self) = @_;
    my $filtered = 0;
    my $doc = $self->{doc};
    my $filter = $self->{filter};
    my $filter_type = $self->{filter_type};
    if ($doc){
        if ($filter && -f $filter){
            if ($filter_type){
                if ( $filter_type eq 'xslt' ){
                    my $xslt = XML::LibXSLT->new();
                    my $style_doc = XML::LibXML->load_xml(location => $filter);
                    my $stylesheet = $xslt->parse_stylesheet($style_doc);
                    if ($stylesheet){
                        my $results = $stylesheet->transform($doc);
                        if ($results){
                            my $root = $results->documentElement;
                            if ($root){
                                my $namespace = $root->namespaceURI;
                                if ($namespace eq "http://www.loc.gov/MARC21/slim"){
                                    #NOTE: Both MARC21 and UNIMARC should be covered by this namespace
                                    my $marcxml = eval { Koha::OAI::Harvester::Import::MARCXML->new({ dom => $results, }) };
                                    if ($@){
                                        warn "Error Koha::OAI::Harvester::Import::MARCXML: $@";
                                        return;
                                    } else {
                                        return $marcxml;
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
    }
    return;
}

sub _find_koha_link {
    my ($self, $args) = @_;
    my $record_type = $args->{record_type} // "biblio";
    my $link_id;
    if ($record_type eq "biblio"){
        my $link = Koha::OAI::Harvester::Biblios->new->find(
            {
                oai_repository => $self->{repository},
                oai_identifier => $self->{header_identifier},
            },
            { key => "oai_record",}
        );
        if ($link && $link->biblionumber){
            $link_id = $link->biblionumber;
        }
    }
    return $link_id;
}

=head2 import_record

    my ($action,$record_id) = $oai_record->import_record({
        filter => $filter,
        framework => $framework,
        record_type => $record_type,
        matcher => $matcher,
    });

    $action eq "added" || "updated" || "deleted" || "not_found" || "error"

=cut

sub import_record {
    my ($self, $args) = @_;
    my $filter = $args->{filter} || 'default';
    my $framework = $args->{framework} || '';
    my $record_type = $args->{record_type} || 'biblio';
    my $matcher = $args->{matcher};

    my $action = "error";

    #Find linkage between OAI-PMH repository-identifier and Koha record id
    my $linked_id = $self->_find_koha_link({
        record_type => $record_type,
    });

    if ($self->is_deleted_upstream){
        #NOTE: If a record is deleted upstream, it will not contain a metadata element
        if ($linked_id){
            $action = $self->delete_koha_record({
                record_id => $linked_id,
                record_type => $record_type,
            });
        }
        else {
            $action = "not_found";
            #NOTE: If there's no OAI-PMH repository-identifier pair in the database,
            #then there's no perfect way to find a linked record to delete.
        }
    }
    else {
        $self->set_filter($filter);


        my $import_record = $self->filter();

        if ($import_record){
            ($action,$linked_id) = $import_record->import_record({
                framework => $framework,
                record_type => $record_type,
                matcher => $matcher,
                koha_id => $linked_id,
            });

            if ($linked_id){
                #Link Koha record ID to OAI-PMH details for this record type,
                #if linkage doesn't already exist.
                $self->link_koha_record({
                    record_type => $record_type,
                    koha_id => $linked_id,
                });
            }
        }
    }

    #Log record details to database
    Koha::OAI::Harvester::History->new({
        header_identifier => $self->{header_identifier},
        header_datestamp => $self->{header_datestamp},
        header_status => $self->{header_status},
        record => $self->{doc}->toString(1),
        repository => $self->{repository},
        status => $action,
        filter => $filter,
        framework => $framework,
        record_type => $record_type,
        matcher_code => $matcher ? $matcher->code : undef,
    })->store();

    return ($action,$linked_id);
}

=head2 link_koha_record

    Link an OAI-PMH record with a Koha record using
    the OAI-PMH repository and OAI-PMH identifier

=cut

sub link_koha_record {
    my ($self, $args) = @_;
    my $record_type = $args->{record_type} // "biblio";
    my $koha_id = $args->{koha_id};
    if ($koha_id){
        if ($record_type eq "biblio"){
            my $import_oai_biblio = Koha::OAI::Harvester::Biblios->new->find_or_create({
                oai_repository => $self->{repository},
                oai_identifier => $self->{header_identifier},
                biblionumber => $koha_id,
            });
            if ( ! $import_oai_biblio->in_storage ){
                $import_oai_biblio->insert;
            }
        }
    }
}

=head2 delete_koha_record

    Delete a Koha record

=cut

sub delete_koha_record {
    my ($self, $args) = @_;
    my $record_type = $args->{record_type} // "biblio";
    my $record_id = $args->{record_id};

    my $action = "error";

    if ($record_type eq "biblio"){
        my $error = C4::Biblio::DelBiblio($record_id);
        if (!$error){
            $action = "deleted";
            #NOTE: If there's no error, a cascading database delete should
            #automatically remove the link between the Koha biblionumber and OAI-PMH record too
        }
    }
    return $action;
}

1;




package Koha::OAI::Harvester::ImportQueue;

# Copyright Prosentient Systems 2018
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;

use Carp;

use base qw(Koha::Object);

=head1 NAME

Koha::OAI::Harvester::ImportQueue

=head1 SYNOPSIS

    use Koha::OAI::Harvester::ImportQueue;
    my $request = Koha::OAI::Harvester::ImportQueue->find($id);

=head1 METHODS

=cut

sub _type {
    return 'OaiHarvesterImportQueue';
}

=head1 AUTHOR

David Cook <dcook@prosentient.com.au>

=cut

1;




package Koha::OAI::Harvester::ImportQueues;

# Copyright Prosentient Systems 2018
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;

use Carp;

use Koha::Database;

use Koha::OAI::Harvester::ImportQueue;

use base qw(Koha::Objects);

=head1 NAME

Koha::OAI::Harvester::ImportQueues

=head2 METHODS

=cut

=head3 _type

=cut

sub _type {
    return 'OaiHarvesterImportQueue';
}

=head3 object_class

=cut

sub object_class {
    return 'Koha::OAI::Harvester::ImportQueue';
}

=head1 AUTHOR

David Cook <dcook@prosentient.com.au>

=cut

1;




package Koha::OAI::Harvester::Listener;

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.
#

use Modern::Perl;
use POE qw(Wheel::SocketFactory Wheel::ReadWrite);
use IO::Socket qw(AF_UNIX);
use JSON;
use URI;
use Encode;

=head1 NAME

Koha::OAI::Harvester::Listener

=head1 SYNOPSIS

    use Koha::OAI::Harvester::Listener;
    my $listener = Koha::OAI::Harvester::Listener->spawn({
        logger => $logger,
        socket => $socket_addr,
    });

=head1 METHODS

=head2 new

    Create object

=cut

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;
    return bless ($args, $class);
}

=head2 spawn

    Create POE session for listener

=cut

sub spawn {
    my ($class, $args) = @_;
    my $self = $class->new($args);
    my $socket = $args->{socket};
    POE::Session->create(
        args => [
            $socket,
        ],
        object_states => [
            $self => {
                _start => "on_start",
                "on_server_success" => "on_server_success",
                "on_server_error" => "on_server_error",
                "on_client_error" => "on_client_error",
                "on_client_input" => "on_client_input",
            },
        ],
    );
}

=head2 on_start

    Internal method for starting listener

=cut

sub on_start {
    my ($kernel,$heap,$socket_uri) = @_[KERNEL,HEAP,ARG0];

    my $uri = URI->new($socket_uri);
    if ($uri && $uri->scheme eq 'unix'){
        my $socket_path = $uri->path;
        unlink $socket_path if -S $socket_path;
        $heap->{server} = POE::Wheel::SocketFactory->new(
            SocketDomain => AF_UNIX,
            BindAddress => $socket_path,
            SuccessEvent => "on_server_success",
            FailureEvent => "on_server_error",
        );

        #Make the socket writeable to other users like Apache
        chmod 0666, $socket_path;
    }
}

=head2 on_server_success

    Internal event handler for successful connection to server

=cut

sub on_server_success {
    my ($self, $client_socket, $server_wheel_id, $heap, $session) = @_[OBJECT, ARG0, ARG3, HEAP,SESSION];
    my $logger = $self->{logger};
    my $null_filter = POE::Filter::Line->new(
         Literal => chr(0),
    );
    my $client_wheel = POE::Wheel::ReadWrite->new(
        Handle => $client_socket,
        InputEvent => "on_client_input",
        ErrorEvent => "on_client_error",
        InputFilter => $null_filter,
        OutputFilter => $null_filter,
    );
    $heap->{client}->{ $client_wheel->ID() } = $client_wheel;
    $logger->info("Connection ".$client_wheel->ID()." started.");
    #TODO: Add basic authentication here?
    $client_wheel->put("HELLO");
}

=head2 on_server_error

    Internal event handler for server errors

=cut

sub on_server_error {
    my ($self, $operation, $errnum, $errstr, $heap, $session) = @_[OBJECT, ARG0, ARG1, ARG2,HEAP, SESSION];
    my $logger = $self->{logger};
    $logger->error("Server $operation error $errnum: $errstr");
    delete $heap->{server};
}

=head2 on_client_error

    Internal event handler for errors relating to the client connection

=cut

sub on_client_error {
    my ($self, $wheel_id,$heap,$session) = @_[OBJECT, ARG3,HEAP,SESSION];
    my $logger = $self->{logger};
    $logger->info("Connection $wheel_id failed or ended.");
    delete $heap->{client}->{$wheel_id};
}

=head2 on_client_input

    Internal event handler for input from clients

=cut

sub on_client_input {
    my ($self, $input, $wheel_id, $session, $kernel, $heap) = @_[OBJECT, ARG0, ARG1, SESSION, KERNEL, HEAP];
    $input = Encode::decode("UTF-8",$input);
    my $logger = $self->{logger};
    $logger->debug("Server input: $input");
    my $server_response = { msg => "fail"};
    eval {
        my $json_input = from_json($input);
        my $command = $json_input->{command};
        my $body = $json_input->{body};
        if ($command){
            if ($command eq "create"){
                my $task = $body->{task};
                if ($task){
                    my $is_created = $kernel->call("harvester","create_task",$task);
                    if ($is_created){
                        $server_response->{msg} = "success";
                    }
                }
            }
            elsif ($command eq "start"){
                my $task = $body->{task};
                if ($task){
                    my $uuid = $task->{uuid};
                    #Fetch from memory now...
                    my $is_started = $kernel->call("harvester","start_task", $uuid);
                    if ($is_started){
                        $server_response->{msg} = "success";
                    }
                }
            }
            elsif ($command eq "stop"){
                my $task = $body->{task};
                if ($task){
                    if ($task->{uuid}){
                        my $is_stopped = $kernel->call("harvester","stop_task",$task->{uuid});
                        if ($is_stopped){
                            $server_response->{msg} = "success";
                        }
                    }
                }
            }
            elsif ($command eq "delete"){
                my $task = $body->{task};
                if ($task){
                    if ($task->{uuid}){
                        my $is_deleted = $kernel->call("harvester","delete_task",$task->{uuid});
                        if ($is_deleted){
                            $server_response->{msg} = "success";
                        }
                    }
                }
            }
            elsif ($command eq "list"){
                my $tasks = $kernel->call("harvester","list_tasks");
                if ($tasks){
                    $server_response->{msg} = "success";
                    $server_response->{data} = $tasks;
                }
            }
        }
    };
    if ($@){
        #NOTE: An error most likely means that something other than a valid JSON string was received
        $logger->error($@);
    }

    if ($server_response){
        eval {
            my $client = $heap->{client}->{$wheel_id};
            my $json_message = to_json($server_response, { pretty => 1 });
            if ($json_message){
                $logger->debug("Server output: $json_message");
                $json_message = Encode::encode("UTF-8",$json_message);
                $client->put($json_message);
            }
        };
        if ($@){
            #NOTE: An error means our response couldn't be serialised as JSON
            $logger->error($@);
        }
    }
}

1;




package Koha::OAI::Harvester::Request;

# Copyright Prosentient Systems 2017
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;

use Carp;

use base qw(Koha::Object);

#For validation
use URI;
use HTTP::OAI;

=head1 NAME

Koha::OAI::Harvester::Request

=head1 SYNOPSIS

    use Koha::OAI::Harvester::Request;
    my $request = Koha::OAI::Harvester::Requests->find($id);

=head1 METHODS

=cut

sub _type {
    return 'OaiHarvesterRequest';
}

=head2 validate

    Method to validate the parameters of an OAI-PMH request

=cut

sub validate {
    my ($self) = @_;
    my $errors = {};

    #Step one: validate URL
    my $uri = URI->new($self->http_url);
    if ( $uri && $uri->scheme && ($uri->scheme eq "http" || $uri->scheme eq "https") ){

        #Step two: validate access and authorization to URL
        my $harvester = $self->_harvester();
        my $identify = $harvester->Identify;
        if ($identify->is_success){

            #Step three: validate OAI-PMH parameters

            #Test Set
            my $set = $self->oai_set;
            if ($set){
                my $set_response = $harvester->ListSets();
                my @server_sets = $set_response->set;
                if ( ! grep {$_->setSpec eq $set} @server_sets ){
                    $errors->{oai_set}->{unavailable} = 1;
                }
            }

            #Test Metadata Prefix
            my $metadataPrefix = $self->oai_metadataPrefix;
            if ($metadataPrefix){
                my $metadata_response = $harvester->ListMetadataFormats();
                my @server_formats = $metadata_response->metadataFormat;
                if ( ! grep { $_->metadataPrefix eq $metadataPrefix } @server_formats ){
                    $errors->{oai_metadataPrefix}->{unavailable} = 1;
                }
            }
            else {
                $errors->{oai_metadataPrefix}->{missing} = 1;
            }

            #Test Granularity and Timestamps
            my $server_granularity = $identify->granularity;
            my $from = $self->oai_from;
            my $until = $self->oai_until;
            if ($from || $until){
                my ($from_granularity,$until_granularity);
                if ($from){
                    $from_granularity = _determine_granularity($from);
                    if ($from_granularity eq "YYYY-MM-DDThh:mm:ssZ"){
                        $errors->{oai_from}->{unavailable} = 1 if $server_granularity ne $from_granularity;
                    } elsif ($from_granularity eq "failed"){
                        $errors->{oai_from}->{malformed} = 1;
                    }
                }
                if ($until){
                    $until_granularity = _determine_granularity($until);
                    if ($until_granularity eq "YYYY-MM-DDThh:mm:ssZ"){
                        $errors->{oai_until}->{unavailable} = 1 if $server_granularity ne $until_granularity;
                    } elsif ($until_granularity eq "failed"){
                        $errors->{oai_until}->{malformed} = 1;
                    }
                }
                if ($from && $until){
                    if ($from_granularity ne $until_granularity){
                        $errors->{oai}->{granularity_mismatch} = 1;
                    }
                }
            }

            #Test if identifier is provided when using GetRecord
            my $verb = $self->oai_verb;
            if ($verb && $verb eq "GetRecord"){
                my $identifier = $self->oai_identifier;
                if (! $identifier){
                    $errors->{oai_identifier}->{missing} = 1;
                }
            }
        }
        elsif ($identify->is_error){
            foreach my $error ($identify->errors){
                if ($error->code =~ /^404$/){
                    $errors->{http}->{404} = 1;
                } elsif ($error->code =~ /^401$/){
                    $errors->{http}->{401} = 1;
                } else {
                    $errors->{http}->{generic} = 1;
                }
            }
        }
        else {
            $errors->{http}->{generic} = 1;
        }
    } else {
        $errors->{http_url}->{malformed} = 1;
    }
    return $errors;
}

sub _harvester {
    my ( $self ) = @_;
    my $harvester;
    if ($self->http_url){
        $harvester = new HTTP::OAI::Harvester( baseURL => $self->http_url );
        my $uri = URI->new($self->http_url);
        if ($uri->scheme && ($uri->scheme eq 'http' || $uri->scheme eq 'https') ){
            my $host = $uri->host;
            my $port = $uri->port;
            $harvester->credentials($host.":".$port, $self->http_realm, $self->http_username, $self->http_password);
        }
    }
    return $harvester;
}

sub _determine_granularity {
    my ($timestamp) = @_;
    my $granularity;
    if ($timestamp =~ /^(\d{4}-\d{2}-\d{2})(T\d{2}:\d{2}:\d{2}Z)?$/){
        if ($1 && $2){
            $granularity = "YYYY-MM-DDThh:mm:ssZ";
        } elsif ($1 && !$2){
            $granularity = "YYYY-MM-DD";
        } else {
            $granularity = "failed";
        }
    } else {
        $granularity = "failed";
    }
    return $granularity;
}

=head1 AUTHOR

David Cook <dcook@prosentient.com.au>

=cut

1;




package Koha::OAI::Harvester::Requests;

# Copyright Prosentient Systems 2017
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;

use Carp;

use Koha::Database;

use Koha::OAI::Harvester::Request;

use base qw(Koha::Objects);

=head1 NAME

Koha::OAI::Harvester::Requests -

=head1 API

=head2 Class Methods

=cut

=head3 _type

=cut

sub _type {
    return 'OaiHarvesterRequest';
}

=head3 object_class

=cut

sub object_class {
    return 'Koha::OAI::Harvester::Request';
}

=head1 AUTHOR

David Cook <dcook@prosentient.com.au>

=cut

1;




package Koha::OAI::Harvester::Worker;

# Copyright Prosentient Systems 2017
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;
use POE;
use DateTime;
use JSON;

=head1 NAME

Koha::OAI::Harvester::Worker

=head1 SYNOPSIS

    This is just a base class to use when writing workers for
    the OAI-PMH harvester.

=head1 METHODS

=head2 new

    Create object

=cut

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;
    $args->{type} = "worker" unless $args->{type};
    return bless ($args, $class);
}

=head2 run

    The entry point to getting a worker to process a task

=cut

sub run {
    my ($self,$args) = @_;
    my $postback = $args->{postback};
    my $task = $args->{task};

    POE::Session->create(
        object_states => [
            $self => {
                _start           => "on_start",
                got_child_stderr => "on_child_stderr",
                got_child_close  => "on_child_close",
                got_child_signal => "on_child_signal",
                got_child_stdout => "on_child_stdout",
                stop_worker      => "stop_worker",
                _stop            => "on_stop",
            },
        ],
        args => [
            $postback,
            $task,
        ],
    );
}

=head2 stop_worker

    Internal method for killing this worker's child processes

=cut

sub stop_worker {
    my ($self,$heap) = @_[OBJECT,HEAP];
    if (my $child_processes = $heap->{children_by_pid}){
        foreach my $child_pid (keys %$child_processes){
            my $child = $child_processes->{$child_pid};
            $child->kill();
        }
    }
}

=head2 on_stop

    Internal event handler for deregistering the worker session
    from the harvester's roster of workers

=cut

sub on_stop {
    my ($self,$kernel) = @_[OBJECT,KERNEL];

    #Deregister the worker session from the harvester's roster of workers
    $kernel->call("harvester","deregister",$self->{type});
}

=head2 on_child_stdout

    Internal event handler for reading output from a child process

=cut

# Wheel event, including the wheel's ID.
sub on_child_stdout {
    my ($self, $stdout_line, $wheel_id) = @_[OBJECT, ARG0, ARG1];
    my $type = $self->{type};
    my $child = $_[HEAP]{children_by_wid}{$wheel_id};
    my $logger = $self->{logger};
    if ($logger){
        $logger->debug("[$type][pid ".$child->PID."][STDOUT] $stdout_line");
    }

    my $postback = $_[HEAP]{postback};
    if ($postback){
        eval {
            my $message = from_json($stdout_line);
            if ($message){
                $postback->($message);
            }
        };
    }
}

=head2 on_child_stderr

    Internal event handler for reading errors from a child process

=cut

# Wheel event, including the wheel's ID.
sub on_child_stderr {
    my ($self,$stderr_line, $wheel_id) = @_[OBJECT, ARG0, ARG1];
    my $type = $self->{type};
    my $child = $_[HEAP]{children_by_wid}{$wheel_id};
    my $logger = $self->{logger};
    if ($logger){
        $logger->debug("[$type][pid ".$child->PID."][STDERR] $stderr_line");
    }
}

=head2 on_child_close

    Internal event handler for when a child closes its pipes

=cut

# Wheel event, including the wheel's ID.
sub on_child_close {
    my ($self,$heap,$wheel_id) = @_[OBJECT,HEAP,ARG0];
    my $type = $self->{type};
    my $logger = $self->{logger};

    my $child = delete $heap->{children_by_wid}->{$wheel_id};

    # May have been reaped by on_child_signal().
    unless (defined $child) {
        if ($logger){
            $logger->debug("[$type][wid $wheel_id] closed all pipes");
        }
        return;
    }
    if ($logger){
        $logger->debug("[$type][pid ".$child->PID."] closed all pipes");
    }
    delete $heap->{children_by_pid}->{$child->PID};
}

=head2 on_child_signal

    Internal event handler for when a child exits

=cut

sub on_child_signal {
    my ($self,$kernel,$pid,$status) = @_[OBJECT,KERNEL,ARG1,ARG2];
    my $type = $self->{type};
    my $logger = $self->{logger};
    if ($logger){
        $logger->debug("[$type][pid $pid] exited with status $status");
    }

    my $child = delete $_[HEAP]{children_by_pid}{$_[ARG1]};

    # May have been reaped by on_child_close().
    return unless defined $child;

    delete $_[HEAP]{children_by_wid}{$child->ID};

    #If the child doesn't complete successfully, we lodge an error
    #and stop the task.
    if ($status != 0){
        my $task = $kernel->call("harvester","get_task");
        if ($task){
            $task->{error} = 1;
            my $uuid = $task->{uuid};
            if ($uuid){
                $kernel->call("harvester","stop_task",$uuid);
            }
        }
    }
}

1;




package Koha::OAI::Harvester::Worker::Download::Stream;

# Copyright Prosentient Systems 2017
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;
use LWP::UserAgent;
use UUID;
use POE;
use JSON;
use File::Path qw/make_path/;

use C4::Context;
use Koha::OAI::Harvester::Downloader;
use parent 'Koha::OAI::Harvester::Worker';

=head1 NAME

Koha::OAI::Harvester::Worker::Download::Stream

=head1 SYNOPSIS

    This is a module used by the OAI-PMH harvester internally.

    As a bare minimum, it must define an "on_start" method.

=head1 METHODS

=head2 new

    Create object

=cut

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;
    $args->{type} = "download" unless $args->{type};
    return bless ($args, $class);
}

=head2 on_start

    Internal event handler for starting the processing of a task

=cut

sub on_start {
    my ($self, $kernel, $heap, $postback,$task,$session) = @_[OBJECT, KERNEL, HEAP, ARG0,ARG1,SESSION];
    #Save postback into heap so other event handlers can use it
    $heap->{postback} = $postback;

    my $task_uuid = $task->{uuid};

    $kernel->sig("cancel" => "stop_worker");
    $kernel->call("harvester","register",$self->{type},$task->{uuid});

    my $child = POE::Wheel::Run->new(
        ProgramArgs => [$task],
        Program => sub {
            my ($args) = @_;
            $self->do_work($args);
        },
        StdoutEvent  => "got_child_stdout",
        StderrEvent  => "got_child_stderr",
        CloseEvent   => "got_child_close",
        NoSetPgrp => 1, #Keep child processes in same group as parent. This is especially useful when using Ctrl+C to kill the whole group.
    );

     $_[KERNEL]->sig_child($child->PID, "got_child_signal");

    # Wheel events include the wheel's ID.
    $_[HEAP]{children_by_wid}{$child->ID} = $child;

    # Signal events include the process ID.
    $_[HEAP]{children_by_pid}{$child->PID} = $child;

    my $logger = $self->{logger};
    if ($logger){
        $logger->debug("Child pid ".$child->PID." started as wheel ".$child->ID);
    }
}

=head2 do_work

    Internal method for processing a download task

=cut

sub do_work {
    my ($self, $task) = @_;
    my $batch = ( $self->{batch} && int($self->{batch}) ) ? $self->{batch} : 100;

    #NOTE: Directory to spool files for processing
    my $spooldir = $task->{spooldir};

    my $task_uuid = $task->{uuid};
    my $task_parameters = $task->{parameters};
    my $interval = $task->{interval};

    my $oai_pmh_parameters = $task_parameters->{oai_pmh};
    my $import_parameters = $task_parameters->{import};

    #NOTE: Overwrite the 'from' and 'until' parameters for repeatable tasks
    if ( $interval && ! $oai_pmh_parameters->{until} ){
        if ($oai_pmh_parameters->{verb} eq "ListRecords"){
            #NOTE: 'effective_from' premiers on the first repetition (ie second request)
            $oai_pmh_parameters->{from} = $task->{effective_from} if $task->{effective_from};
            #NOTE: 'effective_until' appears on the first request
            $oai_pmh_parameters->{until} = $task->{effective_until} if $task->{effective_until};
        }
    }

    my $oai_downloader = Koha::OAI::Harvester::Downloader->new();
    my $url = $oai_downloader->BuildURL($oai_pmh_parameters);

    my $ua = LWP::UserAgent->new();
    #NOTE: setup HTTP Basic Authentication if parameters are supplied
    if($url && $url->host && $url->port){
        my $http_basic_auth = $task_parameters->{http_basic_auth};
        if ($http_basic_auth){
            my $username = $http_basic_auth->{username};
            my $password = $http_basic_auth->{password};
            my $realm = $http_basic_auth->{realm};
            $ua->credentials($url->host.":".$url->port, $realm, $username, $password);
        }
    }

    #NOTE: Prepare database statement handle
    my $dbh = C4::Context->dbh;
    my $sql = "insert into oai_harvester_import_queue (uuid,result) VALUES (?,?)";
    my $sth = $dbh->prepare($sql);

    if($url && $ua){
        #NOTE: You could define the callbacks as object methods instead... that might be nicer...
        #although I suppose it might be a much of a muchness.
        eval {
            my @filename_cache = ();

            $oai_downloader->harvest({
                user_agent => $ua,
                url => $url,
                callback => sub {
                    my ($args) = @_;

                    my $repository = $args->{repository};
                    my $document = $args->{document};

                    #If the spooldir has disappeared, re-create it.
                    if ( ! -d $spooldir ){
                        my $made_spool_directory = make_path($spooldir);
                    }
                    my ($uuid,$uuid_string);
                    UUID::generate($uuid);
                    UUID::unparse($uuid, $uuid_string);
                    my $file_uuid = $uuid_string;
                    my $filename = "$spooldir/$file_uuid";
                    my $state = $document->toFile($filename, 2);
                    if ($state){
                        push(@filename_cache,$filename);
                    }

                    if(scalar @filename_cache == $batch){
                        my $result = {
                            repository => $repository,
                            filenames => \@filename_cache,
                            filter => $import_parameters->{filter},
                            matcher_code => $import_parameters->{matcher_code},
                            frameworkcode => $import_parameters->{frameworkcode},
                            record_type => $import_parameters->{record_type},
                        };
                        eval {
                            my $json_result = to_json($result, { pretty => 1 });
                            $sth->execute($task_uuid,$json_result);
                        };
                        @filename_cache = ();
                    }
                },
                complete_callback => sub {
                    my ($args) = @_;
                    my $repository = $args->{repository};
                    if (@filename_cache){
                        my $result = {
                            repository => "$repository",
                            filenames => \@filename_cache,
                            filter => $import_parameters->{filter},
                            matcher_code => $import_parameters->{matcher_code},
                            frameworkcode => $import_parameters->{frameworkcode},
                            record_type => $import_parameters->{record_type},
                        };
                        eval {
                            my $json_result = to_json($result, { pretty => 1 });
                            $sth->execute($task_uuid,$json_result);
                        };
                    }

                },
            });
        };
        if ($@){
            die "Error during OAI-PMH download";
        }
    }
}

1;




package Koha::OAI::Harvester::Worker::Import;

# Copyright Prosentient Systems 2017
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;
use POE qw(Wheel::Run);
use JSON;
use XML::LibXML;

use C4::Context;
use C4::Matcher;
use Koha::OAI::Harvester::Import::Record;

use parent 'Koha::OAI::Harvester::Worker';

=head1 NAME

Koha::OAI::Harvester::Worker::Import

=head1 SYNOPSIS

    This is a module used by the OAI-PMH harvester internally.

    As a bare minimum, it must define an "on_start" method.

=head1 METHODS

=head2 new

     Create object

=cut

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;
    #NOTE: This type is used for logging and more importantly for registering with the harvester
    $args->{type} = "import" unless $args->{type};
    return bless ($args, $class);
}

=head2 on_start

    Internal event handler for starting the processing of a task

=cut

sub on_start {
    my ($self, $kernel, $heap, $postback,$task) = @_[OBJECT, KERNEL, HEAP, ARG0,ARG1];

    $kernel->call("harvester","register",$self->{type},$task->{uuid});

    $kernel->sig(cancel => "stop_worker");

    my $child = POE::Wheel::Run->new(
        ProgramArgs => [ $task ],
        Program => sub {
            my ($task,$args) = @_;

            my $debug = $args->{debug} // 0;

            if ($task){
                my $json_result = $task->{result};
                my $id = $task->{id};
                my $task_uuid = $task->{uuid};
                eval {
                    my $result = from_json($json_result);
                    if ($result){
                        my $repository = $result->{repository};
                        my $filenames = $result->{filenames};
                        my $filter = $result->{filter};
                        my $matcher_code = $result->{matcher_code};
                        my $frameworkcode = $result->{frameworkcode};
                        my $record_type = $result->{record_type};

                        my $matcher;
                        if ($matcher_code){
                            my $matcher_id = C4::Matcher::GetMatcherId($matcher_code);
                            $matcher = C4::Matcher->fetch($matcher_id);
                        }

                        foreach my $filename (@$filenames){
                            if ($filename){
                                if (-f $filename){
                                    my $dom = XML::LibXML->load_xml(location => $filename, { no_blanks => 1 });
                                    if ($dom){
                                        my $oai_record = Koha::OAI::Harvester::Import::Record->new({
                                            doc => $dom,
                                            repository => $repository,
                                        });
                                        if ($oai_record){
                                            my ($action,$linked_id) = $oai_record->import_record({
                                                filter => $filter,
                                                framework => $frameworkcode,
                                                record_type => $record_type,
                                                matcher => $matcher,
                                            });
                                            $debug && print STDOUT qq({ "import_result": { "task_uuid": "$task_uuid", "action": "$action", "filename": "$filename", "koha_id": "$linked_id" } }\n);
                                        }
                                    }
                                    my $unlinked = unlink $filename;
                                }
                            }
                        }
                    }
                };
                if ($@){
                    warn $@;
                }
                #NOTE: Even if the file doesn't exist, we still need to process the queue item.

                #NOTE: Don't do this via a postback in the parent process, as it's much faster to let the child process handle it.

                #NOTE: It's vital that files are unlinked before deleting from the database,
                #or you could get orphan files if the importer is interrupted.
                my $dbh = C4::Context->dbh;
                my $sql = "delete from oai_harvester_import_queue where id = ?";
                my $sth = $dbh->prepare($sql);
                $sth->execute($id);
            }
        },
        StdoutEvent  => "got_child_stdout",
        StderrEvent  => "got_child_stderr",
        CloseEvent   => "got_child_close",
        NoSetPgrp => 1, #Keep child processes in same group as parent. This is especially useful when using Ctrl+C to kill the whole group.
    );

    $_[KERNEL]->sig_child($child->PID, "got_child_signal");

    # Wheel events include the wheel's ID.
    $_[HEAP]{children_by_wid}{$child->ID} = $child;

    # Signal events include the process ID.
    $_[HEAP]{children_by_pid}{$child->PID} = $child;

    my $logger = $self->{logger};
    if ($logger){
        $logger->debug("Child pid ".$child->PID." started as wheel ".$child->ID);
    }
}

1;

Lines 374-379 my $target_map = { Link Here

(-)a/Makefile.PL (+14 lines)
374	'./skel/var/lib/koha/zebradb/biblios/tmp' => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },	374	'./skel/var/lib/koha/zebradb/biblios/tmp' => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
375	'./skel/var/lock/koha/zebradb/rebuild' => { target => 'ZEBRA_LOCK_DIR', trimdir => 6 },	375	'./skel/var/lock/koha/zebradb/rebuild' => { target => 'ZEBRA_LOCK_DIR', trimdir => 6 },
376	'./skel/var/lib/koha/plugins' => { target => 'PLUGINS_DIR', trimdir => 6 },	376	'./skel/var/lib/koha/plugins' => { target => 'PLUGINS_DIR', trimdir => 6 },
		377	'./skel/var/lib/koha/oai-pmh-harvester' => { target => 'OAI_LIB_DIR', trimdir => 6 },
		378	'./skel/var/run/koha/oai-pmh-harvester' => { target => 'OAI_RUN_DIR', trimdir => 6 },
		379	'./skel/var/spool/koha/oai-pmh-harvester' => { target => 'OAI_SPOOL_DIR', trimdir => 6 },
377	'./sms' => 'INTRANET_CGI_DIR',	380	'./sms' => 'INTRANET_CGI_DIR',
378	'./suggestion' => 'INTRANET_CGI_DIR',	381	'./suggestion' => 'INTRANET_CGI_DIR',
379	'./svc' => 'INTRANET_CGI_DIR',	382	'./svc' => 'INTRANET_CGI_DIR',
Lines 612-617 my $pl_files = { Link Here
612	'blib/KOHA_CONF_DIR/koha-conf.xml',	615	'blib/KOHA_CONF_DIR/koha-conf.xml',
613	'blib/KOHA_CONF_DIR/koha-httpd.conf',	616	'blib/KOHA_CONF_DIR/koha-httpd.conf',
614	'blib/KOHA_CONF_DIR/log4perl.conf',	617	'blib/KOHA_CONF_DIR/log4perl.conf',
		618	'blib/KOHA_CONF_DIR/oai-pmh-harvester.yaml',
615	'blib/ZEBRA_CONF_DIR/etc/default.idx',	619	'blib/ZEBRA_CONF_DIR/etc/default.idx',
616	'blib/MISC_DIR/koha-install-log'	620	'blib/MISC_DIR/koha-install-log'
617	],	621	],
Lines 1398-1403 sub get_target_directories { Link Here
1398	$dirmap{'PLUGINS_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'lib', 'koha', 'plugins');	1402	$dirmap{'PLUGINS_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'lib', 'koha', 'plugins');
1399	$dirmap{'ZEBRA_DATA_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');	1403	$dirmap{'ZEBRA_DATA_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1400	$dirmap{'ZEBRA_RUN_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');	1404	$dirmap{'ZEBRA_RUN_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
		1405	$dirmap{'OAI_RUN_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'run', 'oai-pmh-harvester');
		1406	$dirmap{'OAI_LIB_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'lib', 'oai-pmh-harvester');
		1407	$dirmap{'OAI_SPOOL_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'spool', 'oai-pmh-harvester');
1401	} elsif ($mode eq 'dev') {	1408	} elsif ($mode eq 'dev') {
1402	my $curdir = File::Spec->rel2abs(File::Spec->curdir());	1409	my $curdir = File::Spec->rel2abs(File::Spec->curdir());
1403	$dirmap{'API_CGI_DIR'} = File::Spec->catdir($curdir, 'api');	1410	$dirmap{'API_CGI_DIR'} = File::Spec->catdir($curdir, 'api');
Lines 1434-1439 sub get_target_directories { Link Here
1434	$dirmap{'PLUGINS_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'lib', 'plugins');	1441	$dirmap{'PLUGINS_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'lib', 'plugins');
1435	$dirmap{'ZEBRA_DATA_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');	1442	$dirmap{'ZEBRA_DATA_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1436	$dirmap{'ZEBRA_RUN_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');	1443	$dirmap{'ZEBRA_RUN_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
		1444	$dirmap{'OAI_RUN_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'run', 'oai-pmh-harvester');
		1445	$dirmap{'OAI_LIB_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'lib', 'oai-pmh-harvester');
		1446	$dirmap{'OAI_SPOOL_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'spool', 'oai-pmh-harvester');
		1447
1437	} else {	1448	} else {
1438	# mode is standard, i.e., 'fhs'	1449	# mode is standard, i.e., 'fhs'
1439	$dirmap{'API_CGI_DIR'} = File::Spec->catdir(@basedir, $package, 'api');	1450	$dirmap{'API_CGI_DIR'} = File::Spec->catdir(@basedir, $package, 'api');
Lines 1459-1464 sub get_target_directories { Link Here
1459	$dirmap{'PLUGINS_DIR'} = File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'plugins');	1470	$dirmap{'PLUGINS_DIR'} = File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'plugins');
1460	$dirmap{'ZEBRA_DATA_DIR'} = File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'zebradb');	1471	$dirmap{'ZEBRA_DATA_DIR'} = File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'zebradb');
1461	$dirmap{'ZEBRA_RUN_DIR'} = File::Spec->catdir(File::Spec->rootdir(), 'var', 'run', $package, 'zebradb');	1472	$dirmap{'ZEBRA_RUN_DIR'} = File::Spec->catdir(File::Spec->rootdir(), 'var', 'run', $package, 'zebradb');
		1473	$dirmap{'OAI_RUN_DIR'} = File::Spec->catdir(File::Spec->rootdir(), 'var', 'run', $package, 'oai-pmh-harvester');
		1474	$dirmap{'OAI_LIB_DIR'} = File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'oai-pmh-harvester');
		1475	$dirmap{'OAI_SPOOL_DIR'} = File::Spec->catdir(File::Spec->rootdir(), 'var', 'spool', $package, 'oai-pmh-harvester');
1462	}	1476	}
1463		1477
1464	_get_env_overrides(\%dirmap);	1478	_get_env_overrides(\%dirmap);




          columnname: spent

  tools:
    oai-pmh-harvester-dashboard:
      saved-table:
        -
          columnname: name
        -
          columnname: url
        -
          columnname: set
        -
          columnname: from
        -
          columnname: until
        -
          columnname: interval

      submitted-table:
        -
          columnname: name
        -
          columnname: url
        -
          columnname: set
        -
          columnname: from
        -
          columnname: until
        -
          columnname: interval
        -
          columnname: "effective from"
        -
          columnname: "effective until"
        -
          columnname: "pending imports"
        -
          columnname: status
        -
          columnname: error

      history-table:
        -
          columnname: id
        -
          columnname: repository
        -
          columnname: identifier
        -
          columnname: datestamp
        -
          columnname: "upstream status"
        -
          columnname: "import status"
        -
          columnname: "import timestamp"
        -
          columnname: "imported record"
        -
          columnname: "downloaded record"

    notices:
      lettert:
        -

Lines 87-92 requires 'PDF::API2', '2'; Link Here

(-)a/cpanfile (+2 lines)
87	requires 'PDF::Reuse', '0.36';	87	requires 'PDF::Reuse', '0.36';
88	requires 'PDF::Reuse::Barcode', '0.05';	88	requires 'PDF::Reuse::Barcode', '0.05';
89	requires 'PDF::Table', 'v0.9.3';	89	requires 'PDF::Table', 'v0.9.3';
		90	requires 'POE', '1.354';
		91	requires 'POE::Component::JobQueue', '0.570';
90	requires 'POSIX', '1.09';	92	requires 'POSIX', '1.09';
91	requires 'Plack::Middleware::ReverseProxy', '0.14';	93	requires 'Plack::Middleware::ReverseProxy', '0.14';
92	requires 'Schedule::At', '1.06';	94	requires 'Schedule::At', '1.06';

Lines 117-122 generate_config_file() { Link Here

(-)a/debian/scripts/koha-create (+7 lines)
117	-e "s/__PLUGINS_DIR__/\/var\/lib\/koha\/$name\/plugins/g" \	117	-e "s/__PLUGINS_DIR__/\/var\/lib\/koha\/$name\/plugins/g" \
118	-e "s/__MEMCACHED_NAMESPACE__/$MEMCACHED_NAMESPACE/g" \	118	-e "s/__MEMCACHED_NAMESPACE__/$MEMCACHED_NAMESPACE/g" \
119	-e "s/__MEMCACHED_SERVERS__/$MEMCACHED_SERVERS/g" \	119	-e "s/__MEMCACHED_SERVERS__/$MEMCACHED_SERVERS/g" \
		120	-e "s/__OAI_RUN_DIR__/\/var\/run\/koha\/$name\/oai-pmh-harvester/g" \
		121	-e "s/__OAI_LIB_DIR__/\/var\/lib\/koha\/$name\/oai-pmh-harvester/g" \
		122	-e "s/__OAI_SPOOL_DIR__/\/var\/spool\/koha\/$name\/oai-pmh-harvester/g" \
120	"/etc/koha/$1" > "$2"	123	"/etc/koha/$1" > "$2"
121		124
122	}	125	}
Lines 670-675 eof Link Here
670	generate_config_file zebra.passwd.in \	673	generate_config_file zebra.passwd.in \
671	"/etc/koha/sites/$name/zebra.passwd"	674	"/etc/koha/sites/$name/zebra.passwd"
672		675
		676	# Generate and install OAI-PMH harvester config file
		677	generate_config_file oai-pmh-harvester.yaml.in \
		678	"/etc/koha/sites/$name/oai-pmh-harvester.yaml"
		679
673	# Create a GPG-encrypted file for requesting a DB to be set up.	680	# Create a GPG-encrypted file for requesting a DB to be set up.
674	if [ "$op" = request ]	681	if [ "$op" = request ]
675	then	682	then

Lines 56-66 do Link Here

(-)a/debian/scripts/koha-create-dirs (+3 lines)
56	userdir "$name" "/var/lib/koha/$name/plugins"	56	userdir "$name" "/var/lib/koha/$name/plugins"
57	userdir "$name" "/var/lib/koha/$name/uploads"	57	userdir "$name" "/var/lib/koha/$name/uploads"
58	userdir "$name" "/var/lib/koha/$name/tmp"	58	userdir "$name" "/var/lib/koha/$name/tmp"
		59	userdir "$name" "/var/lib/koha/$name/oai-pmh-harvester"
59	userdir "$name" "/var/lock/koha/$name"	60	userdir "$name" "/var/lock/koha/$name"
60	userdir "$name" "/var/lock/koha/$name/authorities"	61	userdir "$name" "/var/lock/koha/$name/authorities"
61	userdir "$name" "/var/lock/koha/$name/biblios"	62	userdir "$name" "/var/lock/koha/$name/biblios"
62	userdir "$name" "/var/run/koha/$name"	63	userdir "$name" "/var/run/koha/$name"
63	userdir "$name" "/var/run/koha/$name/authorities"	64	userdir "$name" "/var/run/koha/$name/authorities"
64	userdir "$name" "/var/run/koha/$name/biblios"	65	userdir "$name" "/var/run/koha/$name/biblios"
		66	userdir "$name" "/var/run/koha/$name/oai-pmh-harvester"
		67	userdir "$name" "/var/spool/koha/$name/oai-pmh-harvester"
65	done	68	done
66		69

Lines 306-311 __END_SRU_PUBLICSERVER__ Link Here

(-)a/debian/templates/koha-conf-site.xml.in (+1 lines)
306	<zebra_max_record_size>1024</zebra_max_record_size>	306	<zebra_max_record_size>1024</zebra_max_record_size>
307	<queryparser_config>/etc/koha/searchengine/queryparser.yaml</queryparser_config>	307	<queryparser_config>/etc/koha/searchengine/queryparser.yaml</queryparser_config>
308	<log4perl_conf>__KOHA_CONF_DIR__/log4perl.conf</log4perl_conf>	308	<log4perl_conf>__KOHA_CONF_DIR__/log4perl.conf</log4perl_conf>
		309	<oai_pmh_harvester_config>/etc/koha/sites/__KOHASITE__/oai-pmh-harvester.yaml</oai_pmh_harvester_config>
309	<!-- Uncomment/edit next setting if you want to adjust zebra log levels.	310	<!-- Uncomment/edit next setting if you want to adjust zebra log levels.
310	Default is: none,fatal,warn.	311	Default is: none,fatal,warn.
311	You can also include: debug,log,malloc,all,request.	312	You can also include: debug,log,malloc,all,request.




---
#Harvester
socket: 'unix:__OAI_RUN_DIR__/harvesterd.sock'

logfile: '__LOG_DIR__/oai-pmh-harvester.log'
loglevel: 'WARN'

pidfile: '__OAI_RUN_DIR__/harvesterd.pid'

statefile: '__OAI_LIB_DIR__/harvesterd.state'

spooldir: '__OAI_SPOOL_DIR__'

#Downloader
download_workers: 1
download_module: 'Koha::OAI::Harvester::Worker::Download::Stream'
download_batch: 100

#Importer
import_workers: 1
import_poll: 5
import_module: 'Koha::OAI::Harvester::Worker::Import'

Lines 128-133 __PAZPAR2_TOGGLE_XML_POST__ Link Here

(-)a/etc/koha-conf.xml (+1 lines)
128	You can also include: debug,log,malloc,all,request.	128	You can also include: debug,log,malloc,all,request.
129	Use a comma-separated list of levels to include. -->	129	Use a comma-separated list of levels to include. -->
130	<!-- <zebra_loglevels>none,fatal,warn</zebra_loglevels> -->	130	<!-- <zebra_loglevels>none,fatal,warn</zebra_loglevels> -->
		131	<oai_pmh_harvester_config>__KOHA_CONF_DIR__/oai-pmh-harvester.yaml</oai_pmh_harvester_config>
131	<memcached_servers>__MEMCACHED_SERVERS__</memcached_servers>	132	<memcached_servers>__MEMCACHED_SERVERS__</memcached_servers>
132	<memcached_namespace>__MEMCACHED_NAMESPACE__</memcached_namespace>	133	<memcached_namespace>__MEMCACHED_NAMESPACE__</memcached_namespace>
133	<template_cache_dir>__TEMPLATE_CACHE_DIR__</template_cache_dir>	134	<template_cache_dir>__TEMPLATE_CACHE_DIR__</template_cache_dir>




---
#Harvester
socket: 'unix:__OAI_RUN_DIR__/harvesterd.sock'

logfile: '__LOG_DIR__/oai-pmh-harvester.log'
loglevel: 'WARN'

pidfile: '__OAI_RUN_DIR__/harvesterd.pid'

statefile: '__OAI_LIB_DIR__/harvesterd.state'

spooldir: '__OAI_SPOOL_DIR__'

#Downloader
download_workers: 1
download_module: 'Koha::OAI::Harvester::Worker::Download::Stream'
download_batch: 100

#Importer
import_workers: 1
import_poll: 5
import_module: 'Koha::OAI::Harvester::Worker::Import'




--
-- Table structure for table 'oai_harvester_biblios'
--

CREATE TABLE IF NOT EXISTS `oai_harvester_biblios` (
  `import_oai_biblio_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
  `oai_repository` varchar(255) NOT NULL,
  `oai_identifier` varchar(255) DEFAULT NULL,
  `biblionumber` int(11) NOT NULL,
  PRIMARY KEY (`import_oai_biblio_id`),
  UNIQUE KEY `oai_record` (`oai_identifier`,`oai_repository`) USING BTREE,
  KEY `FK_import_oai_biblio_1` (`biblionumber`),
  CONSTRAINT `FK_import_oai_biblio_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE NO ACTION
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

--
-- Table structure for table 'oai_harvester_history'
--

CREATE TABLE IF NOT EXISTS  `oai_harvester_history` (
  `import_oai_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
  `repository` varchar(255) DEFAULT NULL,
  `header_identifier` varchar(255) DEFAULT NULL,
  `header_datestamp` datetime NOT NULL,
  `header_status` varchar(45) DEFAULT NULL,
  `record` longtext NOT NULL,
  `upload_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
  `status` varchar(45) NOT NULL,
  `filter` text NOT NULL,
  `framework` varchar(4) NOT NULL,
  `record_type` enum('biblio','auth','holdings') NOT NULL,
  `matcher_code` varchar(10) DEFAULT NULL,
  PRIMARY KEY (`import_oai_id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

--
-- Table structure for table 'oai_harvester_import_queue'
--

CREATE TABLE IF NOT EXISTS `oai_harvester_import_queue` (
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
  `uuid` varchar(45) CHARACTER SET utf8 NOT NULL,
  `status` varchar(45) CHARACTER SET utf8 NOT NULL DEFAULT 'new',
  `result` text CHARACTER SET utf8 NOT NULL,
  `result_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

--
-- Table structure for table 'oai_harvester_requests'
--

CREATE TABLE IF NOT EXISTS `oai_harvester_requests` (
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
  `uuid` varchar(45) NOT NULL,
  `oai_verb` varchar(45) NOT NULL,
  `oai_metadataPrefix` varchar(255) NOT NULL,
  `oai_identifier` varchar(255) DEFAULT NULL,
  `oai_from` varchar(45) DEFAULT NULL,
  `oai_until` varchar(45) DEFAULT NULL,
  `oai_set` varchar(255) DEFAULT NULL,
  `http_url` varchar(255) DEFAULT NULL,
  `http_username` varchar(255) DEFAULT NULL,
  `http_password` varchar(255) DEFAULT NULL,
  `http_realm` varchar(255) DEFAULT NULL,
  `import_filter` varchar(255) NOT NULL,
  `import_framework_code` varchar(4) NOT NULL,
  `import_record_type` enum('biblio','auth','holdings') NOT NULL,
  `import_matcher_code` varchar(10) DEFAULT NULL,
  `interval` int(10) unsigned NOT NULL,
  `name` varchar(45) NOT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;




    CONSTRAINT `rc_resolved_by_ibfk` FOREIGN KEY (`resolved_by`) REFERENCES `borrowers` (`borrowernumber`) ON DELETE SET NULL ON UPDATE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

--
-- Table structure for table 'oai_harvester_biblios'
--

DROP TABLE IF EXISTS `oai_harvester_biblios`;
CREATE TABLE `oai_harvester_biblios` (
  `import_oai_biblio_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
  `oai_repository` varchar(191) NOT NULL,
  `oai_identifier` varchar(191) DEFAULT NULL,
  `biblionumber` int(11) NOT NULL,
  PRIMARY KEY (`import_oai_biblio_id`),
  UNIQUE KEY `oai_record` (`oai_identifier`,`oai_repository` (191)) USING BTREE,
  KEY `FK_import_oai_biblio_1` (`biblionumber`),
  CONSTRAINT `FK_import_oai_biblio_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE NO ACTION
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

--
-- Table structure for table 'oai_harvester_history'
--

DROP TABLE IF EXISTS `oai_harvester_history`;
CREATE TABLE `oai_harvester_history` (
  `import_oai_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
  `repository` varchar(255) DEFAULT NULL,
  `header_identifier` varchar(255) DEFAULT NULL,
  `header_datestamp` datetime NOT NULL,
  `header_status` varchar(45) DEFAULT NULL,
  `record` longtext NOT NULL,
  `upload_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
  `status` varchar(45) NOT NULL,
  `filter` text NOT NULL,
  `framework` varchar(4) NOT NULL,
  `record_type` enum('biblio','auth','holdings') NOT NULL,
  `matcher_code` varchar(10) DEFAULT NULL,
  PRIMARY KEY (`import_oai_id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

--
-- Table structure for table 'oai_harvester_import_queue'
--

DROP TABLE IF EXISTS `oai_harvester_import_queue`;
CREATE TABLE `oai_harvester_import_queue` (
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
  `uuid` varchar(45) CHARACTER SET utf8 NOT NULL,
  `status` varchar(45) CHARACTER SET utf8 NOT NULL DEFAULT 'new',
  `result` text CHARACTER SET utf8 NOT NULL,
  `result_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

--
-- Table structure for table 'oai_harvester_requests'
--

DROP TABLE IF EXISTS `oai_harvester_requests`;
CREATE TABLE `oai_harvester_requests` (
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
  `uuid` varchar(45) NOT NULL,
  `oai_verb` varchar(45) NOT NULL,
  `oai_metadataPrefix` varchar(255) NOT NULL,
  `oai_identifier` varchar(255) DEFAULT NULL,
  `oai_from` varchar(45) DEFAULT NULL,
  `oai_until` varchar(45) DEFAULT NULL,
  `oai_set` varchar(255) DEFAULT NULL,
  `http_url` varchar(255) DEFAULT NULL,
  `http_username` varchar(255) DEFAULT NULL,
  `http_password` varchar(255) DEFAULT NULL,
  `http_realm` varchar(255) DEFAULT NULL,
  `import_filter` varchar(255) NOT NULL,
  `import_framework_code` varchar(4) NOT NULL,
  `import_record_type` enum('biblio','auth','holdings') NOT NULL,
  `import_matcher_code` varchar(10) DEFAULT NULL,
  `interval` int(10) unsigned NOT NULL,
  `name` varchar(45) NOT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;




<h5>Patrons and circulation</h5>
<ul>
    [% IF ( CAN_user_tools_manage_patron_lists ) %]
    <li><a href="/cgi-bin/koha/patron_lists/lists.pl">Patron lists</a></li>
    [% END %]
    [% IF (CAN_user_clubs) %]
        <li><a href="/cgi-bin/koha/clubs/clubs.pl">Patron clubs</a></li>
    [% END %]
    [% IF ( CAN_user_tools_moderate_comments ) %]
    <li><a href="/cgi-bin/koha/reviews/reviewswaiting.pl">Comments</a></li>
    [% END %]
    [% IF ( CAN_user_tools_import_patrons ) %]
    <li><a href="/cgi-bin/koha/tools/import_borrowers.pl">Import patrons</a></li>
    [% END %]
    [% IF ( CAN_user_tools_edit_notices ) %]
    <li><a href="/cgi-bin/koha/tools/letter.pl">Notices &amp; slips</a></li>
    [% END %]
    [% IF ( CAN_user_tools_edit_notice_status_triggers ) %]
    <li><a href="/cgi-bin/koha/tools/overduerules.pl">Overdue notice/status triggers</a></li>
    [% END %]
    [% IF ( CAN_user_tools_label_creator ) %]
    <li><a href="/cgi-bin/koha/patroncards/home.pl">Patron card creator</a></li>

    <li><a href="/cgi-bin/koha/tags/review.pl">Tag moderation</a></li>
    [% END %]
    [% IF ( CAN_user_tools_batch_upload_patron_images ) %]
    <li><a href="/cgi-bin/koha/tools/picture-upload.pl">Upload patron images</a></li>
    [% END %]
</ul>
[% END %]

<h5>Catalog</h5>
<ul>
    [% IF ( CAN_user_tools_items_batchdel ) %]
    <li><a href="/cgi-bin/koha/tools/batchMod.pl?del=1">Batch item deletion</a></li>
    [% END %]
    [% IF ( CAN_user_tools_items_batchmod ) %]
    <li><a href="/cgi-bin/koha/tools/batchMod.pl">Batch item modification</a></li>
    [% END %]
    [% IF CAN_user_tools_records_batchdel %]
      <li><a href="/cgi-bin/koha/tools/batch_delete_records.pl">Batch record deletion</a></li>

        <li><a href="/cgi-bin/koha/tools/inventory.pl">Inventory</a></li>
    [% END %]
    [% IF ( CAN_user_tools_label_creator ) %]
    <li><a href="/cgi-bin/koha/labels/label-home.pl">Label creator</a></li>
    <li><a href="/cgi-bin/koha/labels/spinelabel-home.pl">Quick spine label creator</a></li>
    [% END %]
    [% IF ( CAN_user_tools_rotating_collections ) %]
    <li><a href="/cgi-bin/koha/rotating_collections/rotatingCollections.pl">Rotating collections</a></li>

        <li><a href="/cgi-bin/koha/tools/marc_modification_templates.pl">Manage MARC modification templates</a></li>
    [% END %]
    [% IF ( CAN_user_tools_stage_marc_import ) %]
    <li><a href="/cgi-bin/koha/tools/stage-marc-import.pl">Stage MARC for import</a></li>
    [% END %]
    [% IF ( CAN_user_tools_manage_staged_marc ) %]
    <li><a href="/cgi-bin/koha/tools/manage-marc-import.pl">Staged MARC management</a></li>
    [% END %]
    [% IF ( CAN_user_tools_upload_local_cover_images ) %]
    <li><a href="/cgi-bin/koha/tools/upload-cover-image.pl">Upload local cover image</a></li>
    [% END %]
    [% IF ( CAN_user_tools_manage_staged_marc ) %]
    <li><a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a></li>
    [% END %]
</ul>
[% END %]
[% IF ( CAN_user_tools_edit_calendar || CAN_user_tools_manage_csv_profiles || CAN_user_tools_view_system_logs || CAN_user_tools_edit_news

<h5>Additional tools</h5>
<ul>
    [% IF ( CAN_user_tools_edit_calendar ) %]
    <li><a href="/cgi-bin/koha/tools/holidays.pl">Calendar</a></li>
    [% END %]
    [% IF ( CAN_user_tools_manage_csv_profiles ) %]
    <li><a href="/cgi-bin/koha/tools/csv-profiles.pl">CSV profiles</a></li>
    [% END %]
    [% IF ( CAN_user_tools_view_system_logs ) %]
    <li><a href="/cgi-bin/koha/tools/viewlog.pl">Log viewer</a></li>
    [% END %]
    [% IF ( CAN_user_tools_edit_news ) %]
    <li><a href="/cgi-bin/koha/tools/koha-news.pl">News</a></li>
    [% END %]
    [% IF ( CAN_user_tools_schedule_tasks ) %]
    <li><a href="/cgi-bin/koha/tools/scheduler.pl">Task scheduler</a></li>
    [% END %]
    [% IF ( CAN_user_tools_edit_quotes ) %]
       <li><a href="/cgi-bin/koha/tools/quotes.pl">Quote editor</a></li>




[% USE ColumnsSettings %]
[% SET footerjs = 1 %]
[% INCLUDE 'doc-head-open.inc' %]
<title>Koha &rsaquo; Tools &rsaquo; OAI-PMH harvester</title>
[% INCLUDE 'doc-head-close.inc' %]
[% dashboard_page = '/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl' %]
[% request_page = '/cgi-bin/koha/tools/oai-pmh-harvester/request.pl' %]
<style type="text/css">
    a.paginate_button {
        padding: 2px;
    }
</style>
</head>
<body id="tools_oai-pmh-harvester" class="tools">
[% INCLUDE 'header.inc' %]
[% INCLUDE 'cat-search.inc' %]
    <div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> &rsaquo; OAI-PMH harvester</div>
    <div id="doc3" class="yui-t2">
        <div id="bd">
            <div id="yui-main">
                <div class="yui-b">
                    <h1>OAI-PMH harvester</h1>
                    <div id="toolbar" class="btn-toolbar">
                        <a id="new-request" class="btn btn-default btn-sm" href="[% request_page | url %]?op=new"><i class="fa fa-plus"></i> New request</a>
                    </div>
                    [% IF ( harvester.offline ) %]
                        <div class="alert">
                            <span>OAI-PMH harvester offline</span>
                        </div>
                    [% END %]
                    <div id="dashboard-items" class="toptabs">
                        <ul>
                            <li>
                                <a href="#saved_requests">Saved requests <span id="saved_count">([% saved_requests.size | html %])</span></a>
                            </li>
                            <li>
                                <a href="#submitted_requests">Submitted requests <span id="submitted_count">([% submitted_requests.size | html %])</span></a>
                            </li>
                            <li>
                                <a href="#imports">Import history <span id="import_count">(0)</span></a>
                            </li>
                        </ul>
                        <div id="submitted_requests">
                            [% IF ( result.defined("start") ) %]
                                <div class="alert">
                                    [% IF ( result.start ) %]
                                        <span>Start succeeded</span>
                                    [% ELSE %]
                                        <span>Start failed</span>
                                    [% END %]
                                </div>
                            [% ELSIF ( result.defined("stop") ) %]
                                 <div class="alert">
                                    [% IF ( result.stop ) %]
                                        <span>Stop succeeded</span>
                                    [% ELSE %]
                                        <span>Stop failed</span>
                                    [% END %]
                                </div>
                            [% ELSIF ( result.defined("delete") ) %]
                                 <div class="alert">
                                    [% IF ( result.delete ) %]
                                        <span>Delete succeeded</span>
                                    [% ELSE %]
                                        <span>Delete failed</span>
                                    [% END %]
                                </div>
                            [% END %]
                            <table id="submitted-table">
                                <thead>
                                    <tr>
                                        <th>Name</th>
                                        <th>URL</th>
                                        <th>Set</th>
                                        <th>From</th>
                                        <th>Until</th>
                                        <th>Interval</th>
                                        <th>Effective from</th>
                                        <th>Effective until</th>
                                        <th>Pending imports</th>
                                        <th>Status</th>
                                        <th>Error</th>
                                        <th></th>
                                    </tr>
                                </thead>
                                <tbody>
                                [% IF ( submitted_requests ) %]
                                    [% FOREACH submitted_request IN submitted_requests %]
                                        <tr>
                                            <td>[% submitted_request.name | html %]</td>
                                            <td>[% submitted_request.parameters.oai_pmh.baseURL | html %]</td>
                                            <td>[% submitted_request.parameters.oai_pmh.set | html %]</td>
                                            <td>[% submitted_request.parameters.oai_pmh.from | html %]</td>
                                            <td>[% submitted_request.parameters.oai_pmh.until | html %]</td>
                                            <td>[% submitted_request.interval | html %]</td>
                                            <td>[% submitted_request.effective_from | html %]</td>
                                            <td>[% submitted_request.effective_until | html %]</td>
                                            <td>[% submitted_request.pending_imports |html %]</td>
                                            <td>
                                                [% IF ( submitted_status = submitted_request.status ) %]
                                                    [% IF ( submitted_status == "new" ) %]
                                                        <span>New</span>
                                                    [% ELSIF ( submitted_status == "active" ) %]
                                                        <span>Active</span>
                                                    [% ELSIF ( submitted_status == "stopped" ) %]
                                                        <span>Stopped</span>
                                                    [% END %]
                                                [% END %]
                                            </td>
                                            <td>
                                                [% IF ( submitted_request.error ) %]
                                                    <span>Harvest failure</span>
                                                [% END %]
                                            </td>
                                            <td>
                                                <div class="dropdown">
                                                    <a class="btn btn-default btn-xs dropdown-toggle" role="button" data-toggle="dropdown" href="#">Actions <span class="caret"></span></a>
                                                    <ul class="dropdown-menu pull-right" role="menu">
                                                          <li>
                                                              <a href="[% dashboard_page | url %]?op=start&uuid=[% submitted_request.uuid | url %]"><i class="fa fa-play"></i> Start</a>
                                                          </li>
                                                          <li>
                                                              <a href="[% dashboard_page | url %]?op=stop&uuid=[% submitted_request.uuid | url %]"><i class="fa fa-stop"></i> Stop</a>
                                                          </li>
                                                          <li>
                                                              <a href="[% dashboard_page | url %]?op=delete&uuid=[% submitted_request.uuid | url %]"><i class="fa fa-trash"></i> Delete</a>
                                                          </li>
                                                    </ul>
                                                </div>
                                            </td>
                                        </tr>
                                    [% END %]
                                [% END %]
                                </tbody>
                            </table>
                        </div>
                        <div id="saved_requests">
                            [% IF ( result.send.defined ) %]
                                <div class="alert">
                                    [% IF ( result.send ) %]
                                        <span>Submit succeeded</span>
                                    [% ELSE %]
                                        <span>Submit failed</span>
                                    [% END %]
                                </div>
                            [% END %]
                            <table id="saved-table">
                                <thead>
                                    <tr>
                                        <th>Name</th>
                                        <th>URL</th>
                                   <!-- <th>Verb</th>
                                        <th>Metadata prefix</th>
                                        <th>Identifier</th> -->
                                        <th>Set</th>
                                        <th>From</th>
                                        <th>Until</th>
                                        <th>Interval</th>
                                   <!-- <th>Filter</th>
                                        <th>Framework code</th>
                                        <th>Record type</th>
                                        <th>Matcher code</th> -->
                                        <th></th>
                                    </tr>
                                </thead>
                                <tbody>
                                [% IF ( saved_requests ) %]
                                    [% FOREACH saved_request IN saved_requests %]
                                        <tr>
                                            <td>[% saved_request.name | html %]</td>
                                            <td>[% saved_request.http_url | html %]</td>
                                       <!-- <td>[% saved_request.oai_verb | html %]</td>
                                            <td>[% saved_request.oai_metadataPrefix | html %]</td>
                                            <td>[% saved_request.oai_identifier | html %]</td> -->
                                            <td>[% saved_request.oai_set | html %]</td>
                                            <td>[% saved_request.oai_from | html %]</td>
                                            <td>[% saved_request.oai_until | html %]</td>
                                            <td>[% saved_request.interval | html %]</td>
                                       <!-- <td>
                                                [% IF ( saved_request.import_filter == "default" ) %]
                                                    <span>Default</span>
                                                [% ELSE %]
                                                    [% saved_request.import_filter | html %]
                                                [% END %]
                                            </td>
                                            <td>
                                                [% display_framework = "" %]
                                                [% FOREACH framework IN frameworks %]
                                                    [% IF ( framework.frameworkcode == saved_request.import_framework_code ) %]
                                                        [% display_framework = framework %]
                                                    [% END %]
                                                [% END %]
                                                [% IF ( display_framework ) %]
                                                    [% display_framework.frameworktext | html %]
                                                [% ELSE %]
                                                    [% saved_request.import_framework_code | html %]
                                                [% END %]
                                            </td>
                                            <td>
                                                [% IF ( saved_request.import_record_type == "biblio" ) %]
                                                    <span>Bibliographic</span>
                                                [% ELSE %]
                                                    [% saved_request.import_record_type | html %]
                                                [% END %]
                                            </td>
                                            <td>
                                                [% display_matcher = "" %]
                                                [% FOREACH matcher IN matchers %]
                                                    [% IF ( matcher.code == saved_request.import_matcher_code ) %]
                                                        [% display_matcher = matcher %]
                                                    [% END %]
                                                [% END %]
                                                [% IF ( display_matcher ) %]
                                                    [% display_matcher.description | html %]
                                                [% ELSE %]
                                                    [% saved_request.import_matcher_code | html %]
                                                [% END %]
                                            </td> -->
                                            <td>
                                                <div class="dropdown">
                                                    <a class="btn btn-default btn-xs dropdown-toggle" role="button" data-toggle="dropdown" href="#">Actions <span class="caret"></span></a>
                                                    <ul class="dropdown-menu pull-right" role="menu">
                                                          <li>
                                                              <a href="[% request_page | url %]?op=edit&id=[% saved_request.id | url %]"><i class="fa fa-pencil"></i> Edit</a>
                                                          </li>
                                                          <li>
                                                              <a href="[% request_page | url %]?op=new&id=[% saved_request.id | url %]"><i class="fa fa-copy"></i> Copy</a>
                                                          </li>
                                                          <li>
                                                              <a href="[% dashboard_page | url %]?op=send&id=[% saved_request.id | url %]"><i class="fa fa-send"></i> Submit</a>
                                                          </li>
                                                          <li>
                                                            <a href="[% request_page | url %]?op=delete&id=[% saved_request.id | url %]"><i class="fa fa-trash"></i> Delete</a>
                                                          </li>
                                                    </ul>
                                                </div>
                                            </td>
                                        </tr>
                                    [% END %]
                                [% END %]
                                </tbody>
                            </table>
                        </div>
                        <div id="imports">
                            <div class="btn-toolbar">
                                <button id="refresh-button" type="button" class="btn btn-default btn-sm">Refresh import history</button>
                            </div>
                            <table id="history-table">
                                <thead>
                                    <tr>
                                        <th>Id</th>
                                        <th>Repository</th>
                                        <th>Identifier</th>
                                        <th>Datestamp</th>
                                        <th>Upstream status</th>
                                        <th>Import status</th>
                                        <th>Import timestamp</th>
                                        <th>Imported record</th>
                                        <th>Downloaded record</th>
                                    </tr>
                                </thead>
                            </table>

                        </div>
                    </div>
                </div>
            </div>
            <div class="yui-b">
                [% INCLUDE 'tools-menu.inc' %]
            </div>
        </div>

[% MACRO jsinclude BLOCK %]
    [% INCLUDE 'datatables.inc' %]
    [% INCLUDE 'columns_settings.inc' %]
    <script type="text/javascript">
        $(document).ready(function() {
            $('#dashboard-items').tabs();
            [% IF ( result.start.defined || result.stop.defined || result.delete.defined ) %]
                $('#dashboard-items').tabs("option","active",1);
            [% END %]

            var saved_table_columns_settings = [% ColumnsSettings.GetColumns( 'tools','oai-pmh-harvester-dashboard', 'saved-table', 'json' ) | $raw %];
            var saved_table = KohaTable("saved-table",{},saved_table_columns_settings);

            var submitted_table_columns_settings = [% ColumnsSettings.GetColumns( 'tools','oai-pmh-harvester-dashboard', 'submitted-table', 'json' ) | $raw %];
            var submitted_table = KohaTable("submitted-table",{},submitted_table_columns_settings);

            var history_table_columns_settings = [% ColumnsSettings.GetColumns( 'tools','oai-pmh-harvester-dashboard', 'history-table', 'json' ) | $raw %];
            var history_table = KohaTable("history-table",{
                serverSide: true,
                searching: true,
                processing: true,
                ajax: {
                    "url": '/cgi-bin/koha/svc/oai-pmh-harvester/history',
                    contentType: 'application/json',
                    type: 'POST',
                    data: function ( d ) {
                        return JSON.stringify( d );
                    },
                    dataSrc: function (json){
                        var recordsTotal = json.recordsTotal;
                        if(recordsTotal){
                            $('#import_count').text( "("+recordsTotal+")" );
                        }
                        return json.data;
                    }
                },
                columns: [
                    { data: 'import_oai_id', },
                    { data: 'repository', },
                    { data: 'header_identifier', },
                    { data: 'header_datestamp', },
                    {
                        data: 'header_status', render: function (data, type, full, meta){
                            var display_string = _("Active");
                            if (data == "deleted"){
                                display_string = _("Deleted");
                            }
                            return display_string;
                        }
                    },
                    {
                        data: 'status', render: function (data, type, full, meta){
                            var display_string = data;
                            if (data == "added"){
                                display_string = _("Added");
                            }
                            else if (data == "error"){
                                display_string = _("Error");
                            }
                            else if (data == "not_found"){
                                display_string = _("Not found");
                            }
                            else if (data == "updated"){
                                display_string = _("Updated");
                            }
                            return display_string;
                        }
                    },
                    { data: 'upload_timestamp', },
                    {
                        data: 'imported_record', render: function (data, type, full, meta){
                            var display_string = data;
                            var record_type = (full.record_type) ? full.record_type : 'biblio';
                            if (data && record_type == 'biblio'){
                                var link_text = _("View biblio record");
                                var link = '<a href="/cgi-bin/koha/catalogue/detail.pl?biblionumber='+data+'">'+link_text+'</a>';
                                display_string = link;
                            }
                            return display_string;
                        }, searchable: false

                    },
                    {
                        data: 'import_oai_id', render: function (data, type, full, meta){
                            var display_string = data;
                            var link_text = _("View record");
                            var link = '<a href="/cgi-bin/koha/tools/oai-pmh-harvester/record.pl?import_oai_id='+data+'">'+link_text+'</a>';
                            display_string = link;
                            return display_string;
                        }, searchable: false
                    },
                ],
                //In theory, it would be nice to sort in descending order, but
                //it causes severe paging issues the data is frequently updated.
                //Caching would make the paging more stable, but the results would be stale.
                order: [
                    [ 0, 'asc' ],
                ]
            },history_table_columns_settings);
            $('#refresh-button').click(function(){
                history_table.dataTable().api().ajax.reload( null, false );
            });
        });
    </script>
[% END %]
[% INCLUDE 'intranet-bottom.inc' %]




[% INCLUDE 'doc-head-open.inc' %]
<title>Koha &rsaquo; Tools &rsaquo; OAI-PMH harvester &rsaquo; Downloaded record</title>
[% INCLUDE 'doc-head-close.inc' %]
</head>
<body id="tools_oai-pmh-harvester_request" class="tools">
[% INCLUDE 'header.inc' %]
[% INCLUDE 'cat-search.inc' %]
    <div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> &rsaquo; <a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a> &rsaquo; Downloaded record</div>
    <div id="doc3" class="yui-t2">
        <div id="bd">
            <div id="yui-main">
                <div class="yui-b">
                    <h1>Downloaded record</h1>
                    [% IF ( record ) %]
                        <div style="white-space:pre">[% record | xml %]</div>
                    [% END %]
                </div>
            </div>
            <div class="yui-b">
                [% INCLUDE 'tools-menu.inc' %]
            </div>
        </div>
[% INCLUDE 'intranet-bottom.inc' %]




[% USE Asset %]
[% USE raw %]
[% SET footerjs = 1 %]
[% INCLUDE 'doc-head-open.inc' %]
<title>Koha &rsaquo; Tools &rsaquo; OAI-PMH harvester &rsaquo; Request</title>
[% INCLUDE 'doc-head-close.inc' %]
<style type="text/css">
    /* Override staff-global.css which hides second, millisecond, and microsecond sliders */
    .ui_tpicker_second {
        display: block;
    }
</style>
</head>
<body id="tools_oai-pmh-harvester_request" class="tools">
[% INCLUDE 'header.inc' %]
[% INCLUDE 'cat-search.inc' %]
    <div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> &rsaquo; <a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a> &rsaquo; Request</div>
    <div id="doc3" class="yui-t2">
        <div id="bd">
            <div id="yui-main">
                <div class="yui-b">
                    [% IF ( op == "edit" ) %]
                        <h1>Edit OAI-PMH request</h1>
                    [% ELSE %]
                        <h1>New OAI-PMH request</h1>
                    [% END %]
                    [% IF ( test_parameters ) %]
                        [% IF ( errors.size ) %]
                            <div class="dialog message"><span class="text-danger">Tests failed!</span></div>
                        [% ELSE %]
                            <div class="dialog message"><span class="text-success">Tests succeeded!</span></div>
                        [% END %]
                    [% END %]
                    <form action="/cgi-bin/koha/tools/oai-pmh-harvester/request.pl" method="post" name="entry-form">
                        [% IF ( op == "new" ) %]
                            <input type="hidden" name="op" value="create" />
                        [% ELSIF ( op == "edit" ) %]
                             <input type="hidden" name="op" value="update" />
                        [% ELSE %]
                            <input type="hidden" name="op" value="[% op | html %]" />
                        [% END %]
                        [% IF ( id ) %]
                            <input type="hidden" name="id" value="[% id | html %]" />
                        [% END %]
                        <fieldset class="rows">
                            <ol>
                                <li>
                                    <label for="name">Name:</label>
                                    <input type="text" size="30" id="name" name="name" value="[% oai_pmh_request.name | html %]"/>
                                    <span class="help">This is just a short name to help in managing requests.</span>
                                </li>
                            </ol>
                        </fieldset>
                        <fieldset class="rows">
                            <legend>HTTP parameters:</legend>
                            <ol>
                                <li>
                                    <label for="http_url">URL:</label>
                                    <input type="text" size="30" id="http_url" name="http_url" value="[% oai_pmh_request.http_url | html %]">
                                    [% IF (errors.http_url.malformed) %]<span class="error">[This must be a properly formatted HTTP or HTTPS URL.]</span>[% END %]
                                    [% IF (errors.http.404) %]<span class="error">[Cannot find address specified by this URL.]</span>[% END %]
                                    [% IF (errors.http.401) %]<span class="error">[Permission denied to access this URL.]</span>[% END %]
                                    [% IF (errors.http.generic) %]<span class="error">[Unable to access this URL.]</span>[% END %]
                                </li>
                            </ol>
                            <span class="help">The following parameters are not required by all OAI-PMH repositories, so they may be optional for this task.</span>
                            <ol>
                                <li>
                                    <label for="http_username">Username:</label>
                                    <input type="text" size="30" id="http_username" name="http_username" value="[% oai_pmh_request.http_username | html %]">
                                </li>
                                <li>
                                    <label for="http_password">Password:</label>
                                    <input type="text" size="30" id="http_password" name="http_password" value="[% oai_pmh_request.http_password | html %]">
                                </li>
                                <li>
                                    <label for="http_realm">Realm:</label>
                                    <input type="text" size="30" id="http_realm" name="http_realm" value="[% oai_pmh_request.http_realm | html %]">
                                </li>
                            </ol>
                        </fieldset>
                        <fieldset class="rows">
                            <legend>OAI-PMH parameters:</legend>
                            <ol>
                                <li>
                                    <label for="oai_verb">Verb:</label>
                                    <select id="oai_verb" name="oai_verb">
                                        [% IF ( oai_pmh_request.oai_verb == "ListRecords" ) %]
                                        <option value="ListRecords" selected="selected">ListRecords</option>
                                        [% ELSE %]
                                        <option value="ListRecords">ListRecords</option>
                                        [% END %]
                                        [% IF ( oai_pmh_request.oai_verb == "GetRecord" ) %]
                                        <option value="GetRecord" selected="selected">GetRecord</option>
                                        [% ELSE %]
                                        <option value="GetRecord">GetRecord</option>
                                        [% END %]
                                    </select>
                                </li>
                                <li>
                                    <label for="oai_metadataPrefix">Metadata prefix:</label>
                                    <input type="text" size="30" id="oai_metadataPrefix" name="oai_metadataPrefix" value="[% oai_pmh_request.oai_metadataPrefix | html %]">
                                    [% IF (errors.oai_metadataPrefix.unavailable) %]<span class="error">[This metadataPrefix is unavailable from this OAI-PMH provider.]</span>[% END %]
                                    [% IF (errors.oai_metadataPrefix.missing) %]<span class="error">[metadataPrefix is a required field for an OAI-PMH request.]</span>[% END %]
                                </li>
                                <li>
                                    <label for="oai_identifier">Identifier:</label>
                                    <input type="text" size="30" id="oai_identifier" name="oai_identifier" value="[% oai_pmh_request.oai_identifier | html %]">
                                    [% IF (errors.oai_identifier.missing) %]<span class="error">[Identifier is a required field when using GetRecord for an OAI-PMH request.]</span>[% END %]
                                </li>
                                <li>
                                    <label for="oai_set">Set:</label>
                                    <input type="text" size="30" id="oai_set" name="oai_set" value="[% oai_pmh_request.oai_set | html %]">
                                    [% IF (errors.oai_set.unavailable) %]<span class="error">[This set is unavailable from this OAI-PMH provider.]</span>[% END %]
                                </li>
                                [% IF (errors.oai.granularity_mismatch) %]<span class="error">[You must specify the same granularity for both From and Until.]</span>[% END %]
                                <li>
                                    <label for="oai_from">From:</label>
                                    <input type="text" size="30" class="datetime_utc" id="oai_from" name="oai_from" value="[% oai_pmh_request.oai_from | html %]">
                                    <span class="help">This value will be treated as UTC time. Note that some repositories only support YYYY-MM-DD datestamps.</span>
                                    [% IF (errors.oai_from.malformed) %]<span class="error">[This must be in YYYY-MM-DD or YYYY-MM-DDThh:mm:ssZ format.]</span>[% END %]
                                    [% IF (errors.oai_from.unavailable) %]<span class="error">[This granularity is unsupported by this OAI-PMH provider.]</span>[% END %]
                                </li>
                                <li>
                                    <label for="oai_until">Until:</label>
                                    <input type="text" size="30" class="datetime_utc" id="oai_until" name="oai_until" value="[% oai_pmh_request.oai_until | html %]">
                                    <span class="help">This value will be treated as UTC time. Note that some repositories only support YYYY-MM-DD datestamps.</span>
                                    [% IF (errors.oai_until.malformed) %]<span class="error">[This must be in YYYY-MM-DD or YYYY-MM-DDThh:mm:ssZ format.]</span>[% END %]
                                    [% IF (errors.oai_until.unavailable) %]<span class="error">[This granularity is unsupported by this OAI-PMH provider.]</span>[% END %]
                                </li>
                            </ol>
                        </fieldset>
                        <fieldset class="rows">
                            <legend>Import parameters:</legend>
                            <ol>
                                <li>
                                    <label for="import_filter">Filter:</label>
                                    [% IF ( oai_pmh_request.import_filter == "default" ) %]
                                        <input type="text" size="30" id="import_filter" name="import_filter" value="default">
                                    [% ELSE %]
                                        <input type="text" size="30" id="import_filter" name="import_filter" value="[% oai_pmh_request.import_filter | html %]">
                                    [% END %]
                                    <span class="help">If no filter is entered, the default filter will be used.</span>
                                </li>
                                <li>
                                    <label for="import_framework_code">Framework code:</label>
                                    <select id="import_framework_code" name="import_framework_code">
                                        <option value="">Default framework</option>
                                        [% FOREACH framework IN frameworks %]
                                            [% IF ( oai_pmh_request.import_framework_code == framework.frameworkcode ) %]
                                                <option selected="selected" value="[% framework.frameworkcode | html %]">[% framework.frameworktext | html %]</option>
                                            [% ELSE %]
                                                <option value="[% framework.frameworkcode | html %]">[% framework.frameworktext | html %]</option>
                                            [% END %]
                                        [% END %]
                                    </select>
                                </li>
                                <li>
                                    <label for="import_record_type">Record type:</label>
                                    <select id="import_record_type" name="import_record_type">
                                        <option value="biblio">Bibliographic</option>
                                    </select>
                                </li>
                                <li>
                                    <label for="import_matcher_code">Record matcher:</label>
                                    <select id="import_matcher_code" name="import_matcher_code">
                                        <option value="">None</option>
                                        [% FOREACH matcher IN matchers %]
                                            [% IF ( oai_pmh_request.import_matcher_code == matcher.code ) %]
                                                <option value="[% matcher.code | html %]" selected="selected">[% matcher.description | html %]</option>
                                            [% ELSE %]
                                                <option value="[% matcher.code | html %]">[% matcher.description | html %]</option>
                                            [% END %]
                                        [% END %]
                                    </select>
                                    <span class="help">See <a href="/cgi-bin/koha/admin/matching-rules.pl">record matching rules</a> to add or edit rules.</span>
                                </li>
                            </ol>
                        </fieldset>
                        <fieldset class="rows">
                            <legend>Download parameters:</legend>
                            <ol>
                                <li>
                                    <label for="interval">Interval (seconds): </label>
                                    <input type="text" id="interval" name="interval" value="[% oai_pmh_request.interval | html %]" size="4">
                                    <span class="help">The download request will be repeated in intervals of this many seconds. Enter "0" if you want the task to only happen once.</span>
                                </li>
                            </ol>
                        </fieldset>
                        <fieldset class="action">
                            <input type="submit" name="test_parameters" value="Test parameters">
                            <input type="submit" name="save" value="Save">
                            <a class="cancel" href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">Cancel</a>
                        </fieldset>
                    </form>
                </div>
            </div>
            <div class="yui-b">
                [% INCLUDE 'tools-menu.inc' %]
            </div>
        </div>
[% MACRO jsinclude BLOCK %]
    [% INCLUDE 'calendar.inc' %]
    [% Asset.js("lib/jquery/plugins/jquery-ui-timepicker-addon.min.js") | $raw %]
    [% INCLUDE 'timepicker.inc' %]
    <script type="text/javascript">
        $(document).ready(function() {
            toggle_identifier();
            $("#oai_verb").on("click",toggle_identifier);
            $(".datetime_utc").datetimepicker({
                separator: "T",
                timeSuffix: 'Z',
                dateFormat: "yy-mm-dd",
                timeFormat: "HH:mm:ss",
                hour: 0,
                minute: 0,
                second: 0,
                showSecond: 1,
                // timezone doesn't work with the "Now" button in v1.4.3 although it appears to in v1.6.1
                // timezone: '+000'
            });
        });
        function toggle_identifier (){
            var verb = $("#oai_verb").find(":selected").val();
            var oai_identifier = $("#oai_identifier");
            var oai_set = $("#oai_set");
            var oai_from = $("#oai_from");
            var oai_until = $("#oai_until");
            if (verb == 'ListRecords'){
                oai_identifier.prop('disabled', true);
                oai_set.prop('disabled', false);
                oai_from.prop('disabled', false);
                oai_until.prop('disabled', false);
            }
            else if (verb == 'GetRecord'){
                oai_identifier.prop('disabled', false);
                oai_set.prop('disabled', true);
                oai_from.prop('disabled', true);
                oai_until.prop('disabled', true);
            }
        }
    </script>
[% END %]
[% INCLUDE 'intranet-bottom.inc' %]

Lines 217-222 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/tools-home.tt (+5 lines)
217	<dd>Utility to upload scanned cover images for display in OPAC</dd>	217	<dd>Utility to upload scanned cover images for display in OPAC</dd>
218	[% END %]	218	[% END %]
219		219
		220	[% IF ( CAN_user_tools_manage_staged_marc ) %]
		221	<dt><a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a></dt>
		222	<dd>Harvest (ie download and import) records from remote sources using the OAI-PMH protocol</dd>
		223	[% END %]
		224
220	</dl>	225	</dl>
221	</div>	226	</div>
222		227




<?xml version="1.0" encoding="UTF-8"?>
<xsl:stylesheet version="1.0"
    xmlns:oai="http://www.openarchives.org/OAI/2.0/"
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>
    <!-- NOTE: This XSLT strips the OAI-PMH wrapper from the metadata. -->

    <!-- Match the root oai:record element -->
    <xsl:template match="oai:record">
        <!-- Apply templates only to the child metadata element(s) -->
        <xsl:apply-templates select="oai:metadata" />
    </xsl:template>

    <!-- Matches an oai:metadata element -->
    <xsl:template match="oai:metadata">
        <!-- Create a copy of child attributes and nodes -->
        <xsl:apply-templates select="@* | node()" />
    </xsl:template>

    <!-- Identity transformation: this template copies attributes and nodes -->
    <xsl:template match="@* | node()">
        <!-- Create a copy of this attribute or node -->
        <xsl:copy>
            <!-- Recursively apply this template to the attributes and child nodes of this element -->
            <xsl:apply-templates select="@* | node()" />
        </xsl:copy>
    </xsl:template>
</xsl:stylesheet>

Lines 26-31 use constant DEFAULT_LOGS_PURGEDAYS => 180; Link Here

(-)a/misc/cronjobs/cleanup_database.pl (-1 / +14 lines)
26	use constant DEFAULT_SEARCHHISTORY_PURGEDAYS => 30;	26	use constant DEFAULT_SEARCHHISTORY_PURGEDAYS => 30;
27	use constant DEFAULT_SHARE_INVITATION_EXPIRY_DAYS => 14;	27	use constant DEFAULT_SHARE_INVITATION_EXPIRY_DAYS => 14;
28	use constant DEFAULT_DEBARMENTS_PURGEDAYS => 30;	28	use constant DEFAULT_DEBARMENTS_PURGEDAYS => 30;
		29	use constant DEFAULT_IMPORTOAI_PURGEDAYS => 30;
29		30
30	BEGIN {	31	BEGIN {
31	# find Koha's Perl modules	32	# find Koha's Perl modules
Lines 45-51 use Koha::UploadedFiles; Link Here
45		46
46	sub usage {	47	sub usage {
47	print STDERR <<USAGE;	48	print STDERR <<USAGE;
48	Usage: $0 [-h\|--help] [--sessions] [--sessdays DAYS] [-v\|--verbose] [--zebraqueue DAYS] [-m\|--mail] [--merged] [--import DAYS] [--logs DAYS] [--searchhistory DAYS] [--restrictions DAYS] [--all-restrictions] [--fees DAYS] [--temp-uploads] [--temp-uploads-days DAYS] [--uploads-missing 0\|1 ] [--statistics DAYS] [--deleted-catalog DAYS] [--deleted-patrons DAYS] [--old-issues DAYS] [--old-reserves DAYS] [--transfers DAYS]	49	Usage: $0 [-h\|--help] [--sessions] [--sessdays DAYS] [-v\|--verbose] [--zebraqueue DAYS] [-m\|--mail] [--merged] [--import DAYS] [--logs DAYS] [--searchhistory DAYS] [--restrictions DAYS] [--all-restrictions] [--fees DAYS] [--temp-uploads] [--temp-uploads-days DAYS] [--uploads-missing 0\|1 ] [--statistics DAYS] [--deleted-catalog DAYS] [--deleted-patrons DAYS] [--old-issues DAYS] [--old-reserves DAYS] [--transfers DAYS] [--importoai DAYS ]
49		50
50	-h --help prints this help message, and exits, ignoring all	51	-h --help prints this help message, and exits, ignoring all
51	other options	52	other options
Lines 90-95 Usage: $0 [-h\|--help] [--sessions] [--sessdays DAYS] [-v\|--verbose] [--zebraqueu Link Here
90	--old-issues DAYS Purge checkouts (old_issues) returned more than DAYS days ago.	91	--old-issues DAYS Purge checkouts (old_issues) returned more than DAYS days ago.
91	--old-reserves DAYS Purge reserves (old_reserves) more than DAYS old.	92	--old-reserves DAYS Purge reserves (old_reserves) more than DAYS old.
92	--transfers DAYS Purge transfers completed more than DAYS day ago.	93	--transfers DAYS Purge transfers completed more than DAYS day ago.
		94	--importoai DAYS purge OAI-PMH records from import_oai table older than DAYS
		95	days. Defaults to 30 days if no days specified.
93	USAGE	96	USAGE
94	exit $_[0];	97	exit $_[0];
95	}	98	}
Lines 122-127 my $pDeletedPatrons; Link Here
122	my $pOldIssues;	125	my $pOldIssues;
123	my $pOldReserves;	126	my $pOldReserves;
124	my $pTransfers;	127	my $pTransfers;
		128	my $importoai_days;
125		129
126	GetOptions(	130	GetOptions(
127	'h\|help' => \$help,	131	'h\|help' => \$help,
Lines 152-157 GetOptions( Link Here
152	'old-issues:i' => \$pOldIssues,	156	'old-issues:i' => \$pOldIssues,
153	'old-reserves:i' => \$pOldReserves,	157	'old-reserves:i' => \$pOldReserves,
154	'transfers:i' => \$pTransfers,	158	'transfers:i' => \$pTransfers,
		159	'importoai:i' => \$importoai_days,
155	) \|\| usage(1);	160	) \|\| usage(1);
156		161
157	# Use default values	162	# Use default values
Lines 163-168 $mail = DEFAULT_MAIL_PURGEDAYS if defined($mail) Link Here
163	$pSearchhistory = DEFAULT_SEARCHHISTORY_PURGEDAYS if defined($pSearchhistory) && $pSearchhistory == 0;	168	$pSearchhistory = DEFAULT_SEARCHHISTORY_PURGEDAYS if defined($pSearchhistory) && $pSearchhistory == 0;
164	$pListShareInvites = DEFAULT_SHARE_INVITATION_EXPIRY_DAYS if defined($pListShareInvites) && $pListShareInvites == 0;	169	$pListShareInvites = DEFAULT_SHARE_INVITATION_EXPIRY_DAYS if defined($pListShareInvites) && $pListShareInvites == 0;
165	$pDebarments = DEFAULT_DEBARMENTS_PURGEDAYS if defined($pDebarments) && $pDebarments == 0;	170	$pDebarments = DEFAULT_DEBARMENTS_PURGEDAYS if defined($pDebarments) && $pDebarments == 0;
		171	$importoai_days = DEFAULT_IMPORTOAI_PURGEDAYS if defined($importoai_days) && $importoai_days == 0;
166		172
167	if ($help) {	173	if ($help) {
168	usage(0);	174	usage(0);
Lines 192-197 unless ( $sessions Link Here
192	\|\| $pOldIssues	198	\|\| $pOldIssues
193	\|\| $pOldReserves	199	\|\| $pOldReserves
194	\|\| $pTransfers	200	\|\| $pTransfers
		201	\|\| $importoai_days
195	) {	202	) {
196	print "You did not specify any cleanup work for the script to do.\n\n";	203	print "You did not specify any cleanup work for the script to do.\n\n";
197	usage(1);	204	usage(1);
Lines 480-485 if ($pTransfers) { Link Here
480	print "Done with purging transfers.\n" if $verbose;	487	print "Done with purging transfers.\n" if $verbose;
481	}	488	}
482		489
		490	if ($importoai_days){
		491	my $sql = "DELETE FROM import_oai WHERE date(upload_timestamp) < (date_sub(curdate(), INTERVAL ? DAY))";
		492	my $sth = $dbh->prepare($sql);
		493	$sth->execute($importoai_days) or die $dbh->errstr;
		494	}
		495
483	exit(0);	496	exit(0);
484		497
485	sub RemoveOldSessions {	498	sub RemoveOldSessions {




#!/usr/bin/perl

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;
use Getopt::Long;
use Pod::Usage;
use Module::Load;
use Log::Log4perl qw(:easy);
use POE;
use YAML;

use C4::Context;
use Koha::Daemon;
use Koha::OAI::Harvester;
use Koha::OAI::Harvester::Listener;

binmode(STDOUT,':encoding(UTF-8)');
$|++;

my $help = 0;
my $daemonize = 0;

my ($socket_addr,$pidfile,$statefile,$spooldir);
my ($download_module,$import_module);
my ($batch,$download_workers,$import_workers,$import_poll);
my ($logfile,$log_level);
my $log_levels = {
    FATAL => $FATAL,
    ERROR => $ERROR,
    WARN => $WARN,
    INFO => $INFO,
    DEBUG => $DEBUG,
    TRACE => $TRACE,
};

my $context = C4::Context->new();
my $config_filename = $context->{config}->{oai_pmh_harvester_config};
if ($config_filename){
    my $config = YAML::LoadFile($config_filename);
    if ($config){
        $socket_addr = $config->{socket};
        $pidfile = $config->{pidfile};
        $statefile = $config->{statefile};
        $spooldir = $config->{spooldir};
        $logfile = $config->{logfile};
        $log_level = $config->{loglevel};
        $download_module = $config->{download_module};
        $import_module = $config->{import_module};
        $batch = $config->{download_batch};
        $download_workers = $config->{download_workers};
        $import_workers = $config->{import_workers};
        $import_poll = $config->{import_poll};
    }
}

GetOptions(
    "help|?"            => \$help,
    "daemon"            => \$daemonize,
    "socket-uri=s"          => \$socket_addr,
    "pid-file=s"             => \$pidfile,
    "state-file=s"      => \$statefile,
    "spool-dir=s"       => \$spooldir,
    "log-file=s"             => \$logfile,
    "log-level=s"       => \$log_level,
    "download-module=s" => \$download_module,
    "import-module=s"   => \$import_module,
) or pod2usage(2);
pod2usage(1) if $help;

my $level = ( $log_level && $log_levels->{$log_level} ) ? $log_levels->{$log_level} : $log_levels->{WARN};
Log::Log4perl->easy_init(
    {
        level => $level,
        file => "STDOUT",
        layout   => '[%d{yyyy-MM-dd HH:mm:ss}][%p] %m%n',
    }
);
my $logger = Log::Log4perl->get_logger();

unless($download_module){
    $download_module = "Koha::OAI::Harvester::Worker::Download::Stream";
}
unless($import_module){
    $import_module = "Koha::OAI::Harvester::Worker::Import";
}

foreach my $module ( $download_module, $import_module ){
    load $module;
}
my $downloader = $download_module->new({
    logger => $logger,
    batch => $batch,
});
my $importer = $import_module->new({
    logger => $logger,
});

my $daemon = Koha::Daemon->new({
    pidfile => $pidfile,
    logfile => $logfile,
    daemonize => $daemonize,
});
$daemon->run();

my $harvester = Koha::OAI::Harvester->spawn({
    Downloader => $downloader,
    DownloaderWorkers => $download_workers,
    Importer => $importer,
    ImporterWorkers => $import_workers,
    ImportQueuePoll => $import_poll,
    logger => $logger,
    state_file => $statefile,
    spooldir => $spooldir,
});

my $listener = Koha::OAI::Harvester::Listener->spawn({
    logger => $logger,
    socket => $socket_addr,
});

$logger->info("OAI-PMH harvester started.");

POE::Kernel->run();

exit;

=head1 NAME

harvesterd.pl - a daemon that asynchronously sends OAI-PMH requests and imports OAI-PMH records

=head1 SYNOPSIS

KOHA_CONF=/path/to/koha-conf.xml ./harvesterd.pl

=head1 OPTIONS

=over 8

=item B<--help>

Print a brief help message and exits.

=item B<--daemon>

Run program as a daemon (ie fork process, setsid, chdir to root, reset umask,
and close STDIN, STDOUT, and STDERR).

=item B<--log-file>

Specify a file to which to log STDOUT and STDERR.

=item B<--pid-file>

Specify a file to store the process id (this prevents multiple copies of the program
from running at the same time).

=item B<--socket-uri>

Specify a URI to use for the UNIX socket used to communicate with the daemon.
(e.g. unix:/path/to/socket.sock)

=item B<--state-file>

Specify a filename to use for storing the harvester's in-memory state.

In the event that the harvester crashes, it can resume from where it stopped.

=item B<--spool-dir>

Specify a directory to store downloaded OAI-PMH records prior to import.

=item B<--log-level>

Specify a log level for logging. The logger uses Log4Perl, which provides
FATAL, ERROR, WARN, INFO, DEBUG, and TRACE in order of descending priority.

Defaults to WARN level.

=item B<--download-module>

Specify a Perl module to use for downloading records. This is a specialty module,
which has particular requirements, so only advanced users should use this option.

=item B<--import-module>

Specify a Perl module to use for importing records. This is a specialty module,
which has particular requirements, so only advanced users should use this option.

=back

=cut

Lines 154-160 $prefix = $ENV{'INSTALL_BASE'} \|\| "/usr"; Link Here

(-)a/rewrite-config.PL (-1 / +4 lines)
154	"__ELASTICSEARCH_SERVERS__" => "localhost:9200",	154	"__ELASTICSEARCH_SERVERS__" => "localhost:9200",
155	"__ELASTICSEARCH_INDEX__" => "koha",	155	"__ELASTICSEARCH_INDEX__" => "koha",
156	"__FONT_DIR__" => "/usr/share/fonts/truetype/ttf-dejavu",	156	"__FONT_DIR__" => "/usr/share/fonts/truetype/ttf-dejavu",
157	"__TEMPLATE_CACHE_DIR__" => "/tmp/koha"	157	"__TEMPLATE_CACHE_DIR__" => "/tmp/koha",
		158	"__OAI_RUN_DIR__" => "",
		159	"__OAI_LIB_DIR__" => "",
		160	"__OAI_SPOOL_DIR__" => "",
158	);	161	);
159		162
160	# Override configuration from the environment	163	# Override configuration from the environment




#!/usr/bin/perl

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;
use C4::Auth qw(check_cookie_auth haspermission get_session);
use JSON;
use Koha::OAI::Harvester::Biblios;
use Koha::OAI::Harvester::Histories;

my $input = new CGI;

my ( $auth_status, $sessionID ) =
  check_cookie_auth( $input->cookie('CGISESSID'));

if ( $auth_status ne "ok" ) {
    exit 0;
}

my $data = {
    data => [],
    recordsTotal => 0,
    recordsFiltered => 0,
    draw => undef,
};

my $length = 10;
my $start = 0;
my @order_by = ();
my @search = ();

if ($input->request_method eq "POST"){
    my $postdata = $input->param('POSTDATA');
    my $request = from_json($postdata);
    $data->{draw} = int( $request->{draw} ) if $request->{draw};
    $length = $request->{length} if $request->{length};
    $start = $request->{start} if $request->{start};
    if (my $search = $request->{search}){
        my $value = $search->{value};
        if ($value){
            foreach my $column (@{$request->{columns}}){
                if ($column->{data} && $column->{searchable}){
                    my $search_element = {
                        $column->{data} => { 'like', "%".$value."%" },
                    };
                    push(@search,$search_element);
                }
            }
        }
    }
    if (my $order = $request->{order}){
        foreach my $element (@$order){
            my $dir = $element->{dir};
            my $column_index = $element->{column};
            my $column = $request->{columns}->[$column_index];
            my $orderable = $column->{orderable};
            if ($orderable){
                my $column_name = $column->{data};
                my $direction;
                if ($dir){
                    if ($dir eq "asc" || $dir eq "desc"){
                        $direction = "-$dir";
                    }
                }
                if ($column_name && $direction){
                    my $single_order = {
                        $direction => $column_name,
                    };
                    push(@order_by,$single_order);
                }
            }
        }
    }
}

my $page = ( $start / $length ) + 1;
my $results = Koha::OAI::Harvester::Histories->new->search(
    \@search,
    {
        page => $page,
        rows => $length,
        order_by => \@order_by,
    },
);
my $count = Koha::OAI::Harvester::Histories->new->count;
my $filtered_count = $results->pager->total_entries;
my @rows = ();
while (my $obj = $results->next){
    my $row = $obj->unblessed;
    $row->{imported_record} = '';
    if ($row->{record_type} eq "biblio"){
        my $harvested_biblio = Koha::OAI::Harvester::Biblios->new->find(
                {
                    oai_repository => $row->{repository},
                    oai_identifier => $row->{header_identifier},
                },
                { key => "oai_record" },
        );
        $row->{imported_record} = $harvested_biblio->biblionumber if $harvested_biblio;
    }
    push(@rows,$row);
}
if ($count){
    $data->{recordsTotal} = $count;
    $data->{recordsFiltered} = $filtered_count;
    $data->{data} = \@rows if @rows;
}

binmode STDOUT, ":encoding(UTF-8)";
print $input->header(
    -type => 'application/json',
    -charset => 'UTF-8'
);
print to_json($data, { pretty => 1, });




#!/usr/bin/perl

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;
use YAML;

use C4::Auth;
use C4::Context;
use C4::Output;
use Koha::OAI::Harvester::Client;
use Koha::OAI::Harvester::Requests;
use Koha::BiblioFrameworks;
use Koha::Database;

my $context = C4::Context->new();
my $config_filename = $context->{config}->{oai_pmh_harvester_config};
my $client_config = {};
if ($config_filename){
    my $config = YAML::LoadFile($config_filename);
    if ($config && $config->{socket}){
        $client_config->{socket_uri} = $config->{socket};
    }
}

my $input = new CGI;

my ($template, $loggedinuser, $cookie) =
    get_template_and_user({template_name => "tools/oai-pmh-harvester/dashboard.tt",
        query => $input,
        type => "intranet",
        authnotrequired => 0,
        flagsrequired => {tools => 'manage_staged_marc'},
    });

my $op = $input->param('op') // 'list';
my $id = $input->param('id');
my $uuid = $input->param('uuid');

my $client = Koha::OAI::Harvester::Client->new($client_config);
my $is_connected = $client->connect;

if ( ($op eq "send") && $id ){
    if ($is_connected){
        my $request = Koha::OAI::Harvester::Requests->find($id);
        if ($request){
            my $task = {
                name => $request->name,
                uuid => $request->uuid,
                interval => $request->interval,
                parameters => {
                    oai_pmh => {
                        baseURL => $request->http_url,
                        verb => $request->oai_verb,
                        metadataPrefix => $request->oai_metadataPrefix,
                        identifier => $request->oai_identifier,
                        set => $request->oai_set,
                        from => $request->oai_from,
                        until => $request->oai_until,
                    },
                    import => {
                        filter => $request->import_filter,
                        frameworkcode => $request->import_framework_code,
                        matcher_code => $request->import_matcher_code,
                        record_type => $request->import_record_type,
                    },
                },
            };
            if ($request->http_username && $request->http_password && $request->http_realm){
                $task->{parameters}->{http_basic_auth} = {
                    username => $request->http_username,
                    password => $request->http_password,
                    realm => $request->http_realm,
                };
            }
            my $is_created = $client->create($task);
            $template->{VARS}->{ result }->{ send } = $is_created;
        }
    }
}
elsif ( ($op eq "start") && ($uuid) ){
    if ($is_connected){
        my $is_started = $client->start($uuid);
        $template->{VARS}->{ result }->{ start } = $is_started;
    }
}
elsif ( ($op eq "stop") && ($uuid) ){
    if ($is_connected){
        my $is_stopped = $client->stop($uuid);
        $template->{VARS}->{ result }->{ stop } = $is_stopped;
    }
}
elsif ( ($op eq "delete") && ($uuid) ){
    if ($is_connected){
        my $is_deleted = $client->delete($uuid);
        $template->{VARS}->{ result }->{ delete } = $is_deleted;
    }
}

my $requests = Koha::OAI::Harvester::Requests->as_list;
$template->{VARS}->{ saved_requests } = $requests;

my $frameworks = Koha::BiblioFrameworks->as_list();
$template->{VARS}->{ frameworks } = $frameworks;

my $schema = Koha::Database->new()->schema();
my $matcher_rs = $schema->resultset("MarcMatcher");
my @matchers = $matcher_rs->all;
$template->{VARS}->{ matchers } = \@matchers;

if ($is_connected){
    my $submitted_requests = $client->list;
    $template->{VARS}->{ submitted_requests } = $submitted_requests;
}
else {
    $template->{VARS}->{ harvester }->{ offline } = 1;
}

output_html_with_http_headers($input, $cookie, $template->output);




#!/usr/bin/perl

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;

use C4::Auth;
use C4::Output;
use Koha::OAI::Harvester::Histories;

my $input = new CGI;

my ($template, $loggedinuser, $cookie) =
    get_template_and_user({template_name => "tools/oai-pmh-harvester/record.tt",
        query => $input,
        type => "intranet",
        authnotrequired => 0,
        flagsrequired => {tools => 'manage_staged_marc'},
    });

my $import_oai_id = $input->param('import_oai_id');
if ($import_oai_id){
    my $history = Koha::OAI::Harvester::Histories->new->find($import_oai_id);
    if ($history){
        my $record = $history->record;
        if ($record){
            $template->{VARS}->{ record } = $record;
        }
    }
}

output_html_with_http_headers($input, $cookie, $template->output);




#!/usr/bin/perl

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;
use UUID;

use C4::Auth;
use C4::Output;
use Koha::OAI::Harvester::Requests;
use Koha::BiblioFrameworks;
use Koha::Database;

my $input = new CGI;

my ($template, $loggedinuser, $cookie) =
    get_template_and_user({template_name => "tools/oai-pmh-harvester/request.tt",
        query => $input,
        type => "intranet",
        authnotrequired => 0,
        flagsrequired => {tools => 'manage_staged_marc'},
    });

my $op = $input->param('op');
my $id = $input->param('id');

my @frameworks = Koha::BiblioFrameworks->as_list();
$template->{VARS}->{ frameworks } = \@frameworks;

my $schema = Koha::Database->new()->schema();
my $rs = $schema->resultset("MarcMatcher");
my @matchers = $rs->all;
$template->{VARS}->{ matchers } = \@matchers;

my $http_url = $input->param('http_url');
my $http_username = $input->param('http_username');
my $http_password = $input->param('http_password');
my $http_realm = $input->param('http_realm');

my $oai_verb = $input->param('oai_verb');
my $oai_metadataPrefix = $input->param('oai_metadataPrefix');
my $oai_identifier = $input->param('oai_identifier');
my $oai_from = $input->param('oai_from');
my $oai_until = $input->param('oai_until');
my $oai_set = $input->param('oai_set');

my $import_filter = $input->param('import_filter') // 'default';
my $import_framework_code = $input->param('import_framework_code');
my $import_record_type = $input->param('import_record_type');
my $import_matcher_code = $input->param('import_matcher_code');

my $interval = $input->param("interval") ? int ( $input->param("interval") ) : 0;
my $name = $input->param("name");

my $save = $input->param('save');
my $test_parameters = $input->param('test_parameters');

my $request = $id ? Koha::OAI::Harvester::Requests->find($id) : Koha::OAI::Harvester::Request->new();
if ($request){
    if ($op eq "create" || $op eq "update"){
        $request->set({
            name => $name,
            http_url => $http_url,
            http_username => $http_username,
            http_password => $http_password,
            http_realm => $http_realm,
            oai_verb => $oai_verb,
            oai_metadataPrefix => $oai_metadataPrefix,
            oai_identifier => $oai_identifier,
            oai_from => $oai_from,
            oai_until => $oai_until,
            oai_set => $oai_set,
            import_filter => $import_filter,
            import_framework_code => $import_framework_code,
            import_record_type => $import_record_type,
            import_matcher_code => $import_matcher_code,
            interval => $interval,
        });
    }
}

if ($test_parameters){
    my $errors = $request->validate();
    $template->{VARS}->{ errors } = $errors;
    $template->{VARS}->{ test_parameters } = 1;
}

if ($op eq "new"){
    #Empty form with some defaults
    $request->import_filter("default") unless $request->import_filter;
    $request->interval(0) unless $request->interval;
}
elsif ($op eq "create"){
    if ($save){
        my ($uuid,$uuid_string);
        UUID::generate($uuid);
        UUID::unparse($uuid, $uuid_string);
        $request->uuid($uuid_string);
        $request->store;
        print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
        exit;
    }
}
elsif ( $op eq "edit"){
    $template->{VARS}->{ id } = $id;
}
elsif ($op eq "update"){
    $template->{VARS}->{ id } = $id;
    if ($save){
        $request->store;
        print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
        exit;
    }
}
elsif ($op eq "delete"){
    if ($request){
        $request->delete;
        print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
    }
}
else {
    print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
}
$template->{VARS}->{ op } = $op;
$template->{VARS}->{ oai_pmh_request } = $request;

output_html_with_http_headers($input, $cookie, $template->output);

Return to bug 10662

Lines 1154-1159 modules: Link Here

(-)a/admin/columns_settings.yml (+59 lines)
1154	columnname: spent	1154	columnname: spent
1155		1155
1156	tools:	1156	tools:
		1157	oai-pmh-harvester-dashboard:
		1158	saved-table:
		1159	-
		1160	columnname: name
		1161	-
		1162	columnname: url
		1163	-
		1164	columnname: set
		1165	-
		1166	columnname: from
		1167	-
		1168	columnname: until
		1169	-
		1170	columnname: interval
		1171
		1172	submitted-table:
		1173	-
		1174	columnname: name
		1175	-
		1176	columnname: url
		1177	-
		1178	columnname: set
		1179	-
		1180	columnname: from
		1181	-
		1182	columnname: until
		1183	-
		1184	columnname: interval
		1185	-
		1186	columnname: "effective from"
		1187	-
		1188	columnname: "effective until"
		1189	-
		1190	columnname: "pending imports"
		1191	-
		1192	columnname: status
		1193	-
		1194	columnname: error
		1195
		1196	history-table:
		1197	-
		1198	columnname: id
		1199	-
		1200	columnname: repository
		1201	-
		1202	columnname: identifier
		1203	-
		1204	columnname: datestamp
		1205	-
		1206	columnname: "upstream status"
		1207	-
		1208	columnname: "import status"
		1209	-
		1210	columnname: "import timestamp"
		1211	-
		1212	columnname: "imported record"
		1213	-
		1214	columnname: "downloaded record"
		1215
1157	notices:	1216	notices:
1158	lettert:	1217	lettert:
1159	-	1218	-

Lines 4415-4420 CREATE TABLE return_claims ( Link Here

(-)a/installer/data/mysql/kohastructure.sql (+78 lines)
4415	CONSTRAINT `rc_resolved_by_ibfk` FOREIGN KEY (`resolved_by`) REFERENCES `borrowers` (`borrowernumber`) ON DELETE SET NULL ON UPDATE CASCADE	4415	CONSTRAINT `rc_resolved_by_ibfk` FOREIGN KEY (`resolved_by`) REFERENCES `borrowers` (`borrowernumber`) ON DELETE SET NULL ON UPDATE CASCADE
4416	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;	4416	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
4417		4417
		4418	--
		4419	-- Table structure for table 'oai_harvester_biblios'
		4420	--
		4421
		4422	DROP TABLE IF EXISTS `oai_harvester_biblios`;
		4423	CREATE TABLE `oai_harvester_biblios` (
		4424	`import_oai_biblio_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
		4425	`oai_repository` varchar(191) NOT NULL,
		4426	`oai_identifier` varchar(191) DEFAULT NULL,
		4427	`biblionumber` int(11) NOT NULL,
		4428	PRIMARY KEY (`import_oai_biblio_id`),
		4429	UNIQUE KEY `oai_record` (`oai_identifier`,`oai_repository` (191)) USING BTREE,
		4430	KEY `FK_import_oai_biblio_1` (`biblionumber`),
		4431	CONSTRAINT `FK_import_oai_biblio_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE NO ACTION
		4432	) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
		4433
		4434	--
		4435	-- Table structure for table 'oai_harvester_history'
		4436	--
		4437
		4438	DROP TABLE IF EXISTS `oai_harvester_history`;
		4439	CREATE TABLE `oai_harvester_history` (
		4440	`import_oai_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
		4441	`repository` varchar(255) DEFAULT NULL,
		4442	`header_identifier` varchar(255) DEFAULT NULL,
		4443	`header_datestamp` datetime NOT NULL,
		4444	`header_status` varchar(45) DEFAULT NULL,
		4445	`record` longtext NOT NULL,
		4446	`upload_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
		4447	`status` varchar(45) NOT NULL,
		4448	`filter` text NOT NULL,
		4449	`framework` varchar(4) NOT NULL,
		4450	`record_type` enum('biblio','auth','holdings') NOT NULL,
		4451	`matcher_code` varchar(10) DEFAULT NULL,
		4452	PRIMARY KEY (`import_oai_id`)
		4453	) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
		4454
		4455	--
		4456	-- Table structure for table 'oai_harvester_import_queue'
		4457	--
		4458
		4459	DROP TABLE IF EXISTS `oai_harvester_import_queue`;
		4460	CREATE TABLE `oai_harvester_import_queue` (
		4461	`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
		4462	`uuid` varchar(45) CHARACTER SET utf8 NOT NULL,
		4463	`status` varchar(45) CHARACTER SET utf8 NOT NULL DEFAULT 'new',
		4464	`result` text CHARACTER SET utf8 NOT NULL,
		4465	`result_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
		4466	PRIMARY KEY (`id`)
		4467	) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
		4468
		4469	--
		4470	-- Table structure for table 'oai_harvester_requests'
		4471	--
		4472
		4473	DROP TABLE IF EXISTS `oai_harvester_requests`;
		4474	CREATE TABLE `oai_harvester_requests` (
		4475	`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
		4476	`uuid` varchar(45) NOT NULL,
		4477	`oai_verb` varchar(45) NOT NULL,
		4478	`oai_metadataPrefix` varchar(255) NOT NULL,
		4479	`oai_identifier` varchar(255) DEFAULT NULL,
		4480	`oai_from` varchar(45) DEFAULT NULL,
		4481	`oai_until` varchar(45) DEFAULT NULL,
4482	`oai_set` varchar(255) DEFAULT NULL,
4483	`http_url` varchar(255) DEFAULT NULL,
4484	`http_username` varchar(255) DEFAULT NULL,
4485	`http_password` varchar(255) DEFAULT NULL,
4486	`http_realm` varchar(255) DEFAULT NULL,
4487	`import_filter` varchar(255) NOT NULL,
4488	`import_framework_code` varchar(4) NOT NULL,
4489	`import_record_type` enum('biblio','auth','holdings') NOT NULL,
4490	`import_matcher_code` varchar(10) DEFAULT NULL,
4491	`interval` int(10) unsigned NOT NULL,
4492	`name` varchar(45) NOT NULL,
4493	PRIMARY KEY (`id`)
4494	) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
4495
4418	/!40103 SET TIME_ZONE=@OLD_TIME_ZONE /;	4496	/!40103 SET TIME_ZONE=@OLD_TIME_ZONE /;
4419	/!40101 SET SQL_MODE=@OLD_SQL_MODE /;	4497	/!40101 SET SQL_MODE=@OLD_SQL_MODE /;
4420	/!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS /;	4498	/!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS /;

Line 0 Link Here

(-)a/Koha/Daemon.pm (+210 lines)
		1	package Koha::Daemon;
		2
		3	# Copyright 2017 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19	#
		20
		21	use Modern::Perl;
		22	use POSIX; #For daemonizing
		23	use Fcntl qw(:flock); #For pidfile
		24
		25	=head1 NAME
		26
		27	Koha::Daemon
		28
		29	=head1 SYNOPSIS
		30
		31	use Koha::Daemon;
		32	my $daemon = Koha::Daemon->new({
		33	pidfile => "/path/to/file.pid",
		34	logfile => "/path/to/file.log",
		35	daemonize => 1,
		36	});
		37	$daemon->run();
		38
		39	=head1 METHODS
		40
		41	=head2 new
		42
		43	Create object
		44
		45	=head2 run
		46
		47	Run the daemon.
		48
		49	This method calls all the internal methods which
		50	are do everything necessary to run the daemon.
		51
		52	=head1 INTERNAL METHODS
		53
		54	=head2 daemonize
		55
		56	Internal function for setting object up as a proper daemon
		57	(e.g. forking, setting permissions, changing directories,
		58	closing file handles, etc.)
		59
		60	=head2 get_pidfile
		61
		62	Internal function to get a filehandle for the pidfile
		63
		64	=head2 lock_pidfile
65
66	Internal function to lock the pidfile, so that only one daemon
67	can run against this pidfile
68
69	=head2 log_to_file
70
71	Internal function to log to a file
72
73	=head2 make_pidfilehandle
74
75	Internal function to make a filehandle for pidfile
76
77	=head2 write_pidfile
78
79	Internal function to write pid to pidfile
80
81	=cut
82
83	sub new {
84	my ($class, $args) = @_;
85	$args = {} unless defined $args;
86	return bless ($args, $class);
87	}
88
89	#######################################################################
90	#NOTE: On Debian, you can use the daemon binary to make a process into a daemon,
91	# but the following can be used if you don't want to use that program.
92
93	sub daemonize {
94	my ($self) = @_;
95
96	my $pid = fork;
97
98	die "Couldn't fork: $!" unless defined($pid);
99	if ($pid){
100	exit; #Parent exit
101	}
102
103	#Become a session leader (ie detach program from controlling terminal)
104	POSIX::setsid() or die "Can't start a new session: $!";
105
106	#Change to known system directory
107	chdir('/');
108
109	#Reset the file creation mask so only the daemon owner can read/write files it creates
110	umask(066);
111
112	#Close inherited file handles, so that you can truly run in the background.
113	open STDIN, '<', '/dev/null';
114	open STDOUT, '>', '/dev/null';
115	open STDERR, '>&STDOUT';
116	}
117
118	sub log_to_file {
119	my ($self,$logfile) = @_;
120
121	#Open a filehandle to append to a log file
122	my $opened = open(my $fh, '>>', $logfile);
123	if ($opened){
124	$fh->autoflush(1); #Make filehandle hot (ie don't buffer)
125	STDOUT = $fh; #Re-assign STDOUT to LOG \| --stdout
126	STDERR = STDOUT; #Re-assign STDERR to STDOUT \| --stderr
127	}
128	else {
129	die "Unable to open a filehandle for $logfile: $!\n"; # --output
130	}
131	}
132
133	sub make_pidfilehandle {
134	my ($self,$pidfile) = @_;
135	if ( ! -f $pidfile ){
136	open(my $fh, '>', $pidfile) or die "Unable to write to $pidfile: $!\n";
137	close($fh);
138	}
139	open(my $pidfilehandle, '+<', $pidfile) or die "Unable to open a filehandle for $pidfile: $!\n";
140	return $pidfilehandle;
141	}
142
143	sub get_pidfile {
144	my ($self,$pidfile) = @_;
145	#NOTE: We need to save the filehandle in the object, so any locks persist for the life of the object
146	my $pidfilehandle = $self->{pidfilehandle} \|\|= $self->make_pidfilehandle($pidfile);
147	return $pidfilehandle;
148	}
149
150	sub lock_pidfile {
151	my ($self,$pidfilehandle) = @_;
152	my $locked;
153	if (flock($pidfilehandle, LOCK_EX\|LOCK_NB)){
154	$locked = 1;
155
156	}
157	return $locked;
158	}
159
160	sub write_pidfile {
161	my ($self,$pidfilehandle) = @_;
162	if ($pidfilehandle){
163	truncate($pidfilehandle, 0);
164	print $pidfilehandle $$."\n" or die $!;
165	#Flush the filehandle so you're not suffering from buffering
166	$pidfilehandle->flush();
167	return 1;
168	}
169	}
170
171	sub run {
172	my ($self) = @_;
173	my $pidfile = $self->{pidfile};
174	my $logfile = $self->{logfile};
175
176	if ($pidfile){
177	my $pidfilehandle = $self->get_pidfile($pidfile);
178	if ($pidfilehandle){
179	my $locked = $self->lock_pidfile($pidfilehandle);
180	if ( ! $locked ) {
181	die "$0 is unable to lock pidfile...\n";
182	}
183	}
184	}
185
186	if (my $configure = $self->{configure}){
187	$configure->($self);
188	}
189
190	if ($self->{daemonize}){
191	$self->daemonize();
192	}
193
194	if ($pidfile){
195	my $pidfilehandle = $self->get_pidfile($pidfile);
196	if ($pidfilehandle){
197	$self->write_pidfile($pidfilehandle);
198	}
199	}
200
201	if ($logfile){
202	$self->log_to_file($logfile);
203	}
204
205	if (my $loop = $self->{loop}){
206	$loop->($self);
207	}
208	}
209
210	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester.pm (+715 lines)
		1	package Koha::OAI::Harvester;
		2
		3	# Copyright 2017 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19	#
		20
		21	use Modern::Perl;
		22	use POE qw(Component::JobQueue);
		23	use JSON;
		24	use Sereal::Encoder;
		25	use Sereal::Decoder;
		26	use IO::Handle;
		27	use File::Copy;
		28	use File::Path qw(make_path remove_tree);
		29	use DateTime;
		30	use DateTime::Format::Strptime;
		31
		32	use C4::Context;
		33	use Koha::OAI::Harvester::ImportQueues;
		34
		35	=head1 NAME
		36
		37	Koha::OAI::Harvester
		38
		39	=head1 SYNOPSIS
		40
		41	my $harvester = Koha::OAI::Harvester->spawn({
		42	Downloader => $downloader,
		43	DownloaderWorkers => $download_workers,
		44	Importer => $importer,
		45	ImporterWorkers => $import_workers,
		46	ImportQueuePoll => $import_poll,
		47	logger => $logger,
		48	state_file => $statefile,
		49	spooldir => $spooldir,
		50	});
		51
		52	=head2 METHODS
		53
		54	=cut
		55
		56	my $day_granularity = DateTime::Format::Strptime->new(
		57	pattern => '%F',
		58	);
		59	my $seconds_granularity = DateTime::Format::Strptime->new(
		60	pattern => '%FT%TZ',
		61	);
		62
		63	=head2 new
		64
65	Create object
66
67	=cut
68
69	sub new {
70	my ($class, $args) = @_;
71	$args = {} unless defined $args;
72	return bless ($args, $class);
73	}
74
75	=head2 spawn
76
77	Creates POE sessions for handling tasks, downloading, and importing
78
79	=cut
80
81	sub spawn {
82	my ($class, $args) = @_;
83	my $self = $class->new($args);
84	my $downloader = $self->{Downloader};
85	my $importer = $self->{Importer};
86	my $download_worker_limit = ( $self->{DownloaderWorkers} && int($self->{DownloaderWorkers}) ) ? $self->{DownloaderWorkers} : 1;
87	my $import_worker_limit = ( $self->{ImporterWorkers} && int($self->{ImporterWorkers}) ) ? $self->{ImporterWorkers} : 1;
88	my $import_queue_poll = ( $self->{ImportQueuePoll} && int($self->{ImportQueuePoll}) ) ? $self->{ImportQueuePoll} : 5;
89
90	#NOTE: This job queue should always be created before the
91	#harvester so that you can start download jobs immediately
92	#upon spawning the harvester.
93	POE::Component::JobQueue->spawn(
94	Alias => 'oai-downloader',
95	WorkerLimit => $download_worker_limit,
96	Worker => sub {
97	my ($postback, $task) = @_;
98	if ($downloader){
99	if ($task->{status} eq "active"){
100	$downloader->run({
101	postback => $postback,
102	task => $task,
103	});
104	}
105	}
106	},
107	Passive => {},
108	);
109
110	POE::Session->create(
111	object_states => [
112	$self => {
113	_start => "on_start",
114	get_task => "get_task",
115	list_tasks => "list_tasks",
116	create_task => "create_task",
117	start_task => "start_task",
118	stop_task => "stop_task",
119	delete_task => "delete_task",
120	repeat_task => "repeat_task",
121	register => "register",
122	deregister => "deregister",
123	restore_state => "restore_state",
124	save_state => "save_state",
125	is_task_finished => "is_task_finished",
126	does_task_repeat => "does_task_repeat",
127	download_postback => "download_postback",
128	reset_imports_status => "reset_imports_status",
129	},
130	],
131	);
132
133	POE::Component::JobQueue->spawn(
134	Alias => 'oai-importer',
135	WorkerLimit => $import_worker_limit,
136	Worker => sub {
137	my $meta_postback = shift;
138
139	#NOTE: We need to only retrieve queue items for active tasks. Otherwise,
140	#the importer will just spin its wheels on inactive tasks and do nothing.
141	my $active_tasks = $poe_kernel->call("harvester","list_tasks","active");
142	my @active_uuids = map { $_->{uuid} } @$active_tasks;
143
144	my $rs = Koha::OAI::Harvester::ImportQueues->new->search({
145	uuid => \@active_uuids,
146	status => "new"
147	},{
148	order_by => { -asc => 'id' },
149	rows => 1,
150	});
151	my $result = $rs->single;
152	if ($result){
153	$result->status("wip");
154	$result->update;
155	my $task = {
156	id => $result->id,
157	uuid => $result->uuid,
158	result => $result->result,
159	};
160
161	my $postback = $meta_postback->($task);
162	$importer->run({
163	postback => $postback,
164	task => $task,
165	});
166	}
167	},
168	Active => {
169	PollInterval => $import_queue_poll,
170	AckAlias => undef,
171	AckState => undef,
172	},
173	);
174
175	return;
176	}
177
178	=head2 on_start
179
180	Internal method that starts the harvester
181
182	=cut
183
184	sub on_start {
185	my ($self, $kernel, $heap) = @_[OBJECT, KERNEL,HEAP];
186	$kernel->alias_set('harvester');
187	$heap->{scoreboard} = {};
188
189	#Reset any 'wip' imports to 'new' so they can be re-tried.
190	$kernel->call("harvester","reset_imports_status");
191
192	#Restore state from state file
193	$kernel->call("harvester","restore_state");
194	}
195
196	=head2 download_postback
197
198	Internal method
199
200	=cut
201
202	#NOTE: This isn't really implemented at the moment, as it's not really necessary.
203	sub download_postback {
204	my ($kernel, $request_packet, $response_packet) = @_[KERNEL, ARG0, ARG1];
205	my $message = $response_packet->[0];
206	}
207
208	=head2 deregister
209
210	Remove the worker session from the harvester's in-memory scoreboard,
211	unset the downloading flag if downloading is completed.
212
213	=cut
214
215	sub deregister {
216	my ($self, $kernel, $heap, $session, $sender, $type) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0];
217
218	my $scoreboard = $heap->{scoreboard};
219
220	my $logger = $self->{logger};
221	$logger->debug("Start deregistering $sender as $type task");
222
223	my $task_uuid = delete $scoreboard->{session}->{$sender};
224	#NOTE: If you don't check each step of the hashref, autovivication can lead to surprises.
225	if ($task_uuid){
226	if ($scoreboard->{task}->{$task_uuid}){
227	if ($scoreboard->{task}->{$task_uuid}->{$type}){
228	delete $scoreboard->{task}->{$task_uuid}->{$type}->{$sender};
229	}
230	}
231	}
232
233	my $task = $heap->{tasks}->{$task_uuid};
234	if ($task && $task->{status} && ($task->{status} eq "active") ){
235	#NOTE: Each task only has 1 download session, so we can now set/unset flags for the task.
236	#NOTE: We should unset the downloading flag, if we're not going to repeat the task.
237	if ($type eq "download"){
238	my $task_repeats = $kernel->call("harvester","does_task_repeat",$task_uuid);
239	if ($task_repeats){
240	my $interval = $task->{interval};
241
242	$task->{effective_from} = delete $task->{effective_until};
243	$task->{download_timer} = $kernel->delay_set("repeat_task", $interval, $task_uuid);
244	}
245	else {
246	$task->{downloading} = 0;
247	$kernel->call("harvester","save_state");
248	$kernel->call("harvester","is_task_finished",$task_uuid);
249	}
250	}
251	elsif ($type eq 'import'){
252	$kernel->call("harvester","is_task_finished",$task_uuid);
253	}
254	}
255	$logger->debug("End deregistering $sender as $type task");
256	}
257
258
259	=head2 is_task_finished
260
261	This event handler checks if the task has finished downloading and importing record.
262	If it is finished downloading and importing, the task is deleted from the harvester.
263
264	This only applies to non-repeating tasks.
265
266	=cut
267
268	sub is_task_finished {
269	my ($self, $kernel, $heap, $session, $uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
270	my $task = $kernel->call("harvester","get_task",$uuid);
271	if ($task && (! $task->{downloading}) ){
272	my $count = $self->get_import_count_for_task($uuid);
273	if ( ! $count ) {
274	#Clear this task out of the harvester as it's finished.
275	$kernel->call("harvester","delete_task",$uuid);
276	return 1;
277	}
278	}
279	return 0;
280	}
281
282	=head2 register
283
284	Internal method for registering a task on the harvester's scoreboard
285
286	=cut
287
288	sub register {
289	my ($self, $kernel, $heap, $session, $sender, $type, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0,ARG1];
290	my $logger = $self->{logger};
291
292	my $scoreboard = $heap->{scoreboard};
293
294
295	if ($type && $task_uuid){
296	$logger->debug("Registering $sender as $type for $task_uuid");
297
298	my $task = $heap->{tasks}->{$task_uuid};
299	if ($task){
300
301	if ($type){
302	#Register the task uuid with the session id as a key for later recall
303	$scoreboard->{session}->{$sender} = $task_uuid;
304
305	#Register the session id as a certain type of session for a task
306	$scoreboard->{task}->{$task_uuid}->{$type}->{$sender} = 1;
307
308	if ($type eq "download"){
309	$task->{downloading} = 1;
310
311	my $task_repeats = $kernel->call("harvester","does_task_repeat",$task_uuid);
312	if ($task_repeats){
313
314	#NOTE: Set an effective until, so we know we're not getting records any newer than
315	#this moment.
316	my $dt = DateTime->now();
317	if ($dt){
318	#NOTE: Ideally, I'd like to make sure that we can use 'seconds' granularity, but
319	#it's valid for 'from' to be null, so it's impossible to know from the data whether
320	#or not the repository will support the seconds granularity.
321	#NOTE: Ideally, it would be good to use either 'day' granularity or 'seconds' granularity,
322	#but at the moment the interval is expressed as seconds only.
323	$dt->set_formatter($seconds_granularity);
324	$task->{effective_until} = "$dt";
325	}
326	}
327
328	$kernel->call("harvester","save_state");
329	}
330	}
331	}
332	}
333	}
334
335	=head2 does_task_repeat
336
337	Internal method for checking if a task is supposed to repeat after it finishes
338
339	=cut
340
341	sub does_task_repeat {
342	my ($self, $kernel, $heap, $session, $uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
343	my $task = $kernel->call("harvester","get_task",$uuid);
344	if ($task){
345	my $interval = $task->{interval};
346	my $parameters = $task->{parameters};
347	if ($parameters){
348	my $oai_pmh = $parameters->{oai_pmh};
349	if ($oai_pmh){
350	if ( $interval && ($oai_pmh->{verb} eq "ListRecords") && (! $oai_pmh->{until}) ){
351	return 1;
352	}
353	}
354	}
355	}
356	return 0;
357	}
358
359	=head2 reset_imports_status
360
361	Internal method that resets the import queue from wip to new
362
363	=cut
364
365	sub reset_imports_status {
366	my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION];
367
368	my $rs = Koha::OAI::Harvester::ImportQueues->new->search({
369	status => "wip",
370	});
371	$rs->update({
372	status => "new",
373	});
374	}
375
376	=head2 restore_state
377
378	Method to restore the harvester to a pre-existing state.
379
380	=cut
381
382	sub restore_state {
383	my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION];
384
385	my $state_file = $self->{state_file};
386	if ($state_file){
387	my $state_backup = "$state_file~";
388
389	#NOTE: If there is a state backup, it means we crashed while saving the state. Otherwise,
390	#let's try the regular state file if it exists.
391	my $file_to_restore = ( -f $state_backup ) ? $state_backup : ( ( -f $state_file ) ? $state_file : undef );
392	if ( $file_to_restore ){
393	my $opened = open( my $fh, '<', $file_to_restore ) or die "Couldn't open state: $!";
394	if ($opened){
395	local $/;
396	my $in = <$fh>;
397	my $decoder = Sereal::Decoder->new;
398	my $state = $decoder->decode($in);
399	if ($state){
400	if ($state->{tasks}){
401	#Restore tasks from our saved state
402	$heap->{tasks} = $state->{tasks};
403	foreach my $uuid ( keys %{$heap->{tasks}} ){
404	my $task = $heap->{tasks}->{$uuid};
405
406	#If tasks were still downloading, restart the task
407	if ( ($task->{status} && $task->{status} eq "active") && $task->{downloading} ){
408	$task->{status} = "new";
409	$kernel->call("harvester","start_task",$task->{uuid});
410	}
411	}
412	}
413	}
414	}
415	}
416	}
417	}
418
419	=head2 save_state
420
421	Method to save the existing state of the harvester
422
423	=cut
424
425	sub save_state {
426	my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION];
427	my $state_file = $self->{state_file};
428	my $state_backup = "$state_file~";
429
430	#Make a backup of existing state record
431	my $moved = move($state_file,$state_backup);
432
433	my $opened = open(my $fh, ">", $state_file) or die "Couldn't save state: $!";
434	if ($opened){
435	$fh->autoflush(1);
436	my $tasks = $heap->{tasks};
437	my $harvester_state = {
438	tasks => $tasks,
439	};
440	my $encoder = Sereal::Encoder->new;
441	my $out = $encoder->encode($harvester_state);
442	local $\;
443	my $printed = print $fh $out;
444	if ($printed){
445	close $fh;
446	unlink($state_backup);
447	return 1;
448	}
449	}
450	return 0;
451	}
452
453	=head2 get_task
454
455	This event handler returns a task from a harvester using the task's
456	uuid as an argument.
457
458	=cut
459
460	sub get_task {
461	my ($self, $kernel, $heap, $session, $uuid, $sender) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0, SENDER];
462
463	if ( ! $uuid && $sender ){
464	my $scoreboard = $heap->{scoreboard};
465	my $uuid_by_session = $scoreboard->{session}->{$sender};
466	if ($uuid_by_session){
467	$uuid = $uuid_by_session;
468	}
469	}
470
471	my $tasks = $heap->{tasks};
472	if ($tasks && $uuid){
473	my $task = $tasks->{$uuid};
474	if ($task){
475	return $task;
476	}
477	}
478	return 0;
479	}
480
481	=head2 get_import_count_for_task
482
483	This gets the count of the number of imports exist for a certain task
484
485	=cut
486
487	sub get_import_count_for_task {
488	my ($self,$uuid) = @_;
489	my $count = undef;
490	if ($uuid){
491	my $items = Koha::OAI::Harvester::ImportQueues->new->search({
492	uuid => $uuid,
493	});
494	$count = $items->count;
495	}
496	return $count;
497	}
498
499	=head2 list_tasks
500
501	This event handler returns a list of tasks that have been submitted
502	to the harvester. It returns data like uuid, status, parameters,
503	number of pending imports, etc.
504
505	=cut
506
507	sub list_tasks {
508	my ($self, $kernel, $heap, $session, $status) = @_[OBJECT, KERNEL,HEAP,SESSION, ARG0];
509	my @tasks = ();
510	foreach my $uuid (sort keys %{$heap->{tasks}}){
511	my $task = $heap->{tasks}->{$uuid};
512	my $items = Koha::OAI::Harvester::ImportQueues->new->search({
513	uuid => $uuid,
514	});
515	my $count = $items->count // 0;
516	$task->{pending_imports} = $count;
517	if ( ( ! $status ) \|\| ( $status && $status eq $task->{status} ) ){
518	push(@tasks, $task);
519	}
520
521	}
522	return \@tasks;
523	}
524
525	=head2 create_task
526
527	This event handler creates a spool directory for the task's imports.
528	It also adds it to the harvester's memory and then saves memory to
529	a persistent datastore.
530
531	Newly created tasks have a status of "new".
532
533	=cut
534
535	sub create_task {
536	my ($self, $kernel, $heap, $session, $incoming_task) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
537	my $logger = $self->{logger};
538	if ($incoming_task){
539	my $uuid = $incoming_task->{uuid};
540	if ( ! $heap->{tasks}->{$uuid} ){
541
542	#Step One: assign a spool directory to this task
543	my $spooldir = $self->{spooldir} // "/tmp";
544	my $task_spooldir = "$spooldir/$uuid";
545	if ( ! -d $task_spooldir ){
546	my $made_spool_directory = make_path($task_spooldir);
547	if ( ! $made_spool_directory ){
548	if ($logger){
549	$logger->warn("Unable to make task-specific spool directory at '$task_spooldir'");
550	}
551	return 0;
552	}
553	}
554	$incoming_task->{spooldir} = $task_spooldir;
555
556	#Step Two: assign new status
557	$incoming_task->{status} = "new";
558
559	#Step Three: add task to harvester's memory
560	$heap->{tasks}->{ $uuid } = $incoming_task;
561
562	#Step Four: save state
563	$kernel->call($session,"save_state");
564	return 1;
565	}
566	}
567	return 0;
568	}
569
570	=head2 start_task
571
572	This event handler marks a task as active in the harvester's memory,
573	save the memory to a persistent datastore, then enqueues the task,
574	so that it can be directed to the next available download worker.
575
576	Newly started tasks have a status of "active".
577
578	=cut
579
580	sub start_task {
581	my ($self, $session,$kernel,$heap,$uuid) = @_[OBJECT, SESSION,KERNEL,HEAP,ARG0];
582	my $task = $heap->{tasks}->{$uuid};
583	if ($task){
584	if ( $task->{status} ne "active" ){
585
586	#Clear any pre-existing error states
587	delete $task->{error} if $task->{error};
588
589	#Step One: mark task as active
590	$task->{status} = "active";
591
592	#Step Two: save state
593	$kernel->call("harvester","save_state");
594
595	#Step Three: enqueue task
596	$kernel->post("oai-downloader",'enqueue','download_postback', $task);
597
598	return 1;
599	}
600	}
601	return 0;
602	}
603
604	=head2 repeat_task
605
606	This method re-queues a task for downloading
607
608	=cut
609
610	sub repeat_task {
611	my ($self, $session,$kernel,$heap,$uuid) = @_[OBJECT, SESSION,KERNEL,HEAP,ARG0];
612	my $task = $heap->{tasks}->{$uuid};
613	if ($task){
614	my $interval = $task->{interval};
615	if ($task->{downloading} && $interval){
616	$kernel->post("oai-downloader",'enqueue','download_postback', $task);
617	}
618	}
619	}
620
621	=head2 stop_task
622
623	This event handler prevents new workers from spawning, kills
624	existing workers, and stops pending imports from being imported.
625
626	Newly stopped tasks have a status of "stopped".
627
628	=cut
629
630	sub stop_task {
631	my ($self, $kernel, $heap, $session, $sender, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0];
632
633	my $task = $heap->{tasks}->{$task_uuid};
634
635	if ($task && $task->{status} && $task->{status} ne "stopped" ){
636
637	#Step One: deactivate task, so no new workers can be started
638	$task->{status} = "stopped";
639	#NOTE: You could also clear timers for new downloads, but that's probably unnecessary because of this step.
640
641	#Step Two: kill workers
642	my $scoreboard = $heap->{scoreboard};
643	my $session_types = $scoreboard->{task}->{$task_uuid};
644	if ($session_types){
645	foreach my $type ( keys %$session_types ){
646	my $sessions = $session_types->{$type};
647	if ($sessions){
648	foreach my $session (keys %$sessions){
649	if ($session){
650	$kernel->signal($session, "cancel");
651	}
652	}
653	}
654	}
655	#Remove the task uuid from the task key of the scoreboard
656	delete $scoreboard->{task}->{$task_uuid};
657	#NOTE: The task uuid will still exist under the session key,
658	#but the sessions will deregister themselves and clean that up for you.
659	}
660
661	#Step Three: stop pending imports for this task
662	my $items = Koha::OAI::Harvester::ImportQueues->new->search({
663	uuid => $task_uuid,
664	});
665	my $rows_updated = $items->update({
666	status => "stopped",
667	});
668
669	#Step Four: save state
670	$kernel->call("harvester","save_state");
671	return 1;
672	}
673	return 0;
674	}
675
676	=head2 delete_task
677
678	Deleted tasks are stopped, pending imports are deleted from the
679	database and file system, and then the task is removed from the harvester.
680
681	=cut
682
683	sub delete_task {
684	my ($self, $kernel, $heap, $session, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
685
686	my $task = $heap->{tasks}->{$task_uuid};
687	if ($task){
688	#Step One: stop task
689	$kernel->call($session,"stop_task",$task_uuid);
690
691	#Step Two: delete pending imports in database
692	my $items = Koha::OAI::Harvester::ImportQueues->new->search({
693	uuid => $task_uuid,
694	});
695	if ($items){
696	my $rows_deleted = $items->delete;
697	#NOTE: shows 0E0 instead of 0
698	}
699
700	#Step Three: remove task specific spool directory and files within it
701	my $spooldir = $task->{spooldir};
702	if ($spooldir){
703	my $files_deleted = remove_tree($spooldir, { safe => 1 });
704	}
705
706	delete $heap->{tasks}->{$task_uuid};
707
708	#Step Four: save state
709	$kernel->call("harvester","save_state");
710	return 1;
711	}
712	return 0;
713	}
714
715	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Biblio.pm (+49 lines)
	1	package Koha::OAI::Harvester::Biblio;
	2
	3	# Copyright Prosentient Systems 2018
	4	#
	5	# This file is part of Koha.
	6	#
	7	# Koha is free software; you can redistribute it and/or modify it under the
	8	# terms of the GNU General Public License as published by the Free Software
	9	# Foundation; either version 3 of the License, or (at your option) any later
	10	# version.
	11	#
	12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
	13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	15	#
	16	# You should have received a copy of the GNU General Public License along
	17	# with Koha; if not, write to the Free Software Foundation, Inc.,
	18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
	19
	20	use Modern::Perl;
	21
	22	use Carp;
	23
	24	use base qw(Koha::Object);
	25
	26	=head1 NAME
	27
	28	Koha::OAI::Harvester::Biblio
	29
	30	=head1 SYNOPSIS
	31
	32	use Koha::OAI::Harvester::Biblio;
	33	my $biblio = Koha::OAI::Harvester::Biblios->find($id);
	34
	35	=head1 METHODS
	36
	37	=cut
	38
	39	sub _type {
	40	return 'OaiHarvesterBiblio';
	41	}
	42
	43	=head1 AUTHOR
	44
	45	David Cook <dcook@prosentient.com.au>
	46
	47	=cut
	48
	49	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Biblios.pm (+60 lines)
	1	package Koha::OAI::Harvester::Biblios;
	2
	3	# Copyright Prosentient Systems 2018
	4	#
	5	# This file is part of Koha.
	6	#
	7	# Koha is free software; you can redistribute it and/or modify it under the
	8	# terms of the GNU General Public License as published by the Free Software
	9	# Foundation; either version 3 of the License, or (at your option) any later
	10	# version.
	11	#
	12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
	13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	15	#
	16	# You should have received a copy of the GNU General Public License along
	17	# with Koha; if not, write to the Free Software Foundation, Inc.,
	18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
	19
	20	use Modern::Perl;
	21
	22	use Carp;
	23
	24	use Koha::Database;
	25
	26	use Koha::OAI::Harvester::Biblio;
	27
	28	use base qw(Koha::Objects);
	29
	30	=head1 NAME
	31
	32	Koha::OAI::Harvester::Biblios
	33
	34	=head2 METHODS
	35
	36	=cut
	37
	38	=head3 _type
	39
	40	=cut
	41
	42	sub _type {
	43	return 'OaiHarvesterBiblio';
	44	}
	45
	46	=head3 object_class
	47
	48	=cut
	49
	50	sub object_class {
	51	return 'Koha::OAI::Harvester::Biblio';
	52	}
	53
	54	=head1 AUTHOR
	55
	56	David Cook <dcook@prosentient.com.au>
	57
	58	=cut
	59
	60	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Client.pm (+233 lines)
		1	package Koha::OAI::Harvester::Client;
		2
		3	# Copyright 2017 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, write to the Free Software Foundation, Inc.,
		18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		19
		20	use Modern::Perl;
		21	use URI;
		22	use IO::Socket::UNIX;
		23	use IO::Select;
		24	use JSON;
		25	use Encode;
		26
		27	=head1 NAME
		28
		29	Koha::OAI::Harvester::Client
		30
		31	=head1 SYNOPSIS
		32
		33	use Koha::OAI::Harvester::Client;
		34	my $client = Koha::OAI::Harvester::Client->new($client_config);
		35
		36	=head1 METHODS
		37
		38	=head2 new
		39
		40	Create object
		41
		42	=cut
		43
		44	sub new {
		45	my ($class, $args) = @_;
		46	$args = {} unless defined $args;
		47	return bless ($args, $class);
		48	}
		49
		50	=head2 connect
		51
		52	Connect to Koha::OAI::Harvester::Listener via a Unix socket
		53
		54	=cut
		55
		56	sub connect {
		57	my ($self) = @_;
		58	my $socket_uri = $self->{socket_uri};
		59	if ($socket_uri){
		60	my $uri = URI->new($socket_uri);
		61	if ($uri && $uri->scheme eq 'unix'){
		62	my $socket_path = $uri->path;
		63	my $socket = IO::Socket::UNIX->new(
		64	Type => IO::Socket::UNIX::SOCK_STREAM(),
65	Peer => $socket_path,
66	);
67	if ($socket){
68	my $select = new IO::Select();
69	$select->add($socket);
70
71	$self->{_select} = $select;
72	$self->{_socket} = $socket;
73	my $message = $self->_read();
74	if ($message){
75	if ($message eq 'HELLO'){
76	$self->{_connected} = 1;
77	return 1;
78	}
79	}
80	}
81	else {
82	warn "Failed to create socket."
83	}
84	}
85	}
86	return 0;
87	}
88
89	=head2 create
90
91	Create a task on the harvester
92
93	=cut
94
95	sub create {
96	my ($self,$task) = @_;
97	my $message = {
98	command => "create",
99	body => {
100	task => $task,
101	}
102	};
103	my ($status) = $self->_exchange($message);
104	return $status;
105	}
106
107	=head2 start
108
109	Start a task on the harvester
110
111	=cut
112
113	sub start {
114	my ($self,$uuid) = @_;
115	my $message = {
116	command => "start",
117	body => {
118	task => {
119	uuid => $uuid,
120	},
121	}
122	};
123	my ($status) = $self->_exchange($message);
124	return $status;
125	}
126
127	=head2 stop
128
129	Stop a task on the harvester
130
131	=cut
132
133	sub stop {
134	my ($self,$uuid) = @_;
135	my $message = {
136	command => "stop",
137	body => {
138	task => {
139	uuid => $uuid,
140	},
141	}
142	};
143	my ($status) = $self->_exchange($message);
144	return $status;
145	}
146
147	=head2 delete
148
149	Delete a task on the harvester
150
151	=cut
152
153	sub delete {
154	my ($self,$uuid) = @_;
155	my $message = {
156	command => "delete",
157	body => {
158	task => {
159	uuid => $uuid,
160	},
161	}
162	};
163	my ($status) = $self->_exchange($message);
164	return $status;
165	}
166
167	=head2 list
168
169	List all tasks on the harvester
170
171	=cut
172
173	sub list {
174	my ($self) = @_;
175	my $message = {
176	command => "list",
177	};
178	my ($status,$tasks) = $self->_exchange($message);
179	return $tasks;
180	}
181
182	sub _exchange {
183	my ($self,$message) = @_;
184	my $status = 0;
185	my $data;
186	if ($message){
187	my $output = to_json($message);
188	if ($output){
189	$self->_write($output);
190	my $json_response = $self->_read();
191	if ($json_response){
192	my $response = from_json($json_response);
193	$data = $response->{data} if $response->{data};
194	$status = 1 if $response->{msg} && $response->{msg} eq "success";
195	}
196	}
197	}
198	return ($status,$data);
199	}
200
201	sub _write {
202	my ($self, $output) = @_;
203	if ($output){
204	if (my $select = $self->{_select}){
205	if (my @filehandles = $select->can_write(5)){
206	foreach my $filehandle (@filehandles){
207	#Localize output record separator as null
208	local $\ = "\x00";
209	$output = Encode::encode("UTF-8",$output);
210	print $filehandle $output;
211	}
212	}
213	}
214	}
215	}
216
217	sub _read {
218	my ($self) = @_;
219	if (my $select = $self->{_select}){
220	if (my @filehandles = $select->can_read(5)){
221	foreach my $filehandle (@filehandles){
222	#Localize input record separator as null
223	local $/ = "\x00";
224	my $message = <$filehandle>;
225	chomp($message) if $message;
226	$message = Encode::decode("UTF-8",$message);
227	return $message;
228	}
229	}
230	}
231	}
232
233	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Downloader.pm (+335 lines)
		1	package Koha::OAI::Harvester::Downloader;
		2
		3	# Copyright 2017 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, write to the Free Software Foundation, Inc.,
		18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		19
		20	use Modern::Perl;
		21	use URI;
		22	use XML::LibXML::Reader;
		23	use IO::Handle;
		24	use JSON;
		25
		26	=head1 NAME
		27
		28	Koha::OAI::Harvester::Downloader
		29
		30	=head1 SYNOPSIS
		31
		32	use Koha::OAI::Harvester::Downloader;
		33	my $oai_downloader = Koha::OAI::Harvester::Downloader->new();
		34
		35	This class is used within a Koha::OAI::Harvester::Work::Download:: module.
		36
		37	=head1 METHODS
		38
		39	=head2 new
		40
		41	Create object
		42
		43	=cut
		44
		45	sub new {
		46	my ($class, $args) = @_;
		47	$args = {} unless defined $args;
		48	return bless ($args, $class);
		49	}
		50
		51	=head2 BuildURL
		52
		53	Takes a baseURL and a mix of required and optional OAI-PMH arguments,
		54	and makes them into a suitable URL for an OAI-PMH request.
		55
		56	=cut
		57
		58	sub BuildURL {
		59	my ($self, $args) = @_;
		60	my $baseURL = $args->{baseURL};
		61	my $url = URI->new($baseURL);
		62	if ($url && $url->isa("URI")){
		63	my $verb = $args->{verb};
		64	if ($verb){
65	my %parameters = (
66	verb => $verb,
67	);
68	if ($verb eq "ListRecords"){
69	my $resumptionToken = $args->{resumptionToken};
70	my $metadataPrefix = $args->{metadataPrefix};
71	if ($resumptionToken){
72	$parameters{resumptionToken} = $resumptionToken;
73	}
74	elsif ($metadataPrefix){
75	$parameters{metadataPrefix} = $metadataPrefix;
76	#Only add optional parameters if they're provided
77	foreach my $param ( qw( from until set ) ){
78	$parameters{$param} = $args->{$param} if $args->{$param};
79	}
80	}
81	else {
82	warn "BuildURL() requires an argument of either resumptionToken or metadataPrefix";
83	return;
84	}
85	}
86	elsif ($verb eq "GetRecord"){
87	my $metadataPrefix = $args->{metadataPrefix};
88	my $identifier = $args->{identifier};
89	if ($metadataPrefix && $identifier){
90	$parameters{metadataPrefix} = $metadataPrefix;
91	$parameters{identifier} = $identifier;
92	}
93	else {
94	warn "BuildURL() requires an argument of metadataPrefix and an argument of identifier";
95	return;
96	}
97	}
98	$url->query_form(%parameters);
99	return $url;
100	}
101	else {
102	warn "BuildURL() requires a verb of GetRecord or ListRecords";
103	return;
104	}
105	}
106	else {
107	warn "BuildURL() requires a base URL of type URI.";
108	return;
109	}
110	}
111
112	=head2 GetXMLStream
113
114	Fork a child process to send the HTTP request, which sends chunks
115	of XML via a pipe to the parent process.
116
117	The parent process creates and returns a XML::LibXML::Reader object,
118	which reads the XML stream coming through the pipe.
119
120	Normally, using a DOM reader, you must wait to read the entire XML document
121	into memory. However, using a stream reader, chunks are read into memory,
122	processed, then discarded. It's faster and more efficient.
123
124	=cut
125
126	sub GetXMLStream {
127	my ($self, $args) = @_;
128	my $url = $args->{url};
129	my $user_agent = $args->{user_agent};
130	if ($url && $user_agent){
131	pipe( CHILD, PARENT ) or die "Cannot created connected pipes: $!";
132	CHILD->autoflush(1);
133	PARENT->autoflush(1);
134	if ( my $pid = fork ){
135	#Parent process
136	close PARENT;
137	return \*CHILD;
138	}
139	else {
140	#Child process
141	close CHILD;
142	my $response = $self->_request({
143	url => $url,
144	user_agent => $user_agent,
145	file_handle => \*PARENT,
146	});
147	if ($response && $response->is_success){
148	#HTTP request has successfully finished, so we close the file handle and exit the process
149	close PARENT;
150	CORE::exit(); #some modules like mod_perl redefine exit
151	}
152	else {
153	warn "[child $$] OAI-PMH unsuccessful. Response status: ".$response->status_line."\n" if $response;
154	CORE::exit();
155	}
156	}
157	}
158	else {
159	warn "GetXMLStream() requires a 'url' argument and a 'user_agent' argument";
160	return;
161	}
162	}
163
164	sub _request {
165	my ($self, $args) = @_;
166	my $url = $args->{url};
167	my $user_agent = $args->{user_agent};
168	my $fh = $args->{file_handle};
169
170	if ($url && $user_agent && $fh){
171	my $request = HTTP::Request->new( GET => $url );
172	my $response = $user_agent->request( $request, sub {
173	my ($chunk_of_data, $ref_to_response, $ref_to_protocol) = @_;
174	print $fh $chunk_of_data;
175	});
176	return $response;
177	}
178	else {
179	warn "_request() requires a 'url' argument, 'user_agent' argument, and 'file_handle' argument.";
180	return;
181	}
182	}
183
184	=head2 ParseXMLStream
185
186	Parse XML using XML::LibXML::Reader from a file handle (e.g. a stream)
187
188	=cut
189
190
191	sub ParseXMLStream {
192	my ($self, $args) = @_;
193
194	my $each_callback = $args->{each_callback};
195	my $fh = $args->{file_handle};
196	if ($fh){
197	my $reader = XML::LibXML::Reader->new( FD => $fh, no_blanks => 1 );
198	my $pattern = XML::LibXML::Pattern->new('oai:OAI-PMH\|/oai:OAI-PMH/*', { 'oai' => "http://www.openarchives.org/OAI/2.0/" });
199
200	my $repository;
201
202	warn "Start parsing...";
203	while (my $rv = $reader->nextPatternMatch($pattern)){
204	#$rv == 1; successful
205	#$rv == 0; end of document reached
206	#$rv == -1; error
207	if ($rv == -1){
208	die "Parser error!";
209	}
210	#NOTE: We do this so we only get the opening tag of the element.
211	next unless $reader->nodeType == XML_READER_TYPE_ELEMENT;
212
213	my $localname = $reader->localName;
214	if ( $localname eq "request" ){
215	my $node = $reader->copyCurrentNode(1);
216	$repository = $node->textContent;
217	}
218	elsif ( $localname eq "error" ){
219	#See https://www.openarchives.org/OAI/openarchivesprotocol.html#ErrorConditions
220	#We should probably die under all circumstances except "noRecordsMatch"
221	my $node = $reader->copyCurrentNode(1);
222	if ($node){
223	my $code = $node->getAttribute("code");
224	if ($code){
225	if ($code ne "noRecordsMatch"){
226	warn "Error code: $code";
227	die;
228	}
229	}
230	}
231	}
232	elsif ( ($localname eq "ListRecords") \|\| ($localname eq "GetRecord") ){
233	my $each_pattern = XML::LibXML::Pattern->new('//oai:record\|oai:resumptionToken', { 'oai' => "http://www.openarchives.org/OAI/2.0/" });
234	while (my $each_rv = $reader->nextPatternMatch($each_pattern)){
235	if ($rv == "-1"){
236	#NOTE: -1 denotes an error state
237	warn "Error getting pattern match";
238	}
239	next unless $reader->nodeType == XML_READER_TYPE_ELEMENT;
240	if ($reader->localName eq "record"){
241	my $node = $reader->copyCurrentNode(1);
242	#NOTE: Without the UTF-8 flag, UTF-8 data will be corrupted.
243	my $document = XML::LibXML::Document->new('1.0', 'UTF-8');
244	$document->setDocumentElement($node);
245
246	#Per record callback
247	if ($each_callback){
248	$each_callback->({
249	repository => $repository,
250	document => $document,
251	});
252	}
253	}
254	elsif ($reader->localName eq "resumptionToken"){
255	my $resumptionToken = $reader->readInnerXml;
256	return ($resumptionToken,$repository);
257
258	}
259	}
260	}
261	} #/OAI-PMH document match
262	}
263	else {
264	warn "ParseXMLStream() requires a 'file_handle' argument.";
265	}
266	}
267
268	=head2 harvest
269
270	Given a URL and a user-agent, start downloading OAI-PMH records from
271	that URL.
272
273	=cut
274
275	sub harvest {
276	my ($self,$args) = @_;
277	my $url = $args->{url};
278	my $ua = $args->{user_agent};
279	my $callback = $args->{callback};
280	my $complete_callback = $args->{complete_callback};
281
282	if ($url && $ua){
283
284	#NOTE: http://search.cpan.org/~shlomif/XML-LibXML-2.0128/lib/XML/LibXML/Parser.pod#ERROR_REPORTING
285	while($url){
286	warn "URL = $url";
287	warn "Creating child process to download and feed parent process parser.";
288	my $stream = $self->GetXMLStream({
289	url => $url,
290	user_agent => $ua,
291	});
292
293	warn "Creating parent process parser.";
294	my ($resumptionToken) = $self->ParseXMLStream({
295	file_handle => $stream,
296	each_callback => $callback,
297	});
298	warn "Finished parsing current XML document.";
299
300	if ($resumptionToken){
301	#If there's a resumptionToken at the end of the stream,
302	#we build a new URL and repeat this process again.
303	$url->query_form({
304	verb => "ListRecords",
305	resumptionToken => $resumptionToken,
306	});
307	}
308	else {
309	warn "Finished harvest.";
310	last;
311	}
312
313	warn "Reap child process downloader.";
314	#Reap the dead child requester process before performing another request,
315	#so we don't fill up the process table with zombie children.
316	while ((my $child = waitpid(-1, 0)) > 0) {
317	warn "Parent $$ reaped child process $child" . ($? ? " with exit code $?" : '') . ".\n";
318	}
319	}
320
321	if ($complete_callback){
322	warn "Run complete callback.";
323
324	#Clear query string
325	$url->query_form({});
326
327	#Run complete callback using the actual URL from the request.
328	$complete_callback->({
329	repository => $url,
330	});
331	}
332	}
333	}
334
335	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Histories.pm (+60 lines)
	1	package Koha::OAI::Harvester::Histories;
	2
	3	# Copyright Prosentient Systems 2018
	4	#
	5	# This file is part of Koha.
	6	#
	7	# Koha is free software; you can redistribute it and/or modify it under the
	8	# terms of the GNU General Public License as published by the Free Software
	9	# Foundation; either version 3 of the License, or (at your option) any later
	10	# version.
	11	#
	12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
	13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	15	#
	16	# You should have received a copy of the GNU General Public License along
	17	# with Koha; if not, write to the Free Software Foundation, Inc.,
	18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
	19
	20	use Modern::Perl;
	21
	22	use Carp;
	23
	24	use Koha::Database;
	25
	26	use Koha::OAI::Harvester::History;
	27
	28	use base qw(Koha::Objects);
	29
	30	=head1 NAME
	31
	32	Koha::OAI::Harvester::Histories
	33
	34	=head2 METHODS
	35
	36	=cut
	37
	38	=head3 _type
	39
	40	=cut
	41
	42	sub _type {
	43	return 'OaiHarvesterHistory';
	44	}
	45
	46	=head3 object_class
	47
	48	=cut
	49
	50	sub object_class {
	51	return 'Koha::OAI::Harvester::History';
	52	}
	53
	54	=head1 AUTHOR
	55
	56	David Cook <dcook@prosentient.com.au>
	57
	58	=cut
	59
	60	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/History.pm (+49 lines)
	1	package Koha::OAI::Harvester::History;
	2
	3	# Copyright Prosentient Systems 2018
	4	#
	5	# This file is part of Koha.
	6	#
	7	# Koha is free software; you can redistribute it and/or modify it under the
	8	# terms of the GNU General Public License as published by the Free Software
	9	# Foundation; either version 3 of the License, or (at your option) any later
	10	# version.
	11	#
	12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
	13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	15	#
	16	# You should have received a copy of the GNU General Public License along
	17	# with Koha; if not, write to the Free Software Foundation, Inc.,
	18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
	19
	20	use Modern::Perl;
	21
	22	use Carp;
	23
	24	use base qw(Koha::Object);
	25
	26	=head1 NAME
	27
	28	Koha::OAI::Harvester::History
	29
	30	=head1 SYNOPSIS
	31
	32	use Koha::OAI::Harvester::History;
	33	my $request = Koha::OAI::Harvester::History->find($id);
	34
	35	=head1 METHODS
	36
	37	=cut
	38
	39	sub _type {
	40	return 'OaiHarvesterHistory';
	41	}
	42
	43	=head1 AUTHOR
	44
	45	David Cook <dcook@prosentient.com.au>
	46
	47	=cut
	48
	49	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Import/MARCXML.pm (+163 lines)
		1	package Koha::OAI::Harvester::Import::MARCXML;
		2
		3	# Copyright 2016 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19	#
		20
		21	use Modern::Perl;
		22	use MARC::Record;
		23
		24	use C4::Context;
		25	use C4::Biblio;
		26
		27	use constant MAX_MATCHES => 99999; #NOTE: This is an arbitrary value. We want to get all matches.
		28
		29	=head1 NAME
		30
		31	Koha::OAI::Harvester::Import::MARCXML
		32
		33	=head1 SYNOPSIS
		34
		35	use Koha::OAI::Harvester::Import::MARCXML;
		36	my $marcxml = eval { Koha::OAI::Harvester::Import::MARCXML->new({ dom => $results, }) };
		37
		38	=head1 METHODS
		39
		40	=head2 new
		41
		42	Create object
		43
		44	=cut
		45
		46	sub new {
		47	my ($class, $args) = @_;
		48	$args = {} unless defined $args;
		49	if ( (! $args->{dom}) && (! $args->{marc_record}) ){
		50	die "You must provide either a dom or marc_record argument to this constructor";
		51	}
		52	if ( $args->{dom} && ( ! $args->{marc_record} ) ){
		53	my $dom = $args->{dom};
		54	my $xml = $dom->toString(2);
		55	my $marcflavour = C4::Context->preference('marcflavour') \|\| 'MARC21';
		56	my $marc_record = eval {MARC::Record::new_from_xml( $xml, "utf8", $marcflavour)};
		57	if ($@){
		58	die "Unable to create MARC::Record object";
		59	}
		60	if ($marc_record){
		61	$args->{marc_record} = $marc_record;
		62	}
		63	}
		64	return bless ($args, $class);
65	}
66
67	=head2 import_record
68
69	Import a record into Koha
70
71	=cut
72
73	sub import_record {
74	my ($self,$args) = @_;
75	my $framework = $args->{framework};
76	my $record_type = $args->{record_type};
77	my $matcher = $args->{matcher};
78	my $koha_id = $args->{koha_id};
79
80	my $action = "error";
81
82	#Try to find a matching Koha MARCXML record via Zebra
83	if (! $koha_id && $matcher){
84	my $matched_id = $self->_try_matcher({
85	matcher => $matcher,
86	});
87	if ($matched_id){
88	$koha_id = $matched_id;
89	}
90	}
91
92	if ($koha_id){
93	#Update
94	($action) = $self->_mod_koha_record({
95	record_type => $record_type,
96	framework => $framework,
97	koha_id => $koha_id,
98	});
99	}
100	else {
101	#Add
102	($action,$koha_id) = $self->_add_koha_record({
103	record_type => $record_type,
104	framework => $framework,
105	});
106	}
107
108	return ($action,$koha_id);
109	}
110
111	sub _try_matcher {
112	my ($self, $args) = @_;
113	my $marc_record = $self->{marc_record};
114	my $matcher = $args->{matcher};
115	my $matched_id;
116	my @matches = $matcher->get_matches($marc_record, MAX_MATCHES);
117	if (@matches){
118	my $bestrecordmatch = shift @matches;
119	if ($bestrecordmatch && $bestrecordmatch->{record_id}){
120	$matched_id = $bestrecordmatch->{record_id};
121	}
122	}
123	return $matched_id;
124	}
125
126	sub _add_koha_record {
127	my ($self, $args) = @_;
128	my $marc_record = $self->{marc_record};
129	my $record_type = $args->{record_type} // "biblio";
130	my $framework = $args->{framework};
131	my $koha_id;
132	my $action = "error";
133	if ($record_type eq "biblio"){
134	#NOTE: Strip item fields to prevent any accidentally getting through.
135	C4::Biblio::_strip_item_fields($marc_record,$framework);
136	my ($biblionumber,$biblioitemnumber) = C4::Biblio::AddBiblio($marc_record,$framework);
137	if ($biblionumber){
138	$action = "added";
139	$koha_id = $biblionumber;
140	}
141	}
142	return ($action,$koha_id);
143	}
144
145	sub _mod_koha_record {
146	my ($self, $args) = @_;
147	my $marc_record = $self->{marc_record};
148	my $record_type = $args->{record_type} // "biblio";
149	my $framework = $args->{framework};
150	my $koha_id = $args->{koha_id};
151	my $action = "error";
152	if ($record_type eq "biblio"){
153	#NOTE: Strip item fields to prevent any accidentally getting through.
154	C4::Biblio::_strip_item_fields($marc_record,$framework);
155	my $updated = C4::Biblio::ModBiblio($marc_record, $koha_id, $framework);
156	if ($updated){
157	$action = "updated";
158	}
159	}
160	return ($action);
161	}
162
163	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Import/Record.pm (+348 lines)
		1	package Koha::OAI::Harvester::Import::Record;
		2
		3	# Copyright 2016 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19	#
		20
		21	use Modern::Perl;
		22	use XML::LibXML;
		23	use XML::LibXSLT;
		24	use URI;
		25	use File::Basename;
		26
		27	use C4::Context;
		28	use C4::Biblio;
		29
		30	use Koha::OAI::Harvester::Import::MARCXML;
		31	use Koha::OAI::Harvester::Biblios;
		32	use Koha::OAI::Harvester::History;
		33
		34	=head1 NAME
		35
		36	Koha::OAI::Harvester::Import::Record
		37
		38	=head1 SYNOPSIS
		39
		40	use Koha::OAI::Harvester::Import::Record;
		41	my $oai_record = Koha::OAI::Harvester::Import::Record->new({
		42	doc => $dom,
		43	repository => $repository,
		44	});
		45
		46	=head1 METHODS
		47
		48	=head2 new
		49
		50	Create object
		51
		52	=cut
		53
		54	sub new {
		55	my ($class, $args) = @_;
		56	$args = {} unless defined $args;
		57
		58	die "You must provide a 'doc' argument to the constructor" unless $args->{doc};
		59	die "You must provide a 'repository' argument to the constructor" unless $args->{repository};
		60
		61	if (my $doc = $args->{doc}){
		62
		63	#Get the root element
		64	my $root = $doc->documentElement;
65
66	#Register namespaces for searching purposes
67	my $xpc = XML::LibXML::XPathContext->new();
68	$xpc->registerNs('oai','http://www.openarchives.org/OAI/2.0/');
69
70	my $xpath_identifier = XML::LibXML::XPathExpression->new("oai:header/oai:identifier");
71	my $identifier = $xpc->findnodes($xpath_identifier,$root)->shift;
72	$args->{header_identifier} = $identifier->textContent;
73
74	my $xpath_datestamp = XML::LibXML::XPathExpression->new("oai:header/oai:datestamp");
75	my $datestamp = $xpc->findnodes($xpath_datestamp,$root)->shift;
76	$args->{header_datestamp} = $datestamp->textContent;
77
78	my $xpath_status = XML::LibXML::XPathExpression->new(q{oai:header/@status});
79	my $status_node = $xpc->findnodes($xpath_status,$root)->shift;
80	$args->{header_status} = $status_node ? $status_node->textContent : "";
81	}
82
83	return bless ($args, $class);
84	}
85
86	=head2 is_deleted_upstream
87
88	Returns true if OAI-PMH record is deleted upstream
89
90	=cut
91
92	sub is_deleted_upstream {
93	my ($self, $args) = @_;
94	if ($self->{header_status}){
95	if ($self->{header_status} eq "deleted"){
96	return 1;
97	}
98	}
99	return 0;
100	}
101
102	=head2 set_filter
103
104	$self->set_filter("/path/to/filter.xsl");
105
106	Set a XSLT to use to filter records on import. This
107	takes a full filepath as an argument.
108
109	=cut
110
111	sub set_filter {
112	my ($self, $filter_definition) = @_;
113
114	#Source a default XSLT to use for filtering
115	my $htdocs = C4::Context->config('intrahtdocs');
116	my $theme = C4::Context->preference("template");
117	$self->{filter} = "$htdocs/$theme/en/xslt/StripOAIPMH.xsl";
118	$self->{filter_type} = "xslt";
119
120	if ($filter_definition && $filter_definition ne "default"){
121	my ($filter_type, $filter) = $self->_parse_filter($filter_definition);
122	if ($filter_type eq "xslt"){
123	if ( -f $filter ){
124	$self->{filter} = $filter;
125	$self->{filter_type} = "xslt";
126	}
127	}
128	}
129	}
130
131	sub _parse_filter {
132	my ($self,$filter_definition) = @_;
133	my ($type,$filter);
134	my $filter_uri = URI->new($filter_definition);
135	if ($filter_uri){
136	my $scheme = $filter_uri->scheme;
137	if ( ($scheme && $scheme eq "file") \|\| ! $scheme ){
138	my $path = $filter_uri->path;
139	#Filters may theoretically be .xsl or .pm files
140	my($filename, $dirs, $suffix) = fileparse($path,(".xsl",".pm"));
141	if ($suffix){
142	if ( $suffix eq ".xsl"){
143	$type = "xslt";
144	$filter = $path;
145	}
146	}
147	}
148	}
149	return ($type,$filter);
150	}
151
152	=head2 filter
153
154	Filters the OAI-PMH record using a filter
155
156	=cut
157
158	sub filter {
159	my ($self) = @_;
160	my $filtered = 0;
161	my $doc = $self->{doc};
162	my $filter = $self->{filter};
163	my $filter_type = $self->{filter_type};
164	if ($doc){
165	if ($filter && -f $filter){
166	if ($filter_type){
167	if ( $filter_type eq 'xslt' ){
168	my $xslt = XML::LibXSLT->new();
169	my $style_doc = XML::LibXML->load_xml(location => $filter);
170	my $stylesheet = $xslt->parse_stylesheet($style_doc);
171	if ($stylesheet){
172	my $results = $stylesheet->transform($doc);
173	if ($results){
174	my $root = $results->documentElement;
175	if ($root){
176	my $namespace = $root->namespaceURI;
177	if ($namespace eq "http://www.loc.gov/MARC21/slim"){
178	#NOTE: Both MARC21 and UNIMARC should be covered by this namespace
179	my $marcxml = eval { Koha::OAI::Harvester::Import::MARCXML->new({ dom => $results, }) };
180	if ($@){
181	warn "Error Koha::OAI::Harvester::Import::MARCXML: $@";
182	return;
183	} else {
184	return $marcxml;
185	}
186	}
187	}
188	}
189	}
190	}
191	}
192	}
193	}
194	return;
195	}
196
197	sub _find_koha_link {
198	my ($self, $args) = @_;
199	my $record_type = $args->{record_type} // "biblio";
200	my $link_id;
201	if ($record_type eq "biblio"){
202	my $link = Koha::OAI::Harvester::Biblios->new->find(
203	{
204	oai_repository => $self->{repository},
205	oai_identifier => $self->{header_identifier},
206	},
207	{ key => "oai_record",}
208	);
209	if ($link && $link->biblionumber){
210	$link_id = $link->biblionumber;
211	}
212	}
213	return $link_id;
214	}
215
216	=head2 import_record
217
218	my ($action,$record_id) = $oai_record->import_record({
219	filter => $filter,
220	framework => $framework,
221	record_type => $record_type,
222	matcher => $matcher,
223	});
224
225	$action eq "added" \|\| "updated" \|\| "deleted" \|\| "not_found" \|\| "error"
226
227	=cut
228
229	sub import_record {
230	my ($self, $args) = @_;
231	my $filter = $args->{filter} \|\| 'default';
232	my $framework = $args->{framework} \|\| '';
233	my $record_type = $args->{record_type} \|\| 'biblio';
234	my $matcher = $args->{matcher};
235
236	my $action = "error";
237
238	#Find linkage between OAI-PMH repository-identifier and Koha record id
239	my $linked_id = $self->_find_koha_link({
240	record_type => $record_type,
241	});
242
243	if ($self->is_deleted_upstream){
244	#NOTE: If a record is deleted upstream, it will not contain a metadata element
245	if ($linked_id){
246	$action = $self->delete_koha_record({
247	record_id => $linked_id,
248	record_type => $record_type,
249	});
250	}
251	else {
252	$action = "not_found";
253	#NOTE: If there's no OAI-PMH repository-identifier pair in the database,
254	#then there's no perfect way to find a linked record to delete.
255	}
256	}
257	else {
258	$self->set_filter($filter);
259
260
261	my $import_record = $self->filter();
262
263	if ($import_record){
264	($action,$linked_id) = $import_record->import_record({
265	framework => $framework,
266	record_type => $record_type,
267	matcher => $matcher,
268	koha_id => $linked_id,
269	});
270
271	if ($linked_id){
272	#Link Koha record ID to OAI-PMH details for this record type,
273	#if linkage doesn't already exist.
274	$self->link_koha_record({
275	record_type => $record_type,
276	koha_id => $linked_id,
277	});
278	}
279	}
280	}
281
282	#Log record details to database
283	Koha::OAI::Harvester::History->new({
284	header_identifier => $self->{header_identifier},
285	header_datestamp => $self->{header_datestamp},
286	header_status => $self->{header_status},
287	record => $self->{doc}->toString(1),
288	repository => $self->{repository},
289	status => $action,
290	filter => $filter,
291	framework => $framework,
292	record_type => $record_type,
293	matcher_code => $matcher ? $matcher->code : undef,
294	})->store();
295
296	return ($action,$linked_id);
297	}
298
299	=head2 link_koha_record
300
301	Link an OAI-PMH record with a Koha record using
302	the OAI-PMH repository and OAI-PMH identifier
303
304	=cut
305
306	sub link_koha_record {
307	my ($self, $args) = @_;
308	my $record_type = $args->{record_type} // "biblio";
309	my $koha_id = $args->{koha_id};
310	if ($koha_id){
311	if ($record_type eq "biblio"){
312	my $import_oai_biblio = Koha::OAI::Harvester::Biblios->new->find_or_create({
313	oai_repository => $self->{repository},
314	oai_identifier => $self->{header_identifier},
315	biblionumber => $koha_id,
316	});
317	if ( ! $import_oai_biblio->in_storage ){
318	$import_oai_biblio->insert;
319	}
320	}
321	}
322	}
323
324	=head2 delete_koha_record
325
326	Delete a Koha record
327
328	=cut
329
330	sub delete_koha_record {
331	my ($self, $args) = @_;
332	my $record_type = $args->{record_type} // "biblio";
333	my $record_id = $args->{record_id};
334
335	my $action = "error";
336
337	if ($record_type eq "biblio"){
338	my $error = C4::Biblio::DelBiblio($record_id);
339	if (!$error){
340	$action = "deleted";
341	#NOTE: If there's no error, a cascading database delete should
342	#automatically remove the link between the Koha biblionumber and OAI-PMH record too
343	}
344	}
345	return $action;
346	}
347
348	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/ImportQueue.pm (+49 lines)
	1	package Koha::OAI::Harvester::ImportQueue;
	2
	3	# Copyright Prosentient Systems 2018
	4	#
	5	# This file is part of Koha.
	6	#
	7	# Koha is free software; you can redistribute it and/or modify it under the
	8	# terms of the GNU General Public License as published by the Free Software
	9	# Foundation; either version 3 of the License, or (at your option) any later
	10	# version.
	11	#
	12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
	13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	15	#
	16	# You should have received a copy of the GNU General Public License along
	17	# with Koha; if not, write to the Free Software Foundation, Inc.,
	18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
	19
	20	use Modern::Perl;
	21
	22	use Carp;
	23
	24	use base qw(Koha::Object);
	25
	26	=head1 NAME
	27
	28	Koha::OAI::Harvester::ImportQueue
	29
	30	=head1 SYNOPSIS
	31
	32	use Koha::OAI::Harvester::ImportQueue;
	33	my $request = Koha::OAI::Harvester::ImportQueue->find($id);
	34
	35	=head1 METHODS
	36
	37	=cut
	38
	39	sub _type {
	40	return 'OaiHarvesterImportQueue';
	41	}
	42
	43	=head1 AUTHOR
	44
	45	David Cook <dcook@prosentient.com.au>
	46
	47	=cut
	48
	49	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/ImportQueues.pm (+60 lines)
	1	package Koha::OAI::Harvester::ImportQueues;
	2
	3	# Copyright Prosentient Systems 2018
	4	#
	5	# This file is part of Koha.
	6	#
	7	# Koha is free software; you can redistribute it and/or modify it under the
	8	# terms of the GNU General Public License as published by the Free Software
	9	# Foundation; either version 3 of the License, or (at your option) any later
	10	# version.
	11	#
	12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
	13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	15	#
	16	# You should have received a copy of the GNU General Public License along
	17	# with Koha; if not, write to the Free Software Foundation, Inc.,
	18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
	19
	20	use Modern::Perl;
	21
	22	use Carp;
	23
	24	use Koha::Database;
	25
	26	use Koha::OAI::Harvester::ImportQueue;
	27
	28	use base qw(Koha::Objects);
	29
	30	=head1 NAME
	31
	32	Koha::OAI::Harvester::ImportQueues
	33
	34	=head2 METHODS
	35
	36	=cut
	37
	38	=head3 _type
	39
	40	=cut
	41
	42	sub _type {
	43	return 'OaiHarvesterImportQueue';
	44	}
	45
	46	=head3 object_class
	47
	48	=cut
	49
	50	sub object_class {
	51	return 'Koha::OAI::Harvester::ImportQueue';
	52	}
	53
	54	=head1 AUTHOR
	55
	56	David Cook <dcook@prosentient.com.au>
	57
	58	=cut
	59
	60	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Listener.pm (+246 lines)
		1	package Koha::OAI::Harvester::Listener;
		2
		3	# Copyright 2017 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19	#
		20
		21	use Modern::Perl;
		22	use POE qw(Wheel::SocketFactory Wheel::ReadWrite);
		23	use IO::Socket qw(AF_UNIX);
		24	use JSON;
		25	use URI;
		26	use Encode;
		27
		28	=head1 NAME
		29
		30	Koha::OAI::Harvester::Listener
		31
		32	=head1 SYNOPSIS
		33
		34	use Koha::OAI::Harvester::Listener;
		35	my $listener = Koha::OAI::Harvester::Listener->spawn({
		36	logger => $logger,
		37	socket => $socket_addr,
		38	});
		39
		40	=head1 METHODS
		41
		42	=head2 new
		43
		44	Create object
		45
		46	=cut
		47
		48	sub new {
		49	my ($class, $args) = @_;
		50	$args = {} unless defined $args;
		51	return bless ($args, $class);
		52	}
		53
		54	=head2 spawn
		55
		56	Create POE session for listener
		57
		58	=cut
		59
		60	sub spawn {
		61	my ($class, $args) = @_;
		62	my $self = $class->new($args);
		63	my $socket = $args->{socket};
		64	POE::Session->create(
65	args => [
66	$socket,
67	],
68	object_states => [
69	$self => {
70	_start => "on_start",
71	"on_server_success" => "on_server_success",
72	"on_server_error" => "on_server_error",
73	"on_client_error" => "on_client_error",
74	"on_client_input" => "on_client_input",
75	},
76	],
77	);
78	}
79
80	=head2 on_start
81
82	Internal method for starting listener
83
84	=cut
85
86	sub on_start {
87	my ($kernel,$heap,$socket_uri) = @_[KERNEL,HEAP,ARG0];
88
89	my $uri = URI->new($socket_uri);
90	if ($uri && $uri->scheme eq 'unix'){
91	my $socket_path = $uri->path;
92	unlink $socket_path if -S $socket_path;
93	$heap->{server} = POE::Wheel::SocketFactory->new(
94	SocketDomain => AF_UNIX,
95	BindAddress => $socket_path,
96	SuccessEvent => "on_server_success",
97	FailureEvent => "on_server_error",
98	);
99
100	#Make the socket writeable to other users like Apache
101	chmod 0666, $socket_path;
102	}
103	}
104
105	=head2 on_server_success
106
107	Internal event handler for successful connection to server
108
109	=cut
110
111	sub on_server_success {
112	my ($self, $client_socket, $server_wheel_id, $heap, $session) = @_[OBJECT, ARG0, ARG3, HEAP,SESSION];
113	my $logger = $self->{logger};
114	my $null_filter = POE::Filter::Line->new(
115	Literal => chr(0),
116	);
117	my $client_wheel = POE::Wheel::ReadWrite->new(
118	Handle => $client_socket,
119	InputEvent => "on_client_input",
120	ErrorEvent => "on_client_error",
121	InputFilter => $null_filter,
122	OutputFilter => $null_filter,
123	);
124	$heap->{client}->{ $client_wheel->ID() } = $client_wheel;
125	$logger->info("Connection ".$client_wheel->ID()." started.");
126	#TODO: Add basic authentication here?
127	$client_wheel->put("HELLO");
128	}
129
130	=head2 on_server_error
131
132	Internal event handler for server errors
133
134	=cut
135
136	sub on_server_error {
137	my ($self, $operation, $errnum, $errstr, $heap, $session) = @_[OBJECT, ARG0, ARG1, ARG2,HEAP, SESSION];
138	my $logger = $self->{logger};
139	$logger->error("Server $operation error $errnum: $errstr");
140	delete $heap->{server};
141	}
142
143	=head2 on_client_error
144
145	Internal event handler for errors relating to the client connection
146
147	=cut
148
149	sub on_client_error {
150	my ($self, $wheel_id,$heap,$session) = @_[OBJECT, ARG3,HEAP,SESSION];
151	my $logger = $self->{logger};
152	$logger->info("Connection $wheel_id failed or ended.");
153	delete $heap->{client}->{$wheel_id};
154	}
155
156	=head2 on_client_input
157
158	Internal event handler for input from clients
159
160	=cut
161
162	sub on_client_input {
163	my ($self, $input, $wheel_id, $session, $kernel, $heap) = @_[OBJECT, ARG0, ARG1, SESSION, KERNEL, HEAP];
164	$input = Encode::decode("UTF-8",$input);
165	my $logger = $self->{logger};
166	$logger->debug("Server input: $input");
167	my $server_response = { msg => "fail"};
168	eval {
169	my $json_input = from_json($input);
170	my $command = $json_input->{command};
171	my $body = $json_input->{body};
172	if ($command){
173	if ($command eq "create"){
174	my $task = $body->{task};
175	if ($task){
176	my $is_created = $kernel->call("harvester","create_task",$task);
177	if ($is_created){
178	$server_response->{msg} = "success";
179	}
180	}
181	}
182	elsif ($command eq "start"){
183	my $task = $body->{task};
184	if ($task){
185	my $uuid = $task->{uuid};
186	#Fetch from memory now...
187	my $is_started = $kernel->call("harvester","start_task", $uuid);
188	if ($is_started){
189	$server_response->{msg} = "success";
190	}
191	}
192	}
193	elsif ($command eq "stop"){
194	my $task = $body->{task};
195	if ($task){
196	if ($task->{uuid}){
197	my $is_stopped = $kernel->call("harvester","stop_task",$task->{uuid});
198	if ($is_stopped){
199	$server_response->{msg} = "success";
200	}
201	}
202	}
203	}
204	elsif ($command eq "delete"){
205	my $task = $body->{task};
206	if ($task){
207	if ($task->{uuid}){
208	my $is_deleted = $kernel->call("harvester","delete_task",$task->{uuid});
209	if ($is_deleted){
210	$server_response->{msg} = "success";
211	}
212	}
213	}
214	}
215	elsif ($command eq "list"){
216	my $tasks = $kernel->call("harvester","list_tasks");
217	if ($tasks){
218	$server_response->{msg} = "success";
219	$server_response->{data} = $tasks;
220	}
221	}
222	}
223	};
224	if ($@){
225	#NOTE: An error most likely means that something other than a valid JSON string was received
226	$logger->error($@);
227	}
228
229	if ($server_response){
230	eval {
231	my $client = $heap->{client}->{$wheel_id};
232	my $json_message = to_json($server_response, { pretty => 1 });
233	if ($json_message){
234	$logger->debug("Server output: $json_message");
235	$json_message = Encode::encode("UTF-8",$json_message);
236	$client->put($json_message);
237	}
238	};
239	if ($@){
240	#NOTE: An error means our response couldn't be serialised as JSON
241	$logger->error($@);
242	}
243	}
244	}
245
246	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Request.pm (+187 lines)
		1	package Koha::OAI::Harvester::Request;
		2
		3	# Copyright Prosentient Systems 2017
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, write to the Free Software Foundation, Inc.,
		18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		19
		20	use Modern::Perl;
		21
		22	use Carp;
		23
		24	use base qw(Koha::Object);
		25
		26	#For validation
		27	use URI;
		28	use HTTP::OAI;
		29
		30	=head1 NAME
		31
		32	Koha::OAI::Harvester::Request
		33
		34	=head1 SYNOPSIS
		35
		36	use Koha::OAI::Harvester::Request;
		37	my $request = Koha::OAI::Harvester::Requests->find($id);
		38
		39	=head1 METHODS
		40
		41	=cut
		42
		43	sub _type {
		44	return 'OaiHarvesterRequest';
		45	}
		46
		47	=head2 validate
		48
		49	Method to validate the parameters of an OAI-PMH request
		50
		51	=cut
		52
		53	sub validate {
		54	my ($self) = @_;
		55	my $errors = {};
		56
		57	#Step one: validate URL
		58	my $uri = URI->new($self->http_url);
		59	if ( $uri && $uri->scheme && ($uri->scheme eq "http" \|\| $uri->scheme eq "https") ){
		60
		61	#Step two: validate access and authorization to URL
		62	my $harvester = $self->_harvester();
		63	my $identify = $harvester->Identify;
		64	if ($identify->is_success){
65
66	#Step three: validate OAI-PMH parameters
67
68	#Test Set
69	my $set = $self->oai_set;
70	if ($set){
71	my $set_response = $harvester->ListSets();
72	my @server_sets = $set_response->set;
73	if ( ! grep {$_->setSpec eq $set} @server_sets ){
74	$errors->{oai_set}->{unavailable} = 1;
75	}
76	}
77
78	#Test Metadata Prefix
79	my $metadataPrefix = $self->oai_metadataPrefix;
80	if ($metadataPrefix){
81	my $metadata_response = $harvester->ListMetadataFormats();
82	my @server_formats = $metadata_response->metadataFormat;
83	if ( ! grep { $_->metadataPrefix eq $metadataPrefix } @server_formats ){
84	$errors->{oai_metadataPrefix}->{unavailable} = 1;
85	}
86	}
87	else {
88	$errors->{oai_metadataPrefix}->{missing} = 1;
89	}
90
91	#Test Granularity and Timestamps
92	my $server_granularity = $identify->granularity;
93	my $from = $self->oai_from;
94	my $until = $self->oai_until;
95	if ($from \|\| $until){
96	my ($from_granularity,$until_granularity);
97	if ($from){
98	$from_granularity = _determine_granularity($from);
99	if ($from_granularity eq "YYYY-MM-DDThh:mm:ssZ"){
100	$errors->{oai_from}->{unavailable} = 1 if $server_granularity ne $from_granularity;
101	} elsif ($from_granularity eq "failed"){
102	$errors->{oai_from}->{malformed} = 1;
103	}
104	}
105	if ($until){
106	$until_granularity = _determine_granularity($until);
107	if ($until_granularity eq "YYYY-MM-DDThh:mm:ssZ"){
108	$errors->{oai_until}->{unavailable} = 1 if $server_granularity ne $until_granularity;
109	} elsif ($until_granularity eq "failed"){
110	$errors->{oai_until}->{malformed} = 1;
111	}
112	}
113	if ($from && $until){
114	if ($from_granularity ne $until_granularity){
115	$errors->{oai}->{granularity_mismatch} = 1;
116	}
117	}
118	}
119
120	#Test if identifier is provided when using GetRecord
121	my $verb = $self->oai_verb;
122	if ($verb && $verb eq "GetRecord"){
123	my $identifier = $self->oai_identifier;
124	if (! $identifier){
125	$errors->{oai_identifier}->{missing} = 1;
126	}
127	}
128	}
129	elsif ($identify->is_error){
130	foreach my $error ($identify->errors){
131	if ($error->code =~ /^404$/){
132	$errors->{http}->{404} = 1;
133	} elsif ($error->code =~ /^401$/){
134	$errors->{http}->{401} = 1;
135	} else {
136	$errors->{http}->{generic} = 1;
137	}
138	}
139	}
140	else {
141	$errors->{http}->{generic} = 1;
142	}
143	} else {
144	$errors->{http_url}->{malformed} = 1;
145	}
146	return $errors;
147	}
148
149	sub _harvester {
150	my ( $self ) = @_;
151	my $harvester;
152	if ($self->http_url){
153	$harvester = new HTTP::OAI::Harvester( baseURL => $self->http_url );
154	my $uri = URI->new($self->http_url);
155	if ($uri->scheme && ($uri->scheme eq 'http' \|\| $uri->scheme eq 'https') ){
156	my $host = $uri->host;
157	my $port = $uri->port;
158	$harvester->credentials($host.":".$port, $self->http_realm, $self->http_username, $self->http_password);
159	}
160	}
161	return $harvester;
162	}
163
164	sub _determine_granularity {
165	my ($timestamp) = @_;
166	my $granularity;
167	if ($timestamp =~ /^(\d{4}-\d{2}-\d{2})(T\d{2}:\d{2}:\d{2}Z)?$/){
168	if ($1 && $2){
169	$granularity = "YYYY-MM-DDThh:mm:ssZ";
170	} elsif ($1 && !$2){
171	$granularity = "YYYY-MM-DD";
172	} else {
173	$granularity = "failed";
174	}
175	} else {
176	$granularity = "failed";
177	}
178	return $granularity;
179	}
180
181	=head1 AUTHOR
182
183	David Cook <dcook@prosentient.com.au>
184
185	=cut
186
187	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Requests.pm (+62 lines)
	1	package Koha::OAI::Harvester::Requests;
	2
	3	# Copyright Prosentient Systems 2017
	4	#
	5	# This file is part of Koha.
	6	#
	7	# Koha is free software; you can redistribute it and/or modify it under the
	8	# terms of the GNU General Public License as published by the Free Software
	9	# Foundation; either version 3 of the License, or (at your option) any later
	10	# version.
	11	#
	12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
	13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	15	#
	16	# You should have received a copy of the GNU General Public License along
	17	# with Koha; if not, write to the Free Software Foundation, Inc.,
	18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
	19
	20	use Modern::Perl;
	21
	22	use Carp;
	23
	24	use Koha::Database;
	25
	26	use Koha::OAI::Harvester::Request;
	27
	28	use base qw(Koha::Objects);
	29
	30	=head1 NAME
	31
	32	Koha::OAI::Harvester::Requests -
	33
	34	=head1 API
	35
	36	=head2 Class Methods
	37
	38	=cut
	39
	40	=head3 _type
	41
	42	=cut
	43
	44	sub _type {
	45	return 'OaiHarvesterRequest';
	46	}
	47
	48	=head3 object_class
	49
	50	=cut
	51
	52	sub object_class {
	53	return 'Koha::OAI::Harvester::Request';
	54	}
	55
	56	=head1 AUTHOR
	57
	58	David Cook <dcook@prosentient.com.au>
	59
	60	=cut
	61
	62	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Worker.pm (+215 lines)
		1	package Koha::OAI::Harvester::Worker;
		2
		3	# Copyright Prosentient Systems 2017
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, write to the Free Software Foundation, Inc.,
		18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		19
		20	use Modern::Perl;
		21	use POE;
		22	use DateTime;
		23	use JSON;
		24
		25	=head1 NAME
		26
		27	Koha::OAI::Harvester::Worker
		28
		29	=head1 SYNOPSIS
		30
		31	This is just a base class to use when writing workers for
		32	the OAI-PMH harvester.
		33
		34	=head1 METHODS
		35
		36	=head2 new
		37
		38	Create object
		39
		40	=cut
		41
		42	sub new {
		43	my ($class, $args) = @_;
		44	$args = {} unless defined $args;
		45	$args->{type} = "worker" unless $args->{type};
		46	return bless ($args, $class);
		47	}
		48
		49	=head2 run
		50
		51	The entry point to getting a worker to process a task
		52
		53	=cut
		54
		55	sub run {
		56	my ($self,$args) = @_;
		57	my $postback = $args->{postback};
		58	my $task = $args->{task};
		59
		60	POE::Session->create(
		61	object_states => [
		62	$self => {
		63	_start => "on_start",
		64	got_child_stderr => "on_child_stderr",
65	got_child_close => "on_child_close",
66	got_child_signal => "on_child_signal",
67	got_child_stdout => "on_child_stdout",
68	stop_worker => "stop_worker",
69	_stop => "on_stop",
70	},
71	],
72	args => [
73	$postback,
74	$task,
75	],
76	);
77	}
78
79	=head2 stop_worker
80
81	Internal method for killing this worker's child processes
82
83	=cut
84
85	sub stop_worker {
86	my ($self,$heap) = @_[OBJECT,HEAP];
87	if (my $child_processes = $heap->{children_by_pid}){
88	foreach my $child_pid (keys %$child_processes){
89	my $child = $child_processes->{$child_pid};
90	$child->kill();
91	}
92	}
93	}
94
95	=head2 on_stop
96
97	Internal event handler for deregistering the worker session
98	from the harvester's roster of workers
99
100	=cut
101
102	sub on_stop {
103	my ($self,$kernel) = @_[OBJECT,KERNEL];
104
105	#Deregister the worker session from the harvester's roster of workers
106	$kernel->call("harvester","deregister",$self->{type});
107	}
108
109	=head2 on_child_stdout
110
111	Internal event handler for reading output from a child process
112
113	=cut
114
115	# Wheel event, including the wheel's ID.
116	sub on_child_stdout {
117	my ($self, $stdout_line, $wheel_id) = @_[OBJECT, ARG0, ARG1];
118	my $type = $self->{type};
119	my $child = $_[HEAP]{children_by_wid}{$wheel_id};
120	my $logger = $self->{logger};
121	if ($logger){
122	$logger->debug("[$type][pid ".$child->PID."][STDOUT] $stdout_line");
123	}
124
125	my $postback = $_[HEAP]{postback};
126	if ($postback){
127	eval {
128	my $message = from_json($stdout_line);
129	if ($message){
130	$postback->($message);
131	}
132	};
133	}
134	}
135
136	=head2 on_child_stderr
137
138	Internal event handler for reading errors from a child process
139
140	=cut
141
142	# Wheel event, including the wheel's ID.
143	sub on_child_stderr {
144	my ($self,$stderr_line, $wheel_id) = @_[OBJECT, ARG0, ARG1];
145	my $type = $self->{type};
146	my $child = $_[HEAP]{children_by_wid}{$wheel_id};
147	my $logger = $self->{logger};
148	if ($logger){
149	$logger->debug("[$type][pid ".$child->PID."][STDERR] $stderr_line");
150	}
151	}
152
153	=head2 on_child_close
154
155	Internal event handler for when a child closes its pipes
156
157	=cut
158
159	# Wheel event, including the wheel's ID.
160	sub on_child_close {
161	my ($self,$heap,$wheel_id) = @_[OBJECT,HEAP,ARG0];
162	my $type = $self->{type};
163	my $logger = $self->{logger};
164
165	my $child = delete $heap->{children_by_wid}->{$wheel_id};
166
167	# May have been reaped by on_child_signal().
168	unless (defined $child) {
169	if ($logger){
170	$logger->debug("[$type][wid $wheel_id] closed all pipes");
171	}
172	return;
173	}
174	if ($logger){
175	$logger->debug("[$type][pid ".$child->PID."] closed all pipes");
176	}
177	delete $heap->{children_by_pid}->{$child->PID};
178	}
179
180	=head2 on_child_signal
181
182	Internal event handler for when a child exits
183
184	=cut
185
186	sub on_child_signal {
187	my ($self,$kernel,$pid,$status) = @_[OBJECT,KERNEL,ARG1,ARG2];
188	my $type = $self->{type};
189	my $logger = $self->{logger};
190	if ($logger){
191	$logger->debug("[$type][pid $pid] exited with status $status");
192	}
193
194	my $child = delete $_[HEAP]{children_by_pid}{$_[ARG1]};
195
196	# May have been reaped by on_child_close().
197	return unless defined $child;
198
199	delete $_[HEAP]{children_by_wid}{$child->ID};
200
201	#If the child doesn't complete successfully, we lodge an error
202	#and stop the task.
203	if ($status != 0){
204	my $task = $kernel->call("harvester","get_task");
205	if ($task){
206	$task->{error} = 1;
207	my $uuid = $task->{uuid};
208	if ($uuid){
209	$kernel->call("harvester","stop_task",$uuid);
210	}
211	}
212	}
213	}
214
215	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Worker/Download/Stream.pm (+220 lines)
		1	package Koha::OAI::Harvester::Worker::Download::Stream;
		2
		3	# Copyright Prosentient Systems 2017
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, write to the Free Software Foundation, Inc.,
		18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		19
		20	use Modern::Perl;
		21	use LWP::UserAgent;
		22	use UUID;
		23	use POE;
		24	use JSON;
		25	use File::Path qw/make_path/;
		26
		27	use C4::Context;
		28	use Koha::OAI::Harvester::Downloader;
		29	use parent 'Koha::OAI::Harvester::Worker';
		30
		31	=head1 NAME
		32
		33	Koha::OAI::Harvester::Worker::Download::Stream
		34
		35	=head1 SYNOPSIS
		36
		37	This is a module used by the OAI-PMH harvester internally.
		38
		39	As a bare minimum, it must define an "on_start" method.
		40
		41	=head1 METHODS
		42
		43	=head2 new
		44
		45	Create object
		46
		47	=cut
		48
		49	sub new {
		50	my ($class, $args) = @_;
		51	$args = {} unless defined $args;
		52	$args->{type} = "download" unless $args->{type};
		53	return bless ($args, $class);
		54	}
		55
		56	=head2 on_start
		57
		58	Internal event handler for starting the processing of a task
		59
		60	=cut
		61
		62	sub on_start {
		63	my ($self, $kernel, $heap, $postback,$task,$session) = @_[OBJECT, KERNEL, HEAP, ARG0,ARG1,SESSION];
		64	#Save postback into heap so other event handlers can use it
65	$heap->{postback} = $postback;
66
67	my $task_uuid = $task->{uuid};
68
69	$kernel->sig("cancel" => "stop_worker");
70	$kernel->call("harvester","register",$self->{type},$task->{uuid});
71
72	my $child = POE::Wheel::Run->new(
73	ProgramArgs => [$task],
74	Program => sub {
75	my ($args) = @_;
76	$self->do_work($args);
77	},
78	StdoutEvent => "got_child_stdout",
79	StderrEvent => "got_child_stderr",
80	CloseEvent => "got_child_close",
81	NoSetPgrp => 1, #Keep child processes in same group as parent. This is especially useful when using Ctrl+C to kill the whole group.
82	);
83
84	$_[KERNEL]->sig_child($child->PID, "got_child_signal");
85
86	# Wheel events include the wheel's ID.
87	$_[HEAP]{children_by_wid}{$child->ID} = $child;
88
89	# Signal events include the process ID.
90	$_[HEAP]{children_by_pid}{$child->PID} = $child;
91
92	my $logger = $self->{logger};
93	if ($logger){
94	$logger->debug("Child pid ".$child->PID." started as wheel ".$child->ID);
95	}
96	}
97
98	=head2 do_work
99
100	Internal method for processing a download task
101
102	=cut
103
104	sub do_work {
105	my ($self, $task) = @_;
106	my $batch = ( $self->{batch} && int($self->{batch}) ) ? $self->{batch} : 100;
107
108	#NOTE: Directory to spool files for processing
109	my $spooldir = $task->{spooldir};
110
111	my $task_uuid = $task->{uuid};
112	my $task_parameters = $task->{parameters};
113	my $interval = $task->{interval};
114
115	my $oai_pmh_parameters = $task_parameters->{oai_pmh};
116	my $import_parameters = $task_parameters->{import};
117
118	#NOTE: Overwrite the 'from' and 'until' parameters for repeatable tasks
119	if ( $interval && ! $oai_pmh_parameters->{until} ){
120	if ($oai_pmh_parameters->{verb} eq "ListRecords"){
121	#NOTE: 'effective_from' premiers on the first repetition (ie second request)
122	$oai_pmh_parameters->{from} = $task->{effective_from} if $task->{effective_from};
123	#NOTE: 'effective_until' appears on the first request
124	$oai_pmh_parameters->{until} = $task->{effective_until} if $task->{effective_until};
125	}
126	}
127
128	my $oai_downloader = Koha::OAI::Harvester::Downloader->new();
129	my $url = $oai_downloader->BuildURL($oai_pmh_parameters);
130
131	my $ua = LWP::UserAgent->new();
132	#NOTE: setup HTTP Basic Authentication if parameters are supplied
133	if($url && $url->host && $url->port){
134	my $http_basic_auth = $task_parameters->{http_basic_auth};
135	if ($http_basic_auth){
136	my $username = $http_basic_auth->{username};
137	my $password = $http_basic_auth->{password};
138	my $realm = $http_basic_auth->{realm};
139	$ua->credentials($url->host.":".$url->port, $realm, $username, $password);
140	}
141	}
142
143	#NOTE: Prepare database statement handle
144	my $dbh = C4::Context->dbh;
145	my $sql = "insert into oai_harvester_import_queue (uuid,result) VALUES (?,?)";
146	my $sth = $dbh->prepare($sql);
147
148	if($url && $ua){
149	#NOTE: You could define the callbacks as object methods instead... that might be nicer...
150	#although I suppose it might be a much of a muchness.
151	eval {
152	my @filename_cache = ();
153
154	$oai_downloader->harvest({
155	user_agent => $ua,
156	url => $url,
157	callback => sub {
158	my ($args) = @_;
159
160	my $repository = $args->{repository};
161	my $document = $args->{document};
162
163	#If the spooldir has disappeared, re-create it.
164	if ( ! -d $spooldir ){
165	my $made_spool_directory = make_path($spooldir);
166	}
167	my ($uuid,$uuid_string);
168	UUID::generate($uuid);
169	UUID::unparse($uuid, $uuid_string);
170	my $file_uuid = $uuid_string;
171	my $filename = "$spooldir/$file_uuid";
172	my $state = $document->toFile($filename, 2);
173	if ($state){
174	push(@filename_cache,$filename);
175	}
176
177	if(scalar @filename_cache == $batch){
178	my $result = {
179	repository => $repository,
180	filenames => \@filename_cache,
181	filter => $import_parameters->{filter},
182	matcher_code => $import_parameters->{matcher_code},
183	frameworkcode => $import_parameters->{frameworkcode},
184	record_type => $import_parameters->{record_type},
185	};
186	eval {
187	my $json_result = to_json($result, { pretty => 1 });
188	$sth->execute($task_uuid,$json_result);
189	};
190	@filename_cache = ();
191	}
192	},
193	complete_callback => sub {
194	my ($args) = @_;
195	my $repository = $args->{repository};
196	if (@filename_cache){
197	my $result = {
198	repository => "$repository",
199	filenames => \@filename_cache,
200	filter => $import_parameters->{filter},
201	matcher_code => $import_parameters->{matcher_code},
202	frameworkcode => $import_parameters->{frameworkcode},
203	record_type => $import_parameters->{record_type},
204	};
205	eval {
206	my $json_result = to_json($result, { pretty => 1 });
207	$sth->execute($task_uuid,$json_result);
208	};
209	}
210
211	},
212	});
213	};
214	if ($@){
215	die "Error during OAI-PMH download";
216	}
217	}
218	}
219
220	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Worker/Import.pm (+157 lines)
		1	package Koha::OAI::Harvester::Worker::Import;
		2
		3	# Copyright Prosentient Systems 2017
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, write to the Free Software Foundation, Inc.,
		18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		19
		20	use Modern::Perl;
		21	use POE qw(Wheel::Run);
		22	use JSON;
		23	use XML::LibXML;
		24
		25	use C4::Context;
		26	use C4::Matcher;
		27	use Koha::OAI::Harvester::Import::Record;
		28
		29	use parent 'Koha::OAI::Harvester::Worker';
		30
		31	=head1 NAME
		32
		33	Koha::OAI::Harvester::Worker::Import
		34
		35	=head1 SYNOPSIS
		36
		37	This is a module used by the OAI-PMH harvester internally.
		38
		39	As a bare minimum, it must define an "on_start" method.
		40
		41	=head1 METHODS
		42
		43	=head2 new
		44
		45	Create object
		46
		47	=cut
		48
		49	sub new {
		50	my ($class, $args) = @_;
		51	$args = {} unless defined $args;
		52	#NOTE: This type is used for logging and more importantly for registering with the harvester
		53	$args->{type} = "import" unless $args->{type};
		54	return bless ($args, $class);
		55	}
		56
		57	=head2 on_start
		58
		59	Internal event handler for starting the processing of a task
		60
		61	=cut
		62
		63	sub on_start {
		64	my ($self, $kernel, $heap, $postback,$task) = @_[OBJECT, KERNEL, HEAP, ARG0,ARG1];
65
66	$kernel->call("harvester","register",$self->{type},$task->{uuid});
67
68	$kernel->sig(cancel => "stop_worker");
69
70	my $child = POE::Wheel::Run->new(
71	ProgramArgs => [ $task ],
72	Program => sub {
73	my ($task,$args) = @_;
74
75	my $debug = $args->{debug} // 0;
76
77	if ($task){
78	my $json_result = $task->{result};
79	my $id = $task->{id};
80	my $task_uuid = $task->{uuid};
81	eval {
82	my $result = from_json($json_result);
83	if ($result){
84	my $repository = $result->{repository};
85	my $filenames = $result->{filenames};
86	my $filter = $result->{filter};
87	my $matcher_code = $result->{matcher_code};
88	my $frameworkcode = $result->{frameworkcode};
89	my $record_type = $result->{record_type};
90
91	my $matcher;
92	if ($matcher_code){
93	my $matcher_id = C4::Matcher::GetMatcherId($matcher_code);
94	$matcher = C4::Matcher->fetch($matcher_id);
95	}
96
97	foreach my $filename (@$filenames){
98	if ($filename){
99	if (-f $filename){
100	my $dom = XML::LibXML->load_xml(location => $filename, { no_blanks => 1 });
101	if ($dom){
102	my $oai_record = Koha::OAI::Harvester::Import::Record->new({
103	doc => $dom,
104	repository => $repository,
105	});
106	if ($oai_record){
107	my ($action,$linked_id) = $oai_record->import_record({
108	filter => $filter,
109	framework => $frameworkcode,
110	record_type => $record_type,
111	matcher => $matcher,
112	});
113	$debug && print STDOUT qq({ "import_result": { "task_uuid": "$task_uuid", "action": "$action", "filename": "$filename", "koha_id": "$linked_id" } }\n);
114	}
115	}
116	my $unlinked = unlink $filename;
117	}
118	}
119	}
120	}
121	};
122	if ($@){
123	warn $@;
124	}
125	#NOTE: Even if the file doesn't exist, we still need to process the queue item.
126
127	#NOTE: Don't do this via a postback in the parent process, as it's much faster to let the child process handle it.
128
129	#NOTE: It's vital that files are unlinked before deleting from the database,
130	#or you could get orphan files if the importer is interrupted.
131	my $dbh = C4::Context->dbh;
132	my $sql = "delete from oai_harvester_import_queue where id = ?";
133	my $sth = $dbh->prepare($sql);
134	$sth->execute($id);
135	}
136	},
137	StdoutEvent => "got_child_stdout",
138	StderrEvent => "got_child_stderr",
139	CloseEvent => "got_child_close",
140	NoSetPgrp => 1, #Keep child processes in same group as parent. This is especially useful when using Ctrl+C to kill the whole group.
141	);
142
143	$_[KERNEL]->sig_child($child->PID, "got_child_signal");
144
145	# Wheel events include the wheel's ID.
146	$_[HEAP]{children_by_wid}{$child->ID} = $child;
147
148	# Signal events include the process ID.
149	$_[HEAP]{children_by_pid}{$child->PID} = $child;
150
151	my $logger = $self->{logger};
152	if ($logger){
153	$logger->debug("Child pid ".$child->PID." started as wheel ".$child->ID);
154	}
155	}
156
157	1;

Line 0 Link Here

(-)a/debian/templates/oai-pmh-harvester-site.yaml.in (+22 lines)
	1	---
	2	#Harvester
	3	socket: 'unix:__OAI_RUN_DIR__/harvesterd.sock'
	4
	5	logfile: '__LOG_DIR__/oai-pmh-harvester.log'
	6	loglevel: 'WARN'
	7
	8	pidfile: '__OAI_RUN_DIR__/harvesterd.pid'
	9
	10	statefile: '__OAI_LIB_DIR__/harvesterd.state'
	11
	12	spooldir: '__OAI_SPOOL_DIR__'
	13
	14	#Downloader
	15	download_workers: 1
	16	download_module: 'Koha::OAI::Harvester::Worker::Download::Stream'
	17	download_batch: 100
	18
	19	#Importer
	20	import_workers: 1
	21	import_poll: 5
	22	import_module: 'Koha::OAI::Harvester::Worker::Import'

Line 0 Link Here

(-)a/etc/oai-pmh-harvester.yaml (+22 lines)
	1	---
	2	#Harvester
	3	socket: 'unix:__OAI_RUN_DIR__/harvesterd.sock'
	4
	5	logfile: '__LOG_DIR__/oai-pmh-harvester.log'
	6	loglevel: 'WARN'
	7
	8	pidfile: '__OAI_RUN_DIR__/harvesterd.pid'
	9
	10	statefile: '__OAI_LIB_DIR__/harvesterd.state'
	11
	12	spooldir: '__OAI_SPOOL_DIR__'
	13
	14	#Downloader
	15	download_workers: 1
	16	download_module: 'Koha::OAI::Harvester::Worker::Download::Stream'
	17	download_batch: 100
	18
	19	#Importer
	20	import_workers: 1
	21	import_poll: 5
	22	import_module: 'Koha::OAI::Harvester::Worker::Import'

Line 0 Link Here

(-)a/installer/data/mysql/atomicupdate/bug_10662.sql (+73 lines)
		1	--
		2	-- Table structure for table 'oai_harvester_biblios'
		3	--
		4
		5	CREATE TABLE IF NOT EXISTS `oai_harvester_biblios` (
		6	`import_oai_biblio_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
		7	`oai_repository` varchar(255) NOT NULL,
		8	`oai_identifier` varchar(255) DEFAULT NULL,
		9	`biblionumber` int(11) NOT NULL,
		10	PRIMARY KEY (`import_oai_biblio_id`),
		11	UNIQUE KEY `oai_record` (`oai_identifier`,`oai_repository`) USING BTREE,
		12	KEY `FK_import_oai_biblio_1` (`biblionumber`),
		13	CONSTRAINT `FK_import_oai_biblio_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE NO ACTION
		14	) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
		15
		16	--
		17	-- Table structure for table 'oai_harvester_history'
		18	--
		19
		20	CREATE TABLE IF NOT EXISTS `oai_harvester_history` (
		21	`import_oai_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
		22	`repository` varchar(255) DEFAULT NULL,
		23	`header_identifier` varchar(255) DEFAULT NULL,
		24	`header_datestamp` datetime NOT NULL,
		25	`header_status` varchar(45) DEFAULT NULL,
		26	`record` longtext NOT NULL,
		27	`upload_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
		28	`status` varchar(45) NOT NULL,
		29	`filter` text NOT NULL,
		30	`framework` varchar(4) NOT NULL,
		31	`record_type` enum('biblio','auth','holdings') NOT NULL,
		32	`matcher_code` varchar(10) DEFAULT NULL,
		33	PRIMARY KEY (`import_oai_id`)
		34	) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
		35
		36	--
		37	-- Table structure for table 'oai_harvester_import_queue'
		38	--
		39
		40	CREATE TABLE IF NOT EXISTS `oai_harvester_import_queue` (
		41	`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
		42	`uuid` varchar(45) CHARACTER SET utf8 NOT NULL,
		43	`status` varchar(45) CHARACTER SET utf8 NOT NULL DEFAULT 'new',
		44	`result` text CHARACTER SET utf8 NOT NULL,
		45	`result_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
		46	PRIMARY KEY (`id`)
		47	) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
		48
		49	--
		50	-- Table structure for table 'oai_harvester_requests'
		51	--
		52
		53	CREATE TABLE IF NOT EXISTS `oai_harvester_requests` (
		54	`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
		55	`uuid` varchar(45) NOT NULL,
		56	`oai_verb` varchar(45) NOT NULL,
		57	`oai_metadataPrefix` varchar(255) NOT NULL,
		58	`oai_identifier` varchar(255) DEFAULT NULL,
		59	`oai_from` varchar(45) DEFAULT NULL,
		60	`oai_until` varchar(45) DEFAULT NULL,
		61	`oai_set` varchar(255) DEFAULT NULL,
		62	`http_url` varchar(255) DEFAULT NULL,
		63	`http_username` varchar(255) DEFAULT NULL,
		64	`http_password` varchar(255) DEFAULT NULL,
65	`http_realm` varchar(255) DEFAULT NULL,
66	`import_filter` varchar(255) NOT NULL,
67	`import_framework_code` varchar(4) NOT NULL,
68	`import_record_type` enum('biblio','auth','holdings') NOT NULL,
69	`import_matcher_code` varchar(10) DEFAULT NULL,
70	`interval` int(10) unsigned NOT NULL,
71	`name` varchar(45) NOT NULL,
72	PRIMARY KEY (`id`)
73	) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

Lines 11-32 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-menu.inc (-16 / +19 lines)
11	<h5>Patrons and circulation</h5>	11	<h5>Patrons and circulation</h5>
12	<ul>	12	<ul>
13	[% IF ( CAN_user_tools_manage_patron_lists ) %]	13	[% IF ( CAN_user_tools_manage_patron_lists ) %]
14	<li><a href="/cgi-bin/koha/patron_lists/lists.pl">Patron lists</a></li>	14	<li><a href="/cgi-bin/koha/patron_lists/lists.pl">Patron lists</a></li>
15	[% END %]	15	[% END %]
16	[% IF (CAN_user_clubs) %]	16	[% IF (CAN_user_clubs) %]
17	<li><a href="/cgi-bin/koha/clubs/clubs.pl">Patron clubs</a></li>	17	<li><a href="/cgi-bin/koha/clubs/clubs.pl">Patron clubs</a></li>
18	[% END %]	18	[% END %]
19	[% IF ( CAN_user_tools_moderate_comments ) %]	19	[% IF ( CAN_user_tools_moderate_comments ) %]
20	<li><a href="/cgi-bin/koha/reviews/reviewswaiting.pl">Comments</a></li>	20	<li><a href="/cgi-bin/koha/reviews/reviewswaiting.pl">Comments</a></li>
21	[% END %]	21	[% END %]
22	[% IF ( CAN_user_tools_import_patrons ) %]	22	[% IF ( CAN_user_tools_import_patrons ) %]
23	<li><a href="/cgi-bin/koha/tools/import_borrowers.pl">Import patrons</a></li>	23	<li><a href="/cgi-bin/koha/tools/import_borrowers.pl">Import patrons</a></li>
24	[% END %]	24	[% END %]
25	[% IF ( CAN_user_tools_edit_notices ) %]	25	[% IF ( CAN_user_tools_edit_notices ) %]
26	<li><a href="/cgi-bin/koha/tools/letter.pl">Notices & slips</a></li>	26	<li><a href="/cgi-bin/koha/tools/letter.pl">Notices & slips</a></li>
27	[% END %]	27	[% END %]
28	[% IF ( CAN_user_tools_edit_notice_status_triggers ) %]	28	[% IF ( CAN_user_tools_edit_notice_status_triggers ) %]
29	<li><a href="/cgi-bin/koha/tools/overduerules.pl">Overdue notice/status triggers</a></li>	29	<li><a href="/cgi-bin/koha/tools/overduerules.pl">Overdue notice/status triggers</a></li>
30	[% END %]	30	[% END %]
31	[% IF ( CAN_user_tools_label_creator ) %]	31	[% IF ( CAN_user_tools_label_creator ) %]
32	<li><a href="/cgi-bin/koha/patroncards/home.pl">Patron card creator</a></li>	32	<li><a href="/cgi-bin/koha/patroncards/home.pl">Patron card creator</a></li>
Lines 41-47 Link Here
41	<li><a href="/cgi-bin/koha/tags/review.pl">Tag moderation</a></li>	41	<li><a href="/cgi-bin/koha/tags/review.pl">Tag moderation</a></li>
42	[% END %]	42	[% END %]
43	[% IF ( CAN_user_tools_batch_upload_patron_images ) %]	43	[% IF ( CAN_user_tools_batch_upload_patron_images ) %]
44	<li><a href="/cgi-bin/koha/tools/picture-upload.pl">Upload patron images</a></li>	44	<li><a href="/cgi-bin/koha/tools/picture-upload.pl">Upload patron images</a></li>
45	[% END %]	45	[% END %]
46	</ul>	46	</ul>
47	[% END %]	47	[% END %]
Lines 54-63 Link Here
54	<h5>Catalog</h5>	54	<h5>Catalog</h5>
55	<ul>	55	<ul>
56	[% IF ( CAN_user_tools_items_batchdel ) %]	56	[% IF ( CAN_user_tools_items_batchdel ) %]
57	<li><a href="/cgi-bin/koha/tools/batchMod.pl?del=1">Batch item deletion</a></li>	57	<li><a href="/cgi-bin/koha/tools/batchMod.pl?del=1">Batch item deletion</a></li>
58	[% END %]	58	[% END %]
59	[% IF ( CAN_user_tools_items_batchmod ) %]	59	[% IF ( CAN_user_tools_items_batchmod ) %]
60	<li><a href="/cgi-bin/koha/tools/batchMod.pl">Batch item modification</a></li>	60	<li><a href="/cgi-bin/koha/tools/batchMod.pl">Batch item modification</a></li>
61	[% END %]	61	[% END %]
62	[% IF CAN_user_tools_records_batchdel %]	62	[% IF CAN_user_tools_records_batchdel %]
63	<li><a href="/cgi-bin/koha/tools/batch_delete_records.pl">Batch record deletion</a></li>	63	<li><a href="/cgi-bin/koha/tools/batch_delete_records.pl">Batch record deletion</a></li>
Lines 75-82 Link Here
75	<li><a href="/cgi-bin/koha/tools/inventory.pl">Inventory</a></li>	75	<li><a href="/cgi-bin/koha/tools/inventory.pl">Inventory</a></li>
76	[% END %]	76	[% END %]
77	[% IF ( CAN_user_tools_label_creator ) %]	77	[% IF ( CAN_user_tools_label_creator ) %]
78	<li><a href="/cgi-bin/koha/labels/label-home.pl">Label creator</a></li>	78	<li><a href="/cgi-bin/koha/labels/label-home.pl">Label creator</a></li>
79	<li><a href="/cgi-bin/koha/labels/spinelabel-home.pl">Quick spine label creator</a></li>	79	<li><a href="/cgi-bin/koha/labels/spinelabel-home.pl">Quick spine label creator</a></li>
80	[% END %]	80	[% END %]
81	[% IF ( CAN_user_tools_rotating_collections ) %]	81	[% IF ( CAN_user_tools_rotating_collections ) %]
82	<li><a href="/cgi-bin/koha/rotating_collections/rotatingCollections.pl">Rotating collections</a></li>	82	<li><a href="/cgi-bin/koha/rotating_collections/rotatingCollections.pl">Rotating collections</a></li>
Lines 88-101 Link Here
88	<li><a href="/cgi-bin/koha/tools/marc_modification_templates.pl">Manage MARC modification templates</a></li>	88	<li><a href="/cgi-bin/koha/tools/marc_modification_templates.pl">Manage MARC modification templates</a></li>
89	[% END %]	89	[% END %]
90	[% IF ( CAN_user_tools_stage_marc_import ) %]	90	[% IF ( CAN_user_tools_stage_marc_import ) %]
91	<li><a href="/cgi-bin/koha/tools/stage-marc-import.pl">Stage MARC for import</a></li>	91	<li><a href="/cgi-bin/koha/tools/stage-marc-import.pl">Stage MARC for import</a></li>
92	[% END %]	92	[% END %]
93	[% IF ( CAN_user_tools_manage_staged_marc ) %]	93	[% IF ( CAN_user_tools_manage_staged_marc ) %]
94	<li><a href="/cgi-bin/koha/tools/manage-marc-import.pl">Staged MARC management</a></li>	94	<li><a href="/cgi-bin/koha/tools/manage-marc-import.pl">Staged MARC management</a></li>
95	[% END %]	95	[% END %]
96	[% IF ( CAN_user_tools_upload_local_cover_images ) %]	96	[% IF ( CAN_user_tools_upload_local_cover_images ) %]
97	<li><a href="/cgi-bin/koha/tools/upload-cover-image.pl">Upload local cover image</a></li>	97	<li><a href="/cgi-bin/koha/tools/upload-cover-image.pl">Upload local cover image</a></li>
98	[% END %]	98	[% END %]
		99	[% IF ( CAN_user_tools_manage_staged_marc ) %]
		100	<li><a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a></li>
		101	[% END %]
99	</ul>	102	</ul>
100	[% END %]	103	[% END %]
101	[% IF ( CAN_user_tools_edit_calendar \|\| CAN_user_tools_manage_csv_profiles \|\| CAN_user_tools_view_system_logs \|\| CAN_user_tools_edit_news	104	[% IF ( CAN_user_tools_edit_calendar \|\| CAN_user_tools_manage_csv_profiles \|\| CAN_user_tools_view_system_logs \|\| CAN_user_tools_edit_news
Lines 104-122 Link Here
104	<h5>Additional tools</h5>	107	<h5>Additional tools</h5>
105	<ul>	108	<ul>
106	[% IF ( CAN_user_tools_edit_calendar ) %]	109	[% IF ( CAN_user_tools_edit_calendar ) %]
107	<li><a href="/cgi-bin/koha/tools/holidays.pl">Calendar</a></li>	110	<li><a href="/cgi-bin/koha/tools/holidays.pl">Calendar</a></li>
108	[% END %]	111	[% END %]
109	[% IF ( CAN_user_tools_manage_csv_profiles ) %]	112	[% IF ( CAN_user_tools_manage_csv_profiles ) %]
110	<li><a href="/cgi-bin/koha/tools/csv-profiles.pl">CSV profiles</a></li>	113	<li><a href="/cgi-bin/koha/tools/csv-profiles.pl">CSV profiles</a></li>
111	[% END %]	114	[% END %]
112	[% IF ( CAN_user_tools_view_system_logs ) %]	115	[% IF ( CAN_user_tools_view_system_logs ) %]
113	<li><a href="/cgi-bin/koha/tools/viewlog.pl">Log viewer</a></li>	116	<li><a href="/cgi-bin/koha/tools/viewlog.pl">Log viewer</a></li>
114	[% END %]	117	[% END %]
115	[% IF ( CAN_user_tools_edit_news ) %]	118	[% IF ( CAN_user_tools_edit_news ) %]
116	<li><a href="/cgi-bin/koha/tools/koha-news.pl">News</a></li>	119	<li><a href="/cgi-bin/koha/tools/koha-news.pl">News</a></li>
117	[% END %]	120	[% END %]
118	[% IF ( CAN_user_tools_schedule_tasks ) %]	121	[% IF ( CAN_user_tools_schedule_tasks ) %]
119	<li><a href="/cgi-bin/koha/tools/scheduler.pl">Task scheduler</a></li>	122	<li><a href="/cgi-bin/koha/tools/scheduler.pl">Task scheduler</a></li>
120	[% END %]	123	[% END %]
121	[% IF ( CAN_user_tools_edit_quotes ) %]	124	[% IF ( CAN_user_tools_edit_quotes ) %]
122	<li><a href="/cgi-bin/koha/tools/quotes.pl">Quote editor</a></li>	125	<li><a href="/cgi-bin/koha/tools/quotes.pl">Quote editor</a></li>

Line 0 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/dashboard.tt (+378 lines)
		1	[% USE ColumnsSettings %]
		2	[% SET footerjs = 1 %]
		3	[% INCLUDE 'doc-head-open.inc' %]
		4	<title>Koha &rsaquo; Tools &rsaquo; OAI-PMH harvester</title>
		5	[% INCLUDE 'doc-head-close.inc' %]
		6	[% dashboard_page = '/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl' %]
		7	[% request_page = '/cgi-bin/koha/tools/oai-pmh-harvester/request.pl' %]
		8	<style type="text/css">
		9	a.paginate_button {
		10	padding: 2px;
		11	}
		12	</style>
		13	</head>
		14	<body id="tools_oai-pmh-harvester" class="tools">
		15	[% INCLUDE 'header.inc' %]
		16	[% INCLUDE 'cat-search.inc' %]
		17	<div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> &rsaquo; OAI-PMH harvester</div>
		18	<div id="doc3" class="yui-t2">
		19	<div id="bd">
		20	<div id="yui-main">
		21	<div class="yui-b">
		22	<h1>OAI-PMH harvester</h1>
		23	<div id="toolbar" class="btn-toolbar">
		24	<a id="new-request" class="btn btn-default btn-sm" href="[% request_page \| url %]?op=new"><i class="fa fa-plus"></i> New request</a>
		25	</div>
		26	[% IF ( harvester.offline ) %]
		27	<div class="alert">
		28	<span>OAI-PMH harvester offline</span>
		29	</div>
		30	[% END %]
		31	<div id="dashboard-items" class="toptabs">
		32	<ul>
		33	<li>
		34	<a href="#saved_requests">Saved requests <span id="saved_count">([% saved_requests.size \| html %])</span></a>
		35	</li>
		36	<li>
		37	<a href="#submitted_requests">Submitted requests <span id="submitted_count">([% submitted_requests.size \| html %])</span></a>
		38	</li>
		39	<li>
		40	<a href="#imports">Import history <span id="import_count">(0)</span></a>
		41	</li>
		42	</ul>
		43	<div id="submitted_requests">
		44	[% IF ( result.defined("start") ) %]
		45	<div class="alert">
		46	[% IF ( result.start ) %]
		47	<span>Start succeeded</span>
		48	[% ELSE %]
		49	<span>Start failed</span>
		50	[% END %]
		51	</div>
		52	[% ELSIF ( result.defined("stop") ) %]
		53	<div class="alert">
		54	[% IF ( result.stop ) %]
		55	<span>Stop succeeded</span>
		56	[% ELSE %]
		57	<span>Stop failed</span>
		58	[% END %]
		59	</div>
		60	[% ELSIF ( result.defined("delete") ) %]
		61	<div class="alert">
		62	[% IF ( result.delete ) %]
		63	<span>Delete succeeded</span>
		64	[% ELSE %]
65	<span>Delete failed</span>
66	[% END %]
67	</div>
68	[% END %]
69	<table id="submitted-table">
70	<thead>
71	<tr>
72	<th>Name</th>
73	<th>URL</th>
74	<th>Set</th>
75	<th>From</th>
76	<th>Until</th>
77	<th>Interval</th>
78	<th>Effective from</th>
79	<th>Effective until</th>
80	<th>Pending imports</th>
81	<th>Status</th>
82	<th>Error</th>
83	<th></th>
84	</tr>
85	</thead>
86	<tbody>
87	[% IF ( submitted_requests ) %]
88	[% FOREACH submitted_request IN submitted_requests %]
89	<tr>
90	<td>[% submitted_request.name \| html %]</td>
91	<td>[% submitted_request.parameters.oai_pmh.baseURL \| html %]</td>
92	<td>[% submitted_request.parameters.oai_pmh.set \| html %]</td>
93	<td>[% submitted_request.parameters.oai_pmh.from \| html %]</td>
94	<td>[% submitted_request.parameters.oai_pmh.until \| html %]</td>
95	<td>[% submitted_request.interval \| html %]</td>
96	<td>[% submitted_request.effective_from \| html %]</td>
97	<td>[% submitted_request.effective_until \| html %]</td>
98	<td>[% submitted_request.pending_imports \|html %]</td>
99	<td>
100	[% IF ( submitted_status = submitted_request.status ) %]
101	[% IF ( submitted_status == "new" ) %]
102	<span>New</span>
103	[% ELSIF ( submitted_status == "active" ) %]
104	<span>Active</span>
105	[% ELSIF ( submitted_status == "stopped" ) %]
106	<span>Stopped</span>
107	[% END %]
108	[% END %]
109	</td>
110	<td>
111	[% IF ( submitted_request.error ) %]
112	<span>Harvest failure</span>
113	[% END %]
114	</td>
115	<td>
116	<div class="dropdown">
117	<a class="btn btn-default btn-xs dropdown-toggle" role="button" data-toggle="dropdown" href="#">Actions <span class="caret"></span></a>
118	<ul class="dropdown-menu pull-right" role="menu">
119	<li>
120	<a href="[% dashboard_page \| url %]?op=start&uuid=[% submitted_request.uuid \| url %]"><i class="fa fa-play"></i> Start</a>
121	</li>
122	<li>
123	<a href="[% dashboard_page \| url %]?op=stop&uuid=[% submitted_request.uuid \| url %]"><i class="fa fa-stop"></i> Stop</a>
124	</li>
125	<li>
126	<a href="[% dashboard_page \| url %]?op=delete&uuid=[% submitted_request.uuid \| url %]"><i class="fa fa-trash"></i> Delete</a>
127	</li>
128	</ul>
129	</div>
130	</td>
131	</tr>
132	[% END %]
133	[% END %]
134	</tbody>
135	</table>
136	</div>
137	<div id="saved_requests">
138	[% IF ( result.send.defined ) %]
139	<div class="alert">
140	[% IF ( result.send ) %]
141	<span>Submit succeeded</span>
142	[% ELSE %]
143	<span>Submit failed</span>
144	[% END %]
145	</div>
146	[% END %]
147	<table id="saved-table">
148	<thead>
149	<tr>
150	<th>Name</th>
151	<th>URL</th>
152	<!-- <th>Verb</th>
153	<th>Metadata prefix</th>
154	<th>Identifier</th> -->
155	<th>Set</th>
156	<th>From</th>
157	<th>Until</th>
158	<th>Interval</th>
159	<!-- <th>Filter</th>
160	<th>Framework code</th>
161	<th>Record type</th>
162	<th>Matcher code</th> -->
163	<th></th>
164	</tr>
165	</thead>
166	<tbody>
167	[% IF ( saved_requests ) %]
168	[% FOREACH saved_request IN saved_requests %]
169	<tr>
170	<td>[% saved_request.name \| html %]</td>
171	<td>[% saved_request.http_url \| html %]</td>
172	<!-- <td>[% saved_request.oai_verb \| html %]</td>
173	<td>[% saved_request.oai_metadataPrefix \| html %]</td>
174	<td>[% saved_request.oai_identifier \| html %]</td> -->
175	<td>[% saved_request.oai_set \| html %]</td>
176	<td>[% saved_request.oai_from \| html %]</td>
177	<td>[% saved_request.oai_until \| html %]</td>
178	<td>[% saved_request.interval \| html %]</td>
179	<!-- <td>
180	[% IF ( saved_request.import_filter == "default" ) %]
181	<span>Default</span>
182	[% ELSE %]
183	[% saved_request.import_filter \| html %]
184	[% END %]
185	</td>
186	<td>
187	[% display_framework = "" %]
188	[% FOREACH framework IN frameworks %]
189	[% IF ( framework.frameworkcode == saved_request.import_framework_code ) %]
190	[% display_framework = framework %]
191	[% END %]
192	[% END %]
193	[% IF ( display_framework ) %]
194	[% display_framework.frameworktext \| html %]
195	[% ELSE %]
196	[% saved_request.import_framework_code \| html %]
197	[% END %]
198	</td>
199	<td>
200	[% IF ( saved_request.import_record_type == "biblio" ) %]
201	<span>Bibliographic</span>
202	[% ELSE %]
203	[% saved_request.import_record_type \| html %]
204	[% END %]
205	</td>
206	<td>
207	[% display_matcher = "" %]
208	[% FOREACH matcher IN matchers %]
209	[% IF ( matcher.code == saved_request.import_matcher_code ) %]
210	[% display_matcher = matcher %]
211	[% END %]
212	[% END %]
213	[% IF ( display_matcher ) %]
214	[% display_matcher.description \| html %]
215	[% ELSE %]
216	[% saved_request.import_matcher_code \| html %]
217	[% END %]
218	</td> -->
219	<td>
220	<div class="dropdown">
221	<a class="btn btn-default btn-xs dropdown-toggle" role="button" data-toggle="dropdown" href="#">Actions <span class="caret"></span></a>
222	<ul class="dropdown-menu pull-right" role="menu">
223	<li>
224	<a href="[% request_page \| url %]?op=edit&id=[% saved_request.id \| url %]"><i class="fa fa-pencil"></i> Edit</a>
225	</li>
226	<li>
227	<a href="[% request_page \| url %]?op=new&id=[% saved_request.id \| url %]"><i class="fa fa-copy"></i> Copy</a>
228	</li>
229	<li>
230	<a href="[% dashboard_page \| url %]?op=send&id=[% saved_request.id \| url %]"><i class="fa fa-send"></i> Submit</a>
231	</li>
232	<li>
233	<a href="[% request_page \| url %]?op=delete&id=[% saved_request.id \| url %]"><i class="fa fa-trash"></i> Delete</a>
234	</li>
235	</ul>
236	</div>
237	</td>
238	</tr>
239	[% END %]
240	[% END %]
241	</tbody>
242	</table>
243	</div>
244	<div id="imports">
245	<div class="btn-toolbar">
246	<button id="refresh-button" type="button" class="btn btn-default btn-sm">Refresh import history</button>
247	</div>
248	<table id="history-table">
249	<thead>
250	<tr>
251	<th>Id</th>
252	<th>Repository</th>
253	<th>Identifier</th>
254	<th>Datestamp</th>
255	<th>Upstream status</th>
256	<th>Import status</th>
257	<th>Import timestamp</th>
258	<th>Imported record</th>
259	<th>Downloaded record</th>
260	</tr>
261	</thead>
262	</table>
263
264	</div>
265	</div>
266	</div>
267	</div>
268	<div class="yui-b">
269	[% INCLUDE 'tools-menu.inc' %]
270	</div>
271	</div>
272
273	[% MACRO jsinclude BLOCK %]
274	[% INCLUDE 'datatables.inc' %]
275	[% INCLUDE 'columns_settings.inc' %]
276	<script type="text/javascript">
277	$(document).ready(function() {
278	$('#dashboard-items').tabs();
279	[% IF ( result.start.defined \|\| result.stop.defined \|\| result.delete.defined ) %]
280	$('#dashboard-items').tabs("option","active",1);
281	[% END %]
282
283	var saved_table_columns_settings = [% ColumnsSettings.GetColumns( 'tools','oai-pmh-harvester-dashboard', 'saved-table', 'json' ) \| $raw %];
284	var saved_table = KohaTable("saved-table",{},saved_table_columns_settings);
285
286	var submitted_table_columns_settings = [% ColumnsSettings.GetColumns( 'tools','oai-pmh-harvester-dashboard', 'submitted-table', 'json' ) \| $raw %];
287	var submitted_table = KohaTable("submitted-table",{},submitted_table_columns_settings);
288
289	var history_table_columns_settings = [% ColumnsSettings.GetColumns( 'tools','oai-pmh-harvester-dashboard', 'history-table', 'json' ) \| $raw %];
290	var history_table = KohaTable("history-table",{
291	serverSide: true,
292	searching: true,
293	processing: true,
294	ajax: {
295	"url": '/cgi-bin/koha/svc/oai-pmh-harvester/history',
296	contentType: 'application/json',
297	type: 'POST',
298	data: function ( d ) {
299	return JSON.stringify( d );
300	},
301	dataSrc: function (json){
302	var recordsTotal = json.recordsTotal;
303	if(recordsTotal){
304	$('#import_count').text( "("+recordsTotal+")" );
305	}
306	return json.data;
307	}
308	},
309	columns: [
310	{ data: 'import_oai_id', },
311	{ data: 'repository', },
312	{ data: 'header_identifier', },
313	{ data: 'header_datestamp', },
314	{
315	data: 'header_status', render: function (data, type, full, meta){
316	var display_string = _("Active");
317	if (data == "deleted"){
318	display_string = _("Deleted");
319	}
320	return display_string;
321	}
322	},
323	{
324	data: 'status', render: function (data, type, full, meta){
325	var display_string = data;
326	if (data == "added"){
327	display_string = _("Added");
328	}
329	else if (data == "error"){
330	display_string = _("Error");
331	}
332	else if (data == "not_found"){
333	display_string = _("Not found");
334	}
335	else if (data == "updated"){
336	display_string = _("Updated");
337	}
338	return display_string;
339	}
340	},
341	{ data: 'upload_timestamp', },
342	{
343	data: 'imported_record', render: function (data, type, full, meta){
344	var display_string = data;
345	var record_type = (full.record_type) ? full.record_type : 'biblio';
346	if (data && record_type == 'biblio'){
347	var link_text = _("View biblio record");
348	var link = '<a href="/cgi-bin/koha/catalogue/detail.pl?biblionumber='+data+'">'+link_text+'</a>';
349	display_string = link;
350	}
351	return display_string;
352	}, searchable: false
353
354	},
355	{
356	data: 'import_oai_id', render: function (data, type, full, meta){
357	var display_string = data;
358	var link_text = _("View record");
359	var link = '<a href="/cgi-bin/koha/tools/oai-pmh-harvester/record.pl?import_oai_id='+data+'">'+link_text+'</a>';
360	display_string = link;
361	return display_string;
362	}, searchable: false
363	},
364	],
365	//In theory, it would be nice to sort in descending order, but
366	//it causes severe paging issues the data is frequently updated.
367	//Caching would make the paging more stable, but the results would be stale.
368	order: [
369	[ 0, 'asc' ],
370	]
371	},history_table_columns_settings);
372	$('#refresh-button').click(function(){
373	history_table.dataTable().api().ajax.reload( null, false );
374	});
375	});
376	</script>
377	[% END %]
378	[% INCLUDE 'intranet-bottom.inc' %]

Line 0 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/request.tt (+244 lines)
		1	[% USE Asset %]
		2	[% USE raw %]
		3	[% SET footerjs = 1 %]
		4	[% INCLUDE 'doc-head-open.inc' %]
		5	<title>Koha &rsaquo; Tools &rsaquo; OAI-PMH harvester &rsaquo; Request</title>
		6	[% INCLUDE 'doc-head-close.inc' %]
		7	<style type="text/css">
		8	/* Override staff-global.css which hides second, millisecond, and microsecond sliders */
		9	.ui_tpicker_second {
		10	display: block;
		11	}
		12	</style>
		13	</head>
		14	<body id="tools_oai-pmh-harvester_request" class="tools">
		15	[% INCLUDE 'header.inc' %]
		16	[% INCLUDE 'cat-search.inc' %]
		17	<div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> &rsaquo; <a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a> &rsaquo; Request</div>
		18	<div id="doc3" class="yui-t2">
		19	<div id="bd">
		20	<div id="yui-main">
		21	<div class="yui-b">
		22	[% IF ( op == "edit" ) %]
		23	<h1>Edit OAI-PMH request</h1>
		24	[% ELSE %]
		25	<h1>New OAI-PMH request</h1>
		26	[% END %]
		27	[% IF ( test_parameters ) %]
		28	[% IF ( errors.size ) %]
		29	<div class="dialog message"><span class="text-danger">Tests failed!</span></div>
		30	[% ELSE %]
		31	<div class="dialog message"><span class="text-success">Tests succeeded!</span></div>
		32	[% END %]
		33	[% END %]
		34	<form action="/cgi-bin/koha/tools/oai-pmh-harvester/request.pl" method="post" name="entry-form">
		35	[% IF ( op == "new" ) %]
		36	<input type="hidden" name="op" value="create" />
		37	[% ELSIF ( op == "edit" ) %]
		38	<input type="hidden" name="op" value="update" />
		39	[% ELSE %]
		40	<input type="hidden" name="op" value="[% op \| html %]" />
		41	[% END %]
		42	[% IF ( id ) %]
		43	<input type="hidden" name="id" value="[% id \| html %]" />
		44	[% END %]
		45	<fieldset class="rows">
		46	<ol>
		47	<li>
		48	<label for="name">Name:</label>
		49	<input type="text" size="30" id="name" name="name" value="[% oai_pmh_request.name \| html %]"/>
		50	<span class="help">This is just a short name to help in managing requests.</span>
		51	</li>
		52	</ol>
		53	</fieldset>
		54	<fieldset class="rows">
		55	<legend>HTTP parameters:</legend>
		56	<ol>
		57	<li>
		58	<label for="http_url">URL:</label>
		59	<input type="text" size="30" id="http_url" name="http_url" value="[% oai_pmh_request.http_url \| html %]">
		60	[% IF (errors.http_url.malformed) %]<span class="error">[This must be a properly formatted HTTP or HTTPS URL.]</span>[% END %]
		61	[% IF (errors.http.404) %]<span class="error">[Cannot find address specified by this URL.]</span>[% END %]
		62	[% IF (errors.http.401) %]<span class="error">[Permission denied to access this URL.]</span>[% END %]
		63	[% IF (errors.http.generic) %]<span class="error">[Unable to access this URL.]</span>[% END %]
		64	</li>
65	</ol>
66	<span class="help">The following parameters are not required by all OAI-PMH repositories, so they may be optional for this task.</span>
67	<ol>
68	<li>
69	<label for="http_username">Username:</label>
70	<input type="text" size="30" id="http_username" name="http_username" value="[% oai_pmh_request.http_username \| html %]">
71	</li>
72	<li>
73	<label for="http_password">Password:</label>
74	<input type="text" size="30" id="http_password" name="http_password" value="[% oai_pmh_request.http_password \| html %]">
75	</li>
76	<li>
77	<label for="http_realm">Realm:</label>
78	<input type="text" size="30" id="http_realm" name="http_realm" value="[% oai_pmh_request.http_realm \| html %]">
79	</li>
80	</ol>
81	</fieldset>
82	<fieldset class="rows">
83	<legend>OAI-PMH parameters:</legend>
84	<ol>
85	<li>
86	<label for="oai_verb">Verb:</label>
87	<select id="oai_verb" name="oai_verb">
88	[% IF ( oai_pmh_request.oai_verb == "ListRecords" ) %]
89	<option value="ListRecords" selected="selected">ListRecords</option>
90	[% ELSE %]
91	<option value="ListRecords">ListRecords</option>
92	[% END %]
93	[% IF ( oai_pmh_request.oai_verb == "GetRecord" ) %]
94	<option value="GetRecord" selected="selected">GetRecord</option>
95	[% ELSE %]
96	<option value="GetRecord">GetRecord</option>
97	[% END %]
98	</select>
99	</li>
100	<li>
101	<label for="oai_metadataPrefix">Metadata prefix:</label>
102	<input type="text" size="30" id="oai_metadataPrefix" name="oai_metadataPrefix" value="[% oai_pmh_request.oai_metadataPrefix \| html %]">
103	[% IF (errors.oai_metadataPrefix.unavailable) %]<span class="error">[This metadataPrefix is unavailable from this OAI-PMH provider.]</span>[% END %]
104	[% IF (errors.oai_metadataPrefix.missing) %]<span class="error">[metadataPrefix is a required field for an OAI-PMH request.]</span>[% END %]
105	</li>
106	<li>
107	<label for="oai_identifier">Identifier:</label>
108	<input type="text" size="30" id="oai_identifier" name="oai_identifier" value="[% oai_pmh_request.oai_identifier \| html %]">
109	[% IF (errors.oai_identifier.missing) %]<span class="error">[Identifier is a required field when using GetRecord for an OAI-PMH request.]</span>[% END %]
110	</li>
111	<li>
112	<label for="oai_set">Set:</label>
113	<input type="text" size="30" id="oai_set" name="oai_set" value="[% oai_pmh_request.oai_set \| html %]">
114	[% IF (errors.oai_set.unavailable) %]<span class="error">[This set is unavailable from this OAI-PMH provider.]</span>[% END %]
115	</li>
116	[% IF (errors.oai.granularity_mismatch) %]<span class="error">[You must specify the same granularity for both From and Until.]</span>[% END %]
117	<li>
118	<label for="oai_from">From:</label>
119	<input type="text" size="30" class="datetime_utc" id="oai_from" name="oai_from" value="[% oai_pmh_request.oai_from \| html %]">
120	<span class="help">This value will be treated as UTC time. Note that some repositories only support YYYY-MM-DD datestamps.</span>
121	[% IF (errors.oai_from.malformed) %]<span class="error">[This must be in YYYY-MM-DD or YYYY-MM-DDThh:mm:ssZ format.]</span>[% END %]
122	[% IF (errors.oai_from.unavailable) %]<span class="error">[This granularity is unsupported by this OAI-PMH provider.]</span>[% END %]
123	</li>
124	<li>
125	<label for="oai_until">Until:</label>
126	<input type="text" size="30" class="datetime_utc" id="oai_until" name="oai_until" value="[% oai_pmh_request.oai_until \| html %]">
127	<span class="help">This value will be treated as UTC time. Note that some repositories only support YYYY-MM-DD datestamps.</span>
128	[% IF (errors.oai_until.malformed) %]<span class="error">[This must be in YYYY-MM-DD or YYYY-MM-DDThh:mm:ssZ format.]</span>[% END %]
129	[% IF (errors.oai_until.unavailable) %]<span class="error">[This granularity is unsupported by this OAI-PMH provider.]</span>[% END %]
130	</li>
131	</ol>
132	</fieldset>
133	<fieldset class="rows">
134	<legend>Import parameters:</legend>
135	<ol>
136	<li>
137	<label for="import_filter">Filter:</label>
138	[% IF ( oai_pmh_request.import_filter == "default" ) %]
139	<input type="text" size="30" id="import_filter" name="import_filter" value="default">
140	[% ELSE %]
141	<input type="text" size="30" id="import_filter" name="import_filter" value="[% oai_pmh_request.import_filter \| html %]">
142	[% END %]
143	<span class="help">If no filter is entered, the default filter will be used.</span>
144	</li>
145	<li>
146	<label for="import_framework_code">Framework code:</label>
147	<select id="import_framework_code" name="import_framework_code">
148	<option value="">Default framework</option>
149	[% FOREACH framework IN frameworks %]
150	[% IF ( oai_pmh_request.import_framework_code == framework.frameworkcode ) %]
151	<option selected="selected" value="[% framework.frameworkcode \| html %]">[% framework.frameworktext \| html %]</option>
152	[% ELSE %]
153	<option value="[% framework.frameworkcode \| html %]">[% framework.frameworktext \| html %]</option>
154	[% END %]
155	[% END %]
156	</select>
157	</li>
158	<li>
159	<label for="import_record_type">Record type:</label>
160	<select id="import_record_type" name="import_record_type">
161	<option value="biblio">Bibliographic</option>
162	</select>
163	</li>
164	<li>
165	<label for="import_matcher_code">Record matcher:</label>
166	<select id="import_matcher_code" name="import_matcher_code">
167	<option value="">None</option>
168	[% FOREACH matcher IN matchers %]
169	[% IF ( oai_pmh_request.import_matcher_code == matcher.code ) %]
170	<option value="[% matcher.code \| html %]" selected="selected">[% matcher.description \| html %]</option>
171	[% ELSE %]
172	<option value="[% matcher.code \| html %]">[% matcher.description \| html %]</option>
173	[% END %]
174	[% END %]
175	</select>
176	<span class="help">See <a href="/cgi-bin/koha/admin/matching-rules.pl">record matching rules</a> to add or edit rules.</span>
177	</li>
178	</ol>
179	</fieldset>
180	<fieldset class="rows">
181	<legend>Download parameters:</legend>
182	<ol>
183	<li>
184	<label for="interval">Interval (seconds): </label>
185	<input type="text" id="interval" name="interval" value="[% oai_pmh_request.interval \| html %]" size="4">
186	<span class="help">The download request will be repeated in intervals of this many seconds. Enter "0" if you want the task to only happen once.</span>
187	</li>
188	</ol>
189	</fieldset>
190	<fieldset class="action">
191	<input type="submit" name="test_parameters" value="Test parameters">
192	<input type="submit" name="save" value="Save">
193	<a class="cancel" href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">Cancel</a>
194	</fieldset>
195	</form>
196	</div>
197	</div>
198	<div class="yui-b">
199	[% INCLUDE 'tools-menu.inc' %]
200	</div>
201	</div>
202	[% MACRO jsinclude BLOCK %]
203	[% INCLUDE 'calendar.inc' %]
204	[% Asset.js("lib/jquery/plugins/jquery-ui-timepicker-addon.min.js") \| $raw %]
205	[% INCLUDE 'timepicker.inc' %]
206	<script type="text/javascript">
207	$(document).ready(function() {
208	toggle_identifier();
209	$("#oai_verb").on("click",toggle_identifier);
210	$(".datetime_utc").datetimepicker({
211	separator: "T",
212	timeSuffix: 'Z',
213	dateFormat: "yy-mm-dd",
214	timeFormat: "HH:mm:ss",
215	hour: 0,
216	minute: 0,
217	second: 0,
218	showSecond: 1,
219	// timezone doesn't work with the "Now" button in v1.4.3 although it appears to in v1.6.1
220	// timezone: '+000'
221	});
222	});
223	function toggle_identifier (){
224	var verb = $("#oai_verb").find(":selected").val();
225	var oai_identifier = $("#oai_identifier");
226	var oai_set = $("#oai_set");
227	var oai_from = $("#oai_from");
228	var oai_until = $("#oai_until");
229	if (verb == 'ListRecords'){
230	oai_identifier.prop('disabled', true);
231	oai_set.prop('disabled', false);
232	oai_from.prop('disabled', false);
233	oai_until.prop('disabled', false);
234	}
235	else if (verb == 'GetRecord'){
236	oai_identifier.prop('disabled', false);
237	oai_set.prop('disabled', true);
238	oai_from.prop('disabled', true);
239	oai_until.prop('disabled', true);
240	}
241	}
242	</script>
243	[% END %]
244	[% INCLUDE 'intranet-bottom.inc' %]

Line 0 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/xslt/StripOAIPMH.xsl (+28 lines)
	1	<?xml version="1.0" encoding="UTF-8"?>
	2	<xsl:stylesheet version="1.0"
	3	xmlns:oai="http://www.openarchives.org/OAI/2.0/"
	4	xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
	5	<xsl:output method="xml" encoding="UTF-8" indent="yes"/>
	6	<!-- NOTE: This XSLT strips the OAI-PMH wrapper from the metadata. -->
	7
	8	<!-- Match the root oai:record element -->
	9	<xsl:template match="oai:record">
	10	<!-- Apply templates only to the child metadata element(s) -->
	11	<xsl:apply-templates select="oai:metadata" />
	12	</xsl:template>
	13
	14	<!-- Matches an oai:metadata element -->
	15	<xsl:template match="oai:metadata">
	16	<!-- Create a copy of child attributes and nodes -->
	17	<xsl:apply-templates select="@* \| node()" />
	18	</xsl:template>
	19
	20	<!-- Identity transformation: this template copies attributes and nodes -->
	21	<xsl:template match="@* \| node()">
	22	<!-- Create a copy of this attribute or node -->
	23	<xsl:copy>
	24	<!-- Recursively apply this template to the attributes and child nodes of this element -->
	25	<xsl:apply-templates select="@* \| node()" />
	26	</xsl:copy>
	27	</xsl:template>
	28	</xsl:stylesheet>

Line 0 Link Here

(-)a/misc/harvesterd.pl (+207 lines)
		1	#!/usr/bin/perl
		2
		3	# Copyright 2017 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19
		20	use Modern::Perl;
		21	use Getopt::Long;
		22	use Pod::Usage;
		23	use Module::Load;
		24	use Log::Log4perl qw(:easy);
		25	use POE;
		26	use YAML;
		27
		28	use C4::Context;
		29	use Koha::Daemon;
		30	use Koha::OAI::Harvester;
		31	use Koha::OAI::Harvester::Listener;
		32
		33	binmode(STDOUT,':encoding(UTF-8)');
		34	$\|++;
		35
		36	my $help = 0;
		37	my $daemonize = 0;
		38
		39	my ($socket_addr,$pidfile,$statefile,$spooldir);
		40	my ($download_module,$import_module);
		41	my ($batch,$download_workers,$import_workers,$import_poll);
		42	my ($logfile,$log_level);
		43	my $log_levels = {
		44	FATAL => $FATAL,
		45	ERROR => $ERROR,
		46	WARN => $WARN,
		47	INFO => $INFO,
		48	DEBUG => $DEBUG,
		49	TRACE => $TRACE,
		50	};
		51
		52	my $context = C4::Context->new();
		53	my $config_filename = $context->{config}->{oai_pmh_harvester_config};
		54	if ($config_filename){
		55	my $config = YAML::LoadFile($config_filename);
		56	if ($config){
		57	$socket_addr = $config->{socket};
		58	$pidfile = $config->{pidfile};
		59	$statefile = $config->{statefile};
		60	$spooldir = $config->{spooldir};
		61	$logfile = $config->{logfile};
		62	$log_level = $config->{loglevel};
		63	$download_module = $config->{download_module};
		64	$import_module = $config->{import_module};
65	$batch = $config->{download_batch};
66	$download_workers = $config->{download_workers};
67	$import_workers = $config->{import_workers};
68	$import_poll = $config->{import_poll};
69	}
70	}
71
72	GetOptions(
73	"help\|?" => \$help,
74	"daemon" => \$daemonize,
75	"socket-uri=s" => \$socket_addr,
76	"pid-file=s" => \$pidfile,
77	"state-file=s" => \$statefile,
78	"spool-dir=s" => \$spooldir,
79	"log-file=s" => \$logfile,
80	"log-level=s" => \$log_level,
81	"download-module=s" => \$download_module,
82	"import-module=s" => \$import_module,
83	) or pod2usage(2);
84	pod2usage(1) if $help;
85
86	my $level = ( $log_level && $log_levels->{$log_level} ) ? $log_levels->{$log_level} : $log_levels->{WARN};
87	Log::Log4perl->easy_init(
88	{
89	level => $level,
90	file => "STDOUT",
91	layout => '[%d{yyyy-MM-dd HH:mm:ss}][%p] %m%n',
92	}
93	);
94	my $logger = Log::Log4perl->get_logger();
95
96	unless($download_module){
97	$download_module = "Koha::OAI::Harvester::Worker::Download::Stream";
98	}
99	unless($import_module){
100	$import_module = "Koha::OAI::Harvester::Worker::Import";
101	}
102
103	foreach my $module ( $download_module, $import_module ){
104	load $module;
105	}
106	my $downloader = $download_module->new({
107	logger => $logger,
108	batch => $batch,
109	});
110	my $importer = $import_module->new({
111	logger => $logger,
112	});
113
114	my $daemon = Koha::Daemon->new({
115	pidfile => $pidfile,
116	logfile => $logfile,
117	daemonize => $daemonize,
118	});
119	$daemon->run();
120
121	my $harvester = Koha::OAI::Harvester->spawn({
122	Downloader => $downloader,
123	DownloaderWorkers => $download_workers,
124	Importer => $importer,
125	ImporterWorkers => $import_workers,
126	ImportQueuePoll => $import_poll,
127	logger => $logger,
128	state_file => $statefile,
129	spooldir => $spooldir,
130	});
131
132	my $listener = Koha::OAI::Harvester::Listener->spawn({
133	logger => $logger,
134	socket => $socket_addr,
135	});
136
137	$logger->info("OAI-PMH harvester started.");
138
139	POE::Kernel->run();
140
141	exit;
142
143	=head1 NAME
144
145	harvesterd.pl - a daemon that asynchronously sends OAI-PMH requests and imports OAI-PMH records
146
147	=head1 SYNOPSIS
148
149	KOHA_CONF=/path/to/koha-conf.xml ./harvesterd.pl
150
151	=head1 OPTIONS
152
153	=over 8
154
155	=item B<--help>
156
157	Print a brief help message and exits.
158
159	=item B<--daemon>
160
161	Run program as a daemon (ie fork process, setsid, chdir to root, reset umask,
162	and close STDIN, STDOUT, and STDERR).
163
164	=item B<--log-file>
165
166	Specify a file to which to log STDOUT and STDERR.
167
168	=item B<--pid-file>
169
170	Specify a file to store the process id (this prevents multiple copies of the program
171	from running at the same time).
172
173	=item B<--socket-uri>
174
175	Specify a URI to use for the UNIX socket used to communicate with the daemon.
176	(e.g. unix:/path/to/socket.sock)
177
178	=item B<--state-file>
179
180	Specify a filename to use for storing the harvester's in-memory state.
181
182	In the event that the harvester crashes, it can resume from where it stopped.
183
184	=item B<--spool-dir>
185
186	Specify a directory to store downloaded OAI-PMH records prior to import.
187
188	=item B<--log-level>
189
190	Specify a log level for logging. The logger uses Log4Perl, which provides
191	FATAL, ERROR, WARN, INFO, DEBUG, and TRACE in order of descending priority.
192
193	Defaults to WARN level.
194
195	=item B<--download-module>
196
197	Specify a Perl module to use for downloading records. This is a specialty module,
198	which has particular requirements, so only advanced users should use this option.
199
200	=item B<--import-module>
201
202	Specify a Perl module to use for importing records. This is a specialty module,
203	which has particular requirements, so only advanced users should use this option.
204
205	=back
206
207	=cut

Line 0 Link Here

(-)a/svc/oai-pmh-harvester/history (+129 lines)
		1	#!/usr/bin/perl
		2
		3	# Copyright 2017 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, write to the Free Software Foundation, Inc.,
		18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		19
		20	use Modern::Perl;
		21	use C4::Auth qw(check_cookie_auth haspermission get_session);
		22	use JSON;
		23	use Koha::OAI::Harvester::Biblios;
		24	use Koha::OAI::Harvester::Histories;
		25
		26	my $input = new CGI;
		27
		28	my ( $auth_status, $sessionID ) =
		29	check_cookie_auth( $input->cookie('CGISESSID'));
		30
		31	if ( $auth_status ne "ok" ) {
		32	exit 0;
		33	}
		34
		35	my $data = {
		36	data => [],
		37	recordsTotal => 0,
		38	recordsFiltered => 0,
		39	draw => undef,
		40	};
		41
		42	my $length = 10;
		43	my $start = 0;
		44	my @order_by = ();
		45	my @search = ();
		46
		47	if ($input->request_method eq "POST"){
		48	my $postdata = $input->param('POSTDATA');
		49	my $request = from_json($postdata);
		50	$data->{draw} = int( $request->{draw} ) if $request->{draw};
		51	$length = $request->{length} if $request->{length};
		52	$start = $request->{start} if $request->{start};
		53	if (my $search = $request->{search}){
		54	my $value = $search->{value};
		55	if ($value){
		56	foreach my $column (@{$request->{columns}}){
		57	if ($column->{data} && $column->{searchable}){
		58	my $search_element = {
		59	$column->{data} => { 'like', "%".$value."%" },
		60	};
		61	push(@search,$search_element);
		62	}
		63	}
		64	}
65	}
66	if (my $order = $request->{order}){
67	foreach my $element (@$order){
68	my $dir = $element->{dir};
69	my $column_index = $element->{column};
70	my $column = $request->{columns}->[$column_index];
71	my $orderable = $column->{orderable};
72	if ($orderable){
73	my $column_name = $column->{data};
74	my $direction;
75	if ($dir){
76	if ($dir eq "asc" \|\| $dir eq "desc"){
77	$direction = "-$dir";
78	}
79	}
80	if ($column_name && $direction){
81	my $single_order = {
82	$direction => $column_name,
83	};
84	push(@order_by,$single_order);
85	}
86	}
87	}
88	}
89	}
90
91	my $page = ( $start / $length ) + 1;
92	my $results = Koha::OAI::Harvester::Histories->new->search(
93	\@search,
94	{
95	page => $page,
96	rows => $length,
97	order_by => \@order_by,
98	},
99	);
100	my $count = Koha::OAI::Harvester::Histories->new->count;
101	my $filtered_count = $results->pager->total_entries;
102	my @rows = ();
103	while (my $obj = $results->next){
104	my $row = $obj->unblessed;
105	$row->{imported_record} = '';
106	if ($row->{record_type} eq "biblio"){
107	my $harvested_biblio = Koha::OAI::Harvester::Biblios->new->find(
108	{
109	oai_repository => $row->{repository},
110	oai_identifier => $row->{header_identifier},
111	},
112	{ key => "oai_record" },
113	);
114	$row->{imported_record} = $harvested_biblio->biblionumber if $harvested_biblio;
115	}
116	push(@rows,$row);
117	}
118	if ($count){
119	$data->{recordsTotal} = $count;
120	$data->{recordsFiltered} = $filtered_count;
121	$data->{data} = \@rows if @rows;
122	}
123
124	binmode STDOUT, ":encoding(UTF-8)";
125	print $input->header(
126	-type => 'application/json',
127	-charset => 'UTF-8'
128	);
129	print to_json($data, { pretty => 1, });

Line 0 Link Here

(-)a/tools/oai-pmh-harvester/dashboard.pl (+134 lines)
		1	#!/usr/bin/perl
		2
		3	# Copyright 2017 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19
		20	use Modern::Perl;
		21	use YAML;
		22
		23	use C4::Auth;
		24	use C4::Context;
		25	use C4::Output;
		26	use Koha::OAI::Harvester::Client;
		27	use Koha::OAI::Harvester::Requests;
		28	use Koha::BiblioFrameworks;
		29	use Koha::Database;
		30
		31	my $context = C4::Context->new();
		32	my $config_filename = $context->{config}->{oai_pmh_harvester_config};
		33	my $client_config = {};
		34	if ($config_filename){
		35	my $config = YAML::LoadFile($config_filename);
		36	if ($config && $config->{socket}){
		37	$client_config->{socket_uri} = $config->{socket};
		38	}
		39	}
		40
		41	my $input = new CGI;
		42
		43	my ($template, $loggedinuser, $cookie) =
		44	get_template_and_user({template_name => "tools/oai-pmh-harvester/dashboard.tt",
		45	query => $input,
		46	type => "intranet",
		47	authnotrequired => 0,
		48	flagsrequired => {tools => 'manage_staged_marc'},
		49	});
		50
		51	my $op = $input->param('op') // 'list';
		52	my $id = $input->param('id');
		53	my $uuid = $input->param('uuid');
		54
		55	my $client = Koha::OAI::Harvester::Client->new($client_config);
		56	my $is_connected = $client->connect;
		57
		58	if ( ($op eq "send") && $id ){
		59	if ($is_connected){
		60	my $request = Koha::OAI::Harvester::Requests->find($id);
		61	if ($request){
		62	my $task = {
		63	name => $request->name,
		64	uuid => $request->uuid,
65	interval => $request->interval,
66	parameters => {
67	oai_pmh => {
68	baseURL => $request->http_url,
69	verb => $request->oai_verb,
70	metadataPrefix => $request->oai_metadataPrefix,
71	identifier => $request->oai_identifier,
72	set => $request->oai_set,
73	from => $request->oai_from,
74	until => $request->oai_until,
75	},
76	import => {
77	filter => $request->import_filter,
78	frameworkcode => $request->import_framework_code,
79	matcher_code => $request->import_matcher_code,
80	record_type => $request->import_record_type,
81	},
82	},
83	};
84	if ($request->http_username && $request->http_password && $request->http_realm){
85	$task->{parameters}->{http_basic_auth} = {
86	username => $request->http_username,
87	password => $request->http_password,
88	realm => $request->http_realm,
89	};
90	}
91	my $is_created = $client->create($task);
92	$template->{VARS}->{ result }->{ send } = $is_created;
93	}
94	}
95	}
96	elsif ( ($op eq "start") && ($uuid) ){
97	if ($is_connected){
98	my $is_started = $client->start($uuid);
99	$template->{VARS}->{ result }->{ start } = $is_started;
100	}
101	}
102	elsif ( ($op eq "stop") && ($uuid) ){
103	if ($is_connected){
104	my $is_stopped = $client->stop($uuid);
105	$template->{VARS}->{ result }->{ stop } = $is_stopped;
106	}
107	}
108	elsif ( ($op eq "delete") && ($uuid) ){
109	if ($is_connected){
110	my $is_deleted = $client->delete($uuid);
111	$template->{VARS}->{ result }->{ delete } = $is_deleted;
112	}
113	}
114
115	my $requests = Koha::OAI::Harvester::Requests->as_list;
116	$template->{VARS}->{ saved_requests } = $requests;
117
118	my $frameworks = Koha::BiblioFrameworks->as_list();
119	$template->{VARS}->{ frameworks } = $frameworks;
120
121	my $schema = Koha::Database->new()->schema();
122	my $matcher_rs = $schema->resultset("MarcMatcher");
123	my @matchers = $matcher_rs->all;
124	$template->{VARS}->{ matchers } = \@matchers;
125
126	if ($is_connected){
127	my $submitted_requests = $client->list;
128	$template->{VARS}->{ submitted_requests } = $submitted_requests;
129	}
130	else {
131	$template->{VARS}->{ harvester }->{ offline } = 1;
132	}
133
134	output_html_with_http_headers($input, $cookie, $template->output);

Line 0 Link Here

(-)a/tools/oai-pmh-harvester/record.pl (+47 lines)
	1	#!/usr/bin/perl
	2
	3	# Copyright 2017 Prosentient Systems
	4	#
	5	# This file is part of Koha.
	6	#
	7	# Koha is free software; you can redistribute it and/or modify it
	8	# under the terms of the GNU General Public License as published by
	9	# the Free Software Foundation; either version 3 of the License, or
	10	# (at your option) any later version.
	11	#
	12	# Koha is distributed in the hope that it will be useful, but
	13	# WITHOUT ANY WARRANTY; without even the implied warranty of
	14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	# GNU General Public License for more details.
	16	#
	17	# You should have received a copy of the GNU General Public License
	18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
	19
	20	use Modern::Perl;
	21
	22	use C4::Auth;
	23	use C4::Output;
	24	use Koha::OAI::Harvester::Histories;
	25
	26	my $input = new CGI;
	27
	28	my ($template, $loggedinuser, $cookie) =
	29	get_template_and_user({template_name => "tools/oai-pmh-harvester/record.tt",
	30	query => $input,
	31	type => "intranet",
	32	authnotrequired => 0,
	33	flagsrequired => {tools => 'manage_staged_marc'},
	34	});
	35
	36	my $import_oai_id = $input->param('import_oai_id');
	37	if ($import_oai_id){
	38	my $history = Koha::OAI::Harvester::Histories->new->find($import_oai_id);
	39	if ($history){
	40	my $record = $history->record;
	41	if ($record){
	42	$template->{VARS}->{ record } = $record;
	43	}
	44	}
	45	}
	46
	47	output_html_with_http_headers($input, $cookie, $template->output);

Line 0 Link Here

(-)a/tools/oai-pmh-harvester/request.pl (-1 / +142 lines)
0	-	1	#!/usr/bin/perl
		2
		3	# Copyright 2017 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19
		20	use Modern::Perl;
		21	use UUID;
		22
		23	use C4::Auth;
		24	use C4::Output;
		25	use Koha::OAI::Harvester::Requests;
		26	use Koha::BiblioFrameworks;
		27	use Koha::Database;
		28
		29	my $input = new CGI;
		30
		31	my ($template, $loggedinuser, $cookie) =
		32	get_template_and_user({template_name => "tools/oai-pmh-harvester/request.tt",
		33	query => $input,
		34	type => "intranet",
		35	authnotrequired => 0,
		36	flagsrequired => {tools => 'manage_staged_marc'},
		37	});
		38
		39	my $op = $input->param('op');
		40	my $id = $input->param('id');
		41
		42	my @frameworks = Koha::BiblioFrameworks->as_list();
		43	$template->{VARS}->{ frameworks } = \@frameworks;
		44
		45	my $schema = Koha::Database->new()->schema();
		46	my $rs = $schema->resultset("MarcMatcher");
		47	my @matchers = $rs->all;
		48	$template->{VARS}->{ matchers } = \@matchers;
		49
		50	my $http_url = $input->param('http_url');
		51	my $http_username = $input->param('http_username');
		52	my $http_password = $input->param('http_password');
		53	my $http_realm = $input->param('http_realm');
		54
		55	my $oai_verb = $input->param('oai_verb');
		56	my $oai_metadataPrefix = $input->param('oai_metadataPrefix');
		57	my $oai_identifier = $input->param('oai_identifier');
		58	my $oai_from = $input->param('oai_from');
		59	my $oai_until = $input->param('oai_until');
		60	my $oai_set = $input->param('oai_set');
		61
		62	my $import_filter = $input->param('import_filter') // 'default';
		63	my $import_framework_code = $input->param('import_framework_code');
		64	my $import_record_type = $input->param('import_record_type');
		65	my $import_matcher_code = $input->param('import_matcher_code');
66
67	my $interval = $input->param("interval") ? int ( $input->param("interval") ) : 0;
68	my $name = $input->param("name");
69
70	my $save = $input->param('save');
71	my $test_parameters = $input->param('test_parameters');
72
73	my $request = $id ? Koha::OAI::Harvester::Requests->find($id) : Koha::OAI::Harvester::Request->new();
74	if ($request){
75	if ($op eq "create" \|\| $op eq "update"){
76	$request->set({
77	name => $name,
78	http_url => $http_url,
79	http_username => $http_username,
80	http_password => $http_password,
81	http_realm => $http_realm,
82	oai_verb => $oai_verb,
83	oai_metadataPrefix => $oai_metadataPrefix,
84	oai_identifier => $oai_identifier,
85	oai_from => $oai_from,
86	oai_until => $oai_until,
87	oai_set => $oai_set,
88	import_filter => $import_filter,
89	import_framework_code => $import_framework_code,
90	import_record_type => $import_record_type,
91	import_matcher_code => $import_matcher_code,
92	interval => $interval,
93	});
94	}
95	}
96
97	if ($test_parameters){
98	my $errors = $request->validate();
99	$template->{VARS}->{ errors } = $errors;
100	$template->{VARS}->{ test_parameters } = 1;
101	}
102
103	if ($op eq "new"){
104	#Empty form with some defaults
105	$request->import_filter("default") unless $request->import_filter;
106	$request->interval(0) unless $request->interval;
107	}
108	elsif ($op eq "create"){
109	if ($save){
110	my ($uuid,$uuid_string);
111	UUID::generate($uuid);
112	UUID::unparse($uuid, $uuid_string);
113	$request->uuid($uuid_string);
114	$request->store;
115	print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
116	exit;
117	}
118	}
119	elsif ( $op eq "edit"){
120	$template->{VARS}->{ id } = $id;
121	}
122	elsif ($op eq "update"){
123	$template->{VARS}->{ id } = $id;
124	if ($save){
125	$request->store;
126	print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
127	exit;
128	}
129	}
130	elsif ($op eq "delete"){
131	if ($request){
132	$request->delete;
133	print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
134	}
135	}
136	else {
137	print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
138	}
139	$template->{VARS}->{ op } = $op;
140	$template->{VARS}->{ oai_pmh_request } = $request;
141
142	output_html_with_http_headers($input, $cookie, $template->output);