Attachment #64479 for bug #10662

View | Details | Raw Unified | Return to bug 10662
Collapse All | Expand All




        'required' => '1',
        'min_ver' => '1.017',
    },
    'POE' => {
        'usage' => 'OAI-PMH harvester',
        'required' => 1,
        'min_ver' => '1.354',
    },
    'POE::Component::JobQueue' => {
        'usage' => 'OAI-PMH harvester',
        'required' => 1,
        'min_ver' => '0.570',
    },
};

1;




package Koha::Daemon;

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.
#

use Modern::Perl;
use POSIX; #For daemonizing
use Fcntl qw(:flock); #For pidfile

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;
    return bless ($args, $class);
}

#######################################################################
#NOTE: On Debian, you can use the daemon binary to make a process into a daemon,
# but the following can be used if you don't want to use that program.

sub daemonize {
    my ($self) = @_;

    my $pid = fork;

    die "Couldn't fork: $!" unless defined($pid);
    if ($pid){
        exit; #Parent exit
    }

    #Become a session leader (ie detach program from controlling terminal)
    POSIX::setsid() or die "Can't start a new session: $!";

    #Change to known system directory
    chdir('/');

    #Reset the file creation mask so only the daemon owner can read/write files it creates
    umask(066);

    #Close inherited file handles, so that you can truly run in the background.
    open STDIN,  '<', '/dev/null';
    open STDOUT, '>', '/dev/null';
    open STDERR, '>&STDOUT';
}

sub log_to_file {
    my ($self,$logfile) = @_;

    #Open a filehandle to append to a log file
    my $opened = open(LOG, '>>', $logfile);
    if ($opened){
        LOG->autoflush(1); #Make filehandle hot (ie don't buffer)
        *STDOUT = *LOG; #Re-assign STDOUT to LOG | --stdout
        *STDERR = *STDOUT; #Re-assign STDERR to STDOUT | --stderr
    }
    else {
        die "Unable to open a filehandle for $logfile: $!\n"; # --output
    }
}

sub make_pidfilehandle {
    my ($self,$pidfile) = @_;
    if ( ! -f $pidfile ){
        open(my $fh, '>', $pidfile) or die "Unable to write to $pidfile: $!\n";
    }
    open(my $pidfilehandle, '+<', $pidfile) or die "Unable to open a filehandle for $pidfile: $!\n";
    return $pidfilehandle;
}

sub get_pidfile {
    my ($self,$pidfile) = @_;
    #NOTE: We need to save the filehandle in the object, so any locks persist for the life of the object
    my $pidfilehandle = $self->{pidfilehandle} ||= $self->make_pidfilehandle($pidfile);
    return $pidfilehandle;
}

sub lock_pidfile {
    my ($self,$pidfilehandle) = @_;
    my $locked;
    if (flock($pidfilehandle, LOCK_EX|LOCK_NB)){
        $locked = 1;

    }
    return $locked;
}

sub write_pidfile {
    my ($self,$pidfilehandle) = @_;
    if ($pidfilehandle){
        truncate($pidfilehandle, 0);
        print $pidfilehandle $$."\n" or die $!;
        #Flush the filehandle so you're not suffering from buffering
        $pidfilehandle->flush();
        return 1;
    }
}

sub run {
    my ($self) = @_;
    my $pidfile = $self->{pidfile};
    my $logfile = $self->{logfile};

    if ($pidfile){
        my $pidfilehandle = $self->get_pidfile($pidfile);
        if ($pidfilehandle){
            my $locked = $self->lock_pidfile($pidfilehandle);
            if ( ! $locked ) {
                die "$0 is unable to lock pidfile...\n";
            }
        }
    }

    if (my $configure = $self->{configure}){
        $configure->($self);
    }

    if ($self->{daemonize}){
        $self->daemonize();
    }

    if ($pidfile){
        my $pidfilehandle = $self->get_pidfile($pidfile);
        if ($pidfilehandle){
            $self->write_pidfile($pidfilehandle);
        }
    }

    if ($logfile){
        $self->log_to_file($logfile);
    }

    if (my $loop = $self->{loop}){
        $loop->($self);
    }
}

1;




package Koha::OAI::Harvester;

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.
#

use Modern::Perl;
use POE qw(Component::JobQueue);
use JSON;
use Sereal::Encoder;
use Sereal::Decoder;
use IO::Handle;
use File::Copy;
use File::Path qw(make_path remove_tree);
use DateTime;
use DateTime::Format::Strptime;

use C4::Context;
use Koha::Database;

my $day_granularity = DateTime::Format::Strptime->new(
    pattern   => '%F',
);
my $seconds_granularity = DateTime::Format::Strptime->new(
    pattern   => '%FT%TZ',
);

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;
    return bless ($args, $class);
}

sub spawn {
    my ($class, $args) = @_;
    my $self = $class->new($args);
    my $downloader = $self->{Downloader};
    my $importer = $self->{Importer};
    my $download_worker_limit = ( $self->{DownloaderWorkers} && int($self->{DownloaderWorkers}) ) ? $self->{DownloaderWorkers} : 1;
    my $import_worker_limit = ( $self->{ImporterWorkers} && int($self->{ImporterWorkers}) ) ? $self->{ImporterWorkers} : 1;
    my $import_queue_poll = ( $self->{ImportQueuePoll} && int($self->{ImportQueuePoll}) ) ? $self->{ImportQueuePoll} : 5;

    #NOTE: This job queue should always be created before the
    #harvester so that you can start download jobs immediately
    #upon spawning the harvester.
    POE::Component::JobQueue->spawn(
        Alias         => 'oai-downloader',
        WorkerLimit   => $download_worker_limit,
        Worker        => sub {
            my ($postback, $task) = @_;
            if ($downloader){
                if ($task->{status} eq "active"){
                    $downloader->run({
                        postback => $postback,
                        task => $task,
                   });
                }
            }
        },
        Passive => {},
    );

    POE::Session->create(
        object_states => [
            $self => {
                _start => "on_start",
                get_task => "get_task",
                list_tasks => "list_tasks",
                create_task => "create_task",
                start_task => "start_task",
                stop_task => "stop_task",
                delete_task => "delete_task",
                repeat_task => "repeat_task",
                register => "register",
                deregister => "deregister",
                restore_state => "restore_state",
                save_state => "save_state",
                is_task_finished => "is_task_finished",
                does_task_repeat => "does_task_repeat",
                download_postback => "download_postback",
                reset_imports_status => "reset_imports_status",
            },
        ],
    );

    POE::Component::JobQueue->spawn(
        Alias         => 'oai-importer',
        WorkerLimit   => $import_worker_limit,
        Worker        => sub {
            my $meta_postback = shift;

            #NOTE: We need to only retrieve queue items for active tasks. Otherwise,
            #the importer will just spin its wheels on inactive tasks and do nothing.
            my $active_tasks = $poe_kernel->call("harvester","list_tasks","active");
            my @active_uuids = map { $_->{uuid} } @$active_tasks;

            my $schema = Koha::Database->new()->schema();
            my $rs = $schema->resultset('OaiHarvesterImportQueue')->search({
                uuid => \@active_uuids,
                status => "new"
            },{
                order_by => { -asc => 'id' },
                rows => 1,
            });
            my $result = $rs->first;
            if ($result){
                $result->status("wip");
                $result->update;
                my $task = {
                    id => $result->id,
                    uuid => $result->uuid,
                    result => $result->result,
                };

                my $postback = $meta_postback->($task);
                $importer->run({
                    postback => $postback,
                    task => $task,
                });
            }
        },
        Active => {
            PollInterval => $import_queue_poll,
            AckAlias => undef,
            AckState => undef,
        },
    );

    return;
}

sub on_start {
    my ($self, $kernel, $heap) = @_[OBJECT, KERNEL,HEAP];
    $kernel->alias_set('harvester');
    $heap->{scoreboard} = {};

    #Reset any 'wip' imports to 'new' so they can be re-tried.
    $kernel->call("harvester","reset_imports_status");

    #Restore state from state file
    $kernel->call("harvester","restore_state");
}

#NOTE: This isn't really implemented at the moment, as it's not really necessary.
sub download_postback {
    my ($kernel, $request_packet, $response_packet) = @_[KERNEL, ARG0, ARG1];
    my $message = $response_packet->[0];
}

=head3 deregister

    Remove the worker session from the harvester's in-memory scoreboard,
    unset the downloading flag if downloading is completed.

=cut

sub deregister {
    my ($self, $kernel, $heap, $session, $sender, $type) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0];

    my $scoreboard = $heap->{scoreboard};

    my $logger = $self->{logger};
    $logger->debug("Start deregistering $sender as $type task");

    my $task_uuid = delete $scoreboard->{session}->{$sender};
    #NOTE: If you don't check each step of the hashref, autovivication can lead to surprises.
    if ($task_uuid){
        if ($scoreboard->{task}->{$task_uuid}){
            if ($scoreboard->{task}->{$task_uuid}->{$type}){
                delete $scoreboard->{task}->{$task_uuid}->{$type}->{$sender};
            }
        }
    }

    my $task = $heap->{tasks}->{$task_uuid};
    if ($task && $task->{status} && ($task->{status} eq "active") ){
        #NOTE: Each task only has 1 download session, so we can now set/unset flags for the task.
        #NOTE: We should unset the downloading flag, if we're not going to repeat the task.
        if ($type eq "download"){
            my $task_repeats = $kernel->call("harvester","does_task_repeat",$task_uuid);
            if ($task_repeats){
                my $interval = $task->{interval};

                $task->{effective_from} = delete $task->{effective_until};
                $task->{download_timer} = $kernel->delay_set("repeat_task", $interval, $task_uuid);
            }
            else {
                $task->{downloading} = 0;
                $kernel->call("harvester","save_state");
                $kernel->call("harvester","is_task_finished",$task_uuid);
            }
        }
        elsif ($type eq 'import'){
            $kernel->call("harvester","is_task_finished",$task_uuid);
        }
    }
    $logger->debug("End deregistering $sender as $type task");
}


=head3 is_task_finished

    This event handler checks if the task has finished downloading and importing record.
    If it is finished downloading and importing, the task is deleted from the harvester.

    This only applies to non-repeating tasks.

=cut

sub is_task_finished {
    my ($self, $kernel, $heap, $session, $uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
    my $task = $kernel->call("harvester","get_task",$uuid);
    if ($task && (! $task->{downloading}) ){
        my $count = $self->get_import_count_for_task($uuid);
        if ( ! $count ) {
            #Clear this task out of the harvester as it's finished.
            $kernel->call("harvester","delete_task",$uuid);
            return 1;
        }
    }
    return 0;
}

sub register {
    my ($self, $kernel, $heap, $session, $sender, $type, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0,ARG1];
    my $logger = $self->{logger};

    my $scoreboard = $heap->{scoreboard};


    if ($type && $task_uuid){
        $logger->debug("Registering $sender as $type for $task_uuid");

        my $task = $heap->{tasks}->{$task_uuid};
        if ($task){

            if ($type){
                #Register the task uuid with the session id as a key for later recall
                $scoreboard->{session}->{$sender} = $task_uuid;

                #Register the session id as a certain type of session for a task
                $scoreboard->{task}->{$task_uuid}->{$type}->{$sender} = 1;

                if ($type eq "download"){
                    $task->{downloading} = 1;

                    my $task_repeats = $kernel->call("harvester","does_task_repeat",$task_uuid);
                    if ($task_repeats){

                        #NOTE: Set an effective until, so we know we're not getting records any newer than
                        #this moment.
                        my $dt = DateTime->now();
                        if ($dt){
                            #NOTE: Ideally, I'd like to make sure that we can use 'seconds' granularity, but
                            #it's valid for 'from' to be null, so it's impossible to know from the data whether
                            #or not the repository will support the seconds granularity.
                            #NOTE: Ideally, it would be good to use either 'day' granularity or 'seconds' granularity,
                            #but at the moment the interval is expressed as seconds only.
                            $dt->set_formatter($seconds_granularity);
                            $task->{effective_until} = "$dt";
                        }
                    }

                    $kernel->call("harvester","save_state");
                }
            }
        }
    }
}

sub does_task_repeat {
    my ($self, $kernel, $heap, $session, $uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
    my $task = $kernel->call("harvester","get_task",$uuid);
    if ($task){
        my $interval = $task->{interval};
        my $parameters = $task->{parameters};
        my $oai_pmh = $parameters->{oai_pmh} if $parameters->{oai_pmh};
        if ( $interval && ($oai_pmh->{verb} eq "ListRecords") && (! $oai_pmh->{until}) ){
            return 1;
        }
    }
    return 0;
}



sub reset_imports_status {
    my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION];

    my $schema = Koha::Database->new()->schema();
    my $rs = $schema->resultset('OaiHarvesterImportQueue')->search({
                status => "wip",
    });
    $rs->update({
        status => "new",
    });
}

sub restore_state {
    my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION];

    my $state_file = $self->{state_file};
    my $state_backup = "$state_file~";

    #NOTE: If there is a state backup, it means we crashed while saving the state. Otherwise,
    #let's try the regular state file if it exists.
    my $file_to_restore = ( -f $state_backup ) ? $state_backup : ( ( -f $state_file ) ? $state_file : undef );
    if ( $file_to_restore ){
        my $opened = open( my $fh, '<', $file_to_restore ) or die "Couldn't open state: $!";
        if ($opened){
            local $/;
            my $in = <$fh>;
            my $decoder = Sereal::Decoder->new;
            my $state = $decoder->decode($in);
            if ($state){
                if ($state->{tasks}){
                    #Restore tasks from our saved state
                    $heap->{tasks} = $state->{tasks};
                    foreach my $uuid ( keys %{$heap->{tasks}} ){
                        my $task = $heap->{tasks}->{$uuid};

                        #If tasks were still downloading, restart the task
                        if ( ($task->{status} && $task->{status} eq "active") && $task->{downloading} ){
                            $task->{status} = "new";
                            $kernel->call("harvester","start_task",$task->{uuid});
                        }
                    }
                }
            }
        }
    }
}

sub save_state {
    my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION];
    my $state_file = $self->{state_file};
    my $state_backup = "$state_file~";

    #Make a backup of existing state record
    my $moved = move($state_file,$state_backup);

    my $opened = open(my $fh, ">", $state_file) or die "Couldn't save state: $!";
    if ($opened){
        $fh->autoflush(1);
        my $tasks = $heap->{tasks};
        my $harvester_state = {
            tasks => $tasks,
        };
        my $encoder = Sereal::Encoder->new;
        my $out = $encoder->encode($harvester_state);
        local $\;
        my $printed = print $fh $out;
        if ($printed){
            close $fh;
            unlink($state_backup);
            return 1;
        }
    }
    return 0;
}

=head3 get_task

    This event handler returns a task from a harvester using the task's
    uuid as an argument.

=cut

sub get_task {
    my ($self, $kernel, $heap, $session, $uuid, $sender) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0, SENDER];

    if ( ! $uuid && $sender ){
        my $scoreboard = $heap->{scoreboard};
        my $uuid_by_session = $scoreboard->{session}->{$sender};
        if ($uuid_by_session){
            $uuid = $uuid_by_session;
        }
    }

    my $tasks = $heap->{tasks};
    if ($tasks && $uuid){
        my $task = $tasks->{$uuid};
        if ($task){
            return $task;
        }
    }
    return 0;
}

=head3 get_import_count_for_task

=cut

sub get_import_count_for_task {
    my ($self,$uuid) = @_;
    my $count = undef;
    if ($uuid){
        my $schema = Koha::Database->new()->schema();
        my $items = $schema->resultset('OaiHarvesterImportQueue')->search({
            uuid => $uuid,
        });
        $count = $items->count;
    }
    return $count;
}

=head3 list_tasks

    This event handler returns a list of tasks that have been submitted
    to the harvester. It returns data like uuid, status, parameters,
    number of pending imports, etc.

=cut

sub list_tasks {
    my ($self, $kernel, $heap, $session, $status) = @_[OBJECT, KERNEL,HEAP,SESSION, ARG0];
    my $schema = Koha::Database->new()->schema();
    my @tasks = ();
    foreach my $uuid (sort keys %{$heap->{tasks}}){
        my $task = $heap->{tasks}->{$uuid};
        my $items = $schema->resultset('OaiHarvesterImportQueue')->search({
            uuid => $uuid,
        });
        my $count = $items->count // 0;
        $task->{pending_imports} = $count;
        if ( ( ! $status ) || ( $status && $status eq $task->{status} ) ){
            push(@tasks, $task);
        }

    }
    return \@tasks;
}

=head3 create_task

    This event handler creates a spool directory for the task's imports.
    It also adds it to the harvester's memory and then saves memory to
    a persistent datastore.

    Newly created tasks have a status of "new".

=cut

sub create_task {
    my ($self, $kernel, $heap, $session, $incoming_task) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
    my $logger = $self->{logger};
    if ($incoming_task){
        my $uuid = $incoming_task->{uuid};
        if ( ! $heap->{tasks}->{$uuid} ){

            #Step One: assign a spool directory to this task
            my $spooldir = $self->{spooldir} // "/tmp";
            my $task_spooldir = "$spooldir/$uuid";
            if ( ! -d $task_spooldir ){
                my $made_spool_directory = make_path($task_spooldir);
                if ( ! $made_spool_directory ){
                    if ($logger){
                        $logger->warn("Unable to make task-specific spool directory at '$task_spooldir'");
                    }
                    return 0;
                }
            }
            $incoming_task->{spooldir} = $task_spooldir;

            #Step Two: assign new status
            $incoming_task->{status} = "new";

            #Step Three: add task to harvester's memory
            $heap->{tasks}->{ $uuid } = $incoming_task;

            #Step Four: save state
            $kernel->call($session,"save_state");
            return 1;
        }
    }
    return 0;
}

=head3 start_task

    This event handler marks a task as active in the harvester's memory,
    save the memory to a persistent datastore, then enqueues the task,
    so that it can be directed to the next available download worker.

    Newly started tasks have a status of "active".

=cut

sub start_task {
    my ($self, $session,$kernel,$heap,$uuid) = @_[OBJECT, SESSION,KERNEL,HEAP,ARG0];
    my $task = $heap->{tasks}->{$uuid};
    if ($task){
        if ( $task->{status} ne "active" ){

            #Clear any pre-existing error states
            delete $task->{error} if $task->{error};

            #Step One: mark task as active
            $task->{status} = "active";

            #Step Two: save state
            $kernel->call("harvester","save_state");

            #Step Three: enqueue task
            $kernel->post("oai-downloader",'enqueue','download_postback', $task);

            return 1;
        }
    }
    return 0;
}

=head3 repeat_task



=cut

sub repeat_task {
    my ($self, $session,$kernel,$heap,$uuid) = @_[OBJECT, SESSION,KERNEL,HEAP,ARG0];
    my $task = $heap->{tasks}->{$uuid};
    if ($task){
        my $interval = $task->{interval};
        if ($task->{downloading} && $interval){
            $kernel->post("oai-downloader",'enqueue','download_postback', $task);
        }
    }
}

=head3 stop_task

    This event handler prevents new workers from spawning, kills
    existing workers, and stops pending imports from being imported.

    Newly stopped tasks have a status of "stopped".

=cut

sub stop_task {
    my ($self, $kernel, $heap, $session, $sender, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0];

    my $task = $heap->{tasks}->{$task_uuid};

    if ($task && $task->{status} && $task->{status} ne "stopped" ){

        #Step One: deactivate task, so no new workers can be started
        $task->{status} = "stopped";
        #NOTE: You could also clear timers for new downloads, but that's probably unnecessary because of this step.

        #Step Two: kill workers
        my $scoreboard = $heap->{scoreboard};
        my $session_types = $scoreboard->{task}->{$task_uuid};
        if ($session_types){
            foreach my $type ( keys %$session_types ){
                my $sessions = $session_types->{$type};
                if ($sessions){
                    foreach my $session (keys %$sessions){
                        if ($session){
                            $kernel->signal($session, "cancel");
                        }
                    }
                }
            }
            #Remove the task uuid from the task key of the scoreboard
            delete $scoreboard->{task}->{$task_uuid};
            #NOTE: The task uuid will still exist under the session key,
            #but the sessions will deregister themselves and clean that up for you.
        }

        #Step Three: stop pending imports for this task
        my $schema = Koha::Database->new()->schema();
        my $items = $schema->resultset('OaiHarvesterImportQueue')->search({
            uuid => $task_uuid,
        });
        my $rows_updated = $items->update({
            status => "stopped",
        });

        #Step Four: save state
        $kernel->call("harvester","save_state");
        return 1;
    }
    return 0;
}

=head3 delete_task

    Deleted tasks are stopped, pending imports are deleted from the
    database and file system, and then the task is removed from the harvester.

=cut

sub delete_task {
    my ($self, $kernel, $heap, $session, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];

    my $task = $heap->{tasks}->{$task_uuid};
    if ($task){
        #Step One: stop task
        $kernel->call($session,"stop_task",$task_uuid);

        #Step Two: delete pending imports in database
        my $schema = Koha::Database->new()->schema();
        my $items = $schema->resultset('OaiHarvesterImportQueue')->search({
            uuid => $task_uuid,
        });
        if ($items){
            my $rows_deleted = $items->delete;
            #NOTE: shows 0E0 instead of 0
        }

        #Step Three: remove task specific spool directory and files within it
        my $spooldir = $task->{spooldir};
        if ($spooldir){
            my $files_deleted = remove_tree($spooldir, { safe => 1 });
        }

        delete $heap->{tasks}->{$task_uuid};

        #Step Four: save state
        $kernel->call("harvester","save_state");
        return 1;
    }
    return 0;
}

1;




package Koha::OAI::Harvester::Client;

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;
use URI;
use IO::Socket::UNIX;
use IO::Select;
use JSON;

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;
    return bless ($args, $class);
}

sub connect {
    my ($self) =  @_;
    my $socket_uri = $self->{socket_uri};
    if ($socket_uri){
        my $uri = URI->new($socket_uri);
        if ($uri && $uri->scheme eq 'unix'){
            my $socket_path = $uri->path;
            my $socket = IO::Socket::UNIX->new(
                Type => IO::Socket::UNIX::SOCK_STREAM(),
                Peer => $socket_path,
            );
            if ($socket){
                my $select = new IO::Select();
                $select->add($socket);

                $self->{_select} = $select;
                $self->{_socket} = $socket;
                my $message = $self->_read();
                if ($message){
                    if ($message eq 'HELLO'){
                        $self->{_connected} = 1;
                        return 1;
                    }
                }
            }
            else {
                warn "Failed to create socket."
            }
        }
    }
    return 0;
}

sub create {
    my ($self,$task) = @_;
    my $message = {
        command => "create",
        body => {
            task => $task,
        }
    };
    my ($status) = $self->_exchange($message);
    return $status;
}

sub start {
    my ($self,$uuid) = @_;
    my $message = {
        command => "start",
        body => {
            task => {
                uuid => $uuid,
            },
        }
    };
    my ($status) = $self->_exchange($message);
    return $status;
}

sub stop {
    my ($self,$uuid) = @_;
    my $message = {
        command => "stop",
        body => {
            task => {
                uuid => $uuid,
            },
        }
    };
    my ($status) = $self->_exchange($message);
    return $status;
}

sub delete {
    my ($self,$uuid) = @_;
    my $message = {
        command => "delete",
        body => {
            task => {
                uuid => $uuid,
            },
        }
    };
    my ($status) = $self->_exchange($message);
    return $status;
}

sub list {
    my ($self) = @_;
    my $message = {
        command => "list",
    };
    my ($status,$tasks) = $self->_exchange($message);
    return $tasks;
}

sub _exchange {
    my ($self,$message) = @_;
    my $status = 0;
    my $data;
    if ($message){
        my $output = to_json($message);
        if ($output){
            $self->_write($output);
            my $json_response = $self->_read();
            if ($json_response){
                my $response = from_json($json_response);
                $data = $response->{data} if $response->{data};
                $status = 1 if $response->{msg} && $response->{msg} eq "success";
            }
        }
    }
    return ($status,$data);
}

sub _write {
    my ($self, $output) = @_;
    if ($output){
        if (my $select = $self->{_select}){
            if (my @filehandles = $select->can_write(5)){
                foreach my $filehandle (@filehandles){
                    #Localize output record separator as null
                    local $\ = "\x00";
                    print $filehandle $output;
                }
            }
        }
    }
}

sub _read {
    my ($self) = @_;
    if (my $select = $self->{_select}){
        if (my @filehandles = $select->can_read(5)){
            foreach my $filehandle (@filehandles){
                #Localize input record separator as null
                local $/ = "\x00";
                my $message = <$filehandle>;
                chomp($message) if $message;
                return $message;
            }
        }
    }
}

1;




package Koha::OAI::Harvester::Downloader;

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;
use URI;
use XML::LibXML::Reader;
use IO::Handle;
use JSON;

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;
    return bless ($args, $class);
}

=head2 BuildURL

    Takes a baseURL and a mix of required and optional OAI-PMH arguments,
    and makes them into a suitable URL for an OAI-PMH request.

=cut

sub BuildURL {
    my ($self, $args) = @_;
    my $baseURL = $args->{baseURL};
    my $url = URI->new($baseURL);
    if ($url && $url->isa("URI")){
        my $verb = $args->{verb};
        if ($verb){
            my %parameters = (
                verb => $verb,
            );
            if ($verb eq "ListRecords"){
                my $resumptionToken = $args->{resumptionToken};
                my $metadataPrefix = $args->{metadataPrefix};
                if ($resumptionToken){
                    $parameters{resumptionToken} = $resumptionToken;
                }
                elsif ($metadataPrefix){
                    $parameters{metadataPrefix} = $metadataPrefix;
                    #Only add optional parameters if they're provided
                    foreach my $param ( qw( from until set ) ){
                        $parameters{$param} = $args->{$param} if $args->{$param};
                    }
                }
                else {
                    warn "BuildURL() requires an argument of either resumptionToken or metadataPrefix";
                    return;
                }
            }
            elsif ($verb eq "GetRecord"){
                my $metadataPrefix = $args->{metadataPrefix};
                my $identifier = $args->{identifier};
                if ($metadataPrefix && $identifier){
                    $parameters{metadataPrefix} = $metadataPrefix;
                    $parameters{identifier} = $identifier;
                }
                else {
                    warn "BuildURL() requires an argument of metadataPrefix and an argument of identifier";
                    return;
                }
            }
            $url->query_form(%parameters);
            return $url;
        }
        else {
            warn "BuildURL() requires a verb of GetRecord or ListRecords";
            return;
        }
    }
    else {
        warn "BuildURL() requires a base URL of type URI.";
        return;
    }
}

=head2 OpenXMLStream

    Fork a child process to send the HTTP request, which sends chunks
    of XML via a pipe to the parent process.

    The parent process creates and returns a XML::LibXML::Reader object,
    which reads the XML stream coming through the pipe.

    Normally, using a DOM reader, you must wait to read the entire XML document
    into memory. However, using a stream reader, chunks are read into memory,
    processed, then discarded. It's faster and more efficient.

=cut

sub GetXMLStream {
    my ($self, $args) = @_;
    my $url = $args->{url};
    my $user_agent = $args->{user_agent};
    if ($url && $user_agent){
        pipe( CHILD, PARENT ) or die "Cannot created connected pipes: $!";
        CHILD->autoflush(1);
        PARENT->autoflush(1);
        if ( my $pid = fork ){
            #Parent process
            close PARENT;
            return \*CHILD;
        }
        else {
            #Child process
            close CHILD;
            my $response = $self->_request({
                url => $url,
                user_agent => $user_agent,
                file_handle => \*PARENT,
            });
            if ($response && $response->is_success){
                #HTTP request has successfully finished, so we close the file handle and exit the process
                close PARENT;
                CORE::exit(); #some modules like mod_perl redefine exit
            }
            else {
                warn "[child $$] OAI-PMH unsuccessful. Response status: ".$response->status_line."\n" if $response;
                CORE::exit();
            }
        }
    }
    else {
        warn "GetXMLStream() requires a 'url' argument and a 'user_agent' argument";
        return;
    }
}

sub _request {
    my ($self, $args) = @_;
    my $url = $args->{url};
    my $user_agent = $args->{user_agent};
    my $fh = $args->{file_handle};

    if ($url && $user_agent && $fh){
        my $request = HTTP::Request->new( GET => $url );
        my $response = $user_agent->request( $request, sub {
                my ($chunk_of_data, $ref_to_response, $ref_to_protocol) = @_;
                print $fh $chunk_of_data;
        });
        return $response;
    }
    else {
        warn "_request() requires a 'url' argument, 'user_agent' argument, and 'file_handle' argument.";
        return;
    }
}

sub ParseXMLStream {
    my ($self, $args) = @_;

    my $each_callback = $args->{each_callback};
    my $fh = $args->{file_handle};
    if ($fh){
        my $reader = XML::LibXML::Reader->new( FD => $fh, no_blanks => 1 );
        my $pattern = XML::LibXML::Pattern->new('oai:OAI-PMH|/oai:OAI-PMH/*', { 'oai' => "http://www.openarchives.org/OAI/2.0/" });

        my $repository;

        warn "Start parsing...";
        while (my $rv = $reader->nextPatternMatch($pattern)){
            #$rv == 1; successful
            #$rv == 0; end of document reached
            #$rv == -1; error
            if ($rv == -1){
                die "Parser error!";
            }
            #NOTE: We do this so we only get the opening tag of the element.
            next unless $reader->nodeType == XML_READER_TYPE_ELEMENT;

            my $localname = $reader->localName;
            if ( $localname eq "request" ){
                my $node = $reader->copyCurrentNode(1);
                $repository = $node->textContent;
            }
            elsif ( $localname eq "error" ){
                #See https://www.openarchives.org/OAI/openarchivesprotocol.html#ErrorConditions
                #We should probably die under all circumstances except "noRecordsMatch"
                my $node = $reader->copyCurrentNode(1);
                if ($node){
                    my $code = $node->getAttribute("code");
                    if ($code){
                        if ($code ne "noRecordsMatch"){
                            warn "Error code: $code";
                            die;
                        }
                    }
                }
            }
            elsif ( ($localname eq "ListRecords") || ($localname eq "GetRecord") ){
                my $each_pattern = XML::LibXML::Pattern->new('//oai:record|oai:resumptionToken', { 'oai' => "http://www.openarchives.org/OAI/2.0/" });
                while (my $each_rv =  $reader->nextPatternMatch($each_pattern)){
                    if ($rv == "-1"){
                        #NOTE: -1 denotes an error state
                        warn "Error getting pattern match";
                    }
                    next unless $reader->nodeType == XML_READER_TYPE_ELEMENT;
                    if ($reader->localName eq "record"){
                        my $node = $reader->copyCurrentNode(1);
                        #NOTE: Without the UTF-8 flag, UTF-8 data will be corrupted.
                        my $document = XML::LibXML::Document->new('1.0', 'UTF-8');
                        $document->setDocumentElement($node);

                       #Per record callback
                        if ($each_callback){
                            $each_callback->({
                                repository => $repository,
                                document => $document,
                            });
                        }
                    }
                    elsif ($reader->localName eq "resumptionToken"){
                        my $resumptionToken = $reader->readInnerXml;
                        return ($resumptionToken,$repository);

                    }
                }
            }
        } #/OAI-PMH document match
    }
    else {
        warn "ParseXMLStream() requires a 'file_handle' argument.";
    }
}

sub harvest {
    my ($self,$args) = @_;
    my $url = $args->{url};
    my $ua = $args->{user_agent};
    my $callback = $args->{callback};
    my $complete_callback = $args->{complete_callback};

    if ($url && $ua){

        #NOTE: http://search.cpan.org/~shlomif/XML-LibXML-2.0128/lib/XML/LibXML/Parser.pod#ERROR_REPORTING
        while($url){
            warn "URL = $url";
            warn "Creating child process to download and feed parent process parser.";
            my $stream = $self->GetXMLStream({
                url => $url,
                user_agent => $ua,
            });

            warn "Creating parent process parser.";
            my ($resumptionToken) = $self->ParseXMLStream({
                file_handle => $stream,
                each_callback => $callback,
            });
            warn "Finished parsing current XML document.";

            if ($resumptionToken){
                #If there's a resumptionToken at the end of the stream,
                #we build a new URL and repeat this process again.
                $url->query_form({
                    verb => "ListRecords",
                    resumptionToken => $resumptionToken,
                });
            }
            else {
                warn "Finished harvest.";
                last;
            }

            warn "Reap child process downloader.";
            #Reap the dead child requester process before performing another request,
            #so we don't fill up the process table with zombie children.
            while ((my $child = waitpid(-1, 0)) > 0) {
                warn "Parent $$ reaped child process $child" . ($? ? " with exit code $?" : '') . ".\n";
            }
        }

        if ($complete_callback){
            warn "Run complete callback.";

            #Clear query string
            $url->query_form({});

            #Run complete callback using the actual URL from the request.
            $complete_callback->({
                repository => $url,
            });
        }
    }
}

1;




package Koha::OAI::Harvester::Import::MARCXML;

# Copyright 2016 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.
#

use Modern::Perl;
use MARC::Record;

use C4::Context;
use C4::Biblio;

use constant MAX_MATCHES => 99999; #NOTE: This is an arbitrary value. We want to get all matches.

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;
    if ( (! $args->{dom}) && (! $args->{marc_record}) ){
        die "You must provide either a dom or marc_record argument to this constructor";
    }
    if ( $args->{dom} && ( ! $args->{marc_record} ) ){
        my $dom = $args->{dom};
        my $xml = $dom->toString(2);
        my $marcflavour = C4::Context->preference('marcflavour') || 'MARC21';
        my $marc_record = eval {MARC::Record::new_from_xml( $xml, "utf8", $marcflavour)};
        if ($@){
            die "Unable to create MARC::Record object";
        }
        if ($marc_record){
            $args->{marc_record} = $marc_record;
        }
    }
    return bless ($args, $class);
}

sub import_record {
    my ($self,$args) = @_;
    my $framework = $args->{framework};
    my $record_type = $args->{record_type};
    my $matcher = $args->{matcher};
    my $koha_id = $args->{koha_id};

    my $action = "error";

    #Try to find a matching Koha MARCXML record via Zebra
    if (! $koha_id && $matcher){
        my $matched_id = $self->_try_matcher({
            matcher => $matcher,
        });
        if ($matched_id){
            $koha_id = $matched_id;
        }
    }

    if ($koha_id){
        #Update
        ($action) = $self->_mod_koha_record({
            record_type => $record_type,
            framework => $framework,
            koha_id => $koha_id,
        });
    }
    else {
        #Add
        ($action,$koha_id) = $self->_add_koha_record({
            record_type => $record_type,
            framework => $framework,
        });
    }

    return ($action,$koha_id);
}

sub _try_matcher {
    my ($self, $args) = @_;
    my $marc_record = $self->{marc_record};
    my $matcher = $args->{matcher};
    my $matched_id;
    my @matches = $matcher->get_matches($marc_record, MAX_MATCHES);
    if (@matches){
        #FIXME: You can remove this sort once Bug 17710 is pushed
        @matches = sort {
            $b->{'score'} cmp $a->{'score'} or
            $b->{'record_id'} cmp $a->{'record_id'}
        } @matches;
        my $bestrecordmatch = shift @matches;
        if ($bestrecordmatch && $bestrecordmatch->{record_id}){
            $matched_id = $bestrecordmatch->{record_id};
        }
    }
    return $matched_id;
}

sub _add_koha_record {
    my ($self, $args) = @_;
    my $marc_record = $self->{marc_record};
    my $record_type = $args->{record_type} // "biblio";
    my $framework = $args->{framework};
    my $koha_id;
    my $action = "error";
    if ($record_type eq "biblio"){
        #NOTE: Strip item fields to prevent any accidentally getting through.
        C4::Biblio::_strip_item_fields($marc_record,$framework);
        my ($biblionumber,$biblioitemnumber) = C4::Biblio::AddBiblio($marc_record,$framework);
        if ($biblionumber){
            $action = "added";
            $koha_id = $biblionumber;
        }
    }
    return ($action,$koha_id);
}

sub _mod_koha_record {
    my ($self, $args) = @_;
    my $marc_record = $self->{marc_record};
    my $record_type = $args->{record_type} // "biblio";
    my $framework = $args->{framework};
    my $koha_id = $args->{koha_id};
    my $action = "error";
    if ($record_type eq "biblio"){
        #NOTE: Strip item fields to prevent any accidentally getting through.
        C4::Biblio::_strip_item_fields($marc_record,$framework);
        my $updated = C4::Biblio::ModBiblio($marc_record, $koha_id, $framework);
        if ($updated){
            $action = "updated";
        }
    }
    return ($action);
}

1;




package Koha::OAI::Harvester::Import::RDFXML;

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.
#

use Modern::Perl;

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;
	return bless ($args, $class);
}

sub import_record {
    my ($self,$args) = @_;
    my $framework = $args->{framework};
    my $record_type = $args->{record_type};
    my $matcher = $args->{matcher};
    my $koha_id = $args->{koha_id};

    my $action = "error";

    if (! $koha_id && $matcher){
        #Create minimal MARCXML record from RDFXML to be used for Zebra-based matching.
    }


    #TODO: add/update triplestore
    return ($action,$koha_id);
}

1;




package Koha::OAI::Harvester::Import::Record;

# Copyright 2016 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.
#

use Modern::Perl;
use XML::LibXML;
use XML::LibXSLT;
use URI;
use File::Basename;

use C4::Context;
use C4::Biblio;

use Koha::Database;
use Koha::OAI::Harvester::Import::MARCXML;
use Koha::OAI::Harvester::Import::RDFXML;

my $schema = Koha::Database->new()->schema();

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;

    die "You must provide a 'doc' argument to the constructor" unless $args->{doc};
    die "You must provide a 'repository' argument to the constructor" unless $args->{repository};

    if (my $doc = $args->{doc}){

        #Get the root element
        my $root = $doc->documentElement;

        #Register namespaces for searching purposes
        my $xpc = XML::LibXML::XPathContext->new();
        $xpc->registerNs('oai','http://www.openarchives.org/OAI/2.0/');

        my $xpath_identifier = XML::LibXML::XPathExpression->new("oai:header/oai:identifier");
        my $identifier = $xpc->findnodes($xpath_identifier,$root)->shift;
        $args->{header_identifier} = $identifier->textContent;

        my $xpath_datestamp = XML::LibXML::XPathExpression->new("oai:header/oai:datestamp");
        my $datestamp = $xpc->findnodes($xpath_datestamp,$root)->shift;
        $args->{header_datestamp} = $datestamp->textContent;

        my $xpath_status = XML::LibXML::XPathExpression->new(q{oai:header/@status});
        my $status_node = $xpc->findnodes($xpath_status,$root)->shift;
        $args->{header_status} = $status_node ? $status_node->textContent : "";
    }

    return bless ($args, $class);
}

sub is_deleted_upstream {
    my ($self, $args) = @_;
    if ($self->{header_status}){
        if ($self->{header_status} eq "deleted"){
            return 1;
        }
    }
    return 0;
}

sub set_filter {
    my ($self, $filter_definition) = @_;

    #Source a default XSLT to use for filtering
    my $htdocs  = C4::Context->config('intrahtdocs');
    my $theme   = C4::Context->preference("template");
    $self->{filter} = "$htdocs/$theme/en/xslt/StripOAIPMH.xsl";
    $self->{filter_type} = "xslt";

    if ($filter_definition && $filter_definition ne "default"){
        my ($filter_type, $filter) = $self->_parse_filter($filter_definition);
        if ($filter_type eq "xslt"){
            if (  -f $filter ){
                $self->{filter} = $filter;
                $self->{filter_type} = "xslt";
            }
        }
    }
}

sub _parse_filter {
    my ($self,$filter_definition) = @_;
    my ($type,$filter);
    my $filter_uri = URI->new($filter_definition);
    if ($filter_uri){
        my $scheme = $filter_uri->scheme;
        if ( ($scheme && $scheme eq "file") || ! $scheme ){
            my $path = $filter_uri->path;
            #Filters may theoretically be .xsl or .pm files
            my($filename, $dirs, $suffix) = fileparse($path,(".xsl",".pm"));
            if ($suffix){
                if ( $suffix eq ".xsl"){
                    $type = "xslt";
                    $filter = $path;
                }
            }
        }
    }
    return ($type,$filter);
}

sub filter {
    my ($self) = @_;
    my $filtered = 0;
    my $doc = $self->{doc};
    my $filter = $self->{filter};
    my $filter_type = $self->{filter_type};
    if ($doc){
        if ($filter && -f $filter){
            if ($filter_type){
                if ( $filter_type eq 'xslt' ){
                    my $xslt = XML::LibXSLT->new();
                    my $style_doc = XML::LibXML->load_xml(location => $filter);
                    my $stylesheet = $xslt->parse_stylesheet($style_doc);
                    if ($stylesheet){
                        my $results = $stylesheet->transform($doc);
						if ($results){
							my $root = $results->documentElement;
							if ($root){
								my $namespace = $root->namespaceURI;
								if ($namespace eq "http://www.loc.gov/MARC21/slim"){
									#NOTE: Both MARC21 and UNIMARC should be covered by this namespace
									my $marcxml = eval { Koha::OAI::Harvester::Import::MARCXML->new({ dom => $results, }) };
                                    if ($@){
                                        warn "Error Koha::OAI::Harvester::Import::MARCXML: $@";
                                        return undef;
                                    } else {
                                        return $marcxml;
                                    }
								} elsif ($namespace eq "http://www.w3.org/1999/02/22-rdf-syntax-ns#"){
									my $rdfxml = Koha::OAI::Harvester::Import::RDFXML->new({ dom => $results, });
									if ($rdfxml){
										return $rdfxml;
									}
								}
							}
						}
                    }
                }
            }
        }
    }
    return;
}

sub _find_koha_link {
    my ($self, $args) = @_;
    my $record_type = $args->{record_type} // "biblio";
    my $link_id;
    if ($record_type eq "biblio"){
        my $link = $schema->resultset('OaiHarvesterBiblio')->find(
            {
                oai_repository => $self->{repository},
                oai_identifier => $self->{header_identifier},
            },
            { key => "oai_record",}
        );
        if ($link && $link->biblionumber){
            $link_id = $link->biblionumber->id;
        }
    }
    return $link_id;
}

=head3

    my ($action,$record_id) = $oai_record->import_record({
        filter => $filter,
        framework => $framework,
        record_type => $record_type,
        matcher => $matcher,
    });

    $action eq "added" || "updated" || "deleted" || "not_found" || "error"

=cut

sub import_record {
    my ($self, $args) = @_;
    my $filter = $args->{filter} || 'default';
    my $framework = $args->{framework} || '';
    my $record_type = $args->{record_type} || 'biblio';
    my $matcher = $args->{matcher};

    my $action = "error";

    #Find linkage between OAI-PMH repository-identifier and Koha record id
    my $linked_id = $self->_find_koha_link({
        record_type => $record_type,
    });

    if ($self->is_deleted_upstream){
        #NOTE: If a record is deleted upstream, it will not contain a metadata element
        if ($linked_id){
            $action = $self->delete_koha_record({
                record_id => $linked_id,
                record_type => $record_type,
            });
            #TODO: How to handle RDF deletions?
            #NOTE: If we delete the Koha record, then we lose the link to our downloaded record, so I don't know if it matters...
            #...although surely we'd want some way of cleaning up records that aren't needed anymore? I suppose that could come up with a cronjob that scans for triples that aren't linked to...
        }
        else {
            $action = "not_found";
            #NOTE: If there's no OAI-PMH repository-identifier pair in the database,
            #then there's no perfect way to find a linked record to delete.
        }
    }
    else {
        $self->set_filter($filter);


        my $import_record = $self->filter();

        if ($import_record){
            ($action,$linked_id) = $import_record->import_record({
                framework => $framework,
                record_type => $record_type,
                matcher => $matcher,
                koha_id => $linked_id,
            });

            if ($linked_id){
                #Link Koha record ID to OAI-PMH details for this record type,
                #if linkage doesn't already exist.
                $self->link_koha_record({
                    record_type => $record_type,
                    koha_id => $linked_id,
                });
            }
        }
    }

    #Log record details to database
    my $importer = $schema->resultset('OaiHarvesterHistory')->create({
        header_identifier => $self->{header_identifier},
        header_datestamp => $self->{header_datestamp},
        header_status => $self->{header_status},
        record => $self->{doc}->toString(1),
        repository => $self->{repository},
        status => $action,
        filter => $filter,
        framework => $framework,
        record_type => $record_type,
        matcher_code => $matcher ? $matcher->code : undef,
    });

    return ($action,$linked_id);
}

sub link_koha_record {
    my ($self, $args) = @_;
    my $record_type = $args->{record_type} // "biblio";
    my $koha_id = $args->{koha_id};
    if ($koha_id){
        if ($record_type eq "biblio"){
            my $import_oai_biblio = $schema->resultset('OaiHarvesterBiblio')->find_or_create({
                oai_repository => $self->{repository},
                oai_identifier => $self->{header_identifier},
                biblionumber => $koha_id,
            });
            if ( ! $import_oai_biblio->in_storage ){
                $import_oai_biblio->insert;
            }
        }
    }
}

sub delete_koha_record {
    my ($self, $args) = @_;
    my $record_type = $args->{record_type} // "biblio";
	my $record_id = $args->{record_id};

    my $action = "error";

	if ($record_type eq "biblio"){
        my $error = C4::Biblio::DelBiblio($record_id);
        if (!$error){
            $action = "deleted";
            #NOTE: If there's no error, a cascading database delete should
            #automatically remove the link between the Koha biblionumber and OAI-PMH record too
        }
    }
    return $action;
}

1;




package Koha::OAI::Harvester::Listener;

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.
#

use Modern::Perl;
use POE qw(Wheel::SocketFactory Wheel::ReadWrite);
use IO::Socket qw(AF_UNIX);
use JSON;
use URI;

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;
    return bless ($args, $class);
}

sub spawn {
    my ($class, $args) = @_;
    my $self = $class->new($args);
    my $socket = $args->{socket};
    POE::Session->create(
        args => [
            $socket,
        ],
        object_states => [
            $self => {
                _start => "on_start",
                "on_server_success" => "on_server_success",
                "on_server_error" => "on_server_error",
                "on_client_error" => "on_client_error",
                "on_client_input" => "on_client_input",
            },
        ],
    );
}

sub on_start {
    my ($kernel,$heap,$socket_uri) = @_[KERNEL,HEAP,ARG0];

    my $uri = URI->new($socket_uri);
    if ($uri && $uri->scheme eq 'unix'){
        my $socket_path = $uri->path;
        unlink $socket_path if -S $socket_path;
        $heap->{server} = POE::Wheel::SocketFactory->new(
            SocketDomain => AF_UNIX,
            BindAddress => $socket_path,
            SuccessEvent => "on_server_success",
            FailureEvent => "on_server_error",
        );

        #Make the socket writeable to other users like Apache
        chmod 0666, $socket_path;
    }
}

sub on_server_success {
    my ($self, $client_socket, $server_wheel_id, $heap, $session) = @_[OBJECT, ARG0, ARG3, HEAP,SESSION];
    my $logger = $self->{logger};
    my $null_filter = POE::Filter::Line->new(
         Literal => chr(0),
    );
    my $client_wheel = POE::Wheel::ReadWrite->new(
        Handle => $client_socket,
        InputEvent => "on_client_input",
        ErrorEvent => "on_client_error",
        InputFilter => $null_filter,
        OutputFilter => $null_filter,
    );
    $heap->{client}->{ $client_wheel->ID() } = $client_wheel;
    $logger->info("Connection ".$client_wheel->ID()." started.");
    #TODO: Add basic authentication here?
    $client_wheel->put("HELLO");
}

sub on_server_error {
    my ($self, $operation, $errnum, $errstr, $heap, $session) = @_[OBJECT, ARG0, ARG1, ARG2,HEAP, SESSION];
    my $logger = $self->{logger};
    $logger->error("Server $operation error $errnum: $errstr");
    delete $heap->{server};
}

sub on_client_error {
    my ($self, $wheel_id,$heap,$session) = @_[OBJECT, ARG3,HEAP,SESSION];
    my $logger = $self->{logger};
    $logger->info("Connection $wheel_id failed or ended.");
    delete $heap->{client}->{$wheel_id};
}

sub on_client_input {
    my ($self, $input, $wheel_id, $session, $kernel, $heap) = @_[OBJECT, ARG0, ARG1, SESSION, KERNEL, HEAP];
    my $logger = $self->{logger};
    $logger->debug("Server input: $input");
    my $server_response = { msg => "fail"};
    eval {
        my $json_input = from_json($input);
        my $command = $json_input->{command};
        my $body = $json_input->{body};
        if ($command){
            if ($command eq "create"){
                my $task = $body->{task};
                if ($task){
                    my $is_created = $kernel->call("harvester","create_task",$task);
                    if ($is_created){
                        $server_response->{msg} = "success";
                    }
                }
            }
            elsif ($command eq "start"){
                my $task = $body->{task};
                if ($task){
                    my $uuid = $task->{uuid};
                    #Fetch from memory now...
                    my $is_started = $kernel->call("harvester","start_task", $uuid);
                    if ($is_started){
                        $server_response->{msg} = "success";
                    }
                }
            }
            elsif ($command eq "stop"){
                my $task = $body->{task};
                if ($task){
                    if ($task->{uuid}){
                        my $is_stopped = $kernel->call("harvester","stop_task",$task->{uuid});
                        if ($is_stopped){
                            $server_response->{msg} = "success";
                        }
                    }
                }
            }
            elsif ($command eq "delete"){
                my $task = $body->{task};
                if ($task){
                    if ($task->{uuid}){
                        my $is_deleted = $kernel->call("harvester","delete_task",$task->{uuid});
                        if ($is_deleted){
                            $server_response->{msg} = "success";
                        }
                    }
                }
            }
            elsif ($command eq "list"){
                my $tasks = $kernel->call("harvester","list_tasks");
                if ($tasks){
                    $server_response->{msg} = "success";
                    $server_response->{data} = $tasks;
                }
            }
        }
    };
    if ($@){
        #NOTE: An error most likely means that something other than a valid JSON string was received
        $logger->error($@);
    }

    if ($server_response){
        eval {
            my $client = $heap->{client}->{$wheel_id};
            my $json_message = to_json($server_response, { pretty => 1 });
            if ($json_message){
                $logger->debug("Server output: $json_message");
                $client->put($json_message);
            }
        };
        if ($@){
            #NOTE: An error means our response couldn't be serialised as JSON
            $logger->error($@);
        }
    }
}

1;




package Koha::OAI::Harvester::Request;

# Copyright Prosentient Systems 2017
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;

use Carp;

use base qw(Koha::Object);

#For validation
use URI;
use HTTP::OAI;

=head1 NAME

Koha::OAI::Harvester::Request -

=head1 API

=head2 Class Methods

=cut



=head3 _type

=cut

sub _type {
    return 'OaiHarvesterRequest';
}

sub validate {
    my ($self) = @_;
    my $errors = {};

    #Step one: validate URL
    my $uri = URI->new($self->http_url);
    if ( $uri && $uri->scheme && ($uri->scheme eq "http" || $uri->scheme eq "https") ){

        #Step two: validate access and authorization to URL
        my $harvester = $self->_harvester();
        my $identify = $harvester->Identify;
        if ($identify->is_success){

            #Step three: validate OAI-PMH parameters

            #Test Set
			my $set = $self->oai_set;
            if ($set){
                my $set_response = $harvester->ListSets();
				my @server_sets = $set_response->set;
				if ( ! grep {$_->setSpec eq $set} @server_sets ){
                    $errors->{oai_set}->{unavailable} = 1;
                }
            }

            #Test Metadata Prefix
			my $metadataPrefix = $self->oai_metadataPrefix;
			if ($metadataPrefix){
				my $metadata_response = $harvester->ListMetadataFormats();
				my @server_formats = $metadata_response->metadataFormat;
				if ( ! grep { $_->metadataPrefix eq $metadataPrefix } @server_formats ){
                    $errors->{oai_metadataPrefix}->{unavailable} = 1;
                }
			}
            else {
                $errors->{oai_metadataPrefix}->{missing} = 1;
            }

            #Test Granularity and Timestamps
			my $server_granularity = $identify->granularity;
			my $from = $self->oai_from;
			my $until = $self->oai_until;
			if ($from || $until){
				my ($from_granularity,$until_granularity);
				if ($from){
					$from_granularity = _determine_granularity($from);
					if ($from_granularity eq "YYYY-MM-DDThh:mm:ssZ"){
						$errors->{oai_from}->{unavailable} = 1 if $server_granularity ne $from_granularity;
					} elsif ($from_granularity eq "failed"){
						$errors->{oai_from}->{malformed} = 1;
					}
				}
				if ($until){
					$until_granularity = _determine_granularity($until);
					if ($until_granularity eq "YYYY-MM-DDThh:mm:ssZ"){
						$errors->{oai_until}->{unavailable} = 1 if $server_granularity ne $until_granularity;
					} elsif ($until_granularity eq "failed"){
						$errors->{oai_until}->{malformed} = 1;
					}
				}
				if ($from && $until){
					if ($from_granularity ne $until_granularity){
						$errors->{oai}->{granularity_mismatch} = 1;
					}
				}
			}

            #Test if identifier is provided when using GetRecord
            my $verb = $self->oai_verb;
            if ($verb && $verb eq "GetRecord"){
                my $identifier = $self->oai_identifier;
                if (! $identifier){
                    $errors->{oai_identifier}->{missing} = 1;
                }
            }
        }
        elsif ($identify->is_error){
            foreach my $error ($identify->errors){
                if ($error->code =~ /^404$/){
                    $errors->{http}->{404} = 1;
                } elsif ($error->code =~ /^401$/){
                    $errors->{http}->{401} = 1;
                } else {
                    $errors->{http}->{generic} = 1;
                }
            }
        }
        else {
            $errors->{http}->{generic} = 1;
        }
    } else {
        $errors->{http_url}->{malformed} = 1;
    }
    return $errors;
}

sub _harvester {
    my ( $self ) = @_;
    my $harvester;
    if ($self->http_url){
        $harvester = new HTTP::OAI::Harvester( baseURL => $self->http_url );
		my $uri = URI->new($self->http_url);
        if ($uri->scheme && ($uri->scheme eq 'http' || $uri->scheme eq 'https') ){
            my $host = $uri->host;
            my $port = $uri->port;
            $harvester->credentials($host.":".$port, $self->http_realm, $self->http_username, $self->http_password);
        }
    }
    return $harvester;
}

sub _determine_granularity {
	my ($timestamp) = @_;
	my $granularity;
	if ($timestamp =~ /^(\d{4}-\d{2}-\d{2})(T\d{2}:\d{2}:\d{2}Z)?$/){
		if ($1 && $2){
			$granularity = "YYYY-MM-DDThh:mm:ssZ";
		} elsif ($1 && !$2){
			$granularity = "YYYY-MM-DD";
		} else {
			$granularity = "failed";
		}
	} else {
		$granularity = "failed";
	}
	return $granularity;
}

=head1 AUTHOR

David Cook <dcook@prosentient.com.au>

=cut

1;




package Koha::OAI::Harvester::Requests;

# Copyright Prosentient Systems 2017
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;

use Carp;

use Koha::Database;

use Koha::OAI::Harvester::Request;

use base qw(Koha::Objects);

=head1 NAME

Koha::OAI::Harvester::Requests -

=head1 API

=head2 Class Methods

=cut

=head3 _type

=cut

sub _type {
    return 'OaiHarvesterRequest';
}

=head3 object_class

=cut

sub object_class {
    return 'Koha::OAI::Harvester::Request';
}

=head1 AUTHOR

David Cook <dcook@prosentient.com.au>

=cut

1;




package Koha::OAI::Harvester::Worker;

# Copyright Prosentient Systems 2017
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;
use POE;
use DateTime;
use JSON;

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;
    $args->{type} = "worker" unless $args->{type};
    return bless ($args, $class);
}

sub run {
    my ($self,$args) = @_;
    my $postback = $args->{postback};
    my $task = $args->{task};

    POE::Session->create(
        object_states => [
            $self => {
                _start           => "on_start",
                got_child_stderr => "on_child_stderr",
                got_child_close  => "on_child_close",
                got_child_signal => "on_child_signal",
                got_child_stdout => "on_child_stdout",
                stop_worker      => "stop_worker",
                _stop            => "on_stop",
            },
        ],
        args => [
            $postback,
            $task,
        ],
    );
}

sub stop_worker {
    my ($self,$heap) = @_[OBJECT,HEAP];
    if (my $child_processes = $heap->{children_by_pid}){
        foreach my $child_pid (keys %$child_processes){
            my $child = $child_processes->{$child_pid};
            $child->kill();
        }
    }
}


sub on_stop {
    my ($self,$kernel) = @_[OBJECT,KERNEL];

    #Deregister the worker session from the harvester's roster of workers
    $kernel->call("harvester","deregister",$self->{type});
}

# Wheel event, including the wheel's ID.
sub on_child_stdout {
    my ($self, $stdout_line, $wheel_id) = @_[OBJECT, ARG0, ARG1];
    my $type = $self->{type};
    my $child = $_[HEAP]{children_by_wid}{$wheel_id};
    my $logger = $self->{logger};
    if ($logger){
        $logger->debug("[$type][pid ".$child->PID."][STDOUT] $stdout_line");
    }

    my $postback = $_[HEAP]{postback};
    if ($postback){
        eval {
            my $message = from_json($stdout_line);
            if ($message){
                $postback->($message);
            }
        };
    }
}

# Wheel event, including the wheel's ID.
sub on_child_stderr {
    my ($self,$stderr_line, $wheel_id) = @_[OBJECT, ARG0, ARG1];
    my $type = $self->{type};
    my $child = $_[HEAP]{children_by_wid}{$wheel_id};
    my $logger = $self->{logger};
    if ($logger){
        $logger->debug("[$type][pid ".$child->PID."][STDERR] $stderr_line");
    }
}

# Wheel event, including the wheel's ID.
sub on_child_close {
    my ($self,$heap,$wheel_id) = @_[OBJECT,HEAP,ARG0];
    my $type = $self->{type};
    my $logger = $self->{logger};

    my $child = delete $heap->{children_by_wid}->{$wheel_id};

    # May have been reaped by on_child_signal().
    unless (defined $child) {
        if ($logger){
            $logger->debug("[$type][wid $wheel_id] closed all pipes");
        }
        return;
    }
    if ($logger){
        $logger->debug("[$type][pid ".$child->PID."] closed all pipes");
    }
    delete $heap->{children_by_pid}->{$child->PID};
}

sub on_child_signal {
    my ($self,$kernel,$pid,$status) = @_[OBJECT,KERNEL,ARG1,ARG2];
    my $type = $self->{type};
    my $logger = $self->{logger};
    if ($logger){
        $logger->debug("[$type][pid $pid] exited with status $status");
    }

    my $child = delete $_[HEAP]{children_by_pid}{$_[ARG1]};

    # May have been reaped by on_child_close().
    return unless defined $child;

    delete $_[HEAP]{children_by_wid}{$child->ID};

    #If the child doesn't complete successfully, we lodge an error
    #and stop the task.
    if ($status != 0){
        my $task = $kernel->call("harvester","get_task");
        if ($task){
            $task->{error} = 1;
            my $uuid = $task->{uuid};
            if ($uuid){
                $kernel->call("harvester","stop_task",$uuid);
            }
        }
    }
}

1;




package Koha::OAI::Harvester::Worker::Download::Stream;

# Copyright Prosentient Systems 2017
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;
use LWP::UserAgent;
use UUID qw/uuid/;
use POE;
use JSON;
use File::Path qw/make_path/;

use C4::Context;
use Koha::OAI::Harvester::Downloader;
use parent 'Koha::OAI::Harvester::Worker';

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;
    $args->{type} = "download" unless $args->{type};
    return bless ($args, $class);
}

sub on_start {
    my ($self, $kernel, $heap, $postback,$task,$session) = @_[OBJECT, KERNEL, HEAP, ARG0,ARG1,SESSION];
    #Save postback into heap so other event handlers can use it
    $heap->{postback} = $postback;

    my $task_uuid = $task->{uuid};

    $kernel->sig("cancel" => "stop_worker");
    $kernel->call("harvester","register",$self->{type},$task->{uuid});

    my $child = POE::Wheel::Run->new(
        ProgramArgs => [$task],
        Program => sub {
            my ($args) = @_;
            $self->do_work($args);
        },
        StdoutEvent  => "got_child_stdout",
        StderrEvent  => "got_child_stderr",
        CloseEvent   => "got_child_close",
        NoSetPgrp => 1, #Keep child processes in same group as parent. This is especially useful when using Ctrl+C to kill the whole group.
    );

     $_[KERNEL]->sig_child($child->PID, "got_child_signal");

    # Wheel events include the wheel's ID.
    $_[HEAP]{children_by_wid}{$child->ID} = $child;

    # Signal events include the process ID.
    $_[HEAP]{children_by_pid}{$child->PID} = $child;

    my $logger = $self->{logger};
    if ($logger){
        $logger->debug("Child pid ".$child->PID." started as wheel ".$child->ID);
    }
}

sub do_work {
    my ($self, $task) = @_;
    my $batch = ( $self->{batch} && int($self->{batch}) ) ? $self->{batch} : 100;

    #NOTE: Directory to spool files for processing
    my $spooldir = $task->{spooldir};

    my $task_uuid = $task->{uuid};
    my $task_parameters = $task->{parameters};
    my $interval = $task->{interval};

    my $oai_pmh_parameters = $task_parameters->{oai_pmh};
    my $import_parameters = $task_parameters->{import};

    #NOTE: Overwrite the 'from' and 'until' parameters for repeatable tasks
    if ( $interval && ! $oai_pmh_parameters->{until} ){
        if ($oai_pmh_parameters->{verb} eq "ListRecords"){
            #NOTE: 'effective_from' premiers on the first repetition (ie second request)
            $oai_pmh_parameters->{from} = $task->{effective_from} if $task->{effective_from};
            #NOTE: 'effective_until' appears on the first request
            $oai_pmh_parameters->{until} = $task->{effective_until} if $task->{effective_until};
        }
    }

    my $oai_downloader = Koha::OAI::Harvester::Downloader->new();
    my $url = $oai_downloader->BuildURL($oai_pmh_parameters);

    my $ua = LWP::UserAgent->new();
    #NOTE: setup HTTP Basic Authentication if parameters are supplied
    if($url && $url->host && $url->port){
        my $http_basic_auth = $task_parameters->{http_basic_auth};
        if ($http_basic_auth){
            my $username = $http_basic_auth->{username};
            my $password = $http_basic_auth->{password};
            my $realm = $http_basic_auth->{realm};
            $ua->credentials($url->host.":".$url->port, $realm, $username, $password);
        }
    }

    #NOTE: Prepare database statement handle
    my $dbh = C4::Context->dbh;
    my $sql = "insert into oai_harvester_import_queue (uuid,result) VALUES (?,?)";
    my $sth = $dbh->prepare($sql);

    if($url && $ua){
        #NOTE: You could define the callbacks as object methods instead... that might be nicer...
        #although I suppose it might be a much of a muchness.
        eval {
            my @filename_cache = ();

            $oai_downloader->harvest({
                user_agent => $ua,
                url => $url,
                callback => sub {
                    my ($args) = @_;

                    my $repository = $args->{repository};
                    my $document = $args->{document};

                    #If the spooldir has disappeared, re-create it.
                    if ( ! -d $spooldir ){
                        my $made_spool_directory = make_path($spooldir);
                    }
                    my $file_uuid = uuid();
                    my $filename = "$spooldir/$file_uuid";
                    my $state = $document->toFile($filename, 2);
                    if ($state){
                        push(@filename_cache,$filename);
                    }

                    if(scalar @filename_cache == $batch){
                        my $result = {
                            repository => $repository,
                            filenames => \@filename_cache,
                            filter => $import_parameters->{filter},
                            matcher_code => $import_parameters->{matcher_code},
                            frameworkcode => $import_parameters->{frameworkcode},
                            record_type => $import_parameters->{record_type},
                        };
                        eval {
                            my $json_result = to_json($result, { pretty => 1 });
                            $sth->execute($task_uuid,$json_result);
                        };
                        @filename_cache = ();
                    }
                },
                complete_callback => sub {
                    my ($args) = @_;
                    my $repository = $args->{repository};
                    if (@filename_cache){
                        my $result = {
                            repository => "$repository",
                            filenames => \@filename_cache,
                            filter => $import_parameters->{filter},
                            matcher_code => $import_parameters->{matcher_code},
                            frameworkcode => $import_parameters->{frameworkcode},
                            record_type => $import_parameters->{record_type},
                        };
                        eval {
                            my $json_result = to_json($result, { pretty => 1 });
                            $sth->execute($task_uuid,$json_result);
                        };
                    }

                },
            });
        };
        if ($@){
            die "Error during OAI-PMH download";
        }
    }
}

1;




package Koha::OAI::Harvester::Worker::Import;

# Copyright Prosentient Systems 2017
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;
use POE qw(Wheel::Run);
use JSON;
use XML::LibXML;

use C4::Context;
use C4::Matcher;
use Koha::OAI::Harvester::Import::Record;

use parent 'Koha::OAI::Harvester::Worker';

sub new {
    my ($class, $args) = @_;
    $args = {} unless defined $args;
    #NOTE: This type is used for logging and more importantly for registering with the harvester
    $args->{type} = "import" unless $args->{type};
    return bless ($args, $class);
}

sub on_start {
    my ($self, $kernel, $heap, $postback,$task) = @_[OBJECT, KERNEL, HEAP, ARG0,ARG1];

    $kernel->call("harvester","register",$self->{type},$task->{uuid});

    $kernel->sig(cancel => "stop_worker");

    my $child = POE::Wheel::Run->new(
        ProgramArgs => [ $task ],
        Program => sub {
            my ($task,$args) = @_;

            my $debug = $args->{debug} // 0;

            if ($task){
                my $json_result = $task->{result};
                my $id = $task->{id};
                my $task_uuid = $task->{uuid};
                eval {
                    my $result = from_json($json_result);
                    if ($result){
                        my $repository = $result->{repository};
                        my $filenames = $result->{filenames};
                        my $filter = $result->{filter};
                        my $matcher_code = $result->{matcher_code};
                        my $frameworkcode = $result->{frameworkcode};
                        my $record_type = $result->{record_type};

                        my $matcher;
                        if ($matcher_code){
                            my $matcher_id = C4::Matcher::GetMatcherId($matcher_code);
                            $matcher = C4::Matcher->fetch($matcher_id);
                        }

                        foreach my $filename (@$filenames){
                            if ($filename){
                                if (-f $filename){
                                    my $dom = XML::LibXML->load_xml(location => $filename, { no_blanks => 1 });
                                    if ($dom){
                                        my $oai_record = Koha::OAI::Harvester::Import::Record->new({
                                            doc => $dom,
                                            repository => $repository,
                                        });
                                        if ($oai_record){
                                            my ($action,$linked_id) = $oai_record->import_record({
                                                filter => $filter,
                                                framework => $frameworkcode,
                                                record_type => $record_type,
                                                matcher => $matcher,
                                            });
                                            $debug && print STDOUT qq({ "import_result": { "task_uuid": "$task_uuid", "action": "$action", "filename": "$filename", "koha_id": "$linked_id" } }\n);
                                        }
                                    }
                                    my $unlinked = unlink $filename;
                                }
                            }
                        }
                    }
                };
                if ($@){
                    warn $@;
                }
                #NOTE: Even if the file doesn't exist, we still need to process the queue item.

                #NOTE: Don't do this via a postback in the parent process, as it's much faster to let the child process handle it.

                #NOTE: It's vital that files are unlinked before deleting from the database,
                #or you could get orphan files if the importer is interrupted.
                my $dbh = C4::Context->dbh;
                my $sql = "delete from oai_harvester_import_queue where id = ?";
                my $sth = $dbh->prepare($sql);
                $sth->execute($id);
            }
        },
        StdoutEvent  => "got_child_stdout",
        StderrEvent  => "got_child_stderr",
        CloseEvent   => "got_child_close",
        NoSetPgrp => 1, #Keep child processes in same group as parent. This is especially useful when using Ctrl+C to kill the whole group.
    );

    $_[KERNEL]->sig_child($child->PID, "got_child_signal");

    # Wheel events include the wheel's ID.
    $_[HEAP]{children_by_wid}{$child->ID} = $child;

    # Signal events include the process ID.
    $_[HEAP]{children_by_pid}{$child->PID} = $child;

    my $logger = $self->{logger};
    if ($logger){
        $logger->debug("Child pid ".$child->PID." started as wheel ".$child->ID);
    }
}

1;




use utf8;
package Koha::Schema::Result::OaiHarvesterBiblio;

# Created by DBIx::Class::Schema::Loader
# DO NOT MODIFY THE FIRST PART OF THIS FILE

=head1 NAME

Koha::Schema::Result::OaiHarvesterBiblio

=cut

use strict;
use warnings;

use base 'DBIx::Class::Core';

=head1 TABLE: C<oai_harvester_biblios>

=cut

__PACKAGE__->table("oai_harvester_biblios");

=head1 ACCESSORS

=head2 import_oai_biblio_id

  data_type: 'integer'
  extra: {unsigned => 1}
  is_auto_increment: 1
  is_nullable: 0

=head2 oai_repository

  data_type: 'varchar'
  is_nullable: 0
  size: 255

=head2 oai_identifier

  data_type: 'varchar'
  is_nullable: 1
  size: 255

=head2 biblionumber

  data_type: 'integer'
  is_foreign_key: 1
  is_nullable: 0

=cut

__PACKAGE__->add_columns(
  "import_oai_biblio_id",
  {
    data_type => "integer",
    extra => { unsigned => 1 },
    is_auto_increment => 1,
    is_nullable => 0,
  },
  "oai_repository",
  { data_type => "varchar", is_nullable => 0, size => 255 },
  "oai_identifier",
  { data_type => "varchar", is_nullable => 1, size => 255 },
  "biblionumber",
  { data_type => "integer", is_foreign_key => 1, is_nullable => 0 },
);

=head1 PRIMARY KEY

=over 4

=item * L</import_oai_biblio_id>

=back

=cut

__PACKAGE__->set_primary_key("import_oai_biblio_id");

=head1 UNIQUE CONSTRAINTS

=head2 C<oai_record>

=over 4

=item * L</oai_identifier>

=item * L</oai_repository>

=back

=cut

__PACKAGE__->add_unique_constraint("oai_record", ["oai_identifier", "oai_repository"]);

=head1 RELATIONS

=head2 biblionumber

Type: belongs_to

Related object: L<Koha::Schema::Result::Biblio>

=cut

__PACKAGE__->belongs_to(
  "biblionumber",
  "Koha::Schema::Result::Biblio",
  { biblionumber => "biblionumber" },
  { is_deferrable => 1, on_delete => "CASCADE", on_update => "NO ACTION" },
);


# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-03-29 12:23:43
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:2URn8tPABMKC+JuIMfGeYw


# You can replace this text with custom code or comments, and it will be preserved on regeneration
1;




use utf8;
package Koha::Schema::Result::OaiHarvesterHistory;

# Created by DBIx::Class::Schema::Loader
# DO NOT MODIFY THE FIRST PART OF THIS FILE

=head1 NAME

Koha::Schema::Result::OaiHarvesterHistory

=cut

use strict;
use warnings;

use base 'DBIx::Class::Core';

=head1 TABLE: C<oai_harvester_history>

=cut

__PACKAGE__->table("oai_harvester_history");

=head1 ACCESSORS

=head2 import_oai_id

  data_type: 'integer'
  extra: {unsigned => 1}
  is_auto_increment: 1
  is_nullable: 0

=head2 repository

  data_type: 'varchar'
  is_nullable: 1
  size: 255

=head2 header_identifier

  data_type: 'varchar'
  is_nullable: 1
  size: 255

=head2 header_datestamp

  data_type: 'datetime'
  datetime_undef_if_invalid: 1
  is_nullable: 0

=head2 header_status

  data_type: 'varchar'
  is_nullable: 1
  size: 45

=head2 record

  data_type: 'longtext'
  is_nullable: 0

=head2 upload_timestamp

  data_type: 'timestamp'
  datetime_undef_if_invalid: 1
  default_value: current_timestamp
  is_nullable: 0

=head2 status

  data_type: 'varchar'
  is_nullable: 0
  size: 45

=head2 filter

  data_type: 'text'
  is_nullable: 0

=head2 framework

  data_type: 'varchar'
  is_nullable: 0
  size: 4

=head2 record_type

  data_type: 'enum'
  extra: {list => ["biblio","auth","holdings"]}
  is_nullable: 0

=head2 matcher_code

  data_type: 'varchar'
  is_nullable: 1
  size: 10

=cut

__PACKAGE__->add_columns(
  "import_oai_id",
  {
    data_type => "integer",
    extra => { unsigned => 1 },
    is_auto_increment => 1,
    is_nullable => 0,
  },
  "repository",
  { data_type => "varchar", is_nullable => 1, size => 255 },
  "header_identifier",
  { data_type => "varchar", is_nullable => 1, size => 255 },
  "header_datestamp",
  {
    data_type => "datetime",
    datetime_undef_if_invalid => 1,
    is_nullable => 0,
  },
  "header_status",
  { data_type => "varchar", is_nullable => 1, size => 45 },
  "record",
  { data_type => "longtext", is_nullable => 0 },
  "upload_timestamp",
  {
    data_type => "timestamp",
    datetime_undef_if_invalid => 1,
    default_value => \"current_timestamp",
    is_nullable => 0,
  },
  "status",
  { data_type => "varchar", is_nullable => 0, size => 45 },
  "filter",
  { data_type => "text", is_nullable => 0 },
  "framework",
  { data_type => "varchar", is_nullable => 0, size => 4 },
  "record_type",
  {
    data_type => "enum",
    extra => { list => ["biblio", "auth", "holdings"] },
    is_nullable => 0,
  },
  "matcher_code",
  { data_type => "varchar", is_nullable => 1, size => 10 },
);

=head1 PRIMARY KEY

=over 4

=item * L</import_oai_id>

=back

=cut

__PACKAGE__->set_primary_key("import_oai_id");


# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-03-29 12:23:43
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Rp/ZEZsKVlLo2vaM3M37ow


# You can replace this text with custom code or comments, and it will be preserved on regeneration
1;




use utf8;
package Koha::Schema::Result::OaiHarvesterImportQueue;

# Created by DBIx::Class::Schema::Loader
# DO NOT MODIFY THE FIRST PART OF THIS FILE

=head1 NAME

Koha::Schema::Result::OaiHarvesterImportQueue

=cut

use strict;
use warnings;

use base 'DBIx::Class::Core';

=head1 TABLE: C<oai_harvester_import_queue>

=cut

__PACKAGE__->table("oai_harvester_import_queue");

=head1 ACCESSORS

=head2 id

  data_type: 'integer'
  extra: {unsigned => 1}
  is_auto_increment: 1
  is_nullable: 0

=head2 uuid

  data_type: 'varchar'
  is_nullable: 0
  size: 45

=head2 status

  data_type: 'varchar'
  default_value: 'new'
  is_nullable: 0
  size: 45

=head2 result

  data_type: 'text'
  is_nullable: 0

=head2 result_timestamp

  data_type: 'timestamp'
  datetime_undef_if_invalid: 1
  default_value: current_timestamp
  is_nullable: 0

=cut

__PACKAGE__->add_columns(
  "id",
  {
    data_type => "integer",
    extra => { unsigned => 1 },
    is_auto_increment => 1,
    is_nullable => 0,
  },
  "uuid",
  { data_type => "varchar", is_nullable => 0, size => 45 },
  "status",
  {
    data_type => "varchar",
    default_value => "new",
    is_nullable => 0,
    size => 45,
  },
  "result",
  { data_type => "text", is_nullable => 0 },
  "result_timestamp",
  {
    data_type => "timestamp",
    datetime_undef_if_invalid => 1,
    default_value => \"current_timestamp",
    is_nullable => 0,
  },
);

=head1 PRIMARY KEY

=over 4

=item * L</id>

=back

=cut

__PACKAGE__->set_primary_key("id");


# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-03-29 12:23:43
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zBD+hMawbvu7sonuLRHnCA


# You can replace this text with custom code or comments, and it will be preserved on regeneration
1;




use utf8;
package Koha::Schema::Result::OaiHarvesterRequest;

# Created by DBIx::Class::Schema::Loader
# DO NOT MODIFY THE FIRST PART OF THIS FILE

=head1 NAME

Koha::Schema::Result::OaiHarvesterRequest

=cut

use strict;
use warnings;

use base 'DBIx::Class::Core';

=head1 TABLE: C<oai_harvester_requests>

=cut

__PACKAGE__->table("oai_harvester_requests");

=head1 ACCESSORS

=head2 id

  data_type: 'integer'
  extra: {unsigned => 1}
  is_auto_increment: 1
  is_nullable: 0

=head2 uuid

  data_type: 'varchar'
  is_nullable: 0
  size: 45

=head2 oai_verb

  data_type: 'varchar'
  is_nullable: 0
  size: 45

=head2 oai_metadataPrefix

  accessor: 'oai_metadata_prefix'
  data_type: 'varchar'
  is_nullable: 0
  size: 255

=head2 oai_identifier

  data_type: 'varchar'
  is_nullable: 1
  size: 255

=head2 oai_from

  data_type: 'varchar'
  is_nullable: 1
  size: 45

=head2 oai_until

  data_type: 'varchar'
  is_nullable: 1
  size: 45

=head2 oai_set

  data_type: 'varchar'
  is_nullable: 1
  size: 255

=head2 http_url

  data_type: 'varchar'
  is_nullable: 1
  size: 255

=head2 http_username

  data_type: 'varchar'
  is_nullable: 1
  size: 255

=head2 http_password

  data_type: 'varchar'
  is_nullable: 1
  size: 255

=head2 http_realm

  data_type: 'varchar'
  is_nullable: 1
  size: 255

=head2 import_filter

  data_type: 'varchar'
  is_nullable: 0
  size: 255

=head2 import_framework_code

  data_type: 'varchar'
  is_nullable: 0
  size: 4

=head2 import_record_type

  data_type: 'enum'
  extra: {list => ["biblio","auth","holdings"]}
  is_nullable: 0

=head2 import_matcher_code

  data_type: 'varchar'
  is_nullable: 1
  size: 10

=head2 interval

  data_type: 'integer'
  extra: {unsigned => 1}
  is_nullable: 0

=head2 name

  data_type: 'varchar'
  is_nullable: 0
  size: 45

=cut

__PACKAGE__->add_columns(
  "id",
  {
    data_type => "integer",
    extra => { unsigned => 1 },
    is_auto_increment => 1,
    is_nullable => 0,
  },
  "uuid",
  { data_type => "varchar", is_nullable => 0, size => 45 },
  "oai_verb",
  { data_type => "varchar", is_nullable => 0, size => 45 },
  "oai_metadataPrefix",
  {
    accessor => "oai_metadata_prefix",
    data_type => "varchar",
    is_nullable => 0,
    size => 255,
  },
  "oai_identifier",
  { data_type => "varchar", is_nullable => 1, size => 255 },
  "oai_from",
  { data_type => "varchar", is_nullable => 1, size => 45 },
  "oai_until",
  { data_type => "varchar", is_nullable => 1, size => 45 },
  "oai_set",
  { data_type => "varchar", is_nullable => 1, size => 255 },
  "http_url",
  { data_type => "varchar", is_nullable => 1, size => 255 },
  "http_username",
  { data_type => "varchar", is_nullable => 1, size => 255 },
  "http_password",
  { data_type => "varchar", is_nullable => 1, size => 255 },
  "http_realm",
  { data_type => "varchar", is_nullable => 1, size => 255 },
  "import_filter",
  { data_type => "varchar", is_nullable => 0, size => 255 },
  "import_framework_code",
  { data_type => "varchar", is_nullable => 0, size => 4 },
  "import_record_type",
  {
    data_type => "enum",
    extra => { list => ["biblio", "auth", "holdings"] },
    is_nullable => 0,
  },
  "import_matcher_code",
  { data_type => "varchar", is_nullable => 1, size => 10 },
  "interval",
  { data_type => "integer", extra => { unsigned => 1 }, is_nullable => 0 },
  "name",
  { data_type => "varchar", is_nullable => 0, size => 45 },
);

=head1 PRIMARY KEY

=over 4

=item * L</id>

=back

=cut

__PACKAGE__->set_primary_key("id");


# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-04-07 11:26:24
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Vm/b4yurmr8WF7z+Fo6KEw


# You can replace this text with custom code or comments, and it will be preserved on regeneration
1;

Lines 360-365 my $target_map = { Link Here

(-)a/Makefile.PL (+14 lines)
360	'./skel/var/lib/koha/zebradb/biblios/tmp' => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },	360	'./skel/var/lib/koha/zebradb/biblios/tmp' => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
361	'./skel/var/lock/koha/zebradb/rebuild' => { target => 'ZEBRA_LOCK_DIR', trimdir => 6 },	361	'./skel/var/lock/koha/zebradb/rebuild' => { target => 'ZEBRA_LOCK_DIR', trimdir => 6 },
362	'./skel/var/lib/koha/plugins' => { target => 'PLUGINS_DIR', trimdir => 6 },	362	'./skel/var/lib/koha/plugins' => { target => 'PLUGINS_DIR', trimdir => 6 },
		363	'./skel/var/lib/koha/oai-pmh-harvester' => { target => 'OAI_LIB_DIR', trimdir => 6 },
		364	'./skel/var/run/koha/oai-pmh-harvester' => { target => 'OAI_RUN_DIR', trimdir => 6 },
		365	'./skel/var/spool/koha/oai-pmh-harvester' => { target => 'OAI_SPOOL_DIR', trimdir => 6 },
363	'./sms' => 'INTRANET_CGI_DIR',	366	'./sms' => 'INTRANET_CGI_DIR',
364	'./suggestion' => 'INTRANET_CGI_DIR',	367	'./suggestion' => 'INTRANET_CGI_DIR',
365	'./svc' => 'INTRANET_CGI_DIR',	368	'./svc' => 'INTRANET_CGI_DIR',
Lines 594-599 my $pl_files = { Link Here
594	'blib/KOHA_CONF_DIR/koha-conf.xml',	597	'blib/KOHA_CONF_DIR/koha-conf.xml',
595	'blib/KOHA_CONF_DIR/koha-httpd.conf',	598	'blib/KOHA_CONF_DIR/koha-httpd.conf',
596	'blib/KOHA_CONF_DIR/log4perl.conf',	599	'blib/KOHA_CONF_DIR/log4perl.conf',
		600	'blib/KOHA_CONF_DIR/oai-pmh-harvester.yaml',
597	'blib/ZEBRA_CONF_DIR/etc/default.idx',	601	'blib/ZEBRA_CONF_DIR/etc/default.idx',
598	'blib/MISC_DIR/koha-install-log'	602	'blib/MISC_DIR/koha-install-log'
599	],	603	],
Lines 1392-1397 sub get_target_directories { Link Here
1392	$dirmap{'PLUGINS_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'lib', 'koha', 'plugins');	1396	$dirmap{'PLUGINS_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'lib', 'koha', 'plugins');
1393	$dirmap{'ZEBRA_DATA_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');	1397	$dirmap{'ZEBRA_DATA_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1394	$dirmap{'ZEBRA_RUN_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');	1398	$dirmap{'ZEBRA_RUN_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
		1399	$dirmap{'OAI_RUN_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'run', 'oai-pmh-harvester');
		1400	$dirmap{'OAI_LIB_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'lib', 'oai-pmh-harvester');
		1401	$dirmap{'OAI_SPOOL_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'spool', 'oai-pmh-harvester');
1395	} elsif ($mode eq 'dev') {	1402	} elsif ($mode eq 'dev') {
1396	my $curdir = File::Spec->rel2abs(File::Spec->curdir());	1403	my $curdir = File::Spec->rel2abs(File::Spec->curdir());
1397	$dirmap{'API_CGI_DIR'} = File::Spec->catdir($curdir, 'api');	1404	$dirmap{'API_CGI_DIR'} = File::Spec->catdir($curdir, 'api');
Lines 1427-1432 sub get_target_directories { Link Here
1427	$dirmap{'PLUGINS_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'lib', 'plugins');	1434	$dirmap{'PLUGINS_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'lib', 'plugins');
1428	$dirmap{'ZEBRA_DATA_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');	1435	$dirmap{'ZEBRA_DATA_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1429	$dirmap{'ZEBRA_RUN_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');	1436	$dirmap{'ZEBRA_RUN_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
		1437	$dirmap{'OAI_RUN_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'run', 'oai-pmh-harvester');
		1438	$dirmap{'OAI_LIB_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'lib', 'oai-pmh-harvester');
		1439	$dirmap{'OAI_SPOOL_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'spool', 'oai-pmh-harvester');
		1440
1430	} else {	1441	} else {
1431	# mode is standard, i.e., 'fhs'	1442	# mode is standard, i.e., 'fhs'
1432	$dirmap{'API_CGI_DIR'} = File::Spec->catdir(@basedir, $package, 'api');	1443	$dirmap{'API_CGI_DIR'} = File::Spec->catdir(@basedir, $package, 'api');
Lines 1451-1456 sub get_target_directories { Link Here
1451	$dirmap{'PLUGINS_DIR'} = File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'plugins');	1462	$dirmap{'PLUGINS_DIR'} = File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'plugins');
1452	$dirmap{'ZEBRA_DATA_DIR'} = File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'zebradb');	1463	$dirmap{'ZEBRA_DATA_DIR'} = File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'zebradb');
1453	$dirmap{'ZEBRA_RUN_DIR'} = File::Spec->catdir(File::Spec->rootdir(), 'var', 'run', $package, 'zebradb');	1464	$dirmap{'ZEBRA_RUN_DIR'} = File::Spec->catdir(File::Spec->rootdir(), 'var', 'run', $package, 'zebradb');
		1465	$dirmap{'OAI_RUN_DIR'} = File::Spec->catdir(File::Spec->rootdir(), 'var', 'run', $package, 'oai-pmh-harvester');
		1466	$dirmap{'OAI_LIB_DIR'} = File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'oai-pmh-harvester');
		1467	$dirmap{'OAI_SPOOL_DIR'} = File::Spec->catdir(File::Spec->rootdir(), 'var', 'spool', $package, 'oai-pmh-harvester');
1454	}	1468	}
1455		1469
1456	_get_env_overrides(\%dirmap);	1470	_get_env_overrides(\%dirmap);

Lines 117-122 generate_config_file() { Link Here

(-)a/debian/scripts/koha-create (+6 lines)
117	-e "s/__PLUGINS_DIR__/\/var\/lib\/koha\/$name\/plugins/g" \	117	-e "s/__PLUGINS_DIR__/\/var\/lib\/koha\/$name\/plugins/g" \
118	-e "s/__MEMCACHED_NAMESPACE__/$MEMCACHED_NAMESPACE/g" \	118	-e "s/__MEMCACHED_NAMESPACE__/$MEMCACHED_NAMESPACE/g" \
119	-e "s/__MEMCACHED_SERVERS__/$MEMCACHED_SERVERS/g" \	119	-e "s/__MEMCACHED_SERVERS__/$MEMCACHED_SERVERS/g" \
		120	-e "s/__OAI_RUN_DIR__/\/var\/run\/koha\/$name\/oai-pmh-harvester/g" \
		121	-e "s/__OAI_LIB_DIR__/\/var\/lib\/koha\/$name\/oai-pmh-harvester/g" \
		122	-e "s/__OAI_SPOOL_DIR__/\/var\/spool\/koha\/$name\/oai-pmh-harvester/g" \
120	"/etc/koha/$1" > "$2"	123	"/etc/koha/$1" > "$2"
121		124
122	}	125	}
Lines 715-720 eof Link Here
715	generate_config_file triplestore.yaml.in \	718	generate_config_file triplestore.yaml.in \
716	"/etc/koha/sites/$name/triplestore.yaml"	719	"/etc/koha/sites/$name/triplestore.yaml"
717		720
		721	# Generate and install OAI-PMH harvester config file
		722	generate_config_file oai-pmh-harvester.yaml.in \
		723	"/etc/koha/sites/$name/oai-pmh-harvester.yaml"
718		724
719	# Create a GPG-encrypted file for requesting a DB to be set up.	725	# Create a GPG-encrypted file for requesting a DB to be set up.
720	if [ "$op" = request ]	726	if [ "$op" = request ]

Lines 53-63 do Link Here

(-)a/debian/scripts/koha-create-dirs (+3 lines)
53	userdir "$name" "/var/lib/koha/$name/biblios/tmp"	53	userdir "$name" "/var/lib/koha/$name/biblios/tmp"
54	userdir "$name" "/var/lib/koha/$name/plugins"	54	userdir "$name" "/var/lib/koha/$name/plugins"
55	userdir "$name" "/var/lib/koha/$name/uploads"	55	userdir "$name" "/var/lib/koha/$name/uploads"
		56	userdir "$name" "/var/lib/koha/$name/oai-pmh-harvester"
56	userdir "$name" "/var/lock/koha/$name"	57	userdir "$name" "/var/lock/koha/$name"
57	userdir "$name" "/var/lock/koha/$name/authorities"	58	userdir "$name" "/var/lock/koha/$name/authorities"
58	userdir "$name" "/var/lock/koha/$name/biblios"	59	userdir "$name" "/var/lock/koha/$name/biblios"
59	userdir "$name" "/var/run/koha/$name"	60	userdir "$name" "/var/run/koha/$name"
60	userdir "$name" "/var/run/koha/$name/authorities"	61	userdir "$name" "/var/run/koha/$name/authorities"
61	userdir "$name" "/var/run/koha/$name/biblios"	62	userdir "$name" "/var/run/koha/$name/biblios"
		63	userdir "$name" "/var/run/koha/$name/oai-pmh-harvester"
		64	userdir "$name" "/var/spool/koha/$name/oai-pmh-harvester"
62	done	65	done
63		66

Lines 284-289 __END_SRU_PUBLICSERVER__ Link Here

(-)a/debian/templates/koha-conf-site.xml.in (+1 lines)
284	<use_zebra_facets>1</use_zebra_facets>	284	<use_zebra_facets>1</use_zebra_facets>
285	<queryparser_config>/etc/koha/searchengine/queryparser.yaml</queryparser_config>	285	<queryparser_config>/etc/koha/searchengine/queryparser.yaml</queryparser_config>
286	<log4perl_conf>__KOHA_CONF_DIR__/log4perl.conf</log4perl_conf>	286	<log4perl_conf>__KOHA_CONF_DIR__/log4perl.conf</log4perl_conf>
		287	<oai_pmh_harvester_config>/etc/koha/sites/__KOHASITE__/oai_pmh_harvester.yaml</oai_pmh_harvester_config>
287	<!-- Uncomment/edit next setting if you want to adjust zebra log levels.	288	<!-- Uncomment/edit next setting if you want to adjust zebra log levels.
288	Default is: none,fatal,warn.	289	Default is: none,fatal,warn.
289	You can also include: debug,log,malloc,all,request.	290	You can also include: debug,log,malloc,all,request.




---
#Harvester
socket: 'unix:__OAI_RUN_DIR__/harvesterd.sock'

logfile: '__LOG_DIR__/oai-pmh-harvester.log'
loglevel: 'WARN'

pidfile: '__OAI_RUN_DIR__/harvesterd.pid'

statefile: '__OAI_LIB_DIR__/harvesterd.state'

spooldir: '__OAI_SPOOL_DIR__'

#Downloader
download_workers: 1
download_module: 'Koha::OAI::Harvester::Worker::Download::Stream'
download_batch: 100

#Importer
import_workers: 1
import_poll: 5
import_module: 'Koha::OAI::Harvester::Worker::Import'

Lines 120-125 __PAZPAR2_TOGGLE_XML_POST__ Link Here

(-)a/etc/koha-conf.xml (+1 lines)
120	<use_zebra_facets>1</use_zebra_facets>	120	<use_zebra_facets>1</use_zebra_facets>
121	<queryparser_config>__KOHA_CONF_DIR__/searchengine/queryparser.yaml</queryparser_config>	121	<queryparser_config>__KOHA_CONF_DIR__/searchengine/queryparser.yaml</queryparser_config>
122	<log4perl_conf>__KOHA_CONF_DIR__/log4perl.conf</log4perl_conf>	122	<log4perl_conf>__KOHA_CONF_DIR__/log4perl.conf</log4perl_conf>
		123	<oai_pmh_harvester_config>__KOHA_CONF_DIR__/oai-pmh-harvester.yaml</oai_pmh_harvester_config>
123	<memcached_servers>__MEMCACHED_SERVERS__</memcached_servers>	124	<memcached_servers>__MEMCACHED_SERVERS__</memcached_servers>
124	<memcached_namespace>__MEMCACHED_NAMESPACE__</memcached_namespace>	125	<memcached_namespace>__MEMCACHED_NAMESPACE__</memcached_namespace>
125	<!-- Uncomment the following line if you want to use template caching	126	<!-- Uncomment the following line if you want to use template caching




---
#Harvester
socket: 'unix:__OAI_RUN_DIR__/harvesterd.sock'

logfile: '__LOG_DIR__/oai-pmh-harvester.log'
loglevel: 'WARN'

pidfile: '__OAI_RUN_DIR__/harvesterd.pid'

statefile: '__OAI_LIB_DIR__/harvesterd.state'

spooldir: '__OAI_SPOOL_DIR__'

#Downloader
download_workers: 1
download_module: 'Koha::OAI::Harvester::Worker::Download::Stream'
download_batch: 100

#Importer
import_workers: 1
import_poll: 5
import_module: 'Koha::OAI::Harvester::Worker::Import'




--
-- Table structure for table 'oai_harvester_biblios'
--

CREATE TABLE IF NOT EXISTS `oai_harvester_biblios` (
  `import_oai_biblio_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
  `oai_repository` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
  `oai_identifier` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
  `biblionumber` int(11) NOT NULL,
  PRIMARY KEY (`import_oai_biblio_id`),
  UNIQUE KEY `oai_record` (`oai_identifier`,`oai_repository`) USING BTREE,
  KEY `FK_import_oai_biblio_1` (`biblionumber`),
  CONSTRAINT `FK_import_oai_biblio_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE NO ACTION
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;

--
-- Table structure for table 'oai_harvester_history'
--

CREATE TABLE IF NOT EXISTS  `oai_harvester_history` (
  `import_oai_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
  `repository` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
  `header_identifier` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
  `header_datestamp` datetime NOT NULL,
  `header_status` varchar(45) COLLATE utf8_unicode_ci DEFAULT NULL,
  `record` longtext COLLATE utf8_unicode_ci NOT NULL,
  `upload_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
  `status` varchar(45) COLLATE utf8_unicode_ci NOT NULL,
  `filter` text COLLATE utf8_unicode_ci NOT NULL,
  `framework` varchar(4) COLLATE utf8_unicode_ci NOT NULL,
  `record_type` enum('biblio','auth','holdings') COLLATE utf8_unicode_ci NOT NULL,
  `matcher_code` varchar(10) COLLATE utf8_unicode_ci DEFAULT NULL,
  PRIMARY KEY (`import_oai_id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;

--
-- Table structure for table 'oai_harvester_import_queue'
--

CREATE TABLE IF NOT EXISTS `oai_harvester_import_queue` (
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
  `uuid` varchar(45) CHARACTER SET utf8 NOT NULL,
  `status` varchar(45) CHARACTER SET utf8 NOT NULL DEFAULT 'new',
  `result` text CHARACTER SET utf8 NOT NULL,
  `result_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;

--
-- Table structure for table 'oai_harvester_requests'
--

CREATE TABLE IF NOT EXISTS `oai_harvester_requests` (
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
  `uuid` varchar(45) NOT NULL,
  `oai_verb` varchar(45) NOT NULL,
  `oai_metadataPrefix` varchar(255) NOT NULL,
  `oai_identifier` varchar(255) DEFAULT NULL,
  `oai_from` varchar(45) DEFAULT NULL,
  `oai_until` varchar(45) DEFAULT NULL,
  `oai_set` varchar(255) DEFAULT NULL,
  `http_url` varchar(255) DEFAULT NULL,
  `http_username` varchar(255) DEFAULT NULL,
  `http_password` varchar(255) DEFAULT NULL,
  `http_realm` varchar(255) DEFAULT NULL,
  `import_filter` varchar(255) NOT NULL,
  `import_framework_code` varchar(4) NOT NULL,
  `import_record_type` enum('biblio','auth','holdings') NOT NULL,
  `import_matcher_code` varchar(10) DEFAULT NULL,
  `interval` int(10) unsigned NOT NULL,
  `name` varchar(45) NOT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;




  CONSTRAINT club_fields_ibfk_4 FOREIGN KEY (club_id) REFERENCES clubs (id) ON DELETE CASCADE ON UPDATE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;

--
-- Table structure for table 'oai_harvester_biblios'
--

CREATE TABLE IF NOT EXISTS `oai_harvester_biblios` (
  `import_oai_biblio_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
  `oai_repository` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
  `oai_identifier` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
  `biblionumber` int(11) NOT NULL,
  PRIMARY KEY (`import_oai_biblio_id`),
  UNIQUE KEY `oai_record` (`oai_identifier`,`oai_repository`) USING BTREE,
  KEY `FK_import_oai_biblio_1` (`biblionumber`),
  CONSTRAINT `FK_import_oai_biblio_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE NO ACTION
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;

--
-- Table structure for table 'oai_harvester_history'
--

CREATE TABLE IF NOT EXISTS  `oai_harvester_history` (
  `import_oai_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
  `repository` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
  `header_identifier` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
  `header_datestamp` datetime NOT NULL,
  `header_status` varchar(45) COLLATE utf8_unicode_ci DEFAULT NULL,
  `record` longtext COLLATE utf8_unicode_ci NOT NULL,
  `upload_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
  `status` varchar(45) COLLATE utf8_unicode_ci NOT NULL,
  `filter` text COLLATE utf8_unicode_ci NOT NULL,
  `framework` varchar(4) COLLATE utf8_unicode_ci NOT NULL,
  `record_type` enum('biblio','auth','holdings') COLLATE utf8_unicode_ci NOT NULL,
  `matcher_code` varchar(10) COLLATE utf8_unicode_ci DEFAULT NULL,
  PRIMARY KEY (`import_oai_id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;

--
-- Table structure for table 'oai_harvester_import_queue'
--

CREATE TABLE IF NOT EXISTS `oai_harvester_import_queue` (
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
  `uuid` varchar(45) CHARACTER SET utf8 NOT NULL,
  `status` varchar(45) CHARACTER SET utf8 NOT NULL DEFAULT 'new',
  `result` text CHARACTER SET utf8 NOT NULL,
  `result_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;

--
-- Table structure for table 'oai_harvester_requests'
--

CREATE TABLE IF NOT EXISTS `oai_harvester_requests` (
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
  `uuid` varchar(45) NOT NULL,
  `oai_verb` varchar(45) NOT NULL,
  `oai_metadataPrefix` varchar(255) NOT NULL,
  `oai_identifier` varchar(255) DEFAULT NULL,
  `oai_from` varchar(45) DEFAULT NULL,
  `oai_until` varchar(45) DEFAULT NULL,
  `oai_set` varchar(255) DEFAULT NULL,
  `http_url` varchar(255) DEFAULT NULL,
  `http_username` varchar(255) DEFAULT NULL,
  `http_password` varchar(255) DEFAULT NULL,
  `http_realm` varchar(255) DEFAULT NULL,
  `import_filter` varchar(255) NOT NULL,
  `import_framework_code` varchar(4) NOT NULL,
  `import_record_type` enum('biblio','auth','holdings') NOT NULL,
  `import_matcher_code` varchar(10) DEFAULT NULL,
  `interval` int(10) unsigned NOT NULL,
  `name` varchar(45) NOT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;

/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;

Lines 98-103 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-menu.inc (+3 lines)
98	[% IF ( CAN_user_tools_upload_local_cover_images ) %]	98	[% IF ( CAN_user_tools_upload_local_cover_images ) %]
99	<li><a href="/cgi-bin/koha/tools/upload-cover-image.pl">Upload local cover image</a></li>	99	<li><a href="/cgi-bin/koha/tools/upload-cover-image.pl">Upload local cover image</a></li>
100	[% END %]	100	[% END %]
		101	[% IF ( CAN_user_tools_manage_staged_marc ) %]
		102	<li><a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a></li>
		103	[% END %]
101	</ul>	104	</ul>
102	<h5>Additional tools</h5>	105	<h5>Additional tools</h5>
103	<ul>	106	<ul>




[% INCLUDE 'doc-head-open.inc' %]
<title>Koha &rsaquo; Tools &rsaquo; OAI-PMH harvester</title>
[% INCLUDE 'doc-head-close.inc' %]
[% INCLUDE 'datatables.inc' %]
[% dashboard_page = '/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl' %]
[% request_page = '/cgi-bin/koha/tools/oai-pmh-harvester/request.pl' %]
<script type="text/javascript">
//<![CDATA[
    $(document).ready(function() {
        $('#dashboard-items').tabs();
        [% IF ( result.start.defined || result.stop.defined || result.delete.defined ) %]
            $('#dashboard-items').tabs("option","active",1);
        [% END %]

        var saved_table = $('#saved-table').DataTable({});
        var submitted_table = $('#submitted-table').DataTable({});
        var history_table = $('#history-table').DataTable({
            serverSide: true,
            searching: true,
            processing: true,
            ajax: {
                "url": '/cgi-bin/koha/svc/oai-pmh-harvester/history',
                contentType: 'application/json',
                type: 'POST',
                data: function ( d ) {
                    return JSON.stringify( d );
                },
                dataSrc: function (json){
                    var recordsTotal = json.recordsTotal;
                    if(recordsTotal){
                        $('#import_count').text( "("+recordsTotal+")" );
                    }
                    return json.data;
                }
            },
            columns: [
                { data: 'import_oai_id', },
                { data: 'repository', },
                { data: 'header_identifier', },
                { data: 'header_datestamp', },
                {
                    data: 'header_status', render: function (data, type, full, meta){
                        var display_string = _("Active");
                        if (data == "deleted"){
                            display_string = _("Deleted");
                        }
                        return display_string;
                    }
                },
                {
                    data: 'status', render: function (data, type, full, meta){
                        var display_string = data;
                        if (data == "added"){
                            display_string = _("Added");
                        }
                        else if (data == "error"){
                            display_string = _("Error");
                        }
                        else if (data == "not_found"){
                            display_string = _("Not found");
                        }
                        else if (data == "updated"){
                            display_string = _("Updated");
                        }
                        return display_string;
                    }
                },
                { data: 'upload_timestamp', },
                {
                    data: 'imported_record', render: function (data, type, full, meta){
                        var display_string = data;
                        var record_type = (full.record_type) ? full.record_type : 'biblio';
                        if (data && record_type == 'biblio'){
                            var link_text = _("View biblio record");
                            var link = '<a href="/cgi-bin/koha/catalogue/detail.pl?biblionumber='+data+'">'+link_text+'</a>';
                            display_string = link;
                        }
                        return display_string;
                    }

                },
                {
                    data: 'import_oai_id', render: function (data, type, full, meta){
                        var display_string = data;
                        var link_text = _("View record");
                        var link = '<a href="/cgi-bin/koha/tools/oai-pmh-harvester/record.pl?import_oai_id='+data+'">'+link_text+'</a>';
                        display_string = link;
                        return display_string;
                    }
                },
            ],
            //In theory, it would be nice to sort in descending order, but
            //it causes severe paging issues the data is frequently updated.
            //Caching would make the paging more stable, but the results would be stale.
            order: [
                [ 0, 'asc' ],
            ]
        });
        $('#refresh-button').click(function(){
            history_table.ajax.reload( null, false );
        });
    });
//]]>
</script>
<style type="text/css">
    a.paginate_button {
        padding: 2px;
    }
</style>
</head>
<body id="tools_oai-pmh-harvester" class="tools">
[% INCLUDE 'header.inc' %]
[% INCLUDE 'cat-search.inc' %]
    <div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> &rsaquo; OAI-PMH harvester</div>
    <div id="doc3" class="yui-t2">
        <div id="bd">
            <div id="yui-main">
                <div class="yui-b">
                    <h1>OAI-PMH harvester</h1>
                    <div id="toolbar" class="btn-toolbar">
                        <a id="new-request" class="btn btn-default btn-sm" href="[% request_page %]?op=new"><i class="fa fa-plus"></i> New request</a>
                    </div>
                    [% IF ( harvester.offline ) %]
                        <div class="alert">
                            <span>OAI-PMH harvester offline</span>
                        </div>
                    [% END %]
                    <div id="dashboard-items" class="toptabs">
                        <ul>
                            <li>
                                <a href="#saved_requests">Saved requests <span id="saved_count">([% saved_requests.size %])</span></a>
                            </li>
                            <li>
                                <a href="#submitted_requests">Submitted requests <span id="submitted_count">([% submitted_requests.size %])</span></a>
                            </li>
                            <li>
                                <a href="#imports">Import history <span id="import_count">(0)</span></a>
                            </li>
                        </ul>
                        <div id="submitted_requests">
                            [% IF ( result.start.defined ) %]
                                <div class="alert">
                                    [% IF ( result.start ) %]
                                        <span>Start succeeded</span>
                                    [% ELSE %]
                                        <span>Start failed</span>
                                    [% END %]
                                </div>
                            [% ELSIF ( result.stop.defined ) %]
                                 <div class="alert">
                                    [% IF ( result.stop ) %]
                                        <span>Stop succeeded</span>
                                    [% ELSE %]
                                        <span>Stop failed</span>
                                    [% END %]
                                </div>
                            [% ELSIF ( result.delete.defined ) %]
                                 <div class="alert">
                                    [% IF ( result.delete ) %]
                                        <span>Delete succeeded</span>
                                    [% ELSE %]
                                        <span>Delete failed</span>
                                    [% END %]
                                </div>
                            [% END %]
                            <table id="submitted-table">
                                <thead>
                                    <tr>
                                        <th>Name</th>
                                        <th>URL</th>
                                        <th>Set</th>
                                        <th>From</th>
                                        <th>Until</th>
                                        <th>Interval</th>
                                        <th>Effective from</th>
                                        <th>Effective until</th>
                                        <th>Pending imports</th>
                                        <th>Status</th>
                                        <th>Error</th>
                                        <th></th>
                                    </tr>
                                </thead>
                                <tbody>
                                [% IF ( submitted_requests ) %]
                                    [% FOREACH submitted_request IN submitted_requests %]
                                        <tr>
                                            <td>[% submitted_request.name %]</td>
                                            <td>[% submitted_request.parameters.oai_pmh.baseURL %]</td>
                                            <td>[% submitted_request.parameters.oai_pmh.set %]</td>
                                            <td>[% submitted_request.parameters.oai_pmh.from %]</td>
                                            <td>[% submitted_request.parameters.oai_pmh.until %]</td>
                                            <td>[% submitted_request.interval %]</td>
                                            <td>[% submitted_request.effective_from %]</td>
                                            <td>[% submitted_request.effective_until %]</td>
                                            <td>[% submitted_request.pending_imports %]</td>
                                            <td>
                                                [% IF ( submitted_status = submitted_request.status ) %]
                                                    [% IF ( submitted_status == "new" ) %]
                                                        <span>New</span>
                                                    [% ELSIF ( submitted_status == "active" ) %]
                                                        <span>Active</span>
                                                    [% ELSIF ( submitted_status == "stopped" ) %]
                                                        <span>Stopped</span>
                                                    [% END %]
                                                [% END %]
                                            </td>
                                            <td>
                                                [% IF ( submitted_request.error ) %]
                                                    <span>Harvest failure</span>
                                                [% END %]
                                            </td>
                                            <td>
                                                <div class="dropdown">
                                                    <a class="btn btn-default btn-xs dropdown-toggle" role="button" data-toggle="dropdown" href="#">Actions <span class="caret"></span></a>
                                                    <ul class="dropdown-menu pull-right" role="menu">
                                                          <li>
                                                              <a href="[% dashboard_page %]?op=start&uuid=[% submitted_request.uuid %]"><i class="fa fa-play"></i> Start</a>
                                                          </li>
                                                          <li>
                                                              <a href="[% dashboard_page %]?op=stop&uuid=[% submitted_request.uuid %]"><i class="fa fa-stop"></i> Stop</a>
                                                          </li>
                                                          <li>
                                                              <a href="[% dashboard_page %]?op=delete&uuid=[% submitted_request.uuid %]"><i class="fa fa-trash"></i> Delete</a>
                                                          </li>
                                                    </ul>
                                                </div>
                                            </td>
                                        </tr>
                                    [% END %]
                                [% END %]
                                </tbody>
                            </table>
                        </div>
                        <div id="saved_requests">
                            [% IF ( result.send.defined ) %]
                                <div class="alert">
                                    [% IF ( result.send ) %]
                                        <span>Send succeeded</span>
                                    [% ELSE %]
                                        <span>Send failed</span>
                                    [% END %]
                                </div>
                            [% END %]
                            <table id="saved-table">
                                <thead>
                                    <tr>
                                        <th>Name</th>
                                        <th>URL</th>
                                   <!-- <th>Verb</th>
                                        <th>Metadata prefix</th>
                                        <th>Identifier</th> -->
                                        <th>Set</th>
                                        <th>From</th>
                                        <th>Until</th>
                                        <th>Interval</th>
                                   <!-- <th>Filter</th>
                                        <th>Framework code</th>
                                        <th>Record type</th>
                                        <th>Matcher code</th> -->
                                        <th></th>
                                    </tr>
                                </thead>
                                <tbody>
                                [% IF ( saved_requests ) %]
                                    [% FOREACH saved_request IN saved_requests %]
                                        <tr>
                                            <td>[% saved_request.name %]</td>
                                            <td>[% saved_request.http_url %]</td>
                                       <!-- <td>[% saved_request.oai_verb %]</td>
                                            <td>[% saved_request.oai_metadataPrefix %]</td>
                                            <td>[% saved_request.oai_identifier %]</td> -->
                                            <td>[% saved_request.oai_set %]</td>
                                            <td>[% saved_request.oai_from %]</td>
                                            <td>[% saved_request.oai_until %]</td>
                                            <td>[% saved_request.interval %]</td>
                                       <!-- <td>
                                                [% IF ( saved_request.import_filter == "default" ) %]
                                                    <span>Default</span>
                                                [% ELSE %]
                                                    [% saved_request.import_filter %]
                                                [% END %]
                                            </td>
                                            <td>
                                                [%  display_framework = "";
                                                    FOREACH framework IN frameworks;
                                                        IF ( framework.frameworkcode == saved_request.import_framework_code );
                                                            display_framework = framework;
                                                        END;
                                                    END;
                                                %]
                                                [% IF ( display_framework ) %]
                                                    [% display_framework.frameworktext %]
                                                [% ELSE %]
                                                    [% saved_request.import_framework_code %]
                                                [% END %]
                                            </td>
                                            <td>
                                                [% IF ( saved_request.import_record_type == "biblio" ) %]
                                                    <span>Bibliographic</span>
                                                [% ELSE %]
                                                    [% saved_request.import_record_type %]
                                                [% END %]
                                            </td>
                                            <td>
                                                [%  display_matcher = "";
                                                    FOREACH matcher IN matchers;
                                                        IF ( matcher.code == saved_request.import_matcher_code );
                                                            display_matcher = matcher;
                                                        END;
                                                    END;
                                                %]
                                                [% IF ( display_matcher ) %]
                                                    [% display_matcher.description %]
                                                [% ELSE %]
                                                    [% saved_request.import_matcher_code %]
                                                [% END %]
                                            </td> -->
                                            <td>
                                                <div class="dropdown">
                                                    <a class="btn btn-default btn-xs dropdown-toggle" role="button" data-toggle="dropdown" href="#">Actions <span class="caret"></span></a>
                                                    <ul class="dropdown-menu pull-right" role="menu">
                                                          <li>
                                                              <a href="[% request_page %]?op=edit&id=[% saved_request.id %]"><i class="fa fa-pencil"></i> Edit</a>
                                                          </li>
                                                          <li>
                                                              <a href="[% request_page %]?op=new&id=[% saved_request.id %]"><i class="fa fa-copy"></i> Copy</a>
                                                          </li>
                                                          <li>
                                                              <a href="[% dashboard_page %]?op=send&id=[% saved_request.id %]"><i class="fa fa-send"></i> Send</a>
                                                          </li>
                                                          <li>
                                                            <a href="[% request_page %]?op=delete&id=[% saved_request.id %]"><i class="fa fa-trash"></i> Delete</a>
                                                          </li>
                                                    </ul>
                                                </div>
                                            </td>
                                        </tr>
                                    [% END %]
                                [% END %]
                                </tbody>
                            </table>
                        </div>
                        <div id="imports">
                            <div class="btn-toolbar">
                                <button id="refresh-button" type="button" class="btn btn-default btn-sm">Refresh import history</button>
                            </div>
                            <table id="history-table">
                                <thead>
                                    <tr>
                                        <th>Id</th>
                                        <th>Repository</th>
                                        <th>Identifier</th>
                                        <th>Datestamp</th>
                                        <th>Upstream status</th>
                                        <th>Import status</th>
                                        <th>Import timestamp</th>
                                        <th>Imported record</th>
                                        <th>Downloaded record</th>
                                    </tr>
                                </thead>
                            </table>

                        </div>
                    </div>
                </div>
            </div>
            <div class="yui-b">
                [% INCLUDE 'tools-menu.inc' %]
            </div>
        </div>
[% INCLUDE 'intranet-bottom.inc' %]




[% INCLUDE 'doc-head-open.inc' %]
<title>Koha &rsaquo; Tools &rsaquo; OAI-PMH harvester &rsaquo; Downloaded record</title>
[% INCLUDE 'doc-head-close.inc' %]
</head>
<body id="tools_oai-pmh-harvester_request" class="tools">
[% INCLUDE 'header.inc' %]
[% INCLUDE 'cat-search.inc' %]
    <div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> &rsaquo; <a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a> &rsaquo; Downloaded record</div>
    <div id="doc3" class="yui-t2">
        <div id="bd">
            <div id="yui-main">
                <div class="yui-b">
                    <h1>Downloaded record</h1>
                    [% IF ( record ) %]
                        <div style="white-space:pre">[% record | xml %]</div>
                    [% END %]
                </div>
            </div>
            <div class="yui-b">
                [% INCLUDE 'tools-menu.inc' %]
            </div>
        </div>
[% INCLUDE 'intranet-bottom.inc' %]




[% INCLUDE 'doc-head-open.inc' %]
<title>Koha &rsaquo; Tools &rsaquo; OAI-PMH harvester &rsaquo; Request</title>
[% INCLUDE 'doc-head-close.inc' %]
[% INCLUDE 'calendar.inc' %]
<script type="text/javascript" src="[% interface %]/lib/jquery/plugins/jquery-ui-timepicker-addon.min.js"></script>
[% INCLUDE 'timepicker.inc' %]
<script type="text/javascript">
//<![CDATA[
    $(document).ready(function() {
        toggle_identifier();
        $("#oai_verb").on("click",toggle_identifier);
        $(".datetime_utc").datetimepicker({
            separator: "T",
            timeSuffix: 'Z',
            dateFormat: "yy-mm-dd",
            timeFormat: "HH:mm:ss",
            hour: 0,
            minute: 0,
            second: 0,
            showSecond: 1,
            // timezone doesn't work with the "Now" button in v1.4.3 although it appears to in v1.6.1
            // timezone: '+000'
        });
    });
    function toggle_identifier (){
        var verb = $("#oai_verb").find(":selected").val();
        var oai_identifier = $("#oai_identifier");
        var oai_set = $("#oai_set");
        var oai_from = $("#oai_from");
        var oai_until = $("#oai_until");
        if (verb == 'ListRecords'){
            oai_identifier.prop('disabled', true);
            oai_set.prop('disabled', false);
            oai_from.prop('disabled', false);
            oai_until.prop('disabled', false);
        }
        else if (verb == 'GetRecord'){
            oai_identifier.prop('disabled', false);
            oai_set.prop('disabled', true);
            oai_from.prop('disabled', true);
            oai_until.prop('disabled', true);
        }
    }
//]]>
</script>
<style type="text/css">
    /* Override staff-global.css which hides second, millisecond, and microsecond sliders */
    .ui_tpicker_second {
        display: block;
    }
</style>
</head>
<body id="tools_oai-pmh-harvester_request" class="tools">
[% INCLUDE 'header.inc' %]
[% INCLUDE 'cat-search.inc' %]
    <div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> &rsaquo; <a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a> &rsaquo; Request</div>
    <div id="doc3" class="yui-t2">
        <div id="bd">
            <div id="yui-main">
                <div class="yui-b">
                    [% IF ( op == "edit" ) %]
                        <h1>Edit OAI-PMH request</h1>
                    [% ELSE %]
                        <h1>New OAI-PMH request</h1>
                    [% END %]
                    [% IF ( test_parameters ) %]
                        [% IF ( errors.size ) %]
                            <span class="text-danger">Tests failed!</span>
                        [% ELSE %]
                            <span class="text-success">Tests successful!</span>
                        [% END %]
                    [% END %]
                    <form action="/cgi-bin/koha/tools/oai-pmh-harvester/request.pl" method="post" name="entry-form">
                        [% IF ( op == "new" ) %]
                            <input type="hidden" name="op" value="create" />
                        [% ELSIF ( op == "edit" ) %]
                             <input type="hidden" name="op" value="update" />
                        [% ELSE %]
                            <input type="hidden" name="op" value="[% op %]" />
                        [% END %]
                        [% IF ( id ) %]
                            <input type="hidden" name="id" value="[% id %]" />
                        [% END %]
                        <fieldset class="rows">
                            <ol>
                                <li>
                                    <label for="name">Name:</label>
                                    <input type="text" size="30" id="name" name="name" value="[% oai_pmh_request.name %]"/>
                                    <span class="help">This is just a short name to help in managing requests.</span>
                                </li>
                            </ol>
                        </fieldset>
                        <fieldset class="rows">
                            <legend>HTTP parameters:</legend>
                            <ol>
                                <li>
                                    <label for="http_url">URL:</label>
                                    <input type="text" size="30" id="http_url" name="http_url" value="[% oai_pmh_request.http_url %]">
                                    [% IF (errors.http_url.malformed) %]<span class="error">[This must be a properly formatted HTTP or HTTPS URL.]</span>[% END %]
                                    [% IF (errors.http.404) %]<span class="error">[Cannot find address specified by this URL.]</span>[% END %]
                                    [% IF (errors.http.401) %]<span class="error">[Permission denied to access this URL.]</span>[% END %]
                                    [% IF (errors.http.generic) %]<span class="error">[Unable to access this URL.]</span>[% END %]
                                </li>
                            </ol>
                            <span class="help">The following parameters are not required by all OAI-PMH repositories, so they may be optional for this task.</span>
                            <ol>
                                <li>
                                    <label for="http_username">Username:</label>
                                    <input type="text" size="30" id="http_username" name="http_username" value="[% oai_pmh_request.http_username %]">
                                </li>
                                <li>
                                    <label for="http_password">Password:</label>
                                    <input type="text" size="30" id="http_password" name="http_password" value="[% oai_pmh_request.http_password %]">
                                </li>
                                <li>
                                    <label for="http_realm">Realm:</label>
                                    <input type="text" size="30" id="http_realm" name="http_realm" value="[% oai_pmh_request.http_realm %]">
                                </li>
                            </ol>
                        </fieldset>
                        <fieldset class="rows">
                            <legend>OAI-PMH parameters:</legend>
                            <ol>
                                <li>
                                    <label for="oai_verb">Verb:</label>
                                    <select id="oai_verb" name="oai_verb">
                                        [% IF ( oai_pmh_request.oai_verb == "ListRecords" ) %]
                                        <option value="ListRecords" selected="selected">ListRecords</option>
                                        [% ELSE %]
                                        <option value="ListRecords">ListRecords</option>
                                        [% END %]
                                        [% IF ( oai_pmh_request.oai_verb == "GetRecord" ) %]
                                        <option value="GetRecord" selected="selected">GetRecord</option>
                                        [% ELSE %]
                                        <option value="GetRecord">GetRecord</option>
                                        [% END %]
                                    </select>
                                </li>
                                <li>
                                    <label for="oai_metadataPrefix">Metadata prefix:</label>
                                    <input type="text" size="30" id="oai_metadataPrefix" name="oai_metadataPrefix" value="[% oai_pmh_request.oai_metadataPrefix %]">
                                    [% IF (errors.oai_metadataPrefix.unavailable) %]<span class="error">[This metadataPrefix is unavailable from this OAI-PMH provider.]</span>[% END %]
                                    [% IF (errors.oai_metadataPrefix.missing) %]<span class="error">[metadataPrefix is a required field for an OAI-PMH request.]</span>[% END %]
                                </li>
                                <li>
                                    <label for="oai_identifier">Identifier:</label>
                                    <input type="text" size="30" id="oai_identifier" name="oai_identifier" value="[% oai_pmh_request.oai_identifier %]">
                                    [% IF (errors.oai_identifier.missing) %]<span class="error">[Identifier is a required field when using GetRecord for an OAI-PMH request.]</span>[% END %]
                                </li>
                                <li>
                                    <label for="oai_set">Set:</label>
                                    <input type="text" size="30" id="oai_set" name="oai_set" value="[% oai_pmh_request.oai_set %]">
                                    [% IF (errors.oai_set.unavailable) %]<span class="error">[This set is unavailable from this OAI-PMH provider.]</span>[% END %]
                                </li>
                                [% IF (errors.oai.granularity_mismatch) %]<span class="error">[You must specify the same granularity for both From and Until.]</span>[% END %]
                                <li>
                                    <label for="oai_from">From:</label>
                                    <input type="text" size="30" class="datetime_utc" id="oai_from" name="oai_from" value="[% oai_pmh_request.oai_from %]">
                                    <span class="help">This value will be treated as UTC time. Note that some repositories only support YYYY-MM-DD datestamps.</span>
                                    [% IF (errors.oai_from.malformed) %]<span class="error">[This must be in YYYY-MM-DD or YYYY-MM-DDThh:mm:ssZ format.]</span>[% END %]
                                    [% IF (errors.oai_from.unavailable) %]<span class="error">[This granularity is unsupported by this OAI-PMH provider.]</span>[% END %]
                                </li>
                                <li>
                                    <label for="oai_until">Until:</label>
                                    <input type="text" size="30" class="datetime_utc" id="oai_until" name="oai_until" value="[% oai_pmh_request.oai_until %]">
                                    <span class="help">This value will be treated as UTC time. Note that some repositories only support YYYY-MM-DD datestamps.</span>
                                    [% IF (errors.oai_until.malformed) %]<span class="error">[This must be in YYYY-MM-DD or YYYY-MM-DDThh:mm:ssZ format.]</span>[% END %]
                                    [% IF (errors.oai_until.unavailable) %]<span class="error">[This granularity is unsupported by this OAI-PMH provider.]</span>[% END %]
                                </li>
                            </ol>
                        </fieldset>
                        <fieldset class="rows">
                            <legend>Import parameters:</legend>
                            <ol>
                                <li>
                                    <label for="import_filter">Filter:</label>
                                    [% IF ( oai_pmh_request.import_filter == "default" ) %]
                                        <input type="text" size="30" id="import_filter" name="import_filter" value="default">
                                    [% ELSE %]
                                        <input type="text" size="30" id="import_filter" name="import_filter" value="[% oai_pmh_request.import_filter %]">
                                    [% END %]
                                    <span class="help">If no filter is entered, the default filter will be used.</span>
                                </li>
                                <li>
                                    <label for="import_framework_code">Framework code:</label>
                                    <select id="import_framework_code" name="import_framework_code">
                                        <option value="">Default framework</option>
                                        [% FOREACH framework IN frameworks %]
                                            [% IF ( oai_pmh_request.import_framework_code == framework.frameworkcode ) %]
                                                <option selected="selected" value="[% framework.frameworkcode %]">[% framework.frameworktext %]</option>
                                            [% ELSE %]
                                                <option value="[% framework.frameworkcode %]">[% framework.frameworktext %]</option>
                                            [% END %]
                                        [% END %]
                                    </select>
                                </li>
                                <li>
                                    <label for="import_record_type">Record type:</label>
                                    <select id="import_record_type" name="import_record_type">
                                        <option value="biblio">Bibliographic</option>
                                    </select>
                                </li>
                                <li>
                                    <label for="import_matcher_code">Record matcher:</label>
                                    <select id="import_matcher_code" name="import_matcher_code">
                                        <option value="">None</option>
                                        [% FOREACH matcher IN matchers %]
                                            [% IF ( oai_pmh_request.import_matcher_code == matcher.code ) %]
                                                <option value="[% matcher.code %]" selected="selected">[% matcher.description %]</option>
                                            [% ELSE %]
                                                <option value="[% matcher.code %]">[% matcher.description %]</option>
                                            [% END %]
                                        [% END %]
                                    </select>
                                    <span class="help">See <a href="/cgi-bin/koha/admin/matching-rules.pl">record matching rules</a> to add or edit rules.</span>
                                </li>
                            </ol>
                        </fieldset>
                        <fieldset class="rows">
                            <legend>Download parameters:</legend>
                            <ol>
                                <li>
                                    <label for="interval">Interval (seconds): </label>
                                    <input type="text" id="interval" name="interval" value="[% oai_pmh_request.interval %]" size="4">
                                    <span class="help">The download request will be repeated in intervals of this many seconds. Enter "0" if you want the task to only happen once.</span>
                                </li>
                            </ol>
                        </fieldset>
                        <fieldset class="action">
                            <input type="submit" name="test_parameters" value="Test parameters">
                            <input type="submit" name="save" value="Save">
                            <a class="cancel" href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">Cancel</a>
                        </fieldset>
                    </form>
                </div>
            </div>
            <div class="yui-b">
                [% INCLUDE 'tools-menu.inc' %]
            </div>
        </div>
[% INCLUDE 'intranet-bottom.inc' %]

Lines 189-194 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/tools-home.tt (+5 lines)
189	<dd>Utility to upload scanned cover images for display in OPAC</dd>	189	<dd>Utility to upload scanned cover images for display in OPAC</dd>
190	[% END %]	190	[% END %]
191		191
		192	[% IF ( CAN_user_tools_manage_staged_marc ) %]
		193	<dt><a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a></dt>
		194	<dd>Harvest (ie download and import) records from remote sources using the OAI-PMH protocol</dd>
		195	[% END %]
		196
192	</dl>	197	</dl>
193	</div>	198	</div>
194		199




<?xml version="1.0" encoding="UTF-8"?>
<xsl:stylesheet version="1.0"
    xmlns:oai="http://www.openarchives.org/OAI/2.0/"
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>
    <!-- NOTE: This XSLT strips the OAI-PMH wrapper from the metadata. -->

    <!-- Match the root oai:record element -->
    <xsl:template match="oai:record">
        <!-- Apply templates only to the child metadata element(s) -->
        <xsl:apply-templates select="oai:metadata" />
    </xsl:template>

    <!-- Matches an oai:metadata element -->
    <xsl:template match="oai:metadata">
        <!-- Create a copy of child attributes and nodes -->
        <xsl:apply-templates select="@* | node()" />
    </xsl:template>

    <!-- Identity transformation: this template copies attributes and nodes -->
    <xsl:template match="@* | node()">
        <!-- Create a copy of this attribute or node -->
        <xsl:copy>
            <!-- Recursively apply this template to the attributes and child nodes of this element -->
            <xsl:apply-templates select="@* | node()" />
        </xsl:copy>
    </xsl:template>
</xsl:stylesheet>

Lines 26-31 use constant DEFAULT_LOGS_PURGEDAYS => 180; Link Here

(-)a/misc/cronjobs/cleanup_database.pl (-1 / +14 lines)
26	use constant DEFAULT_SEARCHHISTORY_PURGEDAYS => 30;	26	use constant DEFAULT_SEARCHHISTORY_PURGEDAYS => 30;
27	use constant DEFAULT_SHARE_INVITATION_EXPIRY_DAYS => 14;	27	use constant DEFAULT_SHARE_INVITATION_EXPIRY_DAYS => 14;
28	use constant DEFAULT_DEBARMENTS_PURGEDAYS => 30;	28	use constant DEFAULT_DEBARMENTS_PURGEDAYS => 30;
		29	use constant DEFAULT_IMPORTOAI_PURGEDAYS => 30;
29		30
30	BEGIN {	31	BEGIN {
31	# find Koha's Perl modules	32	# find Koha's Perl modules
Lines 44-50 use Koha::UploadedFiles; Link Here
44		45
45	sub usage {	46	sub usage {
46	print STDERR <<USAGE;	47	print STDERR <<USAGE;
47	Usage: $0 [-h\|--help] [--sessions] [--sessdays DAYS] [-v\|--verbose] [--zebraqueue DAYS] [-m\|--mail] [--merged] [--import DAYS] [--logs DAYS] [--searchhistory DAYS] [--restrictions DAYS] [--all-restrictions] [--fees DAYS] [--temp-uploads] [--temp-uploads-days DAYS] [--uploads-missing 0\|1 ]	48	Usage: $0 [-h\|--help] [--sessions] [--sessdays DAYS] [-v\|--verbose] [--zebraqueue DAYS] [-m\|--mail] [--merged] [--import DAYS] [--logs DAYS] [--searchhistory DAYS] [--restrictions DAYS] [--all-restrictions] [--fees DAYS] [--temp-uploads] [--temp-uploads-days DAYS] [--uploads-missing 0\|1 ] [--importoai DAYS ]
48		49
49	-h --help prints this help message, and exits, ignoring all	50	-h --help prints this help message, and exits, ignoring all
50	other options	51	other options
Lines 84-89 Usage: $0 [-h\|--help] [--sessions] [--sessdays DAYS] [-v\|--verbose] [--zebraqueu Link Here
84	--temp-uploads Delete temporary uploads.	85	--temp-uploads Delete temporary uploads.
85	--temp-uploads-days DAYS Override the corresponding preference value.	86	--temp-uploads-days DAYS Override the corresponding preference value.
86	--uploads-missing FLAG Delete upload records for missing files when FLAG is true, count them otherwise	87	--uploads-missing FLAG Delete upload records for missing files when FLAG is true, count them otherwise
		88	--importoai DAYS purge OAI-PMH records from import_oai table older than DAYS
		89	days. Defaults to 30 days if no days specified.
87	USAGE	90	USAGE
88	exit $_[0];	91	exit $_[0];
89	}	92	}
Lines 109-114 my $special_holidays_days; Link Here
109	my $temp_uploads;	112	my $temp_uploads;
110	my $temp_uploads_days;	113	my $temp_uploads_days;
111	my $uploads_missing;	114	my $uploads_missing;
		115	my $importoai_days;
112		116
113	GetOptions(	117	GetOptions(
114	'h\|help' => \$help,	118	'h\|help' => \$help,
Lines 132-137 GetOptions( Link Here
132	'temp-uploads' => \$temp_uploads,	136	'temp-uploads' => \$temp_uploads,
133	'temp-uploads-days:i' => \$temp_uploads_days,	137	'temp-uploads-days:i' => \$temp_uploads_days,
134	'uploads-missing:i' => \$uploads_missing,	138	'uploads-missing:i' => \$uploads_missing,
		139	'importoai:i' => \$importoai_days,
135	) \|\| usage(1);	140	) \|\| usage(1);
136		141
137	# Use default values	142	# Use default values
Lines 143-148 $mail = DEFAULT_MAIL_PURGEDAYS if defined($mail) Link Here
143	$pSearchhistory = DEFAULT_SEARCHHISTORY_PURGEDAYS if defined($pSearchhistory) && $pSearchhistory == 0;	148	$pSearchhistory = DEFAULT_SEARCHHISTORY_PURGEDAYS if defined($pSearchhistory) && $pSearchhistory == 0;
144	$pListShareInvites = DEFAULT_SHARE_INVITATION_EXPIRY_DAYS if defined($pListShareInvites) && $pListShareInvites == 0;	149	$pListShareInvites = DEFAULT_SHARE_INVITATION_EXPIRY_DAYS if defined($pListShareInvites) && $pListShareInvites == 0;
145	$pDebarments = DEFAULT_DEBARMENTS_PURGEDAYS if defined($pDebarments) && $pDebarments == 0;	150	$pDebarments = DEFAULT_DEBARMENTS_PURGEDAYS if defined($pDebarments) && $pDebarments == 0;
		151	$importoai_days = DEFAULT_IMPORTOAI_PURGEDAYS if defined($importoai_days) && $importoai_days == 0;
146		152
147	if ($help) {	153	if ($help) {
148	usage(0);	154	usage(0);
Lines 165-170 unless ( $sessions Link Here
165	\|\| $special_holidays_days	171	\|\| $special_holidays_days
166	\|\| $temp_uploads	172	\|\| $temp_uploads
167	\|\| defined $uploads_missing	173	\|\| defined $uploads_missing
		174	\|\| $importoai_days
168	) {	175	) {
169	print "You did not specify any cleanup work for the script to do.\n\n";	176	print "You did not specify any cleanup work for the script to do.\n\n";
170	usage(1);	177	usage(1);
Lines 336-341 if( defined $uploads_missing ) { Link Here
336	}	343	}
337	}	344	}
338		345
		346	if ($importoai_days){
		347	my $sql = "DELETE FROM import_oai WHERE date(upload_timestamp) < (date_sub(curdate(), INTERVAL ? DAY))";
		348	my $sth = $dbh->prepare($sql);
		349	$sth->execute($importoai_days) or die $dbh->errstr;
		350	}
		351
339	exit(0);	352	exit(0);
340		353
341	sub RemoveOldSessions {	354	sub RemoveOldSessions {




#!/usr/bin/perl

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;
use Getopt::Long;
use Pod::Usage;
use Module::Load;
use Log::Log4perl qw(:easy);
use POE;
use YAML;

use C4::Context;
use Koha::Daemon;
use Koha::OAI::Harvester;
use Koha::OAI::Harvester::Listener;

binmode(STDOUT,':encoding(UTF-8)');
$|++;

my $help = 0;
my $daemonize = 0;

my ($socket_addr,$pidfile,$statefile,$spooldir);
my ($download_module,$import_module);
my ($batch,$download_workers,$import_workers,$import_poll);
my ($logfile,$log_level);
my $log_levels = {
    FATAL => $FATAL,
    ERROR => $ERROR,
    WARN => $WARN,
    INFO => $INFO,
    DEBUG => $DEBUG,
    TRACE => $TRACE,
};

my $context = C4::Context->new();
my $config_filename = $context->{config}->{oai_pmh_harvester_config};
if ($config_filename){
    my $config = YAML::LoadFile($config_filename);
    if ($config){
        $socket_addr = $config->{socket};
        $pidfile = $config->{pidfile};
        $statefile = $config->{statefile};
        $spooldir = $config->{spooldir};
        $logfile = $config->{logfile};
        $log_level = $config->{loglevel};
        $download_module = $config->{download_module};
        $import_module = $config->{import_module};
        $batch = $config->{download_batch};
        $download_workers = $config->{download_workers};
        $import_workers = $config->{import_workers};
        $import_poll = $config->{import_poll};
    }
}

GetOptions(
    "help|?"            => \$help,
    "daemon"            => \$daemonize,
    "socket-uri=s"          => \$socket_addr,
    "pid-file=s"             => \$pidfile,
    "state-file=s"      => \$statefile,
    "spool-dir=s"       => \$spooldir,
    "log-file=s"             => \$logfile,
    "log-level=s"       => \$log_level,
    "download-module=s" => \$download_module,
    "import-module=s"   => \$import_module,
) or pod2usage(2);
pod2usage(1) if $help;

my $level = ( $log_level && $log_levels->{$log_level} ) ? $log_levels->{$log_level} : $log_levels->{WARN};
Log::Log4perl->easy_init(
    {
        level => $level,
        file => "STDOUT",
        layout   => '[%d{yyyy-MM-dd HH:mm:ss}][%p] %m%n',
    }
);
my $logger = Log::Log4perl->get_logger();

unless($download_module){
    $download_module = "Koha::OAI::Harvester::Worker::Download::Stream";
}
unless($import_module){
    $import_module = "Koha::OAI::Harvester::Worker::Import";
}

foreach my $module ( $download_module, $import_module ){
    load $module;
}
my $downloader = $download_module->new({
    logger => $logger,
    batch => $batch,
});
my $importer = $import_module->new({
    logger => $logger,
});

my $daemon = Koha::Daemon->new({
    pidfile => $pidfile,
    logfile => $logfile,
    daemonize => $daemonize,
});
$daemon->run();

my $harvester = Koha::OAI::Harvester->spawn({
    Downloader => $downloader,
    DownloaderWorkers => $download_workers,
    Importer => $importer,
    ImporterWorkers => $import_workers,
    ImportQueuePoll => $import_poll,
    logger => $logger,
    state_file => $statefile,
    spooldir => $spooldir,
});

my $listener = Koha::OAI::Harvester::Listener->spawn({
    logger => $logger,
    socket => $socket_addr,
});

$logger->info("OAI-PMH harvester started.");

POE::Kernel->run();

exit;

=head1 NAME

harvesterd.pl - a daemon that asynchronously sends OAI-PMH requests and imports OAI-PMH records

=head1 SYNOPSIS

KOHA_CONF=/path/to/koha-conf.xml ./harvesterd.pl

=head1 OPTIONS

=over 8

=item B<--help>

Print a brief help message and exits.

=item B<--daemon>

Run program as a daemon (ie fork process, setsid, chdir to root, reset umask,
and close STDIN, STDOUT, and STDERR).

=item B<--log-file>

Specify a file to which to log STDOUT and STDERR.

=item B<--pid-file>

Specify a file to store the process id (this prevents multiple copies of the program
from running at the same time).

=item B<--socket-uri>

Specify a URI to use for the UNIX socket used to communicate with the daemon.
(e.g. unix:/path/to/socket.sock)

=item B<--state-file>

Specify a filename to use for storing the harvester's in-memory state.

In the event that the harvester crashes, it can resume from where it stopped.

=item B<--spool-dir>

Specify a directory to store downloaded OAI-PMH records prior to import.

=item B<--log-level>

Specify a log level for logging. The logger uses Log4Perl, which provides
FATAL, ERROR, WARN, INFO, DEBUG, and TRACE in order of descending priority.

Defaults to WARN level.

=item B<--download-module>

Specify a Perl module to use for downloading records. This is a specialty module,
which has particular requirements, so only advanced users should use this option.

=item B<--import-module>

Specify a Perl module to use for importing records. This is a specialty module,
which has particular requirements, so only advanced users should use this option.

=back

=cut

Lines 153-158 $prefix = $ENV{'INSTALL_BASE'} \|\| "/usr"; Link Here

(-)a/rewrite-config.PL (+3 lines)
153	"__MEMCACHED_SERVERS__" => "",	153	"__MEMCACHED_SERVERS__" => "",
154	"__MEMCACHED_NAMESPACE__" => "",	154	"__MEMCACHED_NAMESPACE__" => "",
155	"__FONT_DIR__" => "/usr/share/fonts/truetype/ttf-dejavu",	155	"__FONT_DIR__" => "/usr/share/fonts/truetype/ttf-dejavu",
		156	"__OAI_RUN_DIR__" => "",
		157	"__OAI_LIB_DIR__" => "",
		158	"__OAI_SPOOL_DIR__" => "",
156	);	159	);
157		160
158	# Override configuration from the environment	161	# Override configuration from the environment




#!/usr/bin/perl

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;
use C4::Auth qw(check_cookie_auth haspermission get_session);
use JSON;
use Koha::Database;

my $input = new CGI;

my ( $auth_status, $sessionID ) =
  check_cookie_auth( $input->cookie('CGISESSID'));

if ( $auth_status ne "ok" ) {
    exit 0;
}

my $data = {
    data => [],
    recordsTotal => 0,
    recordsFiltered => 0,
    draw => undef,
};

my $length = 10;
my $start = 0;
my @order_by = ();
my @search = ();

if ($input->request_method eq "POST"){
    my $postdata = $input->param('POSTDATA');
    my $request = from_json($postdata);
    $data->{draw} = int( $request->{draw} ) if $request->{draw};
    $length = $request->{length} if $request->{length};
    $start = $request->{start} if $request->{start};
    if (my $search = $request->{search}){
        my $value = $search->{value};
        if ($value){
            foreach my $column (@{$request->{columns}}){
                if ($column->{data} && $column->{searchable}){
                    my $search_element = {
                        $column->{data} => { 'like', "%".$value."%" },
                    };
                    push(@search,$search_element);
                }
            }
        }
    }
    if (my $order = $request->{order}){
        foreach my $element (@$order){
            my $dir = $element->{dir};
            my $column_index = $element->{column};
            my $column = $request->{columns}->[$column_index];
            my $orderable = $column->{orderable};
            if ($orderable){
                my $column_name = $column->{data};
                my $direction;
                if ($dir){
                    if ($dir eq "asc" || $dir eq "desc"){
                        $direction = "-$dir";
                    }
                }
                if ($column_name && $direction){
                    my $single_order = {
                        $direction => $column_name,
                    };
                    push(@order_by,$single_order);
                }
            }
        }
    }
}

my $page = ( $start / $length ) + 1;
my $schema = Koha::Database->new()->schema();
if ($schema){
    my $rs = $schema->resultset("OaiHarvesterHistory");
    my $results = $rs->search(
        \@search,
        {
            result_class => 'DBIx::Class::ResultClass::HashRefInflator',
            page => $page,
            rows => $length,
            order_by => \@order_by,
        },
    );
    my $count = $rs->count;
    my $filtered_count = $results->pager->total_entries;
    my @rows = ();
    while (my $row = $results->next){
        $row->{imported_record} = '';
        if ($row->{record_type} eq "biblio"){
            my $harvested_biblio = $schema->resultset("OaiHarvesterBiblio")->find(
                    {
                        oai_repository => $row->{repository},
                        oai_identifier => $row->{header_identifier},
                    },
                    { key => "oai_record" },
            );
            $row->{imported_record} = $harvested_biblio->biblionumber->id if $harvested_biblio;
        }
        push(@rows,$row);
    }
    if ($count){
        $data->{recordsTotal} = $count;
        $data->{recordsFiltered} = $filtered_count;
        $data->{data} = \@rows if @rows;
    }
}

binmode STDOUT, ":encoding(UTF-8)";
print $input->header(
    -type => 'application/json',
    -charset => 'UTF-8'
);
print to_json($data, { pretty => 1, });




#!/usr/bin/perl

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;
use YAML;

use C4::Auth;
use C4::Context;
use C4::Output;
use Koha::OAI::Harvester::Client;
use Koha::OAI::Harvester::Requests;
use Koha::BiblioFrameworks;
use Koha::Database;

my $context = C4::Context->new();
my $config_filename = $context->{config}->{oai_pmh_harvester_config};
my $client_config = {};
if ($config_filename){
    my $config = YAML::LoadFile($config_filename);
    if ($config && $config->{socket}){
        $client_config->{socket_uri} = $config->{socket};
    }
}

my $input = new CGI;

my ($template, $loggedinuser, $cookie) =
    get_template_and_user({template_name => "tools/oai-pmh-harvester/dashboard.tt",
        query => $input,
        type => "intranet",
        authnotrequired => 0,
        flagsrequired => {tools => 'manage_staged_marc'},
    });

my $op = $input->param('op') // 'list';
my $id = $input->param('id');
my $uuid = $input->param('uuid');

my $client = Koha::OAI::Harvester::Client->new($client_config);
my $is_connected = $client->connect;

if ( ($op eq "send") && $id ){
    if ($is_connected){
        my $request = Koha::OAI::Harvester::Requests->find($id);
        if ($request){
            my $task = {
                name => $request->name,
                uuid => $request->uuid,
                interval => $request->interval,
                parameters => {
                    oai_pmh => {
                        baseURL => $request->http_url,
                        verb => $request->oai_verb,
                        metadataPrefix => $request->oai_metadataPrefix,
                        identifier => $request->oai_identifier,
                        set => $request->oai_set,
                        from => $request->oai_from,
                        until => $request->oai_until,
                    },
                    import => {
                        filter => $request->import_filter,
                        frameworkcode => $request->import_framework_code,
                        matcher_code => $request->import_matcher_code,
                        record_type => $request->import_record_type,
                    },
                },
            };
            if ($request->http_username && $request->http_password && $request->http_realm){
                $task->{parameters}->{http_basic_auth} = {
                    username => $request->http_username,
                    password => $request->http_password,
                    realm => $request->http_realm,
                };
            }
            my $is_created = $client->create($task);
            $template->{VARS}->{ result }->{ send } = $is_created;
        }
    }
}
elsif ( ($op eq "start") && ($uuid) ){
    if ($is_connected){
        my $is_started = $client->start($uuid);
        $template->{VARS}->{ result }->{ start } = $is_started;
    }
}
elsif ( ($op eq "stop") && ($uuid) ){
    if ($is_connected){
        my $is_stopped = $client->stop($uuid);
        $template->{VARS}->{ result }->{ stop } = $is_stopped;
    }
}
elsif ( ($op eq "delete") && ($uuid) ){
    if ($is_connected){
        my $is_deleted = $client->delete($uuid);
        $template->{VARS}->{ result }->{ delete } = $is_deleted;
    }
}

my $requests = Koha::OAI::Harvester::Requests->as_list;
$template->{VARS}->{ saved_requests } = $requests;

my $frameworks = Koha::BiblioFrameworks->as_list();
$template->{VARS}->{ frameworks } = $frameworks;

my $schema = Koha::Database->new()->schema();
my $matcher_rs = $schema->resultset("MarcMatcher");
my @matchers = $matcher_rs->all;
$template->{VARS}->{ matchers } = \@matchers;

if ($is_connected){
    my $submitted_requests = $client->list;
    $template->{VARS}->{ submitted_requests } = $submitted_requests;
}
else {
    $template->{VARS}->{ harvester }->{ offline } = 1;
}

output_html_with_http_headers($input, $cookie, $template->output);




#!/usr/bin/perl

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;

use C4::Auth;
use C4::Output;
use Koha::Database;

my $input = new CGI;

my ($template, $loggedinuser, $cookie) =
    get_template_and_user({template_name => "tools/oai-pmh-harvester/record.tt",
        query => $input,
        type => "intranet",
        authnotrequired => 0,
        flagsrequired => {tools => 'manage_staged_marc'},
    });

my $import_oai_id = $input->param('import_oai_id');
if ($import_oai_id){
    my $schema = Koha::Database->new()->schema();
    if ($schema){
        my $rs = $schema->resultset("OaiHarvesterHistory");
        if ($rs){
            my $row = $rs->find($import_oai_id);
            if ($row){
                my $record = $row->record;
                if ($record){
                    $template->{VARS}->{ record } = $record;
                }
            }
        }
    }
}

output_html_with_http_headers($input, $cookie, $template->output);




#!/usr/bin/perl

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;
use UUID qw/uuid/;

use C4::Auth;
use C4::Output;
use Koha::OAI::Harvester::Requests;
use Koha::BiblioFrameworks;
use Koha::Database;

my $input = new CGI;

my ($template, $loggedinuser, $cookie) =
    get_template_and_user({template_name => "tools/oai-pmh-harvester/request.tt",
        query => $input,
        type => "intranet",
        authnotrequired => 0,
        flagsrequired => {tools => 'manage_staged_marc'},
    });

my $op = $input->param('op');
my $id = $input->param('id');

my @frameworks = Koha::BiblioFrameworks->as_list();
$template->{VARS}->{ frameworks } = \@frameworks;

my $schema = Koha::Database->new()->schema();
my $rs = $schema->resultset("MarcMatcher");
my @matchers = $rs->all;
$template->{VARS}->{ matchers } = \@matchers;

my $http_url = $input->param('http_url');
my $http_username = $input->param('http_username');
my $http_password = $input->param('http_password');
my $http_realm = $input->param('http_realm');

my $oai_verb = $input->param('oai_verb');
my $oai_metadataPrefix = $input->param('oai_metadataPrefix');
my $oai_identifier = $input->param('oai_identifier');
my $oai_from = $input->param('oai_from');
my $oai_until = $input->param('oai_until');
my $oai_set = $input->param('oai_set');

my $import_filter = $input->param('import_filter') // 'default';
my $import_framework_code = $input->param('import_framework_code');
my $import_record_type = $input->param('import_record_type');
my $import_matcher_code = $input->param('import_matcher_code');

my $interval = $input->param("interval") ? int ( $input->param("interval") ) : 0;
my $name = $input->param("name");

my $save = $input->param('save');
my $test_parameters = $input->param('test_parameters');

my $request = $id ? Koha::OAI::Harvester::Requests->find($id) : Koha::OAI::Harvester::Request->new();
if ($request){
    if ($op eq "create" || $op eq "update"){
        $request->set({
            name => $name,
            http_url => $http_url,
            http_username => $http_username,
            http_password => $http_password,
            http_realm => $http_realm,
            oai_verb => $oai_verb,
            oai_metadataPrefix => $oai_metadataPrefix,
            oai_identifier => $oai_identifier,
            oai_from => $oai_from,
            oai_until => $oai_until,
            oai_set => $oai_set,
            import_filter => $import_filter,
            import_framework_code => $import_framework_code,
            import_record_type => $import_record_type,
            import_matcher_code => $import_matcher_code,
            interval => $interval,
        });
    }
}

if ($test_parameters){
    my $errors = $request->validate();
    $template->{VARS}->{ errors } = $errors;
    $template->{VARS}->{ test_parameters } = 1;
}

if ($op eq "new"){
    #Empty form with some defaults
    $request->import_filter("default") unless $request->import_filter;
    $request->interval(0) unless $request->interval;
}
elsif ($op eq "create"){
    if ($save){
        my $uuid = uuid;
        $request->uuid($uuid);
        $request->store;
        print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
        exit;
    }
}
elsif ( $op eq "edit"){
    $template->{VARS}->{ id } = $id;
}
elsif ($op eq "update"){
    $template->{VARS}->{ id } = $id;
    if ($save){
        $request->store;
        print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
        exit;
    }
}
elsif ($op eq "delete"){
    if ($request){
        $request->delete;
        print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
    }
}
else {
    print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
}
$template->{VARS}->{ op } = $op;
$template->{VARS}->{ oai_pmh_request } = $request;

output_html_with_http_headers($input, $cookie, $template->output);

Return to bug 10662

Lines 867-872 our $PERL_DEPS = { Link Here

(-)a/C4/Installer/PerlDependencies.pm (+10 lines)
867	'required' => '1',	867	'required' => '1',
868	'min_ver' => '1.017',	868	'min_ver' => '1.017',
869	},	869	},
		870	'POE' => {
		871	'usage' => 'OAI-PMH harvester',
		872	'required' => 1,
		873	'min_ver' => '1.354',
		874	},
		875	'POE::Component::JobQueue' => {
		876	'usage' => 'OAI-PMH harvester',
		877	'required' => 1,
		878	'min_ver' => '0.570',
		879	},
870	};	880	};
871		881
872	1;	882	1;

Lines 4100-4105 CREATE TABLE IF NOT EXISTS club_fields ( Link Here

(-)a/installer/data/mysql/kohastructure.sql (+74 lines)
4100	CONSTRAINT club_fields_ibfk_4 FOREIGN KEY (club_id) REFERENCES clubs (id) ON DELETE CASCADE ON UPDATE CASCADE	4100	CONSTRAINT club_fields_ibfk_4 FOREIGN KEY (club_id) REFERENCES clubs (id) ON DELETE CASCADE ON UPDATE CASCADE
4101	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;	4101	) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
4102		4102
		4103	--
		4104	-- Table structure for table 'oai_harvester_biblios'
		4105	--
		4106
		4107	CREATE TABLE IF NOT EXISTS `oai_harvester_biblios` (
		4108	`import_oai_biblio_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
		4109	`oai_repository` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
		4110	`oai_identifier` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
		4111	`biblionumber` int(11) NOT NULL,
		4112	PRIMARY KEY (`import_oai_biblio_id`),
		4113	UNIQUE KEY `oai_record` (`oai_identifier`,`oai_repository`) USING BTREE,
		4114	KEY `FK_import_oai_biblio_1` (`biblionumber`),
		4115	CONSTRAINT `FK_import_oai_biblio_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE NO ACTION
		4116	) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
		4117
		4118	--
		4119	-- Table structure for table 'oai_harvester_history'
		4120	--
		4121
		4122	CREATE TABLE IF NOT EXISTS `oai_harvester_history` (
		4123	`import_oai_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
		4124	`repository` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
		4125	`header_identifier` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
		4126	`header_datestamp` datetime NOT NULL,
		4127	`header_status` varchar(45) COLLATE utf8_unicode_ci DEFAULT NULL,
		4128	`record` longtext COLLATE utf8_unicode_ci NOT NULL,
		4129	`upload_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
		4130	`status` varchar(45) COLLATE utf8_unicode_ci NOT NULL,
		4131	`filter` text COLLATE utf8_unicode_ci NOT NULL,
		4132	`framework` varchar(4) COLLATE utf8_unicode_ci NOT NULL,
		4133	`record_type` enum('biblio','auth','holdings') COLLATE utf8_unicode_ci NOT NULL,
		4134	`matcher_code` varchar(10) COLLATE utf8_unicode_ci DEFAULT NULL,
		4135	PRIMARY KEY (`import_oai_id`)
		4136	) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
		4137
		4138	--
		4139	-- Table structure for table 'oai_harvester_import_queue'
		4140	--
		4141
		4142	CREATE TABLE IF NOT EXISTS `oai_harvester_import_queue` (
		4143	`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
		4144	`uuid` varchar(45) CHARACTER SET utf8 NOT NULL,
		4145	`status` varchar(45) CHARACTER SET utf8 NOT NULL DEFAULT 'new',
		4146	`result` text CHARACTER SET utf8 NOT NULL,
		4147	`result_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
		4148	PRIMARY KEY (`id`)
		4149	) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
		4150
		4151	--
		4152	-- Table structure for table 'oai_harvester_requests'
		4153	--
		4154
		4155	CREATE TABLE IF NOT EXISTS `oai_harvester_requests` (
		4156	`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
		4157	`uuid` varchar(45) NOT NULL,
		4158	`oai_verb` varchar(45) NOT NULL,
		4159	`oai_metadataPrefix` varchar(255) NOT NULL,
		4160	`oai_identifier` varchar(255) DEFAULT NULL,
		4161	`oai_from` varchar(45) DEFAULT NULL,
		4162	`oai_until` varchar(45) DEFAULT NULL,
		4163	`oai_set` varchar(255) DEFAULT NULL,
		4164	`http_url` varchar(255) DEFAULT NULL,
		4165	`http_username` varchar(255) DEFAULT NULL,
		4166	`http_password` varchar(255) DEFAULT NULL,
4167	`http_realm` varchar(255) DEFAULT NULL,
4168	`import_filter` varchar(255) NOT NULL,
4169	`import_framework_code` varchar(4) NOT NULL,
4170	`import_record_type` enum('biblio','auth','holdings') NOT NULL,
4171	`import_matcher_code` varchar(10) DEFAULT NULL,
4172	`interval` int(10) unsigned NOT NULL,
4173	`name` varchar(45) NOT NULL,
4174	PRIMARY KEY (`id`)
4175	) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
4176
4103	/!40103 SET TIME_ZONE=@OLD_TIME_ZONE /;	4177	/!40103 SET TIME_ZONE=@OLD_TIME_ZONE /;
4104	/!40101 SET SQL_MODE=@OLD_SQL_MODE /;	4178	/!40101 SET SQL_MODE=@OLD_SQL_MODE /;
4105	/!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS /;	4179	/!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS /;

Line 0 Link Here

(-)a/Koha/Daemon.pm (+151 lines)
		1	package Koha::Daemon;
		2
		3	# Copyright 2017 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19	#
		20
		21	use Modern::Perl;
		22	use POSIX; #For daemonizing
		23	use Fcntl qw(:flock); #For pidfile
		24
		25	sub new {
		26	my ($class, $args) = @_;
		27	$args = {} unless defined $args;
		28	return bless ($args, $class);
		29	}
		30
		31	#######################################################################
		32	#NOTE: On Debian, you can use the daemon binary to make a process into a daemon,
		33	# but the following can be used if you don't want to use that program.
		34
		35	sub daemonize {
		36	my ($self) = @_;
		37
		38	my $pid = fork;
		39
		40	die "Couldn't fork: $!" unless defined($pid);
		41	if ($pid){
		42	exit; #Parent exit
		43	}
		44
		45	#Become a session leader (ie detach program from controlling terminal)
		46	POSIX::setsid() or die "Can't start a new session: $!";
		47
		48	#Change to known system directory
		49	chdir('/');
		50
		51	#Reset the file creation mask so only the daemon owner can read/write files it creates
		52	umask(066);
		53
		54	#Close inherited file handles, so that you can truly run in the background.
		55	open STDIN, '<', '/dev/null';
		56	open STDOUT, '>', '/dev/null';
		57	open STDERR, '>&STDOUT';
		58	}
		59
		60	sub log_to_file {
		61	my ($self,$logfile) = @_;
		62
		63	#Open a filehandle to append to a log file
		64	my $opened = open(LOG, '>>', $logfile);
65	if ($opened){
66	LOG->autoflush(1); #Make filehandle hot (ie don't buffer)
67	STDOUT = LOG; #Re-assign STDOUT to LOG \| --stdout
68	STDERR = STDOUT; #Re-assign STDERR to STDOUT \| --stderr
69	}
70	else {
71	die "Unable to open a filehandle for $logfile: $!\n"; # --output
72	}
73	}
74
75	sub make_pidfilehandle {
76	my ($self,$pidfile) = @_;
77	if ( ! -f $pidfile ){
78	open(my $fh, '>', $pidfile) or die "Unable to write to $pidfile: $!\n";
79	}
80	open(my $pidfilehandle, '+<', $pidfile) or die "Unable to open a filehandle for $pidfile: $!\n";
81	return $pidfilehandle;
82	}
83
84	sub get_pidfile {
85	my ($self,$pidfile) = @_;
86	#NOTE: We need to save the filehandle in the object, so any locks persist for the life of the object
87	my $pidfilehandle = $self->{pidfilehandle} \|\|= $self->make_pidfilehandle($pidfile);
88	return $pidfilehandle;
89	}
90
91	sub lock_pidfile {
92	my ($self,$pidfilehandle) = @_;
93	my $locked;
94	if (flock($pidfilehandle, LOCK_EX\|LOCK_NB)){
95	$locked = 1;
96
97	}
98	return $locked;
99	}
100
101	sub write_pidfile {
102	my ($self,$pidfilehandle) = @_;
103	if ($pidfilehandle){
104	truncate($pidfilehandle, 0);
105	print $pidfilehandle $$."\n" or die $!;
106	#Flush the filehandle so you're not suffering from buffering
107	$pidfilehandle->flush();
108	return 1;
109	}
110	}
111
112	sub run {
113	my ($self) = @_;
114	my $pidfile = $self->{pidfile};
115	my $logfile = $self->{logfile};
116
117	if ($pidfile){
118	my $pidfilehandle = $self->get_pidfile($pidfile);
119	if ($pidfilehandle){
120	my $locked = $self->lock_pidfile($pidfilehandle);
121	if ( ! $locked ) {
122	die "$0 is unable to lock pidfile...\n";
123	}
124	}
125	}
126
127	if (my $configure = $self->{configure}){
128	$configure->($self);
129	}
130
131	if ($self->{daemonize}){
132	$self->daemonize();
133	}
134
135	if ($pidfile){
136	my $pidfilehandle = $self->get_pidfile($pidfile);
137	if ($pidfilehandle){
138	$self->write_pidfile($pidfilehandle);
139	}
140	}
141
142	if ($logfile){
143	$self->log_to_file($logfile);
144	}
145
146	if (my $loop = $self->{loop}){
147	$loop->($self);
148	}
149	}
150
151	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester.pm (+640 lines)
		1	package Koha::OAI::Harvester;
		2
		3	# Copyright 2017 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19	#
		20
		21	use Modern::Perl;
		22	use POE qw(Component::JobQueue);
		23	use JSON;
		24	use Sereal::Encoder;
		25	use Sereal::Decoder;
		26	use IO::Handle;
		27	use File::Copy;
		28	use File::Path qw(make_path remove_tree);
		29	use DateTime;
		30	use DateTime::Format::Strptime;
		31
		32	use C4::Context;
		33	use Koha::Database;
		34
		35	my $day_granularity = DateTime::Format::Strptime->new(
		36	pattern => '%F',
		37	);
		38	my $seconds_granularity = DateTime::Format::Strptime->new(
		39	pattern => '%FT%TZ',
		40	);
		41
		42	sub new {
		43	my ($class, $args) = @_;
		44	$args = {} unless defined $args;
		45	return bless ($args, $class);
		46	}
		47
		48	sub spawn {
		49	my ($class, $args) = @_;
		50	my $self = $class->new($args);
		51	my $downloader = $self->{Downloader};
		52	my $importer = $self->{Importer};
		53	my $download_worker_limit = ( $self->{DownloaderWorkers} && int($self->{DownloaderWorkers}) ) ? $self->{DownloaderWorkers} : 1;
		54	my $import_worker_limit = ( $self->{ImporterWorkers} && int($self->{ImporterWorkers}) ) ? $self->{ImporterWorkers} : 1;
		55	my $import_queue_poll = ( $self->{ImportQueuePoll} && int($self->{ImportQueuePoll}) ) ? $self->{ImportQueuePoll} : 5;
		56
		57	#NOTE: This job queue should always be created before the
		58	#harvester so that you can start download jobs immediately
		59	#upon spawning the harvester.
		60	POE::Component::JobQueue->spawn(
		61	Alias => 'oai-downloader',
		62	WorkerLimit => $download_worker_limit,
		63	Worker => sub {
		64	my ($postback, $task) = @_;
65	if ($downloader){
66	if ($task->{status} eq "active"){
67	$downloader->run({
68	postback => $postback,
69	task => $task,
70	});
71	}
72	}
73	},
74	Passive => {},
75	);
76
77	POE::Session->create(
78	object_states => [
79	$self => {
80	_start => "on_start",
81	get_task => "get_task",
82	list_tasks => "list_tasks",
83	create_task => "create_task",
84	start_task => "start_task",
85	stop_task => "stop_task",
86	delete_task => "delete_task",
87	repeat_task => "repeat_task",
88	register => "register",
89	deregister => "deregister",
90	restore_state => "restore_state",
91	save_state => "save_state",
92	is_task_finished => "is_task_finished",
93	does_task_repeat => "does_task_repeat",
94	download_postback => "download_postback",
95	reset_imports_status => "reset_imports_status",
96	},
97	],
98	);
99
100	POE::Component::JobQueue->spawn(
101	Alias => 'oai-importer',
102	WorkerLimit => $import_worker_limit,
103	Worker => sub {
104	my $meta_postback = shift;
105
106	#NOTE: We need to only retrieve queue items for active tasks. Otherwise,
107	#the importer will just spin its wheels on inactive tasks and do nothing.
108	my $active_tasks = $poe_kernel->call("harvester","list_tasks","active");
109	my @active_uuids = map { $_->{uuid} } @$active_tasks;
110
111	my $schema = Koha::Database->new()->schema();
112	my $rs = $schema->resultset('OaiHarvesterImportQueue')->search({
113	uuid => \@active_uuids,
114	status => "new"
115	},{
116	order_by => { -asc => 'id' },
117	rows => 1,
118	});
119	my $result = $rs->first;
120	if ($result){
121	$result->status("wip");
122	$result->update;
123	my $task = {
124	id => $result->id,
125	uuid => $result->uuid,
126	result => $result->result,
127	};
128
129	my $postback = $meta_postback->($task);
130	$importer->run({
131	postback => $postback,
132	task => $task,
133	});
134	}
135	},
136	Active => {
137	PollInterval => $import_queue_poll,
138	AckAlias => undef,
139	AckState => undef,
140	},
141	);
142
143	return;
144	}
145
146	sub on_start {
147	my ($self, $kernel, $heap) = @_[OBJECT, KERNEL,HEAP];
148	$kernel->alias_set('harvester');
149	$heap->{scoreboard} = {};
150
151	#Reset any 'wip' imports to 'new' so they can be re-tried.
152	$kernel->call("harvester","reset_imports_status");
153
154	#Restore state from state file
155	$kernel->call("harvester","restore_state");
156	}
157
158	#NOTE: This isn't really implemented at the moment, as it's not really necessary.
159	sub download_postback {
160	my ($kernel, $request_packet, $response_packet) = @_[KERNEL, ARG0, ARG1];
161	my $message = $response_packet->[0];
162	}
163
164	=head3 deregister
165
166	Remove the worker session from the harvester's in-memory scoreboard,
167	unset the downloading flag if downloading is completed.
168
169	=cut
170
171	sub deregister {
172	my ($self, $kernel, $heap, $session, $sender, $type) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0];
173
174	my $scoreboard = $heap->{scoreboard};
175
176	my $logger = $self->{logger};
177	$logger->debug("Start deregistering $sender as $type task");
178
179	my $task_uuid = delete $scoreboard->{session}->{$sender};
180	#NOTE: If you don't check each step of the hashref, autovivication can lead to surprises.
181	if ($task_uuid){
182	if ($scoreboard->{task}->{$task_uuid}){
183	if ($scoreboard->{task}->{$task_uuid}->{$type}){
184	delete $scoreboard->{task}->{$task_uuid}->{$type}->{$sender};
185	}
186	}
187	}
188
189	my $task = $heap->{tasks}->{$task_uuid};
190	if ($task && $task->{status} && ($task->{status} eq "active") ){
191	#NOTE: Each task only has 1 download session, so we can now set/unset flags for the task.
192	#NOTE: We should unset the downloading flag, if we're not going to repeat the task.
193	if ($type eq "download"){
194	my $task_repeats = $kernel->call("harvester","does_task_repeat",$task_uuid);
195	if ($task_repeats){
196	my $interval = $task->{interval};
197
198	$task->{effective_from} = delete $task->{effective_until};
199	$task->{download_timer} = $kernel->delay_set("repeat_task", $interval, $task_uuid);
200	}
201	else {
202	$task->{downloading} = 0;
203	$kernel->call("harvester","save_state");
204	$kernel->call("harvester","is_task_finished",$task_uuid);
205	}
206	}
207	elsif ($type eq 'import'){
208	$kernel->call("harvester","is_task_finished",$task_uuid);
209	}
210	}
211	$logger->debug("End deregistering $sender as $type task");
212	}
213
214
215	=head3 is_task_finished
216
217	This event handler checks if the task has finished downloading and importing record.
218	If it is finished downloading and importing, the task is deleted from the harvester.
219
220	This only applies to non-repeating tasks.
221
222	=cut
223
224	sub is_task_finished {
225	my ($self, $kernel, $heap, $session, $uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
226	my $task = $kernel->call("harvester","get_task",$uuid);
227	if ($task && (! $task->{downloading}) ){
228	my $count = $self->get_import_count_for_task($uuid);
229	if ( ! $count ) {
230	#Clear this task out of the harvester as it's finished.
231	$kernel->call("harvester","delete_task",$uuid);
232	return 1;
233	}
234	}
235	return 0;
236	}
237
238	sub register {
239	my ($self, $kernel, $heap, $session, $sender, $type, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0,ARG1];
240	my $logger = $self->{logger};
241
242	my $scoreboard = $heap->{scoreboard};
243
244
245	if ($type && $task_uuid){
246	$logger->debug("Registering $sender as $type for $task_uuid");
247
248	my $task = $heap->{tasks}->{$task_uuid};
249	if ($task){
250
251	if ($type){
252	#Register the task uuid with the session id as a key for later recall
253	$scoreboard->{session}->{$sender} = $task_uuid;
254
255	#Register the session id as a certain type of session for a task
256	$scoreboard->{task}->{$task_uuid}->{$type}->{$sender} = 1;
257
258	if ($type eq "download"){
259	$task->{downloading} = 1;
260
261	my $task_repeats = $kernel->call("harvester","does_task_repeat",$task_uuid);
262	if ($task_repeats){
263
264	#NOTE: Set an effective until, so we know we're not getting records any newer than
265	#this moment.
266	my $dt = DateTime->now();
267	if ($dt){
268	#NOTE: Ideally, I'd like to make sure that we can use 'seconds' granularity, but
269	#it's valid for 'from' to be null, so it's impossible to know from the data whether
270	#or not the repository will support the seconds granularity.
271	#NOTE: Ideally, it would be good to use either 'day' granularity or 'seconds' granularity,
272	#but at the moment the interval is expressed as seconds only.
273	$dt->set_formatter($seconds_granularity);
274	$task->{effective_until} = "$dt";
275	}
276	}
277
278	$kernel->call("harvester","save_state");
279	}
280	}
281	}
282	}
283	}
284
285	sub does_task_repeat {
286	my ($self, $kernel, $heap, $session, $uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
287	my $task = $kernel->call("harvester","get_task",$uuid);
288	if ($task){
289	my $interval = $task->{interval};
290	my $parameters = $task->{parameters};
291	my $oai_pmh = $parameters->{oai_pmh} if $parameters->{oai_pmh};
292	if ( $interval && ($oai_pmh->{verb} eq "ListRecords") && (! $oai_pmh->{until}) ){
293	return 1;
294	}
295	}
296	return 0;
297	}
298
299
300
301	sub reset_imports_status {
302	my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION];
303
304	my $schema = Koha::Database->new()->schema();
305	my $rs = $schema->resultset('OaiHarvesterImportQueue')->search({
306	status => "wip",
307	});
308	$rs->update({
309	status => "new",
310	});
311	}
312
313	sub restore_state {
314	my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION];
315
316	my $state_file = $self->{state_file};
317	my $state_backup = "$state_file~";
318
319	#NOTE: If there is a state backup, it means we crashed while saving the state. Otherwise,
320	#let's try the regular state file if it exists.
321	my $file_to_restore = ( -f $state_backup ) ? $state_backup : ( ( -f $state_file ) ? $state_file : undef );
322	if ( $file_to_restore ){
323	my $opened = open( my $fh, '<', $file_to_restore ) or die "Couldn't open state: $!";
324	if ($opened){
325	local $/;
326	my $in = <$fh>;
327	my $decoder = Sereal::Decoder->new;
328	my $state = $decoder->decode($in);
329	if ($state){
330	if ($state->{tasks}){
331	#Restore tasks from our saved state
332	$heap->{tasks} = $state->{tasks};
333	foreach my $uuid ( keys %{$heap->{tasks}} ){
334	my $task = $heap->{tasks}->{$uuid};
335
336	#If tasks were still downloading, restart the task
337	if ( ($task->{status} && $task->{status} eq "active") && $task->{downloading} ){
338	$task->{status} = "new";
339	$kernel->call("harvester","start_task",$task->{uuid});
340	}
341	}
342	}
343	}
344	}
345	}
346	}
347
348	sub save_state {
349	my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION];
350	my $state_file = $self->{state_file};
351	my $state_backup = "$state_file~";
352
353	#Make a backup of existing state record
354	my $moved = move($state_file,$state_backup);
355
356	my $opened = open(my $fh, ">", $state_file) or die "Couldn't save state: $!";
357	if ($opened){
358	$fh->autoflush(1);
359	my $tasks = $heap->{tasks};
360	my $harvester_state = {
361	tasks => $tasks,
362	};
363	my $encoder = Sereal::Encoder->new;
364	my $out = $encoder->encode($harvester_state);
365	local $\;
366	my $printed = print $fh $out;
367	if ($printed){
368	close $fh;
369	unlink($state_backup);
370	return 1;
371	}
372	}
373	return 0;
374	}
375
376	=head3 get_task
377
378	This event handler returns a task from a harvester using the task's
379	uuid as an argument.
380
381	=cut
382
383	sub get_task {
384	my ($self, $kernel, $heap, $session, $uuid, $sender) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0, SENDER];
385
386	if ( ! $uuid && $sender ){
387	my $scoreboard = $heap->{scoreboard};
388	my $uuid_by_session = $scoreboard->{session}->{$sender};
389	if ($uuid_by_session){
390	$uuid = $uuid_by_session;
391	}
392	}
393
394	my $tasks = $heap->{tasks};
395	if ($tasks && $uuid){
396	my $task = $tasks->{$uuid};
397	if ($task){
398	return $task;
399	}
400	}
401	return 0;
402	}
403
404	=head3 get_import_count_for_task
405
406	=cut
407
408	sub get_import_count_for_task {
409	my ($self,$uuid) = @_;
410	my $count = undef;
411	if ($uuid){
412	my $schema = Koha::Database->new()->schema();
413	my $items = $schema->resultset('OaiHarvesterImportQueue')->search({
414	uuid => $uuid,
415	});
416	$count = $items->count;
417	}
418	return $count;
419	}
420
421	=head3 list_tasks
422
423	This event handler returns a list of tasks that have been submitted
424	to the harvester. It returns data like uuid, status, parameters,
425	number of pending imports, etc.
426
427	=cut
428
429	sub list_tasks {
430	my ($self, $kernel, $heap, $session, $status) = @_[OBJECT, KERNEL,HEAP,SESSION, ARG0];
431	my $schema = Koha::Database->new()->schema();
432	my @tasks = ();
433	foreach my $uuid (sort keys %{$heap->{tasks}}){
434	my $task = $heap->{tasks}->{$uuid};
435	my $items = $schema->resultset('OaiHarvesterImportQueue')->search({
436	uuid => $uuid,
437	});
438	my $count = $items->count // 0;
439	$task->{pending_imports} = $count;
440	if ( ( ! $status ) \|\| ( $status && $status eq $task->{status} ) ){
441	push(@tasks, $task);
442	}
443
444	}
445	return \@tasks;
446	}
447
448	=head3 create_task
449
450	This event handler creates a spool directory for the task's imports.
451	It also adds it to the harvester's memory and then saves memory to
452	a persistent datastore.
453
454	Newly created tasks have a status of "new".
455
456	=cut
457
458	sub create_task {
459	my ($self, $kernel, $heap, $session, $incoming_task) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
460	my $logger = $self->{logger};
461	if ($incoming_task){
462	my $uuid = $incoming_task->{uuid};
463	if ( ! $heap->{tasks}->{$uuid} ){
464
465	#Step One: assign a spool directory to this task
466	my $spooldir = $self->{spooldir} // "/tmp";
467	my $task_spooldir = "$spooldir/$uuid";
468	if ( ! -d $task_spooldir ){
469	my $made_spool_directory = make_path($task_spooldir);
470	if ( ! $made_spool_directory ){
471	if ($logger){
472	$logger->warn("Unable to make task-specific spool directory at '$task_spooldir'");
473	}
474	return 0;
475	}
476	}
477	$incoming_task->{spooldir} = $task_spooldir;
478
479	#Step Two: assign new status
480	$incoming_task->{status} = "new";
481
482	#Step Three: add task to harvester's memory
483	$heap->{tasks}->{ $uuid } = $incoming_task;
484
485	#Step Four: save state
486	$kernel->call($session,"save_state");
487	return 1;
488	}
489	}
490	return 0;
491	}
492
493	=head3 start_task
494
495	This event handler marks a task as active in the harvester's memory,
496	save the memory to a persistent datastore, then enqueues the task,
497	so that it can be directed to the next available download worker.
498
499	Newly started tasks have a status of "active".
500
501	=cut
502
503	sub start_task {
504	my ($self, $session,$kernel,$heap,$uuid) = @_[OBJECT, SESSION,KERNEL,HEAP,ARG0];
505	my $task = $heap->{tasks}->{$uuid};
506	if ($task){
507	if ( $task->{status} ne "active" ){
508
509	#Clear any pre-existing error states
510	delete $task->{error} if $task->{error};
511
512	#Step One: mark task as active
513	$task->{status} = "active";
514
515	#Step Two: save state
516	$kernel->call("harvester","save_state");
517
518	#Step Three: enqueue task
519	$kernel->post("oai-downloader",'enqueue','download_postback', $task);
520
521	return 1;
522	}
523	}
524	return 0;
525	}
526
527	=head3 repeat_task
528
529
530
531	=cut
532
533	sub repeat_task {
534	my ($self, $session,$kernel,$heap,$uuid) = @_[OBJECT, SESSION,KERNEL,HEAP,ARG0];
535	my $task = $heap->{tasks}->{$uuid};
536	if ($task){
537	my $interval = $task->{interval};
538	if ($task->{downloading} && $interval){
539	$kernel->post("oai-downloader",'enqueue','download_postback', $task);
540	}
541	}
542	}
543
544	=head3 stop_task
545
546	This event handler prevents new workers from spawning, kills
547	existing workers, and stops pending imports from being imported.
548
549	Newly stopped tasks have a status of "stopped".
550
551	=cut
552
553	sub stop_task {
554	my ($self, $kernel, $heap, $session, $sender, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0];
555
556	my $task = $heap->{tasks}->{$task_uuid};
557
558	if ($task && $task->{status} && $task->{status} ne "stopped" ){
559
560	#Step One: deactivate task, so no new workers can be started
561	$task->{status} = "stopped";
562	#NOTE: You could also clear timers for new downloads, but that's probably unnecessary because of this step.
563
564	#Step Two: kill workers
565	my $scoreboard = $heap->{scoreboard};
566	my $session_types = $scoreboard->{task}->{$task_uuid};
567	if ($session_types){
568	foreach my $type ( keys %$session_types ){
569	my $sessions = $session_types->{$type};
570	if ($sessions){
571	foreach my $session (keys %$sessions){
572	if ($session){
573	$kernel->signal($session, "cancel");
574	}
575	}
576	}
577	}
578	#Remove the task uuid from the task key of the scoreboard
579	delete $scoreboard->{task}->{$task_uuid};
580	#NOTE: The task uuid will still exist under the session key,
581	#but the sessions will deregister themselves and clean that up for you.
582	}
583
584	#Step Three: stop pending imports for this task
585	my $schema = Koha::Database->new()->schema();
586	my $items = $schema->resultset('OaiHarvesterImportQueue')->search({
587	uuid => $task_uuid,
588	});
589	my $rows_updated = $items->update({
590	status => "stopped",
591	});
592
593	#Step Four: save state
594	$kernel->call("harvester","save_state");
595	return 1;
596	}
597	return 0;
598	}
599
600	=head3 delete_task
601
602	Deleted tasks are stopped, pending imports are deleted from the
603	database and file system, and then the task is removed from the harvester.
604
605	=cut
606
607	sub delete_task {
608	my ($self, $kernel, $heap, $session, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
609
610	my $task = $heap->{tasks}->{$task_uuid};
611	if ($task){
612	#Step One: stop task
613	$kernel->call($session,"stop_task",$task_uuid);
614
615	#Step Two: delete pending imports in database
616	my $schema = Koha::Database->new()->schema();
617	my $items = $schema->resultset('OaiHarvesterImportQueue')->search({
618	uuid => $task_uuid,
619	});
620	if ($items){
621	my $rows_deleted = $items->delete;
622	#NOTE: shows 0E0 instead of 0
623	}
624
625	#Step Three: remove task specific spool directory and files within it
626	my $spooldir = $task->{spooldir};
627	if ($spooldir){
628	my $files_deleted = remove_tree($spooldir, { safe => 1 });
629	}
630
631	delete $heap->{tasks}->{$task_uuid};
632
633	#Step Four: save state
634	$kernel->call("harvester","save_state");
635	return 1;
636	}
637	return 0;
638	}
639
640	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Client.pm (+177 lines)
		1	package Koha::OAI::Harvester::Client;
		2
		3	# Copyright 2017 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, write to the Free Software Foundation, Inc.,
		18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		19
		20	use Modern::Perl;
		21	use URI;
		22	use IO::Socket::UNIX;
		23	use IO::Select;
		24	use JSON;
		25
		26	sub new {
		27	my ($class, $args) = @_;
		28	$args = {} unless defined $args;
		29	return bless ($args, $class);
		30	}
		31
		32	sub connect {
		33	my ($self) = @_;
		34	my $socket_uri = $self->{socket_uri};
		35	if ($socket_uri){
		36	my $uri = URI->new($socket_uri);
		37	if ($uri && $uri->scheme eq 'unix'){
		38	my $socket_path = $uri->path;
		39	my $socket = IO::Socket::UNIX->new(
		40	Type => IO::Socket::UNIX::SOCK_STREAM(),
		41	Peer => $socket_path,
		42	);
		43	if ($socket){
		44	my $select = new IO::Select();
		45	$select->add($socket);
		46
		47	$self->{_select} = $select;
		48	$self->{_socket} = $socket;
		49	my $message = $self->_read();
		50	if ($message){
		51	if ($message eq 'HELLO'){
		52	$self->{_connected} = 1;
		53	return 1;
		54	}
		55	}
		56	}
		57	else {
		58	warn "Failed to create socket."
		59	}
		60	}
		61	}
		62	return 0;
		63	}
		64
65	sub create {
66	my ($self,$task) = @_;
67	my $message = {
68	command => "create",
69	body => {
70	task => $task,
71	}
72	};
73	my ($status) = $self->_exchange($message);
74	return $status;
75	}
76
77	sub start {
78	my ($self,$uuid) = @_;
79	my $message = {
80	command => "start",
81	body => {
82	task => {
83	uuid => $uuid,
84	},
85	}
86	};
87	my ($status) = $self->_exchange($message);
88	return $status;
89	}
90
91	sub stop {
92	my ($self,$uuid) = @_;
93	my $message = {
94	command => "stop",
95	body => {
96	task => {
97	uuid => $uuid,
98	},
99	}
100	};
101	my ($status) = $self->_exchange($message);
102	return $status;
103	}
104
105	sub delete {
106	my ($self,$uuid) = @_;
107	my $message = {
108	command => "delete",
109	body => {
110	task => {
111	uuid => $uuid,
112	},
113	}
114	};
115	my ($status) = $self->_exchange($message);
116	return $status;
117	}
118
119	sub list {
120	my ($self) = @_;
121	my $message = {
122	command => "list",
123	};
124	my ($status,$tasks) = $self->_exchange($message);
125	return $tasks;
126	}
127
128	sub _exchange {
129	my ($self,$message) = @_;
130	my $status = 0;
131	my $data;
132	if ($message){
133	my $output = to_json($message);
134	if ($output){
135	$self->_write($output);
136	my $json_response = $self->_read();
137	if ($json_response){
138	my $response = from_json($json_response);
139	$data = $response->{data} if $response->{data};
140	$status = 1 if $response->{msg} && $response->{msg} eq "success";
141	}
142	}
143	}
144	return ($status,$data);
145	}
146
147	sub _write {
148	my ($self, $output) = @_;
149	if ($output){
150	if (my $select = $self->{_select}){
151	if (my @filehandles = $select->can_write(5)){
152	foreach my $filehandle (@filehandles){
153	#Localize output record separator as null
154	local $\ = "\x00";
155	print $filehandle $output;
156	}
157	}
158	}
159	}
160	}
161
162	sub _read {
163	my ($self) = @_;
164	if (my $select = $self->{_select}){
165	if (my @filehandles = $select->can_read(5)){
166	foreach my $filehandle (@filehandles){
167	#Localize input record separator as null
168	local $/ = "\x00";
169	my $message = <$filehandle>;
170	chomp($message) if $message;
171	return $message;
172	}
173	}
174	}
175	}
176
177	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Downloader.pm (+302 lines)
		1	package Koha::OAI::Harvester::Downloader;
		2
		3	# Copyright 2017 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, write to the Free Software Foundation, Inc.,
		18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		19
		20	use Modern::Perl;
		21	use URI;
		22	use XML::LibXML::Reader;
		23	use IO::Handle;
		24	use JSON;
		25
		26	sub new {
		27	my ($class, $args) = @_;
		28	$args = {} unless defined $args;
		29	return bless ($args, $class);
		30	}
		31
		32	=head2 BuildURL
		33
		34	Takes a baseURL and a mix of required and optional OAI-PMH arguments,
		35	and makes them into a suitable URL for an OAI-PMH request.
		36
		37	=cut
		38
		39	sub BuildURL {
		40	my ($self, $args) = @_;
		41	my $baseURL = $args->{baseURL};
		42	my $url = URI->new($baseURL);
		43	if ($url && $url->isa("URI")){
		44	my $verb = $args->{verb};
		45	if ($verb){
		46	my %parameters = (
		47	verb => $verb,
		48	);
		49	if ($verb eq "ListRecords"){
		50	my $resumptionToken = $args->{resumptionToken};
		51	my $metadataPrefix = $args->{metadataPrefix};
		52	if ($resumptionToken){
		53	$parameters{resumptionToken} = $resumptionToken;
		54	}
		55	elsif ($metadataPrefix){
		56	$parameters{metadataPrefix} = $metadataPrefix;
		57	#Only add optional parameters if they're provided
		58	foreach my $param ( qw( from until set ) ){
		59	$parameters{$param} = $args->{$param} if $args->{$param};
		60	}
		61	}
		62	else {
		63	warn "BuildURL() requires an argument of either resumptionToken or metadataPrefix";
		64	return;
65	}
66	}
67	elsif ($verb eq "GetRecord"){
68	my $metadataPrefix = $args->{metadataPrefix};
69	my $identifier = $args->{identifier};
70	if ($metadataPrefix && $identifier){
71	$parameters{metadataPrefix} = $metadataPrefix;
72	$parameters{identifier} = $identifier;
73	}
74	else {
75	warn "BuildURL() requires an argument of metadataPrefix and an argument of identifier";
76	return;
77	}
78	}
79	$url->query_form(%parameters);
80	return $url;
81	}
82	else {
83	warn "BuildURL() requires a verb of GetRecord or ListRecords";
84	return;
85	}
86	}
87	else {
88	warn "BuildURL() requires a base URL of type URI.";
89	return;
90	}
91	}
92
93	=head2 OpenXMLStream
94
95	Fork a child process to send the HTTP request, which sends chunks
96	of XML via a pipe to the parent process.
97
98	The parent process creates and returns a XML::LibXML::Reader object,
99	which reads the XML stream coming through the pipe.
100
101	Normally, using a DOM reader, you must wait to read the entire XML document
102	into memory. However, using a stream reader, chunks are read into memory,
103	processed, then discarded. It's faster and more efficient.
104
105	=cut
106
107	sub GetXMLStream {
108	my ($self, $args) = @_;
109	my $url = $args->{url};
110	my $user_agent = $args->{user_agent};
111	if ($url && $user_agent){
112	pipe( CHILD, PARENT ) or die "Cannot created connected pipes: $!";
113	CHILD->autoflush(1);
114	PARENT->autoflush(1);
115	if ( my $pid = fork ){
116	#Parent process
117	close PARENT;
118	return \*CHILD;
119	}
120	else {
121	#Child process
122	close CHILD;
123	my $response = $self->_request({
124	url => $url,
125	user_agent => $user_agent,
126	file_handle => \*PARENT,
127	});
128	if ($response && $response->is_success){
129	#HTTP request has successfully finished, so we close the file handle and exit the process
130	close PARENT;
131	CORE::exit(); #some modules like mod_perl redefine exit
132	}
133	else {
134	warn "[child $$] OAI-PMH unsuccessful. Response status: ".$response->status_line."\n" if $response;
135	CORE::exit();
136	}
137	}
138	}
139	else {
140	warn "GetXMLStream() requires a 'url' argument and a 'user_agent' argument";
141	return;
142	}
143	}
144
145	sub _request {
146	my ($self, $args) = @_;
147	my $url = $args->{url};
148	my $user_agent = $args->{user_agent};
149	my $fh = $args->{file_handle};
150
151	if ($url && $user_agent && $fh){
152	my $request = HTTP::Request->new( GET => $url );
153	my $response = $user_agent->request( $request, sub {
154	my ($chunk_of_data, $ref_to_response, $ref_to_protocol) = @_;
155	print $fh $chunk_of_data;
156	});
157	return $response;
158	}
159	else {
160	warn "_request() requires a 'url' argument, 'user_agent' argument, and 'file_handle' argument.";
161	return;
162	}
163	}
164
165	sub ParseXMLStream {
166	my ($self, $args) = @_;
167
168	my $each_callback = $args->{each_callback};
169	my $fh = $args->{file_handle};
170	if ($fh){
171	my $reader = XML::LibXML::Reader->new( FD => $fh, no_blanks => 1 );
172	my $pattern = XML::LibXML::Pattern->new('oai:OAI-PMH\|/oai:OAI-PMH/*', { 'oai' => "http://www.openarchives.org/OAI/2.0/" });
173
174	my $repository;
175
176	warn "Start parsing...";
177	while (my $rv = $reader->nextPatternMatch($pattern)){
178	#$rv == 1; successful
179	#$rv == 0; end of document reached
180	#$rv == -1; error
181	if ($rv == -1){
182	die "Parser error!";
183	}
184	#NOTE: We do this so we only get the opening tag of the element.
185	next unless $reader->nodeType == XML_READER_TYPE_ELEMENT;
186
187	my $localname = $reader->localName;
188	if ( $localname eq "request" ){
189	my $node = $reader->copyCurrentNode(1);
190	$repository = $node->textContent;
191	}
192	elsif ( $localname eq "error" ){
193	#See https://www.openarchives.org/OAI/openarchivesprotocol.html#ErrorConditions
194	#We should probably die under all circumstances except "noRecordsMatch"
195	my $node = $reader->copyCurrentNode(1);
196	if ($node){
197	my $code = $node->getAttribute("code");
198	if ($code){
199	if ($code ne "noRecordsMatch"){
200	warn "Error code: $code";
201	die;
202	}
203	}
204	}
205	}
206	elsif ( ($localname eq "ListRecords") \|\| ($localname eq "GetRecord") ){
207	my $each_pattern = XML::LibXML::Pattern->new('//oai:record\|oai:resumptionToken', { 'oai' => "http://www.openarchives.org/OAI/2.0/" });
208	while (my $each_rv = $reader->nextPatternMatch($each_pattern)){
209	if ($rv == "-1"){
210	#NOTE: -1 denotes an error state
211	warn "Error getting pattern match";
212	}
213	next unless $reader->nodeType == XML_READER_TYPE_ELEMENT;
214	if ($reader->localName eq "record"){
215	my $node = $reader->copyCurrentNode(1);
216	#NOTE: Without the UTF-8 flag, UTF-8 data will be corrupted.
217	my $document = XML::LibXML::Document->new('1.0', 'UTF-8');
218	$document->setDocumentElement($node);
219
220	#Per record callback
221	if ($each_callback){
222	$each_callback->({
223	repository => $repository,
224	document => $document,
225	});
226	}
227	}
228	elsif ($reader->localName eq "resumptionToken"){
229	my $resumptionToken = $reader->readInnerXml;
230	return ($resumptionToken,$repository);
231
232	}
233	}
234	}
235	} #/OAI-PMH document match
236	}
237	else {
238	warn "ParseXMLStream() requires a 'file_handle' argument.";
239	}
240	}
241
242	sub harvest {
243	my ($self,$args) = @_;
244	my $url = $args->{url};
245	my $ua = $args->{user_agent};
246	my $callback = $args->{callback};
247	my $complete_callback = $args->{complete_callback};
248
249	if ($url && $ua){
250
251	#NOTE: http://search.cpan.org/~shlomif/XML-LibXML-2.0128/lib/XML/LibXML/Parser.pod#ERROR_REPORTING
252	while($url){
253	warn "URL = $url";
254	warn "Creating child process to download and feed parent process parser.";
255	my $stream = $self->GetXMLStream({
256	url => $url,
257	user_agent => $ua,
258	});
259
260	warn "Creating parent process parser.";
261	my ($resumptionToken) = $self->ParseXMLStream({
262	file_handle => $stream,
263	each_callback => $callback,
264	});
265	warn "Finished parsing current XML document.";
266
267	if ($resumptionToken){
268	#If there's a resumptionToken at the end of the stream,
269	#we build a new URL and repeat this process again.
270	$url->query_form({
271	verb => "ListRecords",
272	resumptionToken => $resumptionToken,
273	});
274	}
275	else {
276	warn "Finished harvest.";
277	last;
278	}
279
280	warn "Reap child process downloader.";
281	#Reap the dead child requester process before performing another request,
282	#so we don't fill up the process table with zombie children.
283	while ((my $child = waitpid(-1, 0)) > 0) {
284	warn "Parent $$ reaped child process $child" . ($? ? " with exit code $?" : '') . ".\n";
285	}
286	}
287
288	if ($complete_callback){
289	warn "Run complete callback.";
290
291	#Clear query string
292	$url->query_form({});
293
294	#Run complete callback using the actual URL from the request.
295	$complete_callback->({
296	repository => $url,
297	});
298	}
299	}
300	}
301
302	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Import/MARCXML.pm (+145 lines)
		1	package Koha::OAI::Harvester::Import::MARCXML;
		2
		3	# Copyright 2016 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19	#
		20
		21	use Modern::Perl;
		22	use MARC::Record;
		23
		24	use C4::Context;
		25	use C4::Biblio;
		26
		27	use constant MAX_MATCHES => 99999; #NOTE: This is an arbitrary value. We want to get all matches.
		28
		29	sub new {
		30	my ($class, $args) = @_;
		31	$args = {} unless defined $args;
		32	if ( (! $args->{dom}) && (! $args->{marc_record}) ){
		33	die "You must provide either a dom or marc_record argument to this constructor";
		34	}
		35	if ( $args->{dom} && ( ! $args->{marc_record} ) ){
		36	my $dom = $args->{dom};
		37	my $xml = $dom->toString(2);
		38	my $marcflavour = C4::Context->preference('marcflavour') \|\| 'MARC21';
		39	my $marc_record = eval {MARC::Record::new_from_xml( $xml, "utf8", $marcflavour)};
		40	if ($@){
		41	die "Unable to create MARC::Record object";
		42	}
		43	if ($marc_record){
		44	$args->{marc_record} = $marc_record;
		45	}
		46	}
		47	return bless ($args, $class);
		48	}
		49
		50	sub import_record {
		51	my ($self,$args) = @_;
		52	my $framework = $args->{framework};
		53	my $record_type = $args->{record_type};
		54	my $matcher = $args->{matcher};
		55	my $koha_id = $args->{koha_id};
		56
		57	my $action = "error";
		58
		59	#Try to find a matching Koha MARCXML record via Zebra
		60	if (! $koha_id && $matcher){
		61	my $matched_id = $self->_try_matcher({
		62	matcher => $matcher,
		63	});
		64	if ($matched_id){
65	$koha_id = $matched_id;
66	}
67	}
68
69	if ($koha_id){
70	#Update
71	($action) = $self->_mod_koha_record({
72	record_type => $record_type,
73	framework => $framework,
74	koha_id => $koha_id,
75	});
76	}
77	else {
78	#Add
79	($action,$koha_id) = $self->_add_koha_record({
80	record_type => $record_type,
81	framework => $framework,
82	});
83	}
84
85	return ($action,$koha_id);
86	}
87
88	sub _try_matcher {
89	my ($self, $args) = @_;
90	my $marc_record = $self->{marc_record};
91	my $matcher = $args->{matcher};
92	my $matched_id;
93	my @matches = $matcher->get_matches($marc_record, MAX_MATCHES);
94	if (@matches){
95	#FIXME: You can remove this sort once Bug 17710 is pushed
96	@matches = sort {
97	$b->{'score'} cmp $a->{'score'} or
98	$b->{'record_id'} cmp $a->{'record_id'}
99	} @matches;
100	my $bestrecordmatch = shift @matches;
101	if ($bestrecordmatch && $bestrecordmatch->{record_id}){
102	$matched_id = $bestrecordmatch->{record_id};
103	}
104	}
105	return $matched_id;
106	}
107
108	sub _add_koha_record {
109	my ($self, $args) = @_;
110	my $marc_record = $self->{marc_record};
111	my $record_type = $args->{record_type} // "biblio";
112	my $framework = $args->{framework};
113	my $koha_id;
114	my $action = "error";
115	if ($record_type eq "biblio"){
116	#NOTE: Strip item fields to prevent any accidentally getting through.
117	C4::Biblio::_strip_item_fields($marc_record,$framework);
118	my ($biblionumber,$biblioitemnumber) = C4::Biblio::AddBiblio($marc_record,$framework);
119	if ($biblionumber){
120	$action = "added";
121	$koha_id = $biblionumber;
122	}
123	}
124	return ($action,$koha_id);
125	}
126
127	sub _mod_koha_record {
128	my ($self, $args) = @_;
129	my $marc_record = $self->{marc_record};
130	my $record_type = $args->{record_type} // "biblio";
131	my $framework = $args->{framework};
132	my $koha_id = $args->{koha_id};
133	my $action = "error";
134	if ($record_type eq "biblio"){
135	#NOTE: Strip item fields to prevent any accidentally getting through.
136	C4::Biblio::_strip_item_fields($marc_record,$framework);
137	my $updated = C4::Biblio::ModBiblio($marc_record, $koha_id, $framework);
138	if ($updated){
139	$action = "updated";
140	}
141	}
142	return ($action);
143	}
144
145	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Import/RDFXML.pm (+47 lines)
	1	package Koha::OAI::Harvester::Import::RDFXML;
	2
	3	# Copyright 2017 Prosentient Systems
	4	#
	5	# This file is part of Koha.
	6	#
	7	# Koha is free software; you can redistribute it and/or modify it
	8	# under the terms of the GNU General Public License as published by
	9	# the Free Software Foundation; either version 3 of the License, or
	10	# (at your option) any later version.
	11	#
	12	# Koha is distributed in the hope that it will be useful, but
	13	# WITHOUT ANY WARRANTY; without even the implied warranty of
	14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	# GNU General Public License for more details.
	16	#
	17	# You should have received a copy of the GNU General Public License
	18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
	19	#
	20
	21	use Modern::Perl;
	22
	23	sub new {
	24	my ($class, $args) = @_;
	25	$args = {} unless defined $args;
	26	return bless ($args, $class);
	27	}
	28
	29	sub import_record {
	30	my ($self,$args) = @_;
	31	my $framework = $args->{framework};
	32	my $record_type = $args->{record_type};
	33	my $matcher = $args->{matcher};
	34	my $koha_id = $args->{koha_id};
	35
	36	my $action = "error";
	37
	38	if (! $koha_id && $matcher){
	39	#Create minimal MARCXML record from RDFXML to be used for Zebra-based matching.
	40	}
	41
	42
	43	#TODO: add/update triplestore
	44	return ($action,$koha_id);
	45	}
	46
	47	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Import/Record.pm (+304 lines)
		1	package Koha::OAI::Harvester::Import::Record;
		2
		3	# Copyright 2016 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19	#
		20
		21	use Modern::Perl;
		22	use XML::LibXML;
		23	use XML::LibXSLT;
		24	use URI;
		25	use File::Basename;
		26
		27	use C4::Context;
		28	use C4::Biblio;
		29
		30	use Koha::Database;
		31	use Koha::OAI::Harvester::Import::MARCXML;
		32	use Koha::OAI::Harvester::Import::RDFXML;
		33
		34	my $schema = Koha::Database->new()->schema();
		35
		36	sub new {
		37	my ($class, $args) = @_;
		38	$args = {} unless defined $args;
		39
		40	die "You must provide a 'doc' argument to the constructor" unless $args->{doc};
		41	die "You must provide a 'repository' argument to the constructor" unless $args->{repository};
		42
		43	if (my $doc = $args->{doc}){
		44
		45	#Get the root element
		46	my $root = $doc->documentElement;
		47
		48	#Register namespaces for searching purposes
		49	my $xpc = XML::LibXML::XPathContext->new();
		50	$xpc->registerNs('oai','http://www.openarchives.org/OAI/2.0/');
		51
		52	my $xpath_identifier = XML::LibXML::XPathExpression->new("oai:header/oai:identifier");
		53	my $identifier = $xpc->findnodes($xpath_identifier,$root)->shift;
		54	$args->{header_identifier} = $identifier->textContent;
		55
		56	my $xpath_datestamp = XML::LibXML::XPathExpression->new("oai:header/oai:datestamp");
		57	my $datestamp = $xpc->findnodes($xpath_datestamp,$root)->shift;
		58	$args->{header_datestamp} = $datestamp->textContent;
		59
		60	my $xpath_status = XML::LibXML::XPathExpression->new(q{oai:header/@status});
		61	my $status_node = $xpc->findnodes($xpath_status,$root)->shift;
		62	$args->{header_status} = $status_node ? $status_node->textContent : "";
		63	}
		64
65	return bless ($args, $class);
66	}
67
68	sub is_deleted_upstream {
69	my ($self, $args) = @_;
70	if ($self->{header_status}){
71	if ($self->{header_status} eq "deleted"){
72	return 1;
73	}
74	}
75	return 0;
76	}
77
78	sub set_filter {
79	my ($self, $filter_definition) = @_;
80
81	#Source a default XSLT to use for filtering
82	my $htdocs = C4::Context->config('intrahtdocs');
83	my $theme = C4::Context->preference("template");
84	$self->{filter} = "$htdocs/$theme/en/xslt/StripOAIPMH.xsl";
85	$self->{filter_type} = "xslt";
86
87	if ($filter_definition && $filter_definition ne "default"){
88	my ($filter_type, $filter) = $self->_parse_filter($filter_definition);
89	if ($filter_type eq "xslt"){
90	if ( -f $filter ){
91	$self->{filter} = $filter;
92	$self->{filter_type} = "xslt";
93	}
94	}
95	}
96	}
97
98	sub _parse_filter {
99	my ($self,$filter_definition) = @_;
100	my ($type,$filter);
101	my $filter_uri = URI->new($filter_definition);
102	if ($filter_uri){
103	my $scheme = $filter_uri->scheme;
104	if ( ($scheme && $scheme eq "file") \|\| ! $scheme ){
105	my $path = $filter_uri->path;
106	#Filters may theoretically be .xsl or .pm files
107	my($filename, $dirs, $suffix) = fileparse($path,(".xsl",".pm"));
108	if ($suffix){
109	if ( $suffix eq ".xsl"){
110	$type = "xslt";
111	$filter = $path;
112	}
113	}
114	}
115	}
116	return ($type,$filter);
117	}
118
119	sub filter {
120	my ($self) = @_;
121	my $filtered = 0;
122	my $doc = $self->{doc};
123	my $filter = $self->{filter};
124	my $filter_type = $self->{filter_type};
125	if ($doc){
126	if ($filter && -f $filter){
127	if ($filter_type){
128	if ( $filter_type eq 'xslt' ){
129	my $xslt = XML::LibXSLT->new();
130	my $style_doc = XML::LibXML->load_xml(location => $filter);
131	my $stylesheet = $xslt->parse_stylesheet($style_doc);
132	if ($stylesheet){
133	my $results = $stylesheet->transform($doc);
134	if ($results){
135	my $root = $results->documentElement;
136	if ($root){
137	my $namespace = $root->namespaceURI;
138	if ($namespace eq "http://www.loc.gov/MARC21/slim"){
139	#NOTE: Both MARC21 and UNIMARC should be covered by this namespace
140	my $marcxml = eval { Koha::OAI::Harvester::Import::MARCXML->new({ dom => $results, }) };
141	if ($@){
142	warn "Error Koha::OAI::Harvester::Import::MARCXML: $@";
143	return undef;
144	} else {
145	return $marcxml;
146	}
147	} elsif ($namespace eq "http://www.w3.org/1999/02/22-rdf-syntax-ns#"){
148	my $rdfxml = Koha::OAI::Harvester::Import::RDFXML->new({ dom => $results, });
149	if ($rdfxml){
150	return $rdfxml;
151	}
152	}
153	}
154	}
155	}
156	}
157	}
158	}
159	}
160	return;
161	}
162
163	sub _find_koha_link {
164	my ($self, $args) = @_;
165	my $record_type = $args->{record_type} // "biblio";
166	my $link_id;
167	if ($record_type eq "biblio"){
168	my $link = $schema->resultset('OaiHarvesterBiblio')->find(
169	{
170	oai_repository => $self->{repository},
171	oai_identifier => $self->{header_identifier},
172	},
173	{ key => "oai_record",}
174	);
175	if ($link && $link->biblionumber){
176	$link_id = $link->biblionumber->id;
177	}
178	}
179	return $link_id;
180	}
181
182	=head3
183
184	my ($action,$record_id) = $oai_record->import_record({
185	filter => $filter,
186	framework => $framework,
187	record_type => $record_type,
188	matcher => $matcher,
189	});
190
191	$action eq "added" \|\| "updated" \|\| "deleted" \|\| "not_found" \|\| "error"
192
193	=cut
194
195	sub import_record {
196	my ($self, $args) = @_;
197	my $filter = $args->{filter} \|\| 'default';
198	my $framework = $args->{framework} \|\| '';
199	my $record_type = $args->{record_type} \|\| 'biblio';
200	my $matcher = $args->{matcher};
201
202	my $action = "error";
203
204	#Find linkage between OAI-PMH repository-identifier and Koha record id
205	my $linked_id = $self->_find_koha_link({
206	record_type => $record_type,
207	});
208
209	if ($self->is_deleted_upstream){
210	#NOTE: If a record is deleted upstream, it will not contain a metadata element
211	if ($linked_id){
212	$action = $self->delete_koha_record({
213	record_id => $linked_id,
214	record_type => $record_type,
215	});
216	#TODO: How to handle RDF deletions?
217	#NOTE: If we delete the Koha record, then we lose the link to our downloaded record, so I don't know if it matters...
218	#...although surely we'd want some way of cleaning up records that aren't needed anymore? I suppose that could come up with a cronjob that scans for triples that aren't linked to...
219	}
220	else {
221	$action = "not_found";
222	#NOTE: If there's no OAI-PMH repository-identifier pair in the database,
223	#then there's no perfect way to find a linked record to delete.
224	}
225	}
226	else {
227	$self->set_filter($filter);
228
229
230	my $import_record = $self->filter();
231
232	if ($import_record){
233	($action,$linked_id) = $import_record->import_record({
234	framework => $framework,
235	record_type => $record_type,
236	matcher => $matcher,
237	koha_id => $linked_id,
238	});
239
240	if ($linked_id){
241	#Link Koha record ID to OAI-PMH details for this record type,
242	#if linkage doesn't already exist.
243	$self->link_koha_record({
244	record_type => $record_type,
245	koha_id => $linked_id,
246	});
247	}
248	}
249	}
250
251	#Log record details to database
252	my $importer = $schema->resultset('OaiHarvesterHistory')->create({
253	header_identifier => $self->{header_identifier},
254	header_datestamp => $self->{header_datestamp},
255	header_status => $self->{header_status},
256	record => $self->{doc}->toString(1),
257	repository => $self->{repository},
258	status => $action,
259	filter => $filter,
260	framework => $framework,
261	record_type => $record_type,
262	matcher_code => $matcher ? $matcher->code : undef,
263	});
264
265	return ($action,$linked_id);
266	}
267
268	sub link_koha_record {
269	my ($self, $args) = @_;
270	my $record_type = $args->{record_type} // "biblio";
271	my $koha_id = $args->{koha_id};
272	if ($koha_id){
273	if ($record_type eq "biblio"){
274	my $import_oai_biblio = $schema->resultset('OaiHarvesterBiblio')->find_or_create({
275	oai_repository => $self->{repository},
276	oai_identifier => $self->{header_identifier},
277	biblionumber => $koha_id,
278	});
279	if ( ! $import_oai_biblio->in_storage ){
280	$import_oai_biblio->insert;
281	}
282	}
283	}
284	}
285
286	sub delete_koha_record {
287	my ($self, $args) = @_;
288	my $record_type = $args->{record_type} // "biblio";
289	my $record_id = $args->{record_id};
290
291	my $action = "error";
292
293	if ($record_type eq "biblio"){
294	my $error = C4::Biblio::DelBiblio($record_id);
295	if (!$error){
296	$action = "deleted";
297	#NOTE: If there's no error, a cascading database delete should
298	#automatically remove the link between the Koha biblionumber and OAI-PMH record too
299	}
300	}
301	return $action;
302	}
303
304	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Listener.pm (+187 lines)
		1	package Koha::OAI::Harvester::Listener;
		2
		3	# Copyright 2017 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19	#
		20
		21	use Modern::Perl;
		22	use POE qw(Wheel::SocketFactory Wheel::ReadWrite);
		23	use IO::Socket qw(AF_UNIX);
		24	use JSON;
		25	use URI;
		26
		27	sub new {
		28	my ($class, $args) = @_;
		29	$args = {} unless defined $args;
		30	return bless ($args, $class);
		31	}
		32
		33	sub spawn {
		34	my ($class, $args) = @_;
		35	my $self = $class->new($args);
		36	my $socket = $args->{socket};
		37	POE::Session->create(
		38	args => [
		39	$socket,
		40	],
		41	object_states => [
		42	$self => {
		43	_start => "on_start",
		44	"on_server_success" => "on_server_success",
		45	"on_server_error" => "on_server_error",
		46	"on_client_error" => "on_client_error",
		47	"on_client_input" => "on_client_input",
		48	},
		49	],
		50	);
		51	}
		52
		53	sub on_start {
		54	my ($kernel,$heap,$socket_uri) = @_[KERNEL,HEAP,ARG0];
		55
		56	my $uri = URI->new($socket_uri);
		57	if ($uri && $uri->scheme eq 'unix'){
		58	my $socket_path = $uri->path;
		59	unlink $socket_path if -S $socket_path;
		60	$heap->{server} = POE::Wheel::SocketFactory->new(
		61	SocketDomain => AF_UNIX,
		62	BindAddress => $socket_path,
		63	SuccessEvent => "on_server_success",
		64	FailureEvent => "on_server_error",
65	);
66
67	#Make the socket writeable to other users like Apache
68	chmod 0666, $socket_path;
69	}
70	}
71
72	sub on_server_success {
73	my ($self, $client_socket, $server_wheel_id, $heap, $session) = @_[OBJECT, ARG0, ARG3, HEAP,SESSION];
74	my $logger = $self->{logger};
75	my $null_filter = POE::Filter::Line->new(
76	Literal => chr(0),
77	);
78	my $client_wheel = POE::Wheel::ReadWrite->new(
79	Handle => $client_socket,
80	InputEvent => "on_client_input",
81	ErrorEvent => "on_client_error",
82	InputFilter => $null_filter,
83	OutputFilter => $null_filter,
84	);
85	$heap->{client}->{ $client_wheel->ID() } = $client_wheel;
86	$logger->info("Connection ".$client_wheel->ID()." started.");
87	#TODO: Add basic authentication here?
88	$client_wheel->put("HELLO");
89	}
90
91	sub on_server_error {
92	my ($self, $operation, $errnum, $errstr, $heap, $session) = @_[OBJECT, ARG0, ARG1, ARG2,HEAP, SESSION];
93	my $logger = $self->{logger};
94	$logger->error("Server $operation error $errnum: $errstr");
95	delete $heap->{server};
96	}
97
98	sub on_client_error {
99	my ($self, $wheel_id,$heap,$session) = @_[OBJECT, ARG3,HEAP,SESSION];
100	my $logger = $self->{logger};
101	$logger->info("Connection $wheel_id failed or ended.");
102	delete $heap->{client}->{$wheel_id};
103	}
104
105	sub on_client_input {
106	my ($self, $input, $wheel_id, $session, $kernel, $heap) = @_[OBJECT, ARG0, ARG1, SESSION, KERNEL, HEAP];
107	my $logger = $self->{logger};
108	$logger->debug("Server input: $input");
109	my $server_response = { msg => "fail"};
110	eval {
111	my $json_input = from_json($input);
112	my $command = $json_input->{command};
113	my $body = $json_input->{body};
114	if ($command){
115	if ($command eq "create"){
116	my $task = $body->{task};
117	if ($task){
118	my $is_created = $kernel->call("harvester","create_task",$task);
119	if ($is_created){
120	$server_response->{msg} = "success";
121	}
122	}
123	}
124	elsif ($command eq "start"){
125	my $task = $body->{task};
126	if ($task){
127	my $uuid = $task->{uuid};
128	#Fetch from memory now...
129	my $is_started = $kernel->call("harvester","start_task", $uuid);
130	if ($is_started){
131	$server_response->{msg} = "success";
132	}
133	}
134	}
135	elsif ($command eq "stop"){
136	my $task = $body->{task};
137	if ($task){
138	if ($task->{uuid}){
139	my $is_stopped = $kernel->call("harvester","stop_task",$task->{uuid});
140	if ($is_stopped){
141	$server_response->{msg} = "success";
142	}
143	}
144	}
145	}
146	elsif ($command eq "delete"){
147	my $task = $body->{task};
148	if ($task){
149	if ($task->{uuid}){
150	my $is_deleted = $kernel->call("harvester","delete_task",$task->{uuid});
151	if ($is_deleted){
152	$server_response->{msg} = "success";
153	}
154	}
155	}
156	}
157	elsif ($command eq "list"){
158	my $tasks = $kernel->call("harvester","list_tasks");
159	if ($tasks){
160	$server_response->{msg} = "success";
161	$server_response->{data} = $tasks;
162	}
163	}
164	}
165	};
166	if ($@){
167	#NOTE: An error most likely means that something other than a valid JSON string was received
168	$logger->error($@);
169	}
170
171	if ($server_response){
172	eval {
173	my $client = $heap->{client}->{$wheel_id};
174	my $json_message = to_json($server_response, { pretty => 1 });
175	if ($json_message){
176	$logger->debug("Server output: $json_message");
177	$client->put($json_message);
178	}
179	};
180	if ($@){
181	#NOTE: An error means our response couldn't be serialised as JSON
182	$logger->error($@);
183	}
184	}
185	}
186
187	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Request.pm (+184 lines)
		1	package Koha::OAI::Harvester::Request;
		2
		3	# Copyright Prosentient Systems 2017
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, write to the Free Software Foundation, Inc.,
		18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		19
		20	use Modern::Perl;
		21
		22	use Carp;
		23
		24	use base qw(Koha::Object);
		25
		26	#For validation
		27	use URI;
		28	use HTTP::OAI;
		29
		30	=head1 NAME
		31
		32	Koha::OAI::Harvester::Request -
		33
		34	=head1 API
		35
		36	=head2 Class Methods
		37
		38	=cut
		39
		40
		41
		42	=head3 _type
		43
		44	=cut
		45
		46	sub _type {
		47	return 'OaiHarvesterRequest';
		48	}
		49
		50	sub validate {
		51	my ($self) = @_;
		52	my $errors = {};
		53
		54	#Step one: validate URL
		55	my $uri = URI->new($self->http_url);
		56	if ( $uri && $uri->scheme && ($uri->scheme eq "http" \|\| $uri->scheme eq "https") ){
		57
		58	#Step two: validate access and authorization to URL
		59	my $harvester = $self->_harvester();
		60	my $identify = $harvester->Identify;
		61	if ($identify->is_success){
		62
		63	#Step three: validate OAI-PMH parameters
		64
65	#Test Set
66	my $set = $self->oai_set;
67	if ($set){
68	my $set_response = $harvester->ListSets();
69	my @server_sets = $set_response->set;
70	if ( ! grep {$_->setSpec eq $set} @server_sets ){
71	$errors->{oai_set}->{unavailable} = 1;
72	}
73	}
74
75	#Test Metadata Prefix
76	my $metadataPrefix = $self->oai_metadataPrefix;
77	if ($metadataPrefix){
78	my $metadata_response = $harvester->ListMetadataFormats();
79	my @server_formats = $metadata_response->metadataFormat;
80	if ( ! grep { $_->metadataPrefix eq $metadataPrefix } @server_formats ){
81	$errors->{oai_metadataPrefix}->{unavailable} = 1;
82	}
83	}
84	else {
85	$errors->{oai_metadataPrefix}->{missing} = 1;
86	}
87
88	#Test Granularity and Timestamps
89	my $server_granularity = $identify->granularity;
90	my $from = $self->oai_from;
91	my $until = $self->oai_until;
92	if ($from \|\| $until){
93	my ($from_granularity,$until_granularity);
94	if ($from){
95	$from_granularity = _determine_granularity($from);
96	if ($from_granularity eq "YYYY-MM-DDThh:mm:ssZ"){
97	$errors->{oai_from}->{unavailable} = 1 if $server_granularity ne $from_granularity;
98	} elsif ($from_granularity eq "failed"){
99	$errors->{oai_from}->{malformed} = 1;
100	}
101	}
102	if ($until){
103	$until_granularity = _determine_granularity($until);
104	if ($until_granularity eq "YYYY-MM-DDThh:mm:ssZ"){
105	$errors->{oai_until}->{unavailable} = 1 if $server_granularity ne $until_granularity;
106	} elsif ($until_granularity eq "failed"){
107	$errors->{oai_until}->{malformed} = 1;
108	}
109	}
110	if ($from && $until){
111	if ($from_granularity ne $until_granularity){
112	$errors->{oai}->{granularity_mismatch} = 1;
113	}
114	}
115	}
116
117	#Test if identifier is provided when using GetRecord
118	my $verb = $self->oai_verb;
119	if ($verb && $verb eq "GetRecord"){
120	my $identifier = $self->oai_identifier;
121	if (! $identifier){
122	$errors->{oai_identifier}->{missing} = 1;
123	}
124	}
125	}
126	elsif ($identify->is_error){
127	foreach my $error ($identify->errors){
128	if ($error->code =~ /^404$/){
129	$errors->{http}->{404} = 1;
130	} elsif ($error->code =~ /^401$/){
131	$errors->{http}->{401} = 1;
132	} else {
133	$errors->{http}->{generic} = 1;
134	}
135	}
136	}
137	else {
138	$errors->{http}->{generic} = 1;
139	}
140	} else {
141	$errors->{http_url}->{malformed} = 1;
142	}
143	return $errors;
144	}
145
146	sub _harvester {
147	my ( $self ) = @_;
148	my $harvester;
149	if ($self->http_url){
150	$harvester = new HTTP::OAI::Harvester( baseURL => $self->http_url );
151	my $uri = URI->new($self->http_url);
152	if ($uri->scheme && ($uri->scheme eq 'http' \|\| $uri->scheme eq 'https') ){
153	my $host = $uri->host;
154	my $port = $uri->port;
155	$harvester->credentials($host.":".$port, $self->http_realm, $self->http_username, $self->http_password);
156	}
157	}
158	return $harvester;
159	}
160
161	sub _determine_granularity {
162	my ($timestamp) = @_;
163	my $granularity;
164	if ($timestamp =~ /^(\d{4}-\d{2}-\d{2})(T\d{2}:\d{2}:\d{2}Z)?$/){
165	if ($1 && $2){
166	$granularity = "YYYY-MM-DDThh:mm:ssZ";
167	} elsif ($1 && !$2){
168	$granularity = "YYYY-MM-DD";
169	} else {
170	$granularity = "failed";
171	}
172	} else {
173	$granularity = "failed";
174	}
175	return $granularity;
176	}
177
178	=head1 AUTHOR
179
180	David Cook <dcook@prosentient.com.au>
181
182	=cut
183
184	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Requests.pm (+62 lines)
	1	package Koha::OAI::Harvester::Requests;
	2
	3	# Copyright Prosentient Systems 2017
	4	#
	5	# This file is part of Koha.
	6	#
	7	# Koha is free software; you can redistribute it and/or modify it under the
	8	# terms of the GNU General Public License as published by the Free Software
	9	# Foundation; either version 3 of the License, or (at your option) any later
	10	# version.
	11	#
	12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
	13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	15	#
	16	# You should have received a copy of the GNU General Public License along
	17	# with Koha; if not, write to the Free Software Foundation, Inc.,
	18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
	19
	20	use Modern::Perl;
	21
	22	use Carp;
	23
	24	use Koha::Database;
	25
	26	use Koha::OAI::Harvester::Request;
	27
	28	use base qw(Koha::Objects);
	29
	30	=head1 NAME
	31
	32	Koha::OAI::Harvester::Requests -
	33
	34	=head1 API
	35
	36	=head2 Class Methods
	37
	38	=cut
	39
	40	=head3 _type
	41
	42	=cut
	43
	44	sub _type {
	45	return 'OaiHarvesterRequest';
	46	}
	47
	48	=head3 object_class
	49
	50	=cut
	51
	52	sub object_class {
	53	return 'Koha::OAI::Harvester::Request';
	54	}
	55
	56	=head1 AUTHOR
	57
	58	David Cook <dcook@prosentient.com.au>
	59
	60	=cut
	61
	62	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Worker.pm (+156 lines)
		1	package Koha::OAI::Harvester::Worker;
		2
		3	# Copyright Prosentient Systems 2017
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, write to the Free Software Foundation, Inc.,
		18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		19
		20	use Modern::Perl;
		21	use POE;
		22	use DateTime;
		23	use JSON;
		24
		25	sub new {
		26	my ($class, $args) = @_;
		27	$args = {} unless defined $args;
		28	$args->{type} = "worker" unless $args->{type};
		29	return bless ($args, $class);
		30	}
		31
		32	sub run {
		33	my ($self,$args) = @_;
		34	my $postback = $args->{postback};
		35	my $task = $args->{task};
		36
		37	POE::Session->create(
		38	object_states => [
		39	$self => {
		40	_start => "on_start",
		41	got_child_stderr => "on_child_stderr",
		42	got_child_close => "on_child_close",
		43	got_child_signal => "on_child_signal",
		44	got_child_stdout => "on_child_stdout",
		45	stop_worker => "stop_worker",
		46	_stop => "on_stop",
		47	},
		48	],
		49	args => [
		50	$postback,
		51	$task,
		52	],
		53	);
		54	}
		55
		56	sub stop_worker {
		57	my ($self,$heap) = @_[OBJECT,HEAP];
		58	if (my $child_processes = $heap->{children_by_pid}){
		59	foreach my $child_pid (keys %$child_processes){
		60	my $child = $child_processes->{$child_pid};
		61	$child->kill();
		62	}
		63	}
		64	}
65
66
67	sub on_stop {
68	my ($self,$kernel) = @_[OBJECT,KERNEL];
69
70	#Deregister the worker session from the harvester's roster of workers
71	$kernel->call("harvester","deregister",$self->{type});
72	}
73
74	# Wheel event, including the wheel's ID.
75	sub on_child_stdout {
76	my ($self, $stdout_line, $wheel_id) = @_[OBJECT, ARG0, ARG1];
77	my $type = $self->{type};
78	my $child = $_[HEAP]{children_by_wid}{$wheel_id};
79	my $logger = $self->{logger};
80	if ($logger){
81	$logger->debug("[$type][pid ".$child->PID."][STDOUT] $stdout_line");
82	}
83
84	my $postback = $_[HEAP]{postback};
85	if ($postback){
86	eval {
87	my $message = from_json($stdout_line);
88	if ($message){
89	$postback->($message);
90	}
91	};
92	}
93	}
94
95	# Wheel event, including the wheel's ID.
96	sub on_child_stderr {
97	my ($self,$stderr_line, $wheel_id) = @_[OBJECT, ARG0, ARG1];
98	my $type = $self->{type};
99	my $child = $_[HEAP]{children_by_wid}{$wheel_id};
100	my $logger = $self->{logger};
101	if ($logger){
102	$logger->debug("[$type][pid ".$child->PID."][STDERR] $stderr_line");
103	}
104	}
105
106	# Wheel event, including the wheel's ID.
107	sub on_child_close {
108	my ($self,$heap,$wheel_id) = @_[OBJECT,HEAP,ARG0];
109	my $type = $self->{type};
110	my $logger = $self->{logger};
111
112	my $child = delete $heap->{children_by_wid}->{$wheel_id};
113
114	# May have been reaped by on_child_signal().
115	unless (defined $child) {
116	if ($logger){
117	$logger->debug("[$type][wid $wheel_id] closed all pipes");
118	}
119	return;
120	}
121	if ($logger){
122	$logger->debug("[$type][pid ".$child->PID."] closed all pipes");
123	}
124	delete $heap->{children_by_pid}->{$child->PID};
125	}
126
127	sub on_child_signal {
128	my ($self,$kernel,$pid,$status) = @_[OBJECT,KERNEL,ARG1,ARG2];
129	my $type = $self->{type};
130	my $logger = $self->{logger};
131	if ($logger){
132	$logger->debug("[$type][pid $pid] exited with status $status");
133	}
134
135	my $child = delete $_[HEAP]{children_by_pid}{$_[ARG1]};
136
137	# May have been reaped by on_child_close().
138	return unless defined $child;
139
140	delete $_[HEAP]{children_by_wid}{$child->ID};
141
142	#If the child doesn't complete successfully, we lodge an error
143	#and stop the task.
144	if ($status != 0){
145	my $task = $kernel->call("harvester","get_task");
146	if ($task){
147	$task->{error} = 1;
148	my $uuid = $task->{uuid};
149	if ($uuid){
150	$kernel->call("harvester","stop_task",$uuid);
151	}
152	}
153	}
154	}
155
156	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Worker/Download/Stream.pm (+187 lines)
		1	package Koha::OAI::Harvester::Worker::Download::Stream;
		2
		3	# Copyright Prosentient Systems 2017
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, write to the Free Software Foundation, Inc.,
		18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		19
		20	use Modern::Perl;
		21	use LWP::UserAgent;
		22	use UUID qw/uuid/;
		23	use POE;
		24	use JSON;
		25	use File::Path qw/make_path/;
		26
		27	use C4::Context;
		28	use Koha::OAI::Harvester::Downloader;
		29	use parent 'Koha::OAI::Harvester::Worker';
		30
		31	sub new {
		32	my ($class, $args) = @_;
		33	$args = {} unless defined $args;
		34	$args->{type} = "download" unless $args->{type};
		35	return bless ($args, $class);
		36	}
		37
		38	sub on_start {
		39	my ($self, $kernel, $heap, $postback,$task,$session) = @_[OBJECT, KERNEL, HEAP, ARG0,ARG1,SESSION];
		40	#Save postback into heap so other event handlers can use it
		41	$heap->{postback} = $postback;
		42
		43	my $task_uuid = $task->{uuid};
		44
		45	$kernel->sig("cancel" => "stop_worker");
		46	$kernel->call("harvester","register",$self->{type},$task->{uuid});
		47
		48	my $child = POE::Wheel::Run->new(
		49	ProgramArgs => [$task],
		50	Program => sub {
		51	my ($args) = @_;
		52	$self->do_work($args);
		53	},
		54	StdoutEvent => "got_child_stdout",
		55	StderrEvent => "got_child_stderr",
		56	CloseEvent => "got_child_close",
		57	NoSetPgrp => 1, #Keep child processes in same group as parent. This is especially useful when using Ctrl+C to kill the whole group.
		58	);
		59
		60	$_[KERNEL]->sig_child($child->PID, "got_child_signal");
		61
		62	# Wheel events include the wheel's ID.
		63	$_[HEAP]{children_by_wid}{$child->ID} = $child;
		64
65	# Signal events include the process ID.
66	$_[HEAP]{children_by_pid}{$child->PID} = $child;
67
68	my $logger = $self->{logger};
69	if ($logger){
70	$logger->debug("Child pid ".$child->PID." started as wheel ".$child->ID);
71	}
72	}
73
74	sub do_work {
75	my ($self, $task) = @_;
76	my $batch = ( $self->{batch} && int($self->{batch}) ) ? $self->{batch} : 100;
77
78	#NOTE: Directory to spool files for processing
79	my $spooldir = $task->{spooldir};
80
81	my $task_uuid = $task->{uuid};
82	my $task_parameters = $task->{parameters};
83	my $interval = $task->{interval};
84
85	my $oai_pmh_parameters = $task_parameters->{oai_pmh};
86	my $import_parameters = $task_parameters->{import};
87
88	#NOTE: Overwrite the 'from' and 'until' parameters for repeatable tasks
89	if ( $interval && ! $oai_pmh_parameters->{until} ){
90	if ($oai_pmh_parameters->{verb} eq "ListRecords"){
91	#NOTE: 'effective_from' premiers on the first repetition (ie second request)
92	$oai_pmh_parameters->{from} = $task->{effective_from} if $task->{effective_from};
93	#NOTE: 'effective_until' appears on the first request
94	$oai_pmh_parameters->{until} = $task->{effective_until} if $task->{effective_until};
95	}
96	}
97
98	my $oai_downloader = Koha::OAI::Harvester::Downloader->new();
99	my $url = $oai_downloader->BuildURL($oai_pmh_parameters);
100
101	my $ua = LWP::UserAgent->new();
102	#NOTE: setup HTTP Basic Authentication if parameters are supplied
103	if($url && $url->host && $url->port){
104	my $http_basic_auth = $task_parameters->{http_basic_auth};
105	if ($http_basic_auth){
106	my $username = $http_basic_auth->{username};
107	my $password = $http_basic_auth->{password};
108	my $realm = $http_basic_auth->{realm};
109	$ua->credentials($url->host.":".$url->port, $realm, $username, $password);
110	}
111	}
112
113	#NOTE: Prepare database statement handle
114	my $dbh = C4::Context->dbh;
115	my $sql = "insert into oai_harvester_import_queue (uuid,result) VALUES (?,?)";
116	my $sth = $dbh->prepare($sql);
117
118	if($url && $ua){
119	#NOTE: You could define the callbacks as object methods instead... that might be nicer...
120	#although I suppose it might be a much of a muchness.
121	eval {
122	my @filename_cache = ();
123
124	$oai_downloader->harvest({
125	user_agent => $ua,
126	url => $url,
127	callback => sub {
128	my ($args) = @_;
129
130	my $repository = $args->{repository};
131	my $document = $args->{document};
132
133	#If the spooldir has disappeared, re-create it.
134	if ( ! -d $spooldir ){
135	my $made_spool_directory = make_path($spooldir);
136	}
137	my $file_uuid = uuid();
138	my $filename = "$spooldir/$file_uuid";
139	my $state = $document->toFile($filename, 2);
140	if ($state){
141	push(@filename_cache,$filename);
142	}
143
144	if(scalar @filename_cache == $batch){
145	my $result = {
146	repository => $repository,
147	filenames => \@filename_cache,
148	filter => $import_parameters->{filter},
149	matcher_code => $import_parameters->{matcher_code},
150	frameworkcode => $import_parameters->{frameworkcode},
151	record_type => $import_parameters->{record_type},
152	};
153	eval {
154	my $json_result = to_json($result, { pretty => 1 });
155	$sth->execute($task_uuid,$json_result);
156	};
157	@filename_cache = ();
158	}
159	},
160	complete_callback => sub {
161	my ($args) = @_;
162	my $repository = $args->{repository};
163	if (@filename_cache){
164	my $result = {
165	repository => "$repository",
166	filenames => \@filename_cache,
167	filter => $import_parameters->{filter},
168	matcher_code => $import_parameters->{matcher_code},
169	frameworkcode => $import_parameters->{frameworkcode},
170	record_type => $import_parameters->{record_type},
171	};
172	eval {
173	my $json_result = to_json($result, { pretty => 1 });
174	$sth->execute($task_uuid,$json_result);
175	};
176	}
177
178	},
179	});
180	};
181	if ($@){
182	die "Error during OAI-PMH download";
183	}
184	}
185	}
186
187	1;

Line 0 Link Here

(-)a/Koha/OAI/Harvester/Worker/Import.pm (+133 lines)
		1	package Koha::OAI::Harvester::Worker::Import;
		2
		3	# Copyright Prosentient Systems 2017
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, write to the Free Software Foundation, Inc.,
		18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		19
		20	use Modern::Perl;
		21	use POE qw(Wheel::Run);
		22	use JSON;
		23	use XML::LibXML;
		24
		25	use C4::Context;
		26	use C4::Matcher;
		27	use Koha::OAI::Harvester::Import::Record;
		28
		29	use parent 'Koha::OAI::Harvester::Worker';
		30
		31	sub new {
		32	my ($class, $args) = @_;
		33	$args = {} unless defined $args;
		34	#NOTE: This type is used for logging and more importantly for registering with the harvester
		35	$args->{type} = "import" unless $args->{type};
		36	return bless ($args, $class);
		37	}
		38
		39	sub on_start {
		40	my ($self, $kernel, $heap, $postback,$task) = @_[OBJECT, KERNEL, HEAP, ARG0,ARG1];
		41
		42	$kernel->call("harvester","register",$self->{type},$task->{uuid});
		43
		44	$kernel->sig(cancel => "stop_worker");
		45
		46	my $child = POE::Wheel::Run->new(
		47	ProgramArgs => [ $task ],
		48	Program => sub {
		49	my ($task,$args) = @_;
		50
		51	my $debug = $args->{debug} // 0;
		52
		53	if ($task){
		54	my $json_result = $task->{result};
		55	my $id = $task->{id};
		56	my $task_uuid = $task->{uuid};
		57	eval {
		58	my $result = from_json($json_result);
		59	if ($result){
		60	my $repository = $result->{repository};
		61	my $filenames = $result->{filenames};
		62	my $filter = $result->{filter};
		63	my $matcher_code = $result->{matcher_code};
		64	my $frameworkcode = $result->{frameworkcode};
65	my $record_type = $result->{record_type};
66
67	my $matcher;
68	if ($matcher_code){
69	my $matcher_id = C4::Matcher::GetMatcherId($matcher_code);
70	$matcher = C4::Matcher->fetch($matcher_id);
71	}
72
73	foreach my $filename (@$filenames){
74	if ($filename){
75	if (-f $filename){
76	my $dom = XML::LibXML->load_xml(location => $filename, { no_blanks => 1 });
77	if ($dom){
78	my $oai_record = Koha::OAI::Harvester::Import::Record->new({
79	doc => $dom,
80	repository => $repository,
81	});
82	if ($oai_record){
83	my ($action,$linked_id) = $oai_record->import_record({
84	filter => $filter,
85	framework => $frameworkcode,
86	record_type => $record_type,
87	matcher => $matcher,
88	});
89	$debug && print STDOUT qq({ "import_result": { "task_uuid": "$task_uuid", "action": "$action", "filename": "$filename", "koha_id": "$linked_id" } }\n);
90	}
91	}
92	my $unlinked = unlink $filename;
93	}
94	}
95	}
96	}
97	};
98	if ($@){
99	warn $@;
100	}
101	#NOTE: Even if the file doesn't exist, we still need to process the queue item.
102
103	#NOTE: Don't do this via a postback in the parent process, as it's much faster to let the child process handle it.
104
105	#NOTE: It's vital that files are unlinked before deleting from the database,
106	#or you could get orphan files if the importer is interrupted.
107	my $dbh = C4::Context->dbh;
108	my $sql = "delete from oai_harvester_import_queue where id = ?";
109	my $sth = $dbh->prepare($sql);
110	$sth->execute($id);
111	}
112	},
113	StdoutEvent => "got_child_stdout",
114	StderrEvent => "got_child_stderr",
115	CloseEvent => "got_child_close",
116	NoSetPgrp => 1, #Keep child processes in same group as parent. This is especially useful when using Ctrl+C to kill the whole group.
117	);
118
119	$_[KERNEL]->sig_child($child->PID, "got_child_signal");
120
121	# Wheel events include the wheel's ID.
122	$_[HEAP]{children_by_wid}{$child->ID} = $child;
123
124	# Signal events include the process ID.
125	$_[HEAP]{children_by_pid}{$child->PID} = $child;
126
127	my $logger = $self->{logger};
128	if ($logger){
129	$logger->debug("Child pid ".$child->PID." started as wheel ".$child->ID);
130	}
131	}
132
133	1;

Line 0 Link Here

(-)a/Koha/Schema/Result/OaiHarvesterBiblio.pm (+120 lines)
		1	use utf8;
		2	package Koha::Schema::Result::OaiHarvesterBiblio;
		3
		4	# Created by DBIx::Class::Schema::Loader
		5	# DO NOT MODIFY THE FIRST PART OF THIS FILE
		6
		7	=head1 NAME
		8
		9	Koha::Schema::Result::OaiHarvesterBiblio
		10
		11	=cut
		12
		13	use strict;
		14	use warnings;
		15
		16	use base 'DBIx::Class::Core';
		17
		18	=head1 TABLE: C<oai_harvester_biblios>
		19
		20	=cut
		21
		22	__PACKAGE__->table("oai_harvester_biblios");
		23
		24	=head1 ACCESSORS
		25
		26	=head2 import_oai_biblio_id
		27
		28	data_type: 'integer'
		29	extra: {unsigned => 1}
		30	is_auto_increment: 1
		31	is_nullable: 0
		32
		33	=head2 oai_repository
		34
		35	data_type: 'varchar'
		36	is_nullable: 0
		37	size: 255
		38
		39	=head2 oai_identifier
		40
		41	data_type: 'varchar'
		42	is_nullable: 1
		43	size: 255
		44
		45	=head2 biblionumber
		46
		47	data_type: 'integer'
		48	is_foreign_key: 1
		49	is_nullable: 0
		50
		51	=cut
		52
		53	__PACKAGE__->add_columns(
		54	"import_oai_biblio_id",
		55	{
		56	data_type => "integer",
		57	extra => { unsigned => 1 },
		58	is_auto_increment => 1,
		59	is_nullable => 0,
		60	},
		61	"oai_repository",
		62	{ data_type => "varchar", is_nullable => 0, size => 255 },
		63	"oai_identifier",
		64	{ data_type => "varchar", is_nullable => 1, size => 255 },
65	"biblionumber",
66	{ data_type => "integer", is_foreign_key => 1, is_nullable => 0 },
67	);
68
69	=head1 PRIMARY KEY
70
71	=over 4
72
73	=item * L</import_oai_biblio_id>
74
75	=back
76
77	=cut
78
79	__PACKAGE__->set_primary_key("import_oai_biblio_id");
80
81	=head1 UNIQUE CONSTRAINTS
82
83	=head2 C<oai_record>
84
85	=over 4
86
87	=item * L</oai_identifier>
88
89	=item * L</oai_repository>
90
91	=back
92
93	=cut
94
95	__PACKAGE__->add_unique_constraint("oai_record", ["oai_identifier", "oai_repository"]);
96
97	=head1 RELATIONS
98
99	=head2 biblionumber
100
101	Type: belongs_to
102
103	Related object: L<Koha::Schema::Result::Biblio>
104
105	=cut
106
107	__PACKAGE__->belongs_to(
108	"biblionumber",
109	"Koha::Schema::Result::Biblio",
110	{ biblionumber => "biblionumber" },
111	{ is_deferrable => 1, on_delete => "CASCADE", on_update => "NO ACTION" },
112	);
113
114
115	# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-03-29 12:23:43
116	# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:2URn8tPABMKC+JuIMfGeYw
117
118
119	# You can replace this text with custom code or comments, and it will be preserved on regeneration
120	1;

Line 0 Link Here

(-)a/Koha/Schema/Result/OaiHarvesterHistory.pm (+163 lines)
		1	use utf8;
		2	package Koha::Schema::Result::OaiHarvesterHistory;
		3
		4	# Created by DBIx::Class::Schema::Loader
		5	# DO NOT MODIFY THE FIRST PART OF THIS FILE
		6
		7	=head1 NAME
		8
		9	Koha::Schema::Result::OaiHarvesterHistory
		10
		11	=cut
		12
		13	use strict;
		14	use warnings;
		15
		16	use base 'DBIx::Class::Core';
		17
		18	=head1 TABLE: C<oai_harvester_history>
		19
		20	=cut
		21
		22	__PACKAGE__->table("oai_harvester_history");
		23
		24	=head1 ACCESSORS
		25
		26	=head2 import_oai_id
		27
		28	data_type: 'integer'
		29	extra: {unsigned => 1}
		30	is_auto_increment: 1
		31	is_nullable: 0
		32
		33	=head2 repository
		34
		35	data_type: 'varchar'
		36	is_nullable: 1
		37	size: 255
		38
		39	=head2 header_identifier
		40
		41	data_type: 'varchar'
		42	is_nullable: 1
		43	size: 255
		44
		45	=head2 header_datestamp
		46
		47	data_type: 'datetime'
		48	datetime_undef_if_invalid: 1
		49	is_nullable: 0
		50
		51	=head2 header_status
		52
		53	data_type: 'varchar'
		54	is_nullable: 1
		55	size: 45
		56
		57	=head2 record
		58
		59	data_type: 'longtext'
		60	is_nullable: 0
		61
		62	=head2 upload_timestamp
		63
		64	data_type: 'timestamp'
65	datetime_undef_if_invalid: 1
66	default_value: current_timestamp
67	is_nullable: 0
68
69	=head2 status
70
71	data_type: 'varchar'
72	is_nullable: 0
73	size: 45
74
75	=head2 filter
76
77	data_type: 'text'
78	is_nullable: 0
79
80	=head2 framework
81
82	data_type: 'varchar'
83	is_nullable: 0
84	size: 4
85
86	=head2 record_type
87
88	data_type: 'enum'
89	extra: {list => ["biblio","auth","holdings"]}
90	is_nullable: 0
91
92	=head2 matcher_code
93
94	data_type: 'varchar'
95	is_nullable: 1
96	size: 10
97
98	=cut
99
100	__PACKAGE__->add_columns(
101	"import_oai_id",
102	{
103	data_type => "integer",
104	extra => { unsigned => 1 },
105	is_auto_increment => 1,
106	is_nullable => 0,
107	},
108	"repository",
109	{ data_type => "varchar", is_nullable => 1, size => 255 },
110	"header_identifier",
111	{ data_type => "varchar", is_nullable => 1, size => 255 },
112	"header_datestamp",
113	{
114	data_type => "datetime",
115	datetime_undef_if_invalid => 1,
116	is_nullable => 0,
117	},
118	"header_status",
119	{ data_type => "varchar", is_nullable => 1, size => 45 },
120	"record",
121	{ data_type => "longtext", is_nullable => 0 },
122	"upload_timestamp",
123	{
124	data_type => "timestamp",
125	datetime_undef_if_invalid => 1,
126	default_value => \"current_timestamp",
127	is_nullable => 0,
128	},
129	"status",
130	{ data_type => "varchar", is_nullable => 0, size => 45 },
131	"filter",
132	{ data_type => "text", is_nullable => 0 },
133	"framework",
134	{ data_type => "varchar", is_nullable => 0, size => 4 },
135	"record_type",
136	{
137	data_type => "enum",
138	extra => { list => ["biblio", "auth", "holdings"] },
139	is_nullable => 0,
140	},
141	"matcher_code",
142	{ data_type => "varchar", is_nullable => 1, size => 10 },
143	);
144
145	=head1 PRIMARY KEY
146
147	=over 4
148
149	=item * L</import_oai_id>
150
151	=back
152
153	=cut
154
155	__PACKAGE__->set_primary_key("import_oai_id");
156
157
158	# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-03-29 12:23:43
159	# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Rp/ZEZsKVlLo2vaM3M37ow
160
161
162	# You can replace this text with custom code or comments, and it will be preserved on regeneration
163	1;

Line 0 Link Here

(-)a/Koha/Schema/Result/OaiHarvesterImportQueue.pm (+106 lines)
		1	use utf8;
		2	package Koha::Schema::Result::OaiHarvesterImportQueue;
		3
		4	# Created by DBIx::Class::Schema::Loader
		5	# DO NOT MODIFY THE FIRST PART OF THIS FILE
		6
		7	=head1 NAME
		8
		9	Koha::Schema::Result::OaiHarvesterImportQueue
		10
		11	=cut
		12
		13	use strict;
		14	use warnings;
		15
		16	use base 'DBIx::Class::Core';
		17
		18	=head1 TABLE: C<oai_harvester_import_queue>
		19
		20	=cut
		21
		22	__PACKAGE__->table("oai_harvester_import_queue");
		23
		24	=head1 ACCESSORS
		25
		26	=head2 id
		27
		28	data_type: 'integer'
		29	extra: {unsigned => 1}
		30	is_auto_increment: 1
		31	is_nullable: 0
		32
		33	=head2 uuid
		34
		35	data_type: 'varchar'
		36	is_nullable: 0
		37	size: 45
		38
		39	=head2 status
		40
		41	data_type: 'varchar'
		42	default_value: 'new'
		43	is_nullable: 0
		44	size: 45
		45
		46	=head2 result
		47
		48	data_type: 'text'
		49	is_nullable: 0
		50
		51	=head2 result_timestamp
		52
		53	data_type: 'timestamp'
		54	datetime_undef_if_invalid: 1
		55	default_value: current_timestamp
		56	is_nullable: 0
		57
		58	=cut
		59
		60	__PACKAGE__->add_columns(
		61	"id",
		62	{
		63	data_type => "integer",
		64	extra => { unsigned => 1 },
65	is_auto_increment => 1,
66	is_nullable => 0,
67	},
68	"uuid",
69	{ data_type => "varchar", is_nullable => 0, size => 45 },
70	"status",
71	{
72	data_type => "varchar",
73	default_value => "new",
74	is_nullable => 0,
75	size => 45,
76	},
77	"result",
78	{ data_type => "text", is_nullable => 0 },
79	"result_timestamp",
80	{
81	data_type => "timestamp",
82	datetime_undef_if_invalid => 1,
83	default_value => \"current_timestamp",
84	is_nullable => 0,
85	},
86	);
87
88	=head1 PRIMARY KEY
89
90	=over 4
91
92	=item * L</id>
93
94	=back
95
96	=cut
97
98	__PACKAGE__->set_primary_key("id");
99
100
101	# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-03-29 12:23:43
102	# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zBD+hMawbvu7sonuLRHnCA
103
104
105	# You can replace this text with custom code or comments, and it will be preserved on regeneration
106	1;

Line 0 Link Here

(-)a/Koha/Schema/Result/OaiHarvesterRequest.pm (+209 lines)
		1	use utf8;
		2	package Koha::Schema::Result::OaiHarvesterRequest;
		3
		4	# Created by DBIx::Class::Schema::Loader
		5	# DO NOT MODIFY THE FIRST PART OF THIS FILE
		6
		7	=head1 NAME
		8
		9	Koha::Schema::Result::OaiHarvesterRequest
		10
		11	=cut
		12
		13	use strict;
		14	use warnings;
		15
		16	use base 'DBIx::Class::Core';
		17
		18	=head1 TABLE: C<oai_harvester_requests>
		19
		20	=cut
		21
		22	__PACKAGE__->table("oai_harvester_requests");
		23
		24	=head1 ACCESSORS
		25
		26	=head2 id
		27
		28	data_type: 'integer'
		29	extra: {unsigned => 1}
		30	is_auto_increment: 1
		31	is_nullable: 0
		32
		33	=head2 uuid
		34
		35	data_type: 'varchar'
		36	is_nullable: 0
		37	size: 45
		38
		39	=head2 oai_verb
		40
		41	data_type: 'varchar'
		42	is_nullable: 0
		43	size: 45
		44
		45	=head2 oai_metadataPrefix
		46
		47	accessor: 'oai_metadata_prefix'
		48	data_type: 'varchar'
		49	is_nullable: 0
		50	size: 255
		51
		52	=head2 oai_identifier
		53
		54	data_type: 'varchar'
		55	is_nullable: 1
		56	size: 255
		57
		58	=head2 oai_from
		59
		60	data_type: 'varchar'
		61	is_nullable: 1
		62	size: 45
		63
		64	=head2 oai_until
65
66	data_type: 'varchar'
67	is_nullable: 1
68	size: 45
69
70	=head2 oai_set
71
72	data_type: 'varchar'
73	is_nullable: 1
74	size: 255
75
76	=head2 http_url
77
78	data_type: 'varchar'
79	is_nullable: 1
80	size: 255
81
82	=head2 http_username
83
84	data_type: 'varchar'
85	is_nullable: 1
86	size: 255
87
88	=head2 http_password
89
90	data_type: 'varchar'
91	is_nullable: 1
92	size: 255
93
94	=head2 http_realm
95
96	data_type: 'varchar'
97	is_nullable: 1
98	size: 255
99
100	=head2 import_filter
101
102	data_type: 'varchar'
103	is_nullable: 0
104	size: 255
105
106	=head2 import_framework_code
107
108	data_type: 'varchar'
109	is_nullable: 0
110	size: 4
111
112	=head2 import_record_type
113
114	data_type: 'enum'
115	extra: {list => ["biblio","auth","holdings"]}
116	is_nullable: 0
117
118	=head2 import_matcher_code
119
120	data_type: 'varchar'
121	is_nullable: 1
122	size: 10
123
124	=head2 interval
125
126	data_type: 'integer'
127	extra: {unsigned => 1}
128	is_nullable: 0
129
130	=head2 name
131
132	data_type: 'varchar'
133	is_nullable: 0
134	size: 45
135
136	=cut
137
138	__PACKAGE__->add_columns(
139	"id",
140	{
141	data_type => "integer",
142	extra => { unsigned => 1 },
143	is_auto_increment => 1,
144	is_nullable => 0,
145	},
146	"uuid",
147	{ data_type => "varchar", is_nullable => 0, size => 45 },
148	"oai_verb",
149	{ data_type => "varchar", is_nullable => 0, size => 45 },
150	"oai_metadataPrefix",
151	{
152	accessor => "oai_metadata_prefix",
153	data_type => "varchar",
154	is_nullable => 0,
155	size => 255,
156	},
157	"oai_identifier",
158	{ data_type => "varchar", is_nullable => 1, size => 255 },
159	"oai_from",
160	{ data_type => "varchar", is_nullable => 1, size => 45 },
161	"oai_until",
162	{ data_type => "varchar", is_nullable => 1, size => 45 },
163	"oai_set",
164	{ data_type => "varchar", is_nullable => 1, size => 255 },
165	"http_url",
166	{ data_type => "varchar", is_nullable => 1, size => 255 },
167	"http_username",
168	{ data_type => "varchar", is_nullable => 1, size => 255 },
169	"http_password",
170	{ data_type => "varchar", is_nullable => 1, size => 255 },
171	"http_realm",
172	{ data_type => "varchar", is_nullable => 1, size => 255 },
173	"import_filter",
174	{ data_type => "varchar", is_nullable => 0, size => 255 },
175	"import_framework_code",
176	{ data_type => "varchar", is_nullable => 0, size => 4 },
177	"import_record_type",
178	{
179	data_type => "enum",
180	extra => { list => ["biblio", "auth", "holdings"] },
181	is_nullable => 0,
182	},
183	"import_matcher_code",
184	{ data_type => "varchar", is_nullable => 1, size => 10 },
185	"interval",
186	{ data_type => "integer", extra => { unsigned => 1 }, is_nullable => 0 },
187	"name",
188	{ data_type => "varchar", is_nullable => 0, size => 45 },
189	);
190
191	=head1 PRIMARY KEY
192
193	=over 4
194
195	=item * L</id>
196
197	=back
198
199	=cut
200
201	__PACKAGE__->set_primary_key("id");
202
203
204	# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-04-07 11:26:24
205	# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Vm/b4yurmr8WF7z+Fo6KEw
206
207
208	# You can replace this text with custom code or comments, and it will be preserved on regeneration
209	1;

Line 0 Link Here

(-)a/debian/templates/oai-pmh-harvester-site.yaml.in (+22 lines)
	1	---
	2	#Harvester
	3	socket: 'unix:__OAI_RUN_DIR__/harvesterd.sock'
	4
	5	logfile: '__LOG_DIR__/oai-pmh-harvester.log'
	6	loglevel: 'WARN'
	7
	8	pidfile: '__OAI_RUN_DIR__/harvesterd.pid'
	9
	10	statefile: '__OAI_LIB_DIR__/harvesterd.state'
	11
	12	spooldir: '__OAI_SPOOL_DIR__'
	13
	14	#Downloader
	15	download_workers: 1
	16	download_module: 'Koha::OAI::Harvester::Worker::Download::Stream'
	17	download_batch: 100
	18
	19	#Importer
	20	import_workers: 1
	21	import_poll: 5
	22	import_module: 'Koha::OAI::Harvester::Worker::Import'

Line 0 Link Here

(-)a/etc/oai-pmh-harvester.yaml (+22 lines)
	1	---
	2	#Harvester
	3	socket: 'unix:__OAI_RUN_DIR__/harvesterd.sock'
	4
	5	logfile: '__LOG_DIR__/oai-pmh-harvester.log'
	6	loglevel: 'WARN'
	7
	8	pidfile: '__OAI_RUN_DIR__/harvesterd.pid'
	9
	10	statefile: '__OAI_LIB_DIR__/harvesterd.state'
	11
	12	spooldir: '__OAI_SPOOL_DIR__'
	13
	14	#Downloader
	15	download_workers: 1
	16	download_module: 'Koha::OAI::Harvester::Worker::Download::Stream'
	17	download_batch: 100
	18
	19	#Importer
	20	import_workers: 1
	21	import_poll: 5
	22	import_module: 'Koha::OAI::Harvester::Worker::Import'

Line 0 Link Here

(-)a/installer/data/mysql/atomicupdate/bug_10662.sql (+73 lines)
		1	--
		2	-- Table structure for table 'oai_harvester_biblios'
		3	--
		4
		5	CREATE TABLE IF NOT EXISTS `oai_harvester_biblios` (
		6	`import_oai_biblio_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
		7	`oai_repository` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
		8	`oai_identifier` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
		9	`biblionumber` int(11) NOT NULL,
		10	PRIMARY KEY (`import_oai_biblio_id`),
		11	UNIQUE KEY `oai_record` (`oai_identifier`,`oai_repository`) USING BTREE,
		12	KEY `FK_import_oai_biblio_1` (`biblionumber`),
		13	CONSTRAINT `FK_import_oai_biblio_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE NO ACTION
		14	) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
		15
		16	--
		17	-- Table structure for table 'oai_harvester_history'
		18	--
		19
		20	CREATE TABLE IF NOT EXISTS `oai_harvester_history` (
		21	`import_oai_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
		22	`repository` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
		23	`header_identifier` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
		24	`header_datestamp` datetime NOT NULL,
		25	`header_status` varchar(45) COLLATE utf8_unicode_ci DEFAULT NULL,
		26	`record` longtext COLLATE utf8_unicode_ci NOT NULL,
		27	`upload_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
		28	`status` varchar(45) COLLATE utf8_unicode_ci NOT NULL,
		29	`filter` text COLLATE utf8_unicode_ci NOT NULL,
		30	`framework` varchar(4) COLLATE utf8_unicode_ci NOT NULL,
		31	`record_type` enum('biblio','auth','holdings') COLLATE utf8_unicode_ci NOT NULL,
		32	`matcher_code` varchar(10) COLLATE utf8_unicode_ci DEFAULT NULL,
		33	PRIMARY KEY (`import_oai_id`)
		34	) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
		35
		36	--
		37	-- Table structure for table 'oai_harvester_import_queue'
		38	--
		39
		40	CREATE TABLE IF NOT EXISTS `oai_harvester_import_queue` (
		41	`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
		42	`uuid` varchar(45) CHARACTER SET utf8 NOT NULL,
		43	`status` varchar(45) CHARACTER SET utf8 NOT NULL DEFAULT 'new',
		44	`result` text CHARACTER SET utf8 NOT NULL,
		45	`result_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
		46	PRIMARY KEY (`id`)
		47	) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
		48
		49	--
		50	-- Table structure for table 'oai_harvester_requests'
		51	--
		52
		53	CREATE TABLE IF NOT EXISTS `oai_harvester_requests` (
		54	`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
		55	`uuid` varchar(45) NOT NULL,
		56	`oai_verb` varchar(45) NOT NULL,
		57	`oai_metadataPrefix` varchar(255) NOT NULL,
		58	`oai_identifier` varchar(255) DEFAULT NULL,
		59	`oai_from` varchar(45) DEFAULT NULL,
		60	`oai_until` varchar(45) DEFAULT NULL,
		61	`oai_set` varchar(255) DEFAULT NULL,
		62	`http_url` varchar(255) DEFAULT NULL,
		63	`http_username` varchar(255) DEFAULT NULL,
		64	`http_password` varchar(255) DEFAULT NULL,
65	`http_realm` varchar(255) DEFAULT NULL,
66	`import_filter` varchar(255) NOT NULL,
67	`import_framework_code` varchar(4) NOT NULL,
68	`import_record_type` enum('biblio','auth','holdings') NOT NULL,
69	`import_matcher_code` varchar(10) DEFAULT NULL,
70	`interval` int(10) unsigned NOT NULL,
71	`name` varchar(45) NOT NULL,
72	PRIMARY KEY (`id`)
73	) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;

Line 0 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/dashboard.tt (+371 lines)
		1	[% INCLUDE 'doc-head-open.inc' %]
		2	<title>Koha &rsaquo; Tools &rsaquo; OAI-PMH harvester</title>
		3	[% INCLUDE 'doc-head-close.inc' %]
		4	[% INCLUDE 'datatables.inc' %]
		5	[% dashboard_page = '/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl' %]
		6	[% request_page = '/cgi-bin/koha/tools/oai-pmh-harvester/request.pl' %]
		7	<script type="text/javascript">
		8	//<![CDATA[
		9	$(document).ready(function() {
		10	$('#dashboard-items').tabs();
		11	[% IF ( result.start.defined \|\| result.stop.defined \|\| result.delete.defined ) %]
		12	$('#dashboard-items').tabs("option","active",1);
		13	[% END %]
		14
		15	var saved_table = $('#saved-table').DataTable({});
		16	var submitted_table = $('#submitted-table').DataTable({});
		17	var history_table = $('#history-table').DataTable({
		18	serverSide: true,
		19	searching: true,
		20	processing: true,
		21	ajax: {
		22	"url": '/cgi-bin/koha/svc/oai-pmh-harvester/history',
		23	contentType: 'application/json',
		24	type: 'POST',
		25	data: function ( d ) {
		26	return JSON.stringify( d );
		27	},
		28	dataSrc: function (json){
		29	var recordsTotal = json.recordsTotal;
		30	if(recordsTotal){
		31	$('#import_count').text( "("+recordsTotal+")" );
		32	}
		33	return json.data;
		34	}
		35	},
		36	columns: [
		37	{ data: 'import_oai_id', },
		38	{ data: 'repository', },
		39	{ data: 'header_identifier', },
		40	{ data: 'header_datestamp', },
		41	{
		42	data: 'header_status', render: function (data, type, full, meta){
		43	var display_string = _("Active");
		44	if (data == "deleted"){
		45	display_string = _("Deleted");
		46	}
		47	return display_string;
		48	}
		49	},
		50	{
		51	data: 'status', render: function (data, type, full, meta){
		52	var display_string = data;
		53	if (data == "added"){
		54	display_string = _("Added");
		55	}
		56	else if (data == "error"){
		57	display_string = _("Error");
		58	}
		59	else if (data == "not_found"){
		60	display_string = _("Not found");
		61	}
		62	else if (data == "updated"){
		63	display_string = _("Updated");
		64	}
65	return display_string;
66	}
67	},
68	{ data: 'upload_timestamp', },
69	{
70	data: 'imported_record', render: function (data, type, full, meta){
71	var display_string = data;
72	var record_type = (full.record_type) ? full.record_type : 'biblio';
73	if (data && record_type == 'biblio'){
74	var link_text = _("View biblio record");
75	var link = '<a href="/cgi-bin/koha/catalogue/detail.pl?biblionumber='+data+'">'+link_text+'</a>';
76	display_string = link;
77	}
78	return display_string;
79	}
80
81	},
82	{
83	data: 'import_oai_id', render: function (data, type, full, meta){
84	var display_string = data;
85	var link_text = _("View record");
86	var link = '<a href="/cgi-bin/koha/tools/oai-pmh-harvester/record.pl?import_oai_id='+data+'">'+link_text+'</a>';
87	display_string = link;
88	return display_string;
89	}
90	},
91	],
92	//In theory, it would be nice to sort in descending order, but
93	//it causes severe paging issues the data is frequently updated.
94	//Caching would make the paging more stable, but the results would be stale.
95	order: [
96	[ 0, 'asc' ],
97	]
98	});
99	$('#refresh-button').click(function(){
100	history_table.ajax.reload( null, false );
101	});
102	});
103	//]]>
104	</script>
105	<style type="text/css">
106	a.paginate_button {
107	padding: 2px;
108	}
109	</style>
110	</head>
111	<body id="tools_oai-pmh-harvester" class="tools">
112	[% INCLUDE 'header.inc' %]
113	[% INCLUDE 'cat-search.inc' %]
114	<div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> &rsaquo; OAI-PMH harvester</div>
115	<div id="doc3" class="yui-t2">
116	<div id="bd">
117	<div id="yui-main">
118	<div class="yui-b">
119	<h1>OAI-PMH harvester</h1>
120	<div id="toolbar" class="btn-toolbar">
121	<a id="new-request" class="btn btn-default btn-sm" href="[% request_page %]?op=new"><i class="fa fa-plus"></i> New request</a>
122	</div>
123	[% IF ( harvester.offline ) %]
124	<div class="alert">
125	<span>OAI-PMH harvester offline</span>
126	</div>
127	[% END %]
128	<div id="dashboard-items" class="toptabs">
129	<ul>
130	<li>
131	<a href="#saved_requests">Saved requests <span id="saved_count">([% saved_requests.size %])</span></a>
132	</li>
133	<li>
134	<a href="#submitted_requests">Submitted requests <span id="submitted_count">([% submitted_requests.size %])</span></a>
135	</li>
136	<li>
137	<a href="#imports">Import history <span id="import_count">(0)</span></a>
138	</li>
139	</ul>
140	<div id="submitted_requests">
141	[% IF ( result.start.defined ) %]
142	<div class="alert">
143	[% IF ( result.start ) %]
144	<span>Start succeeded</span>
145	[% ELSE %]
146	<span>Start failed</span>
147	[% END %]
148	</div>
149	[% ELSIF ( result.stop.defined ) %]
150	<div class="alert">
151	[% IF ( result.stop ) %]
152	<span>Stop succeeded</span>
153	[% ELSE %]
154	<span>Stop failed</span>
155	[% END %]
156	</div>
157	[% ELSIF ( result.delete.defined ) %]
158	<div class="alert">
159	[% IF ( result.delete ) %]
160	<span>Delete succeeded</span>
161	[% ELSE %]
162	<span>Delete failed</span>
163	[% END %]
164	</div>
165	[% END %]
166	<table id="submitted-table">
167	<thead>
168	<tr>
169	<th>Name</th>
170	<th>URL</th>
171	<th>Set</th>
172	<th>From</th>
173	<th>Until</th>
174	<th>Interval</th>
175	<th>Effective from</th>
176	<th>Effective until</th>
177	<th>Pending imports</th>
178	<th>Status</th>
179	<th>Error</th>
180	<th></th>
181	</tr>
182	</thead>
183	<tbody>
184	[% IF ( submitted_requests ) %]
185	[% FOREACH submitted_request IN submitted_requests %]
186	<tr>
187	<td>[% submitted_request.name %]</td>
188	<td>[% submitted_request.parameters.oai_pmh.baseURL %]</td>
189	<td>[% submitted_request.parameters.oai_pmh.set %]</td>
190	<td>[% submitted_request.parameters.oai_pmh.from %]</td>
191	<td>[% submitted_request.parameters.oai_pmh.until %]</td>
192	<td>[% submitted_request.interval %]</td>
193	<td>[% submitted_request.effective_from %]</td>
194	<td>[% submitted_request.effective_until %]</td>
195	<td>[% submitted_request.pending_imports %]</td>
196	<td>
197	[% IF ( submitted_status = submitted_request.status ) %]
198	[% IF ( submitted_status == "new" ) %]
199	<span>New</span>
200	[% ELSIF ( submitted_status == "active" ) %]
201	<span>Active</span>
202	[% ELSIF ( submitted_status == "stopped" ) %]
203	<span>Stopped</span>
204	[% END %]
205	[% END %]
206	</td>
207	<td>
208	[% IF ( submitted_request.error ) %]
209	<span>Harvest failure</span>
210	[% END %]
211	</td>
212	<td>
213	<div class="dropdown">
214	<a class="btn btn-default btn-xs dropdown-toggle" role="button" data-toggle="dropdown" href="#">Actions <span class="caret"></span></a>
215	<ul class="dropdown-menu pull-right" role="menu">
216	<li>
217	<a href="[% dashboard_page %]?op=start&uuid=[% submitted_request.uuid %]"><i class="fa fa-play"></i> Start</a>
218	</li>
219	<li>
220	<a href="[% dashboard_page %]?op=stop&uuid=[% submitted_request.uuid %]"><i class="fa fa-stop"></i> Stop</a>
221	</li>
222	<li>
223	<a href="[% dashboard_page %]?op=delete&uuid=[% submitted_request.uuid %]"><i class="fa fa-trash"></i> Delete</a>
224	</li>
225	</ul>
226	</div>
227	</td>
228	</tr>
229	[% END %]
230	[% END %]
231	</tbody>
232	</table>
233	</div>
234	<div id="saved_requests">
235	[% IF ( result.send.defined ) %]
236	<div class="alert">
237	[% IF ( result.send ) %]
238	<span>Send succeeded</span>
239	[% ELSE %]
240	<span>Send failed</span>
241	[% END %]
242	</div>
243	[% END %]
244	<table id="saved-table">
245	<thead>
246	<tr>
247	<th>Name</th>
248	<th>URL</th>
249	<!-- <th>Verb</th>
250	<th>Metadata prefix</th>
251	<th>Identifier</th> -->
252	<th>Set</th>
253	<th>From</th>
254	<th>Until</th>
255	<th>Interval</th>
256	<!-- <th>Filter</th>
257	<th>Framework code</th>
258	<th>Record type</th>
259	<th>Matcher code</th> -->
260	<th></th>
261	</tr>
262	</thead>
263	<tbody>
264	[% IF ( saved_requests ) %]
265	[% FOREACH saved_request IN saved_requests %]
266	<tr>
267	<td>[% saved_request.name %]</td>
268	<td>[% saved_request.http_url %]</td>
269	<!-- <td>[% saved_request.oai_verb %]</td>
270	<td>[% saved_request.oai_metadataPrefix %]</td>
271	<td>[% saved_request.oai_identifier %]</td> -->
272	<td>[% saved_request.oai_set %]</td>
273	<td>[% saved_request.oai_from %]</td>
274	<td>[% saved_request.oai_until %]</td>
275	<td>[% saved_request.interval %]</td>
276	<!-- <td>
277	[% IF ( saved_request.import_filter == "default" ) %]
278	<span>Default</span>
279	[% ELSE %]
280	[% saved_request.import_filter %]
281	[% END %]
282	</td>
283	<td>
284	[% display_framework = "";
285	FOREACH framework IN frameworks;
286	IF ( framework.frameworkcode == saved_request.import_framework_code );
287	display_framework = framework;
288	END;
289	END;
290	%]
291	[% IF ( display_framework ) %]
292	[% display_framework.frameworktext %]
293	[% ELSE %]
294	[% saved_request.import_framework_code %]
295	[% END %]
296	</td>
297	<td>
298	[% IF ( saved_request.import_record_type == "biblio" ) %]
299	<span>Bibliographic</span>
300	[% ELSE %]
301	[% saved_request.import_record_type %]
302	[% END %]
303	</td>
304	<td>
305	[% display_matcher = "";
306	FOREACH matcher IN matchers;
307	IF ( matcher.code == saved_request.import_matcher_code );
308	display_matcher = matcher;
309	END;
310	END;
311	%]
312	[% IF ( display_matcher ) %]
313	[% display_matcher.description %]
314	[% ELSE %]
315	[% saved_request.import_matcher_code %]
316	[% END %]
317	</td> -->
318	<td>
319	<div class="dropdown">
320	<a class="btn btn-default btn-xs dropdown-toggle" role="button" data-toggle="dropdown" href="#">Actions <span class="caret"></span></a>
321	<ul class="dropdown-menu pull-right" role="menu">
322	<li>
323	<a href="[% request_page %]?op=edit&id=[% saved_request.id %]"><i class="fa fa-pencil"></i> Edit</a>
324	</li>
325	<li>
326	<a href="[% request_page %]?op=new&id=[% saved_request.id %]"><i class="fa fa-copy"></i> Copy</a>
327	</li>
328	<li>
329	<a href="[% dashboard_page %]?op=send&id=[% saved_request.id %]"><i class="fa fa-send"></i> Send</a>
330	</li>
331	<li>
332	<a href="[% request_page %]?op=delete&id=[% saved_request.id %]"><i class="fa fa-trash"></i> Delete</a>
333	</li>
334	</ul>
335	</div>
336	</td>
337	</tr>
338	[% END %]
339	[% END %]
340	</tbody>
341	</table>
342	</div>
343	<div id="imports">
344	<div class="btn-toolbar">
345	<button id="refresh-button" type="button" class="btn btn-default btn-sm">Refresh import history</button>
346	</div>
347	<table id="history-table">
348	<thead>
349	<tr>
350	<th>Id</th>
351	<th>Repository</th>
352	<th>Identifier</th>
353	<th>Datestamp</th>
354	<th>Upstream status</th>
355	<th>Import status</th>
356	<th>Import timestamp</th>
357	<th>Imported record</th>
358	<th>Downloaded record</th>
359	</tr>
360	</thead>
361	</table>
362
363	</div>
364	</div>
365	</div>
366	</div>
367	<div class="yui-b">
368	[% INCLUDE 'tools-menu.inc' %]
369	</div>
370	</div>
371	[% INCLUDE 'intranet-bottom.inc' %]

Line 0 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/xslt/StripOAIPMH.xsl (+28 lines)
	1	<?xml version="1.0" encoding="UTF-8"?>
	2	<xsl:stylesheet version="1.0"
	3	xmlns:oai="http://www.openarchives.org/OAI/2.0/"
	4	xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
	5	<xsl:output method="xml" encoding="UTF-8" indent="yes"/>
	6	<!-- NOTE: This XSLT strips the OAI-PMH wrapper from the metadata. -->
	7
	8	<!-- Match the root oai:record element -->
	9	<xsl:template match="oai:record">
	10	<!-- Apply templates only to the child metadata element(s) -->
	11	<xsl:apply-templates select="oai:metadata" />
	12	</xsl:template>
	13
	14	<!-- Matches an oai:metadata element -->
	15	<xsl:template match="oai:metadata">
	16	<!-- Create a copy of child attributes and nodes -->
	17	<xsl:apply-templates select="@* \| node()" />
	18	</xsl:template>
	19
	20	<!-- Identity transformation: this template copies attributes and nodes -->
	21	<xsl:template match="@* \| node()">
	22	<!-- Create a copy of this attribute or node -->
	23	<xsl:copy>
	24	<!-- Recursively apply this template to the attributes and child nodes of this element -->
	25	<xsl:apply-templates select="@* \| node()" />
	26	</xsl:copy>
	27	</xsl:template>
	28	</xsl:stylesheet>

Line 0 Link Here

(-)a/misc/harvesterd.pl (+207 lines)
		1	#!/usr/bin/perl
		2
		3	# Copyright 2017 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19
		20	use Modern::Perl;
		21	use Getopt::Long;
		22	use Pod::Usage;
		23	use Module::Load;
		24	use Log::Log4perl qw(:easy);
		25	use POE;
		26	use YAML;
		27
		28	use C4::Context;
		29	use Koha::Daemon;
		30	use Koha::OAI::Harvester;
		31	use Koha::OAI::Harvester::Listener;
		32
		33	binmode(STDOUT,':encoding(UTF-8)');
		34	$\|++;
		35
		36	my $help = 0;
		37	my $daemonize = 0;
		38
		39	my ($socket_addr,$pidfile,$statefile,$spooldir);
		40	my ($download_module,$import_module);
		41	my ($batch,$download_workers,$import_workers,$import_poll);
		42	my ($logfile,$log_level);
		43	my $log_levels = {
		44	FATAL => $FATAL,
		45	ERROR => $ERROR,
		46	WARN => $WARN,
		47	INFO => $INFO,
		48	DEBUG => $DEBUG,
		49	TRACE => $TRACE,
		50	};
		51
		52	my $context = C4::Context->new();
		53	my $config_filename = $context->{config}->{oai_pmh_harvester_config};
		54	if ($config_filename){
		55	my $config = YAML::LoadFile($config_filename);
		56	if ($config){
		57	$socket_addr = $config->{socket};
		58	$pidfile = $config->{pidfile};
		59	$statefile = $config->{statefile};
		60	$spooldir = $config->{spooldir};
		61	$logfile = $config->{logfile};
		62	$log_level = $config->{loglevel};
		63	$download_module = $config->{download_module};
		64	$import_module = $config->{import_module};
65	$batch = $config->{download_batch};
66	$download_workers = $config->{download_workers};
67	$import_workers = $config->{import_workers};
68	$import_poll = $config->{import_poll};
69	}
70	}
71
72	GetOptions(
73	"help\|?" => \$help,
74	"daemon" => \$daemonize,
75	"socket-uri=s" => \$socket_addr,
76	"pid-file=s" => \$pidfile,
77	"state-file=s" => \$statefile,
78	"spool-dir=s" => \$spooldir,
79	"log-file=s" => \$logfile,
80	"log-level=s" => \$log_level,
81	"download-module=s" => \$download_module,
82	"import-module=s" => \$import_module,
83	) or pod2usage(2);
84	pod2usage(1) if $help;
85
86	my $level = ( $log_level && $log_levels->{$log_level} ) ? $log_levels->{$log_level} : $log_levels->{WARN};
87	Log::Log4perl->easy_init(
88	{
89	level => $level,
90	file => "STDOUT",
91	layout => '[%d{yyyy-MM-dd HH:mm:ss}][%p] %m%n',
92	}
93	);
94	my $logger = Log::Log4perl->get_logger();
95
96	unless($download_module){
97	$download_module = "Koha::OAI::Harvester::Worker::Download::Stream";
98	}
99	unless($import_module){
100	$import_module = "Koha::OAI::Harvester::Worker::Import";
101	}
102
103	foreach my $module ( $download_module, $import_module ){
104	load $module;
105	}
106	my $downloader = $download_module->new({
107	logger => $logger,
108	batch => $batch,
109	});
110	my $importer = $import_module->new({
111	logger => $logger,
112	});
113
114	my $daemon = Koha::Daemon->new({
115	pidfile => $pidfile,
116	logfile => $logfile,
117	daemonize => $daemonize,
118	});
119	$daemon->run();
120
121	my $harvester = Koha::OAI::Harvester->spawn({
122	Downloader => $downloader,
123	DownloaderWorkers => $download_workers,
124	Importer => $importer,
125	ImporterWorkers => $import_workers,
126	ImportQueuePoll => $import_poll,
127	logger => $logger,
128	state_file => $statefile,
129	spooldir => $spooldir,
130	});
131
132	my $listener = Koha::OAI::Harvester::Listener->spawn({
133	logger => $logger,
134	socket => $socket_addr,
135	});
136
137	$logger->info("OAI-PMH harvester started.");
138
139	POE::Kernel->run();
140
141	exit;
142
143	=head1 NAME
144
145	harvesterd.pl - a daemon that asynchronously sends OAI-PMH requests and imports OAI-PMH records
146
147	=head1 SYNOPSIS
148
149	KOHA_CONF=/path/to/koha-conf.xml ./harvesterd.pl
150
151	=head1 OPTIONS
152
153	=over 8
154
155	=item B<--help>
156
157	Print a brief help message and exits.
158
159	=item B<--daemon>
160
161	Run program as a daemon (ie fork process, setsid, chdir to root, reset umask,
162	and close STDIN, STDOUT, and STDERR).
163
164	=item B<--log-file>
165
166	Specify a file to which to log STDOUT and STDERR.
167
168	=item B<--pid-file>
169
170	Specify a file to store the process id (this prevents multiple copies of the program
171	from running at the same time).
172
173	=item B<--socket-uri>
174
175	Specify a URI to use for the UNIX socket used to communicate with the daemon.
176	(e.g. unix:/path/to/socket.sock)
177
178	=item B<--state-file>
179
180	Specify a filename to use for storing the harvester's in-memory state.
181
182	In the event that the harvester crashes, it can resume from where it stopped.
183
184	=item B<--spool-dir>
185
186	Specify a directory to store downloaded OAI-PMH records prior to import.
187
188	=item B<--log-level>
189
190	Specify a log level for logging. The logger uses Log4Perl, which provides
191	FATAL, ERROR, WARN, INFO, DEBUG, and TRACE in order of descending priority.
192
193	Defaults to WARN level.
194
195	=item B<--download-module>
196
197	Specify a Perl module to use for downloading records. This is a specialty module,
198	which has particular requirements, so only advanced users should use this option.
199
200	=item B<--import-module>
201
202	Specify a Perl module to use for importing records. This is a specialty module,
203	which has particular requirements, so only advanced users should use this option.
204
205	=back
206
207	=cut

Line 0 Link Here

(-)a/svc/oai-pmh-harvester/history (+132 lines)
		1	#!/usr/bin/perl
		2
		3	# Copyright 2017 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, write to the Free Software Foundation, Inc.,
		18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		19
		20	use Modern::Perl;
		21	use C4::Auth qw(check_cookie_auth haspermission get_session);
		22	use JSON;
		23	use Koha::Database;
		24
		25	my $input = new CGI;
		26
		27	my ( $auth_status, $sessionID ) =
		28	check_cookie_auth( $input->cookie('CGISESSID'));
		29
		30	if ( $auth_status ne "ok" ) {
		31	exit 0;
		32	}
		33
		34	my $data = {
		35	data => [],
		36	recordsTotal => 0,
		37	recordsFiltered => 0,
		38	draw => undef,
		39	};
		40
		41	my $length = 10;
		42	my $start = 0;
		43	my @order_by = ();
		44	my @search = ();
		45
		46	if ($input->request_method eq "POST"){
		47	my $postdata = $input->param('POSTDATA');
		48	my $request = from_json($postdata);
		49	$data->{draw} = int( $request->{draw} ) if $request->{draw};
		50	$length = $request->{length} if $request->{length};
		51	$start = $request->{start} if $request->{start};
		52	if (my $search = $request->{search}){
		53	my $value = $search->{value};
		54	if ($value){
		55	foreach my $column (@{$request->{columns}}){
		56	if ($column->{data} && $column->{searchable}){
		57	my $search_element = {
		58	$column->{data} => { 'like', "%".$value."%" },
		59	};
		60	push(@search,$search_element);
		61	}
		62	}
		63	}
		64	}
65	if (my $order = $request->{order}){
66	foreach my $element (@$order){
67	my $dir = $element->{dir};
68	my $column_index = $element->{column};
69	my $column = $request->{columns}->[$column_index];
70	my $orderable = $column->{orderable};
71	if ($orderable){
72	my $column_name = $column->{data};
73	my $direction;
74	if ($dir){
75	if ($dir eq "asc" \|\| $dir eq "desc"){
76	$direction = "-$dir";
77	}
78	}
79	if ($column_name && $direction){
80	my $single_order = {
81	$direction => $column_name,
82	};
83	push(@order_by,$single_order);
84	}
85	}
86	}
87	}
88	}
89
90	my $page = ( $start / $length ) + 1;
91	my $schema = Koha::Database->new()->schema();
92	if ($schema){
93	my $rs = $schema->resultset("OaiHarvesterHistory");
94	my $results = $rs->search(
95	\@search,
96	{
97	result_class => 'DBIx::Class::ResultClass::HashRefInflator',
98	page => $page,
99	rows => $length,
100	order_by => \@order_by,
101	},
102	);
103	my $count = $rs->count;
104	my $filtered_count = $results->pager->total_entries;
105	my @rows = ();
106	while (my $row = $results->next){
107	$row->{imported_record} = '';
108	if ($row->{record_type} eq "biblio"){
109	my $harvested_biblio = $schema->resultset("OaiHarvesterBiblio")->find(
110	{
111	oai_repository => $row->{repository},
112	oai_identifier => $row->{header_identifier},
113	},
114	{ key => "oai_record" },
115	);
116	$row->{imported_record} = $harvested_biblio->biblionumber->id if $harvested_biblio;
117	}
118	push(@rows,$row);
119	}
120	if ($count){
121	$data->{recordsTotal} = $count;
122	$data->{recordsFiltered} = $filtered_count;
123	$data->{data} = \@rows if @rows;
124	}
125	}
126
127	binmode STDOUT, ":encoding(UTF-8)";
128	print $input->header(
129	-type => 'application/json',
130	-charset => 'UTF-8'
131	);
132	print to_json($data, { pretty => 1, });

Line 0 Link Here

(-)a/tools/oai-pmh-harvester/dashboard.pl (+134 lines)
		1	#!/usr/bin/perl
		2
		3	# Copyright 2017 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19
		20	use Modern::Perl;
		21	use YAML;
		22
		23	use C4::Auth;
		24	use C4::Context;
		25	use C4::Output;
		26	use Koha::OAI::Harvester::Client;
		27	use Koha::OAI::Harvester::Requests;
		28	use Koha::BiblioFrameworks;
		29	use Koha::Database;
		30
		31	my $context = C4::Context->new();
		32	my $config_filename = $context->{config}->{oai_pmh_harvester_config};
		33	my $client_config = {};
		34	if ($config_filename){
		35	my $config = YAML::LoadFile($config_filename);
		36	if ($config && $config->{socket}){
		37	$client_config->{socket_uri} = $config->{socket};
		38	}
		39	}
		40
		41	my $input = new CGI;
		42
		43	my ($template, $loggedinuser, $cookie) =
		44	get_template_and_user({template_name => "tools/oai-pmh-harvester/dashboard.tt",
		45	query => $input,
		46	type => "intranet",
		47	authnotrequired => 0,
		48	flagsrequired => {tools => 'manage_staged_marc'},
		49	});
		50
		51	my $op = $input->param('op') // 'list';
		52	my $id = $input->param('id');
		53	my $uuid = $input->param('uuid');
		54
		55	my $client = Koha::OAI::Harvester::Client->new($client_config);
		56	my $is_connected = $client->connect;
		57
		58	if ( ($op eq "send") && $id ){
		59	if ($is_connected){
		60	my $request = Koha::OAI::Harvester::Requests->find($id);
		61	if ($request){
		62	my $task = {
		63	name => $request->name,
		64	uuid => $request->uuid,
65	interval => $request->interval,
66	parameters => {
67	oai_pmh => {
68	baseURL => $request->http_url,
69	verb => $request->oai_verb,
70	metadataPrefix => $request->oai_metadataPrefix,
71	identifier => $request->oai_identifier,
72	set => $request->oai_set,
73	from => $request->oai_from,
74	until => $request->oai_until,
75	},
76	import => {
77	filter => $request->import_filter,
78	frameworkcode => $request->import_framework_code,
79	matcher_code => $request->import_matcher_code,
80	record_type => $request->import_record_type,
81	},
82	},
83	};
84	if ($request->http_username && $request->http_password && $request->http_realm){
85	$task->{parameters}->{http_basic_auth} = {
86	username => $request->http_username,
87	password => $request->http_password,
88	realm => $request->http_realm,
89	};
90	}
91	my $is_created = $client->create($task);
92	$template->{VARS}->{ result }->{ send } = $is_created;
93	}
94	}
95	}
96	elsif ( ($op eq "start") && ($uuid) ){
97	if ($is_connected){
98	my $is_started = $client->start($uuid);
99	$template->{VARS}->{ result }->{ start } = $is_started;
100	}
101	}
102	elsif ( ($op eq "stop") && ($uuid) ){
103	if ($is_connected){
104	my $is_stopped = $client->stop($uuid);
105	$template->{VARS}->{ result }->{ stop } = $is_stopped;
106	}
107	}
108	elsif ( ($op eq "delete") && ($uuid) ){
109	if ($is_connected){
110	my $is_deleted = $client->delete($uuid);
111	$template->{VARS}->{ result }->{ delete } = $is_deleted;
112	}
113	}
114
115	my $requests = Koha::OAI::Harvester::Requests->as_list;
116	$template->{VARS}->{ saved_requests } = $requests;
117
118	my $frameworks = Koha::BiblioFrameworks->as_list();
119	$template->{VARS}->{ frameworks } = $frameworks;
120
121	my $schema = Koha::Database->new()->schema();
122	my $matcher_rs = $schema->resultset("MarcMatcher");
123	my @matchers = $matcher_rs->all;
124	$template->{VARS}->{ matchers } = \@matchers;
125
126	if ($is_connected){
127	my $submitted_requests = $client->list;
128	$template->{VARS}->{ submitted_requests } = $submitted_requests;
129	}
130	else {
131	$template->{VARS}->{ harvester }->{ offline } = 1;
132	}
133
134	output_html_with_http_headers($input, $cookie, $template->output);

Line 0 Link Here

(-)a/tools/oai-pmh-harvester/record.pl (+53 lines)
	1	#!/usr/bin/perl
	2
	3	# Copyright 2017 Prosentient Systems
	4	#
	5	# This file is part of Koha.
	6	#
	7	# Koha is free software; you can redistribute it and/or modify it
	8	# under the terms of the GNU General Public License as published by
	9	# the Free Software Foundation; either version 3 of the License, or
	10	# (at your option) any later version.
	11	#
	12	# Koha is distributed in the hope that it will be useful, but
	13	# WITHOUT ANY WARRANTY; without even the implied warranty of
	14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	# GNU General Public License for more details.
	16	#
	17	# You should have received a copy of the GNU General Public License
	18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
	19
	20	use Modern::Perl;
	21
	22	use C4::Auth;
	23	use C4::Output;
	24	use Koha::Database;
	25
	26	my $input = new CGI;
	27
	28	my ($template, $loggedinuser, $cookie) =
	29	get_template_and_user({template_name => "tools/oai-pmh-harvester/record.tt",
	30	query => $input,
	31	type => "intranet",
	32	authnotrequired => 0,
	33	flagsrequired => {tools => 'manage_staged_marc'},
	34	});
	35
	36	my $import_oai_id = $input->param('import_oai_id');
	37	if ($import_oai_id){
	38	my $schema = Koha::Database->new()->schema();
	39	if ($schema){
	40	my $rs = $schema->resultset("OaiHarvesterHistory");
	41	if ($rs){
	42	my $row = $rs->find($import_oai_id);
	43	if ($row){
	44	my $record = $row->record;
	45	if ($record){
	46	$template->{VARS}->{ record } = $record;
	47	}
	48	}
	49	}
	50	}
	51	}
	52
	53	output_html_with_http_headers($input, $cookie, $template->output);

Line 0 Link Here

(-)a/tools/oai-pmh-harvester/request.pl (-1 / +140 lines)
0	-	1	#!/usr/bin/perl
		2
		3	# Copyright 2017 Prosentient Systems
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19
		20	use Modern::Perl;
		21	use UUID qw/uuid/;
		22
		23	use C4::Auth;
		24	use C4::Output;
		25	use Koha::OAI::Harvester::Requests;
		26	use Koha::BiblioFrameworks;
		27	use Koha::Database;
		28
		29	my $input = new CGI;
		30
		31	my ($template, $loggedinuser, $cookie) =
		32	get_template_and_user({template_name => "tools/oai-pmh-harvester/request.tt",
		33	query => $input,
		34	type => "intranet",
		35	authnotrequired => 0,
		36	flagsrequired => {tools => 'manage_staged_marc'},
		37	});
		38
		39	my $op = $input->param('op');
		40	my $id = $input->param('id');
		41
		42	my @frameworks = Koha::BiblioFrameworks->as_list();
		43	$template->{VARS}->{ frameworks } = \@frameworks;
		44
		45	my $schema = Koha::Database->new()->schema();
		46	my $rs = $schema->resultset("MarcMatcher");
		47	my @matchers = $rs->all;
		48	$template->{VARS}->{ matchers } = \@matchers;
		49
		50	my $http_url = $input->param('http_url');
		51	my $http_username = $input->param('http_username');
		52	my $http_password = $input->param('http_password');
		53	my $http_realm = $input->param('http_realm');
		54
		55	my $oai_verb = $input->param('oai_verb');
		56	my $oai_metadataPrefix = $input->param('oai_metadataPrefix');
		57	my $oai_identifier = $input->param('oai_identifier');
		58	my $oai_from = $input->param('oai_from');
		59	my $oai_until = $input->param('oai_until');
		60	my $oai_set = $input->param('oai_set');
		61
		62	my $import_filter = $input->param('import_filter') // 'default';
		63	my $import_framework_code = $input->param('import_framework_code');
		64	my $import_record_type = $input->param('import_record_type');
		65	my $import_matcher_code = $input->param('import_matcher_code');
66
67	my $interval = $input->param("interval") ? int ( $input->param("interval") ) : 0;
68	my $name = $input->param("name");
69
70	my $save = $input->param('save');
71	my $test_parameters = $input->param('test_parameters');
72
73	my $request = $id ? Koha::OAI::Harvester::Requests->find($id) : Koha::OAI::Harvester::Request->new();
74	if ($request){
75	if ($op eq "create" \|\| $op eq "update"){
76	$request->set({
77	name => $name,
78	http_url => $http_url,
79	http_username => $http_username,
80	http_password => $http_password,
81	http_realm => $http_realm,
82	oai_verb => $oai_verb,
83	oai_metadataPrefix => $oai_metadataPrefix,
84	oai_identifier => $oai_identifier,
85	oai_from => $oai_from,
86	oai_until => $oai_until,
87	oai_set => $oai_set,
88	import_filter => $import_filter,
89	import_framework_code => $import_framework_code,
90	import_record_type => $import_record_type,
91	import_matcher_code => $import_matcher_code,
92	interval => $interval,
93	});
94	}
95	}
96
97	if ($test_parameters){
98	my $errors = $request->validate();
99	$template->{VARS}->{ errors } = $errors;
100	$template->{VARS}->{ test_parameters } = 1;
101	}
102
103	if ($op eq "new"){
104	#Empty form with some defaults
105	$request->import_filter("default") unless $request->import_filter;
106	$request->interval(0) unless $request->interval;
107	}
108	elsif ($op eq "create"){
109	if ($save){
110	my $uuid = uuid;
111	$request->uuid($uuid);
112	$request->store;
113	print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
114	exit;
115	}
116	}
117	elsif ( $op eq "edit"){
118	$template->{VARS}->{ id } = $id;
119	}
120	elsif ($op eq "update"){
121	$template->{VARS}->{ id } = $id;
122	if ($save){
123	$request->store;
124	print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
125	exit;
126	}
127	}
128	elsif ($op eq "delete"){
129	if ($request){
130	$request->delete;
131	print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
132	}
133	}
134	else {
135	print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
136	}
137	$template->{VARS}->{ op } = $op;
138	$template->{VARS}->{ oai_pmh_request } = $request;
139
140	output_html_with_http_headers($input, $cookie, $template->output);