From d267731a50b7b343aa291772f74af2e6a4506d64 Mon Sep 17 00:00:00 2001 From: David Cook Date: Wed, 31 Oct 2018 03:14:46 +0000 Subject: [PATCH] Bug 10662: (QA follow-up) addressing qa test tool output I've done the following in this follow-up: - Added missing POD - Replaced DBIC use with Koha::Object[s] - Replaced DataTables with KohaTable - Fixed database charset and collation - Moved Javascript to bottom of templates - Fixed a syntax error in rewrite-config.PL --- Koha/Daemon.pm | 58 +++++ Koha/OAI/Harvester.pm | 117 +++++++-- Koha/OAI/Harvester/Biblio.pm | 49 ++++ Koha/OAI/Harvester/Biblios.pm | 60 +++++ Koha/OAI/Harvester/Client.pm | 55 ++++- Koha/OAI/Harvester/Downloader.pm | 33 ++- Koha/OAI/Harvester/Histories.pm | 60 +++++ Koha/OAI/Harvester/History.pm | 49 ++++ Koha/OAI/Harvester/Import/MARCXML.pm | 23 ++ Koha/OAI/Harvester/Import/Record.pm | 69 +++++- Koha/OAI/Harvester/ImportQueue.pm | 49 ++++ Koha/OAI/Harvester/ImportQueues.pm | 60 +++++ Koha/OAI/Harvester/Listener.pm | 56 +++++ Koha/OAI/Harvester/Request.pm | 19 +- Koha/OAI/Harvester/Worker.pm | 59 +++++ Koha/OAI/Harvester/Worker/Download/Stream.pm | 30 +++ Koha/OAI/Harvester/Worker/Import.pm | 24 ++ Koha/Schema/Result/Biblio.pm | 19 +- Koha/Schema/Result/OaiHarvesterBiblio.pm | 4 +- Koha/Schema/Result/OaiHarvesterHistory.pm | 4 +- Koha/Schema/Result/OaiHarvesterImportQueue.pm | 4 +- Koha/Schema/Result/OaiHarvesterRequest.pm | 4 +- admin/columns_settings.yml | 59 +++++ installer/data/mysql/atomicupdate/bug_10662.sql | 8 +- installer/data/mysql/kohastructure.sql | 8 +- .../modules/tools/oai-pmh-harvester/dashboard.tt | 275 +++++++++++---------- .../en/modules/tools/oai-pmh-harvester/request.tt | 123 ++++----- rewrite-config.PL | 2 +- svc/oai-pmh-harvester/history | 65 +++-- tools/oai-pmh-harvester/record.pl | 18 +- 30 files changed, 1155 insertions(+), 308 deletions(-) create mode 100644 Koha/OAI/Harvester/Biblio.pm create mode 100644 Koha/OAI/Harvester/Biblios.pm create mode 100644 Koha/OAI/Harvester/Histories.pm create mode 100644 Koha/OAI/Harvester/History.pm create mode 100644 Koha/OAI/Harvester/ImportQueue.pm create mode 100644 Koha/OAI/Harvester/ImportQueues.pm diff --git a/Koha/Daemon.pm b/Koha/Daemon.pm index cdae369..19010f0 100644 --- a/Koha/Daemon.pm +++ b/Koha/Daemon.pm @@ -22,6 +22,64 @@ use Modern::Perl; use POSIX; #For daemonizing use Fcntl qw(:flock); #For pidfile +=head1 NAME + +Koha::Daemon + +=head1 SYNOPSIS + + use Koha::Daemon; + my $daemon = Koha::Daemon->new({ + pidfile => "/path/to/file.pid", + logfile => "/path/to/file.log", + daemonize => 1, + }); + $daemon->run(); + +=head1 METHODS + +=head2 new + +Create object + +=head2 run + +Run the daemon. + +This method calls all the internal methods which +are do everything necessary to run the daemon. + +=head1 INTERNAL METHODS + +=head2 daemonize + +Internal function for setting object up as a proper daemon +(e.g. forking, setting permissions, changing directories, + closing file handles, etc.) + +=head2 get_pidfile + +Internal function to get a filehandle for the pidfile + +=head2 lock_pidfile + +Internal function to lock the pidfile, so that only one daemon +can run against this pidfile + +=head2 log_to_file + +Internal function to log to a file + +=head2 make_pidfilehandle + +Internal function to make a filehandle for pidfile + +=head2 write_pidfile + +Internal function to write pid to pidfile + +=cut + sub new { my ($class, $args) = @_; $args = {} unless defined $args; diff --git a/Koha/OAI/Harvester.pm b/Koha/OAI/Harvester.pm index 86619fa..7a374fe 100644 --- a/Koha/OAI/Harvester.pm +++ b/Koha/OAI/Harvester.pm @@ -30,11 +30,26 @@ use DateTime; use DateTime::Format::Strptime; use C4::Context; -use Koha::Database; +use Koha::OAI::Harvester::ImportQueues; -=head1 API +=head1 NAME -=head2 Class Methods +Koha::OAI::Harvester + +=head1 SYNOPSIS + +my $harvester = Koha::OAI::Harvester->spawn({ + Downloader => $downloader, + DownloaderWorkers => $download_workers, + Importer => $importer, + ImporterWorkers => $import_workers, + ImportQueuePoll => $import_poll, + logger => $logger, + state_file => $statefile, + spooldir => $spooldir, +}); + +=head2 METHODS =cut @@ -45,12 +60,24 @@ my $seconds_granularity = DateTime::Format::Strptime->new( pattern => '%FT%TZ', ); +=head2 new + +Create object + +=cut + sub new { my ($class, $args) = @_; $args = {} unless defined $args; return bless ($args, $class); } +=head2 spawn + + Creates POE sessions for handling tasks, downloading, and importing + +=cut + sub spawn { my ($class, $args) = @_; my $self = $class->new($args); @@ -114,15 +141,14 @@ sub spawn { my $active_tasks = $poe_kernel->call("harvester","list_tasks","active"); my @active_uuids = map { $_->{uuid} } @$active_tasks; - my $schema = Koha::Database->new()->schema(); - my $rs = $schema->resultset('OaiHarvesterImportQueue')->search({ + my $rs = Koha::OAI::Harvester::ImportQueues->new->search({ uuid => \@active_uuids, status => "new" },{ order_by => { -asc => 'id' }, rows => 1, }); - my $result = $rs->first; + my $result = $rs->single; if ($result){ $result->status("wip"); $result->update; @@ -149,6 +175,12 @@ sub spawn { return; } +=head2 on_start + + Internal method that starts the harvester + +=cut + sub on_start { my ($self, $kernel, $heap) = @_[OBJECT, KERNEL,HEAP]; $kernel->alias_set('harvester'); @@ -161,13 +193,19 @@ sub on_start { $kernel->call("harvester","restore_state"); } +=head2 download_postback + + Internal method + +=cut + #NOTE: This isn't really implemented at the moment, as it's not really necessary. sub download_postback { my ($kernel, $request_packet, $response_packet) = @_[KERNEL, ARG0, ARG1]; my $message = $response_packet->[0]; } -=head3 deregister +=head2 deregister Remove the worker session from the harvester's in-memory scoreboard, unset the downloading flag if downloading is completed. @@ -218,7 +256,7 @@ sub deregister { } -=head3 is_task_finished +=head2 is_task_finished This event handler checks if the task has finished downloading and importing record. If it is finished downloading and importing, the task is deleted from the harvester. @@ -241,6 +279,12 @@ sub is_task_finished { return 0; } +=head2 register + + Internal method for registering a task on the harvester's scoreboard + +=cut + sub register { my ($self, $kernel, $heap, $session, $sender, $type, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0,ARG1]; my $logger = $self->{logger}; @@ -288,6 +332,12 @@ sub register { } } +=head2 does_task_repeat + + Internal method for checking if a task is supposed to repeat after it finishes + +=cut + sub does_task_repeat { my ($self, $kernel, $heap, $session, $uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0]; my $task = $kernel->call("harvester","get_task",$uuid); @@ -306,13 +356,16 @@ sub does_task_repeat { return 0; } +=head2 reset_imports_status + Internal method that resets the import queue from wip to new + +=cut sub reset_imports_status { my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION]; - my $schema = Koha::Database->new()->schema(); - my $rs = $schema->resultset('OaiHarvesterImportQueue')->search({ + my $rs = Koha::OAI::Harvester::ImportQueues->new->search({ status => "wip", }); $rs->update({ @@ -320,6 +373,12 @@ sub reset_imports_status { }); } +=head2 restore_state + + Method to restore the harvester to a pre-existing state. + +=cut + sub restore_state { my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION]; @@ -357,6 +416,12 @@ sub restore_state { } } +=head2 save_state + + Method to save the existing state of the harvester + +=cut + sub save_state { my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION]; my $state_file = $self->{state_file}; @@ -385,7 +450,7 @@ sub save_state { return 0; } -=head3 get_task +=head2 get_task This event handler returns a task from a harvester using the task's uuid as an argument. @@ -413,7 +478,9 @@ sub get_task { return 0; } -=head3 get_import_count_for_task +=head2 get_import_count_for_task + + This gets the count of the number of imports exist for a certain task =cut @@ -421,8 +488,7 @@ sub get_import_count_for_task { my ($self,$uuid) = @_; my $count = undef; if ($uuid){ - my $schema = Koha::Database->new()->schema(); - my $items = $schema->resultset('OaiHarvesterImportQueue')->search({ + my $items = Koha::OAI::Harvester::ImportQueues->new->search({ uuid => $uuid, }); $count = $items->count; @@ -430,7 +496,7 @@ sub get_import_count_for_task { return $count; } -=head3 list_tasks +=head2 list_tasks This event handler returns a list of tasks that have been submitted to the harvester. It returns data like uuid, status, parameters, @@ -440,11 +506,10 @@ sub get_import_count_for_task { sub list_tasks { my ($self, $kernel, $heap, $session, $status) = @_[OBJECT, KERNEL,HEAP,SESSION, ARG0]; - my $schema = Koha::Database->new()->schema(); my @tasks = (); foreach my $uuid (sort keys %{$heap->{tasks}}){ my $task = $heap->{tasks}->{$uuid}; - my $items = $schema->resultset('OaiHarvesterImportQueue')->search({ + my $items = Koha::OAI::Harvester::ImportQueues->new->search({ uuid => $uuid, }); my $count = $items->count // 0; @@ -457,7 +522,7 @@ sub list_tasks { return \@tasks; } -=head3 create_task +=head2 create_task This event handler creates a spool directory for the task's imports. It also adds it to the harvester's memory and then saves memory to @@ -502,7 +567,7 @@ sub create_task { return 0; } -=head3 start_task +=head2 start_task This event handler marks a task as active in the harvester's memory, save the memory to a persistent datastore, then enqueues the task, @@ -536,9 +601,9 @@ sub start_task { return 0; } -=head3 repeat_task - +=head2 repeat_task + This method re-queues a task for downloading =cut @@ -553,7 +618,7 @@ sub repeat_task { } } -=head3 stop_task +=head2 stop_task This event handler prevents new workers from spawning, kills existing workers, and stops pending imports from being imported. @@ -594,8 +659,7 @@ sub stop_task { } #Step Three: stop pending imports for this task - my $schema = Koha::Database->new()->schema(); - my $items = $schema->resultset('OaiHarvesterImportQueue')->search({ + my $items = Koha::OAI::Harvester::ImportQueues->new->search({ uuid => $task_uuid, }); my $rows_updated = $items->update({ @@ -609,7 +673,7 @@ sub stop_task { return 0; } -=head3 delete_task +=head2 delete_task Deleted tasks are stopped, pending imports are deleted from the database and file system, and then the task is removed from the harvester. @@ -625,8 +689,7 @@ sub delete_task { $kernel->call($session,"stop_task",$task_uuid); #Step Two: delete pending imports in database - my $schema = Koha::Database->new()->schema(); - my $items = $schema->resultset('OaiHarvesterImportQueue')->search({ + my $items = Koha::OAI::Harvester::ImportQueues->new->search({ uuid => $task_uuid, }); if ($items){ diff --git a/Koha/OAI/Harvester/Biblio.pm b/Koha/OAI/Harvester/Biblio.pm new file mode 100644 index 0000000..b84aeca --- /dev/null +++ b/Koha/OAI/Harvester/Biblio.pm @@ -0,0 +1,49 @@ +package Koha::OAI::Harvester::Biblio; + +# Copyright Prosentient Systems 2018 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; + +use Carp; + +use base qw(Koha::Object); + +=head1 NAME + +Koha::OAI::Harvester::Biblio + +=head1 SYNOPSIS + + use Koha::OAI::Harvester::Biblio; + my $biblio = Koha::OAI::Harvester::Biblios->find($id); + +=head1 METHODS + +=cut + +sub _type { + return 'OaiHarvesterBiblio'; +} + +=head1 AUTHOR + +David Cook + +=cut + +1; diff --git a/Koha/OAI/Harvester/Biblios.pm b/Koha/OAI/Harvester/Biblios.pm new file mode 100644 index 0000000..a1a9aec --- /dev/null +++ b/Koha/OAI/Harvester/Biblios.pm @@ -0,0 +1,60 @@ +package Koha::OAI::Harvester::Biblios; + +# Copyright Prosentient Systems 2018 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; + +use Carp; + +use Koha::Database; + +use Koha::OAI::Harvester::Biblio; + +use base qw(Koha::Objects); + +=head1 NAME + +Koha::OAI::Harvester::Biblios + +=head2 METHODS + +=cut + +=head3 _type + +=cut + +sub _type { + return 'OaiHarvesterBiblio'; +} + +=head3 object_class + +=cut + +sub object_class { + return 'Koha::OAI::Harvester::Biblio'; +} + +=head1 AUTHOR + +David Cook + +=cut + +1; diff --git a/Koha/OAI/Harvester/Client.pm b/Koha/OAI/Harvester/Client.pm index d5c5351..abfc276 100644 --- a/Koha/OAI/Harvester/Client.pm +++ b/Koha/OAI/Harvester/Client.pm @@ -23,12 +23,35 @@ use IO::Socket::UNIX; use IO::Select; use JSON; +=head1 NAME + +Koha::OAI::Harvester::Client + +=head1 SYNOPSIS + + use Koha::OAI::Harvester::Client; + my $client = Koha::OAI::Harvester::Client->new($client_config); + +=head1 METHODS + +=head2 new + +Create object + +=cut + sub new { my ($class, $args) = @_; $args = {} unless defined $args; return bless ($args, $class); } +=head2 connect + +Connect to Koha::OAI::Harvester::Listener via a Unix socket + +=cut + sub connect { my ($self) = @_; my $socket_uri = $self->{socket_uri}; @@ -62,6 +85,12 @@ sub connect { return 0; } +=head2 create + + Create a task on the harvester + +=cut + sub create { my ($self,$task) = @_; my $message = { @@ -74,6 +103,12 @@ sub create { return $status; } +=head2 start + + Start a task on the harvester + +=cut + sub start { my ($self,$uuid) = @_; my $message = { @@ -88,6 +123,12 @@ sub start { return $status; } +=head2 stop + + Stop a task on the harvester + +=cut + sub stop { my ($self,$uuid) = @_; my $message = { @@ -102,6 +143,12 @@ sub stop { return $status; } +=head2 delete + + Delete a task on the harvester + +=cut + sub delete { my ($self,$uuid) = @_; my $message = { @@ -116,6 +163,12 @@ sub delete { return $status; } +=head2 list + + List all tasks on the harvester + +=cut + sub list { my ($self) = @_; my $message = { @@ -174,4 +227,4 @@ sub _read { } } -1; \ No newline at end of file +1; diff --git a/Koha/OAI/Harvester/Downloader.pm b/Koha/OAI/Harvester/Downloader.pm index 4ca8be6..3137b10 100644 --- a/Koha/OAI/Harvester/Downloader.pm +++ b/Koha/OAI/Harvester/Downloader.pm @@ -23,9 +23,22 @@ use XML::LibXML::Reader; use IO::Handle; use JSON; -=head1 API +=head1 NAME -=head2 Class Methods +Koha::OAI::Harvester::Downloader + +=head1 SYNOPSIS + + use Koha::OAI::Harvester::Downloader; + my $oai_downloader = Koha::OAI::Harvester::Downloader->new(); + + This class is used within a Koha::OAI::Harvester::Work::Download:: module. + +=head1 METHODS + +=head2 new + + Create object =cut @@ -96,7 +109,7 @@ sub BuildURL { } } -=head2 OpenXMLStream +=head2 GetXMLStream Fork a child process to send the HTTP request, which sends chunks of XML via a pipe to the parent process. @@ -168,6 +181,13 @@ sub _request { } } +=head2 ParseXMLStream + + Parse XML using XML::LibXML::Reader from a file handle (e.g. a stream) + +=cut + + sub ParseXMLStream { my ($self, $args) = @_; @@ -245,6 +265,13 @@ sub ParseXMLStream { } } +=head2 harvest + + Given a URL and a user-agent, start downloading OAI-PMH records from + that URL. + +=cut + sub harvest { my ($self,$args) = @_; my $url = $args->{url}; diff --git a/Koha/OAI/Harvester/Histories.pm b/Koha/OAI/Harvester/Histories.pm new file mode 100644 index 0000000..c3f870c --- /dev/null +++ b/Koha/OAI/Harvester/Histories.pm @@ -0,0 +1,60 @@ +package Koha::OAI::Harvester::Histories; + +# Copyright Prosentient Systems 2018 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; + +use Carp; + +use Koha::Database; + +use Koha::OAI::Harvester::History; + +use base qw(Koha::Objects); + +=head1 NAME + +Koha::OAI::Harvester::Histories + +=head2 METHODS + +=cut + +=head3 _type + +=cut + +sub _type { + return 'OaiHarvesterHistory'; +} + +=head3 object_class + +=cut + +sub object_class { + return 'Koha::OAI::Harvester::History'; +} + +=head1 AUTHOR + +David Cook + +=cut + +1; diff --git a/Koha/OAI/Harvester/History.pm b/Koha/OAI/Harvester/History.pm new file mode 100644 index 0000000..5ad490c --- /dev/null +++ b/Koha/OAI/Harvester/History.pm @@ -0,0 +1,49 @@ +package Koha::OAI::Harvester::History; + +# Copyright Prosentient Systems 2018 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; + +use Carp; + +use base qw(Koha::Object); + +=head1 NAME + +Koha::OAI::Harvester::History + +=head1 SYNOPSIS + + use Koha::OAI::Harvester::History; + my $request = Koha::OAI::Harvester::History->find($id); + +=head1 METHODS + +=cut + +sub _type { + return 'OaiHarvesterHistory'; +} + +=head1 AUTHOR + +David Cook + +=cut + +1; diff --git a/Koha/OAI/Harvester/Import/MARCXML.pm b/Koha/OAI/Harvester/Import/MARCXML.pm index 6accb78..6eb7c5b 100755 --- a/Koha/OAI/Harvester/Import/MARCXML.pm +++ b/Koha/OAI/Harvester/Import/MARCXML.pm @@ -26,6 +26,23 @@ use C4::Biblio; use constant MAX_MATCHES => 99999; #NOTE: This is an arbitrary value. We want to get all matches. +=head1 NAME + +Koha::OAI::Harvester::Import::MARCXML + +=head1 SYNOPSIS + + use Koha::OAI::Harvester::Import::MARCXML; + my $marcxml = eval { Koha::OAI::Harvester::Import::MARCXML->new({ dom => $results, }) }; + +=head1 METHODS + +=head2 new + + Create object + +=cut + sub new { my ($class, $args) = @_; $args = {} unless defined $args; @@ -47,6 +64,12 @@ sub new { return bless ($args, $class); } +=head2 import_record + + Import a record into Koha + +=cut + sub import_record { my ($self,$args) = @_; my $framework = $args->{framework}; diff --git a/Koha/OAI/Harvester/Import/Record.pm b/Koha/OAI/Harvester/Import/Record.pm index 9a67eb8..bd2b8b9 100755 --- a/Koha/OAI/Harvester/Import/Record.pm +++ b/Koha/OAI/Harvester/Import/Record.pm @@ -27,16 +27,29 @@ use File::Basename; use C4::Context; use C4::Biblio; -use Koha::Database; use Koha::OAI::Harvester::Import::MARCXML; +use Koha::OAI::Harvester::Biblios; +use Koha::OAI::Harvester::History; -=head1 API +=head1 NAME -=head2 Class Methods +Koha::OAI::Harvester::Import::Record -=cut +=head1 SYNOPSIS + + use Koha::OAI::Harvester::Import::Record; + my $oai_record = Koha::OAI::Harvester::Import::Record->new({ + doc => $dom, + repository => $repository, + }); + +=head1 METHODS -my $schema = Koha::Database->new()->schema(); +=head2 new + + Create object + +=cut sub new { my ($class, $args) = @_; @@ -70,6 +83,12 @@ sub new { return bless ($args, $class); } +=head2 is_deleted_upstream + + Returns true if OAI-PMH record is deleted upstream + +=cut + sub is_deleted_upstream { my ($self, $args) = @_; if ($self->{header_status}){ @@ -80,6 +99,15 @@ sub is_deleted_upstream { return 0; } +=head2 set_filter + + $self->set_filter("/path/to/filter.xsl"); + + Set a XSLT to use to filter records on import. This + takes a full filepath as an argument. + +=cut + sub set_filter { my ($self, $filter_definition) = @_; @@ -121,6 +149,12 @@ sub _parse_filter { return ($type,$filter); } +=head2 filter + + Filters the OAI-PMH record using a filter + +=cut + sub filter { my ($self) = @_; my $filtered = 0; @@ -165,7 +199,7 @@ sub _find_koha_link { my $record_type = $args->{record_type} // "biblio"; my $link_id; if ($record_type eq "biblio"){ - my $link = $schema->resultset('OaiHarvesterBiblio')->find( + my $link = Koha::OAI::Harvester::Biblios->new->find( { oai_repository => $self->{repository}, oai_identifier => $self->{header_identifier}, @@ -173,13 +207,13 @@ sub _find_koha_link { { key => "oai_record",} ); if ($link && $link->biblionumber){ - $link_id = $link->biblionumber->id; + $link_id = $link->biblionumber; } } return $link_id; } -=head3 import_record +=head2 import_record my ($action,$record_id) = $oai_record->import_record({ filter => $filter, @@ -246,7 +280,7 @@ sub import_record { } #Log record details to database - my $importer = $schema->resultset('OaiHarvesterHistory')->create({ + Koha::OAI::Harvester::History->new({ header_identifier => $self->{header_identifier}, header_datestamp => $self->{header_datestamp}, header_status => $self->{header_status}, @@ -257,18 +291,25 @@ sub import_record { framework => $framework, record_type => $record_type, matcher_code => $matcher ? $matcher->code : undef, - }); + })->store(); return ($action,$linked_id); } +=head2 link_koha_record + + Link an OAI-PMH record with a Koha record using + the OAI-PMH repository and OAI-PMH identifier + +=cut + sub link_koha_record { my ($self, $args) = @_; my $record_type = $args->{record_type} // "biblio"; my $koha_id = $args->{koha_id}; if ($koha_id){ if ($record_type eq "biblio"){ - my $import_oai_biblio = $schema->resultset('OaiHarvesterBiblio')->find_or_create({ + my $import_oai_biblio = Koha::OAI::Harvester::Biblios->new->find_or_create({ oai_repository => $self->{repository}, oai_identifier => $self->{header_identifier}, biblionumber => $koha_id, @@ -280,6 +321,12 @@ sub link_koha_record { } } +=head2 delete_koha_record + + Delete a Koha record + +=cut + sub delete_koha_record { my ($self, $args) = @_; my $record_type = $args->{record_type} // "biblio"; diff --git a/Koha/OAI/Harvester/ImportQueue.pm b/Koha/OAI/Harvester/ImportQueue.pm new file mode 100644 index 0000000..065170f --- /dev/null +++ b/Koha/OAI/Harvester/ImportQueue.pm @@ -0,0 +1,49 @@ +package Koha::OAI::Harvester::ImportQueue; + +# Copyright Prosentient Systems 2018 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; + +use Carp; + +use base qw(Koha::Object); + +=head1 NAME + +Koha::OAI::Harvester::ImportQueue + +=head1 SYNOPSIS + + use Koha::OAI::Harvester::ImportQueue; + my $request = Koha::OAI::Harvester::ImportQueue->find($id); + +=head1 METHODS + +=cut + +sub _type { + return 'OaiHarvesterImportQueue'; +} + +=head1 AUTHOR + +David Cook + +=cut + +1; diff --git a/Koha/OAI/Harvester/ImportQueues.pm b/Koha/OAI/Harvester/ImportQueues.pm new file mode 100644 index 0000000..378027d --- /dev/null +++ b/Koha/OAI/Harvester/ImportQueues.pm @@ -0,0 +1,60 @@ +package Koha::OAI::Harvester::ImportQueues; + +# Copyright Prosentient Systems 2018 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; + +use Carp; + +use Koha::Database; + +use Koha::OAI::Harvester::ImportQueue; + +use base qw(Koha::Objects); + +=head1 NAME + +Koha::OAI::Harvester::ImportQueues + +=head2 METHODS + +=cut + +=head3 _type + +=cut + +sub _type { + return 'OaiHarvesterImportQueue'; +} + +=head3 object_class + +=cut + +sub object_class { + return 'Koha::OAI::Harvester::ImportQueue'; +} + +=head1 AUTHOR + +David Cook + +=cut + +1; diff --git a/Koha/OAI/Harvester/Listener.pm b/Koha/OAI/Harvester/Listener.pm index 03ec391..a40cf70 100644 --- a/Koha/OAI/Harvester/Listener.pm +++ b/Koha/OAI/Harvester/Listener.pm @@ -24,12 +24,38 @@ use IO::Socket qw(AF_UNIX); use JSON; use URI; +=head1 NAME + +Koha::OAI::Harvester::Listener + +=head1 SYNOPSIS + + use Koha::OAI::Harvester::Listener; + my $listener = Koha::OAI::Harvester::Listener->spawn({ + logger => $logger, + socket => $socket_addr, + }); + +=head1 METHODS + +=head2 new + + Create object + +=cut + sub new { my ($class, $args) = @_; $args = {} unless defined $args; return bless ($args, $class); } +=head2 spawn + + Create POE session for listener + +=cut + sub spawn { my ($class, $args) = @_; my $self = $class->new($args); @@ -50,6 +76,12 @@ sub spawn { ); } +=head2 on_start + + Internal method for starting listener + +=cut + sub on_start { my ($kernel,$heap,$socket_uri) = @_[KERNEL,HEAP,ARG0]; @@ -69,6 +101,12 @@ sub on_start { } } +=head2 on_server_success + + Internal event handler for successful connection to server + +=cut + sub on_server_success { my ($self, $client_socket, $server_wheel_id, $heap, $session) = @_[OBJECT, ARG0, ARG3, HEAP,SESSION]; my $logger = $self->{logger}; @@ -88,6 +126,12 @@ sub on_server_success { $client_wheel->put("HELLO"); } +=head2 on_server_error + + Internal event handler for server errors + +=cut + sub on_server_error { my ($self, $operation, $errnum, $errstr, $heap, $session) = @_[OBJECT, ARG0, ARG1, ARG2,HEAP, SESSION]; my $logger = $self->{logger}; @@ -95,6 +139,12 @@ sub on_server_error { delete $heap->{server}; } +=head2 on_client_error + + Internal event handler for errors relating to the client connection + +=cut + sub on_client_error { my ($self, $wheel_id,$heap,$session) = @_[OBJECT, ARG3,HEAP,SESSION]; my $logger = $self->{logger}; @@ -102,6 +152,12 @@ sub on_client_error { delete $heap->{client}->{$wheel_id}; } +=head2 on_client_input + + Internal event handler for input from clients + +=cut + sub on_client_input { my ($self, $input, $wheel_id, $session, $kernel, $heap) = @_[OBJECT, ARG0, ARG1, SESSION, KERNEL, HEAP]; my $logger = $self->{logger}; diff --git a/Koha/OAI/Harvester/Request.pm b/Koha/OAI/Harvester/Request.pm index 3824947..f2fdea3 100644 --- a/Koha/OAI/Harvester/Request.pm +++ b/Koha/OAI/Harvester/Request.pm @@ -29,17 +29,14 @@ use HTTP::OAI; =head1 NAME -Koha::OAI::Harvester::Request - +Koha::OAI::Harvester::Request -=head1 API +=head1 SYNOPSIS -=head2 Class Methods + use Koha::OAI::Harvester::Request; + my $request = Koha::OAI::Harvester::Requests->find($id); -=cut - - - -=head3 _type +=head1 METHODS =cut @@ -47,6 +44,12 @@ sub _type { return 'OaiHarvesterRequest'; } +=head2 validate + + Method to validate the parameters of an OAI-PMH request + +=cut + sub validate { my ($self) = @_; my $errors = {}; diff --git a/Koha/OAI/Harvester/Worker.pm b/Koha/OAI/Harvester/Worker.pm index 5374a29..f65d853 100644 --- a/Koha/OAI/Harvester/Worker.pm +++ b/Koha/OAI/Harvester/Worker.pm @@ -22,6 +22,23 @@ use POE; use DateTime; use JSON; +=head1 NAME + +Koha::OAI::Harvester::Worker + +=head1 SYNOPSIS + + This is just a base class to use when writing workers for + the OAI-PMH harvester. + +=head1 METHODS + +=head2 new + + Create object + +=cut + sub new { my ($class, $args) = @_; $args = {} unless defined $args; @@ -29,6 +46,12 @@ sub new { return bless ($args, $class); } +=head2 run + + The entry point to getting a worker to process a task + +=cut + sub run { my ($self,$args) = @_; my $postback = $args->{postback}; @@ -53,6 +76,12 @@ sub run { ); } +=head2 stop_worker + + Internal method for killing this worker's child processes + +=cut + sub stop_worker { my ($self,$heap) = @_[OBJECT,HEAP]; if (my $child_processes = $heap->{children_by_pid}){ @@ -63,6 +92,12 @@ sub stop_worker { } } +=head2 on_stop + + Internal event handler for deregistering the worker session + from the harvester's roster of workers + +=cut sub on_stop { my ($self,$kernel) = @_[OBJECT,KERNEL]; @@ -71,6 +106,12 @@ sub on_stop { $kernel->call("harvester","deregister",$self->{type}); } +=head2 on_child_stdout + + Internal event handler for reading output from a child process + +=cut + # Wheel event, including the wheel's ID. sub on_child_stdout { my ($self, $stdout_line, $wheel_id) = @_[OBJECT, ARG0, ARG1]; @@ -92,6 +133,12 @@ sub on_child_stdout { } } +=head2 on_child_stderr + + Internal event handler for reading errors from a child process + +=cut + # Wheel event, including the wheel's ID. sub on_child_stderr { my ($self,$stderr_line, $wheel_id) = @_[OBJECT, ARG0, ARG1]; @@ -103,6 +150,12 @@ sub on_child_stderr { } } +=head2 on_child_close + + Internal event handler for when a child closes its pipes + +=cut + # Wheel event, including the wheel's ID. sub on_child_close { my ($self,$heap,$wheel_id) = @_[OBJECT,HEAP,ARG0]; @@ -124,6 +177,12 @@ sub on_child_close { delete $heap->{children_by_pid}->{$child->PID}; } +=head2 on_child_signal + + Internal event handler for when a child exits + +=cut + sub on_child_signal { my ($self,$kernel,$pid,$status) = @_[OBJECT,KERNEL,ARG1,ARG2]; my $type = $self->{type}; diff --git a/Koha/OAI/Harvester/Worker/Download/Stream.pm b/Koha/OAI/Harvester/Worker/Download/Stream.pm index c87580a..442511a 100644 --- a/Koha/OAI/Harvester/Worker/Download/Stream.pm +++ b/Koha/OAI/Harvester/Worker/Download/Stream.pm @@ -28,6 +28,24 @@ use C4::Context; use Koha::OAI::Harvester::Downloader; use parent 'Koha::OAI::Harvester::Worker'; +=head1 NAME + +Koha::OAI::Harvester::Worker::Download::Stream + +=head1 SYNOPSIS + + This is a module used by the OAI-PMH harvester internally. + + As a bare minimum, it must define an "on_start" method. + +=head1 METHODS + +=head2 new + + Create object + +=cut + sub new { my ($class, $args) = @_; $args = {} unless defined $args; @@ -35,6 +53,12 @@ sub new { return bless ($args, $class); } +=head2 on_start + + Internal event handler for starting the processing of a task + +=cut + sub on_start { my ($self, $kernel, $heap, $postback,$task,$session) = @_[OBJECT, KERNEL, HEAP, ARG0,ARG1,SESSION]; #Save postback into heap so other event handlers can use it @@ -71,6 +95,12 @@ sub on_start { } } +=head2 do_work + + Internal method for processing a download task + +=cut + sub do_work { my ($self, $task) = @_; my $batch = ( $self->{batch} && int($self->{batch}) ) ? $self->{batch} : 100; diff --git a/Koha/OAI/Harvester/Worker/Import.pm b/Koha/OAI/Harvester/Worker/Import.pm index ec86736..e57279a 100644 --- a/Koha/OAI/Harvester/Worker/Import.pm +++ b/Koha/OAI/Harvester/Worker/Import.pm @@ -28,6 +28,24 @@ use Koha::OAI::Harvester::Import::Record; use parent 'Koha::OAI::Harvester::Worker'; +=head1 NAME + +Koha::OAI::Harvester::Worker::Import + +=head1 SYNOPSIS + + This is a module used by the OAI-PMH harvester internally. + + As a bare minimum, it must define an "on_start" method. + +=head1 METHODS + +=head2 new + + Create object + +=cut + sub new { my ($class, $args) = @_; $args = {} unless defined $args; @@ -36,6 +54,12 @@ sub new { return bless ($args, $class); } +=head2 on_start + + Internal event handler for starting the processing of a task + +=cut + sub on_start { my ($self, $kernel, $heap, $postback,$task) = @_[OBJECT, KERNEL, HEAP, ARG0,ARG1]; diff --git a/Koha/Schema/Result/Biblio.pm b/Koha/Schema/Result/Biblio.pm index b118011..89bef66 100644 --- a/Koha/Schema/Result/Biblio.pm +++ b/Koha/Schema/Result/Biblio.pm @@ -242,6 +242,21 @@ __PACKAGE__->has_many( { cascade_copy => 0, cascade_delete => 0 }, ); +=head2 oai_harvester_biblios + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "oai_harvester_biblios", + "Koha::Schema::Result::OaiHarvesterBiblio", + { "foreign.biblionumber" => "self.biblionumber" }, + { cascade_copy => 0, cascade_delete => 0 }, +); + =head2 old_reserves Type: has_many @@ -348,7 +363,7 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.07042 @ 2018-02-16 17:54:53 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:bUv00JjY09Hj2Zj4klqyxA +# Created by DBIx::Class::Schema::Loader v0.07042 @ 2018-10-31 05:45:15 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ptt942qUu2GcGqOTRePNWQ 1; diff --git a/Koha/Schema/Result/OaiHarvesterBiblio.pm b/Koha/Schema/Result/OaiHarvesterBiblio.pm index 85bb64a..90f46e5 100755 --- a/Koha/Schema/Result/OaiHarvesterBiblio.pm +++ b/Koha/Schema/Result/OaiHarvesterBiblio.pm @@ -112,8 +112,8 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-03-29 12:23:43 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:2URn8tPABMKC+JuIMfGeYw +# Created by DBIx::Class::Schema::Loader v0.07042 @ 2018-10-31 05:53:48 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:awQgyVvOqnfBrOhl6tecBQ # You can replace this text with custom code or comments, and it will be preserved on regeneration diff --git a/Koha/Schema/Result/OaiHarvesterHistory.pm b/Koha/Schema/Result/OaiHarvesterHistory.pm index 3e3d1d5..be1d619 100755 --- a/Koha/Schema/Result/OaiHarvesterHistory.pm +++ b/Koha/Schema/Result/OaiHarvesterHistory.pm @@ -155,8 +155,8 @@ __PACKAGE__->add_columns( __PACKAGE__->set_primary_key("import_oai_id"); -# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-03-29 12:23:43 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Rp/ZEZsKVlLo2vaM3M37ow +# Created by DBIx::Class::Schema::Loader v0.07042 @ 2018-10-31 05:53:48 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:C/ETSPdx0O4xfQnu09kGeQ # You can replace this text with custom code or comments, and it will be preserved on regeneration diff --git a/Koha/Schema/Result/OaiHarvesterImportQueue.pm b/Koha/Schema/Result/OaiHarvesterImportQueue.pm index 54188e3..4ff5e18 100755 --- a/Koha/Schema/Result/OaiHarvesterImportQueue.pm +++ b/Koha/Schema/Result/OaiHarvesterImportQueue.pm @@ -98,8 +98,8 @@ __PACKAGE__->add_columns( __PACKAGE__->set_primary_key("id"); -# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-03-29 12:23:43 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zBD+hMawbvu7sonuLRHnCA +# Created by DBIx::Class::Schema::Loader v0.07042 @ 2018-10-31 05:53:48 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:1+ziAxPsvC8ftKLEAJ2keA # You can replace this text with custom code or comments, and it will be preserved on regeneration diff --git a/Koha/Schema/Result/OaiHarvesterRequest.pm b/Koha/Schema/Result/OaiHarvesterRequest.pm index e322281..6ca35e3 100644 --- a/Koha/Schema/Result/OaiHarvesterRequest.pm +++ b/Koha/Schema/Result/OaiHarvesterRequest.pm @@ -201,8 +201,8 @@ __PACKAGE__->add_columns( __PACKAGE__->set_primary_key("id"); -# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-04-07 11:26:24 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Vm/b4yurmr8WF7z+Fo6KEw +# Created by DBIx::Class::Schema::Loader v0.07042 @ 2018-10-31 05:53:48 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:adGx1Jo2gvt5PH1eX31y2Q # You can replace this text with custom code or comments, and it will be preserved on regeneration diff --git a/admin/columns_settings.yml b/admin/columns_settings.yml index 1480019..1dbb701 100644 --- a/admin/columns_settings.yml +++ b/admin/columns_settings.yml @@ -675,6 +675,65 @@ modules: columnname: spent tools: + oai-pmh-harvester-dashboard: + saved-table: + - + columnname: name + - + columnname: url + - + columnname: set + - + columnname: from + - + columnname: until + - + columnname: interval + + submitted-table: + - + columnname: name + - + columnname: url + - + columnname: set + - + columnname: from + - + columnname: until + - + columnname: interval + - + columnname: "effective from" + - + columnname: "effective until" + - + columnname: "pending imports" + - + columnname: status + - + columnname: error + + history-table: + - + columnname: id + - + columnname: repository + - + columnname: identifier + - + columnname: datestamp + - + columnname: "upstream status" + - + columnname: "import status" + - + columnname: "import timestamp" + - + columnname: "imported record" + - + columnname: "downloaded record" + notices: lettert: - diff --git a/installer/data/mysql/atomicupdate/bug_10662.sql b/installer/data/mysql/atomicupdate/bug_10662.sql index f42b732..30f1d18 100644 --- a/installer/data/mysql/atomicupdate/bug_10662.sql +++ b/installer/data/mysql/atomicupdate/bug_10662.sql @@ -11,7 +11,7 @@ CREATE TABLE IF NOT EXISTS `oai_harvester_biblios` ( UNIQUE KEY `oai_record` (`oai_identifier`,`oai_repository`) USING BTREE, KEY `FK_import_oai_biblio_1` (`biblionumber`), CONSTRAINT `FK_import_oai_biblio_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE NO ACTION -) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table 'oai_harvester_history' @@ -31,7 +31,7 @@ CREATE TABLE IF NOT EXISTS `oai_harvester_history` ( `record_type` enum('biblio','auth','holdings') COLLATE utf8_unicode_ci NOT NULL, `matcher_code` varchar(10) COLLATE utf8_unicode_ci DEFAULT NULL, PRIMARY KEY (`import_oai_id`) -) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table 'oai_harvester_import_queue' @@ -44,7 +44,7 @@ CREATE TABLE IF NOT EXISTS `oai_harvester_import_queue` ( `result` text CHARACTER SET utf8 NOT NULL, `result_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table 'oai_harvester_requests' @@ -70,4 +70,4 @@ CREATE TABLE IF NOT EXISTS `oai_harvester_requests` ( `interval` int(10) unsigned NOT NULL, `name` varchar(45) NOT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql index e66cdfa..87cd359 100644 --- a/installer/data/mysql/kohastructure.sql +++ b/installer/data/mysql/kohastructure.sql @@ -4331,7 +4331,7 @@ CREATE TABLE IF NOT EXISTS `oai_harvester_biblios` ( UNIQUE KEY `oai_record` (`oai_identifier`,`oai_repository`) USING BTREE, KEY `FK_import_oai_biblio_1` (`biblionumber`), CONSTRAINT `FK_import_oai_biblio_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE NO ACTION -) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table 'oai_harvester_history' @@ -4351,7 +4351,7 @@ CREATE TABLE IF NOT EXISTS `oai_harvester_history` ( `record_type` enum('biblio','auth','holdings') COLLATE utf8_unicode_ci NOT NULL, `matcher_code` varchar(10) COLLATE utf8_unicode_ci DEFAULT NULL, PRIMARY KEY (`import_oai_id`) -) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table 'oai_harvester_import_queue' @@ -4364,7 +4364,7 @@ CREATE TABLE IF NOT EXISTS `oai_harvester_import_queue` ( `result` text CHARACTER SET utf8 NOT NULL, `result_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table 'oai_harvester_requests' @@ -4390,7 +4390,7 @@ CREATE TABLE IF NOT EXISTS `oai_harvester_requests` ( `interval` int(10) unsigned NOT NULL, `name` varchar(45) NOT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/dashboard.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/dashboard.tt index 36fdccf..e939db6 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/dashboard.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/dashboard.tt @@ -1,107 +1,10 @@ +[% USE ColumnsSettings %] +[% SET footerjs = 1 %] [% INCLUDE 'doc-head-open.inc' %] Koha › Tools › OAI-PMH harvester [% INCLUDE 'doc-head-close.inc' %] -[% INCLUDE 'datatables.inc' %] [% dashboard_page = '/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl' %] [% request_page = '/cgi-bin/koha/tools/oai-pmh-harvester/request.pl' %] -