View | Details | Raw Unified | Return to bug 10662
Collapse All | Expand All

(-)a/C4/Installer/PerlDependencies.pm (+10 lines)
Lines 893-898 our $PERL_DEPS = { Link Here
893
        required => '1',
893
        required => '1',
894
        min_ver  => '0.37',
894
        min_ver  => '0.37',
895
    },
895
    },
896
    'POE' => {
897
        'usage' => 'OAI-PMH harvester',
898
        'required' => 1,
899
        'min_ver' => '1.354',
900
    },
901
    'POE::Component::JobQueue' => {
902
        'usage' => 'OAI-PMH harvester',
903
        'required' => 1,
904
        'min_ver' => '0.570',
905
    },
896
};
906
};
897
907
898
1;
908
1;
(-)a/Koha/Daemon.pm (+210 lines)
Line 0 Link Here
1
package Koha::Daemon;
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
use POSIX; #For daemonizing
23
use Fcntl qw(:flock); #For pidfile
24
25
=head1 NAME
26
27
Koha::Daemon
28
29
=head1 SYNOPSIS
30
31
  use Koha::Daemon;
32
  my $daemon = Koha::Daemon->new({
33
    pidfile => "/path/to/file.pid",
34
    logfile => "/path/to/file.log",
35
    daemonize => 1,
36
  });
37
  $daemon->run();
38
39
=head1 METHODS
40
41
=head2 new
42
43
Create object
44
45
=head2 run
46
47
Run the daemon.
48
49
This method calls all the internal methods which
50
are do everything necessary to run the daemon.
51
52
=head1 INTERNAL METHODS
53
54
=head2 daemonize
55
56
Internal function for setting object up as a proper daemon
57
(e.g. forking, setting permissions, changing directories,
58
 closing file handles, etc.)
59
60
=head2 get_pidfile
61
62
Internal function to get a filehandle for the pidfile
63
64
=head2 lock_pidfile
65
66
Internal function to lock the pidfile, so that only one daemon
67
can run against this pidfile
68
69
=head2 log_to_file
70
71
Internal function to log to a file
72
73
=head2 make_pidfilehandle
74
75
Internal function to make a filehandle for pidfile
76
77
=head2 write_pidfile
78
79
Internal function to write pid to pidfile
80
81
=cut
82
83
sub new {
84
    my ($class, $args) = @_;
85
    $args = {} unless defined $args;
86
    return bless ($args, $class);
87
}
88
89
#######################################################################
90
#NOTE: On Debian, you can use the daemon binary to make a process into a daemon,
91
# but the following can be used if you don't want to use that program.
92
93
sub daemonize {
94
    my ($self) = @_;
95
96
    my $pid = fork;
97
98
    die "Couldn't fork: $!" unless defined($pid);
99
    if ($pid){
100
        exit; #Parent exit
101
    }
102
103
    #Become a session leader (ie detach program from controlling terminal)
104
    POSIX::setsid() or die "Can't start a new session: $!";
105
106
    #Change to known system directory
107
    chdir('/');
108
109
    #Reset the file creation mask so only the daemon owner can read/write files it creates
110
    umask(066);
111
112
    #Close inherited file handles, so that you can truly run in the background.
113
    open STDIN,  '<', '/dev/null';
114
    open STDOUT, '>', '/dev/null';
115
    open STDERR, '>&STDOUT';
116
}
117
118
sub log_to_file {
119
    my ($self,$logfile) = @_;
120
121
    #Open a filehandle to append to a log file
122
    my $opened = open(my $fh, '>>', $logfile);
123
    if ($opened){
124
        $fh->autoflush(1); #Make filehandle hot (ie don't buffer)
125
        *STDOUT = *$fh; #Re-assign STDOUT to LOG | --stdout
126
        *STDERR = *STDOUT; #Re-assign STDERR to STDOUT | --stderr
127
    }
128
    else {
129
        die "Unable to open a filehandle for $logfile: $!\n"; # --output
130
    }
131
}
132
133
sub make_pidfilehandle {
134
    my ($self,$pidfile) = @_;
135
    if ( ! -f $pidfile ){
136
        open(my $fh, '>', $pidfile) or die "Unable to write to $pidfile: $!\n";
137
        close($fh);
138
    }
139
    open(my $pidfilehandle, '+<', $pidfile) or die "Unable to open a filehandle for $pidfile: $!\n";
140
    return $pidfilehandle;
141
}
142
143
sub get_pidfile {
144
    my ($self,$pidfile) = @_;
145
    #NOTE: We need to save the filehandle in the object, so any locks persist for the life of the object
146
    my $pidfilehandle = $self->{pidfilehandle} ||= $self->make_pidfilehandle($pidfile);
147
    return $pidfilehandle;
148
}
149
150
sub lock_pidfile {
151
    my ($self,$pidfilehandle) = @_;
152
    my $locked;
153
    if (flock($pidfilehandle, LOCK_EX|LOCK_NB)){
154
        $locked = 1;
155
156
    }
157
    return $locked;
158
}
159
160
sub write_pidfile {
161
    my ($self,$pidfilehandle) = @_;
162
    if ($pidfilehandle){
163
        truncate($pidfilehandle, 0);
164
        print $pidfilehandle $$."\n" or die $!;
165
        #Flush the filehandle so you're not suffering from buffering
166
        $pidfilehandle->flush();
167
        return 1;
168
    }
169
}
170
171
sub run {
172
    my ($self) = @_;
173
    my $pidfile = $self->{pidfile};
174
    my $logfile = $self->{logfile};
175
176
    if ($pidfile){
177
        my $pidfilehandle = $self->get_pidfile($pidfile);
178
        if ($pidfilehandle){
179
            my $locked = $self->lock_pidfile($pidfilehandle);
180
            if ( ! $locked ) {
181
                die "$0 is unable to lock pidfile...\n";
182
            }
183
        }
184
    }
185
186
    if (my $configure = $self->{configure}){
187
        $configure->($self);
188
    }
189
190
    if ($self->{daemonize}){
191
        $self->daemonize();
192
    }
193
194
    if ($pidfile){
195
        my $pidfilehandle = $self->get_pidfile($pidfile);
196
        if ($pidfilehandle){
197
            $self->write_pidfile($pidfilehandle);
198
        }
199
    }
200
201
    if ($logfile){
202
        $self->log_to_file($logfile);
203
    }
204
205
    if (my $loop = $self->{loop}){
206
        $loop->($self);
207
    }
208
}
209
210
1;
(-)a/Koha/OAI/Harvester.pm (+715 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester;
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
use POE qw(Component::JobQueue);
23
use JSON;
24
use Sereal::Encoder;
25
use Sereal::Decoder;
26
use IO::Handle;
27
use File::Copy;
28
use File::Path qw(make_path remove_tree);
29
use DateTime;
30
use DateTime::Format::Strptime;
31
32
use C4::Context;
33
use Koha::OAI::Harvester::ImportQueues;
34
35
=head1 NAME
36
37
Koha::OAI::Harvester
38
39
=head1 SYNOPSIS
40
41
my $harvester = Koha::OAI::Harvester->spawn({
42
    Downloader => $downloader,
43
    DownloaderWorkers => $download_workers,
44
    Importer => $importer,
45
    ImporterWorkers => $import_workers,
46
    ImportQueuePoll => $import_poll,
47
    logger => $logger,
48
    state_file => $statefile,
49
    spooldir => $spooldir,
50
});
51
52
=head2 METHODS
53
54
=cut
55
56
my $day_granularity = DateTime::Format::Strptime->new(
57
    pattern   => '%F',
58
);
59
my $seconds_granularity = DateTime::Format::Strptime->new(
60
    pattern   => '%FT%TZ',
61
);
62
63
=head2 new
64
65
Create object
66
67
=cut
68
69
sub new {
70
    my ($class, $args) = @_;
71
    $args = {} unless defined $args;
72
    return bless ($args, $class);
73
}
74
75
=head2 spawn
76
77
    Creates POE sessions for handling tasks, downloading, and importing
78
79
=cut
80
81
sub spawn {
82
    my ($class, $args) = @_;
83
    my $self = $class->new($args);
84
    my $downloader = $self->{Downloader};
85
    my $importer = $self->{Importer};
86
    my $download_worker_limit = ( $self->{DownloaderWorkers} && int($self->{DownloaderWorkers}) ) ? $self->{DownloaderWorkers} : 1;
87
    my $import_worker_limit = ( $self->{ImporterWorkers} && int($self->{ImporterWorkers}) ) ? $self->{ImporterWorkers} : 1;
88
    my $import_queue_poll = ( $self->{ImportQueuePoll} && int($self->{ImportQueuePoll}) ) ? $self->{ImportQueuePoll} : 5;
89
90
    #NOTE: This job queue should always be created before the
91
    #harvester so that you can start download jobs immediately
92
    #upon spawning the harvester.
93
    POE::Component::JobQueue->spawn(
94
        Alias         => 'oai-downloader',
95
        WorkerLimit   => $download_worker_limit,
96
        Worker        => sub {
97
            my ($postback, $task) = @_;
98
            if ($downloader){
99
                if ($task->{status} eq "active"){
100
                    $downloader->run({
101
                        postback => $postback,
102
                        task => $task,
103
                   });
104
                }
105
            }
106
        },
107
        Passive => {},
108
    );
109
110
    POE::Session->create(
111
        object_states => [
112
            $self => {
113
                _start => "on_start",
114
                get_task => "get_task",
115
                list_tasks => "list_tasks",
116
                create_task => "create_task",
117
                start_task => "start_task",
118
                stop_task => "stop_task",
119
                delete_task => "delete_task",
120
                repeat_task => "repeat_task",
121
                register => "register",
122
                deregister => "deregister",
123
                restore_state => "restore_state",
124
                save_state => "save_state",
125
                is_task_finished => "is_task_finished",
126
                does_task_repeat => "does_task_repeat",
127
                download_postback => "download_postback",
128
                reset_imports_status => "reset_imports_status",
129
            },
130
        ],
131
    );
132
133
    POE::Component::JobQueue->spawn(
134
        Alias         => 'oai-importer',
135
        WorkerLimit   => $import_worker_limit,
136
        Worker        => sub {
137
            my $meta_postback = shift;
138
139
            #NOTE: We need to only retrieve queue items for active tasks. Otherwise,
140
            #the importer will just spin its wheels on inactive tasks and do nothing.
141
            my $active_tasks = $poe_kernel->call("harvester","list_tasks","active");
142
            my @active_uuids = map { $_->{uuid} } @$active_tasks;
143
144
            my $rs = Koha::OAI::Harvester::ImportQueues->new->search({
145
                uuid => \@active_uuids,
146
                status => "new"
147
            },{
148
                order_by => { -asc => 'id' },
149
                rows => 1,
150
            });
151
            my $result = $rs->single;
152
            if ($result){
153
                $result->status("wip");
154
                $result->update;
155
                my $task = {
156
                    id => $result->id,
157
                    uuid => $result->uuid,
158
                    result => $result->result,
159
                };
160
161
                my $postback = $meta_postback->($task);
162
                $importer->run({
163
                    postback => $postback,
164
                    task => $task,
165
                });
166
            }
167
        },
168
        Active => {
169
            PollInterval => $import_queue_poll,
170
            AckAlias => undef,
171
            AckState => undef,
172
        },
173
    );
174
175
    return;
176
}
177
178
=head2 on_start
179
180
    Internal method that starts the harvester
181
182
=cut
183
184
sub on_start {
185
    my ($self, $kernel, $heap) = @_[OBJECT, KERNEL,HEAP];
186
    $kernel->alias_set('harvester');
187
    $heap->{scoreboard} = {};
188
189
    #Reset any 'wip' imports to 'new' so they can be re-tried.
190
    $kernel->call("harvester","reset_imports_status");
191
192
    #Restore state from state file
193
    $kernel->call("harvester","restore_state");
194
}
195
196
=head2 download_postback
197
198
    Internal method
199
200
=cut
201
202
#NOTE: This isn't really implemented at the moment, as it's not really necessary.
203
sub download_postback {
204
    my ($kernel, $request_packet, $response_packet) = @_[KERNEL, ARG0, ARG1];
205
    my $message = $response_packet->[0];
206
}
207
208
=head2 deregister
209
210
    Remove the worker session from the harvester's in-memory scoreboard,
211
    unset the downloading flag if downloading is completed.
212
213
=cut
214
215
sub deregister {
216
    my ($self, $kernel, $heap, $session, $sender, $type) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0];
217
218
    my $scoreboard = $heap->{scoreboard};
219
220
    my $logger = $self->{logger};
221
    $logger->debug("Start deregistering $sender as $type task");
222
223
    my $task_uuid = delete $scoreboard->{session}->{$sender};
224
    #NOTE: If you don't check each step of the hashref, autovivication can lead to surprises.
225
    if ($task_uuid){
226
        if ($scoreboard->{task}->{$task_uuid}){
227
            if ($scoreboard->{task}->{$task_uuid}->{$type}){
228
                delete $scoreboard->{task}->{$task_uuid}->{$type}->{$sender};
229
            }
230
        }
231
    }
232
233
    my $task = $heap->{tasks}->{$task_uuid};
234
    if ($task && $task->{status} && ($task->{status} eq "active") ){
235
        #NOTE: Each task only has 1 download session, so we can now set/unset flags for the task.
236
        #NOTE: We should unset the downloading flag, if we're not going to repeat the task.
237
        if ($type eq "download"){
238
            my $task_repeats = $kernel->call("harvester","does_task_repeat",$task_uuid);
239
            if ($task_repeats){
240
                my $interval = $task->{interval};
241
242
                $task->{effective_from} = delete $task->{effective_until};
243
                $task->{download_timer} = $kernel->delay_set("repeat_task", $interval, $task_uuid);
244
            }
245
            else {
246
                $task->{downloading} = 0;
247
                $kernel->call("harvester","save_state");
248
                $kernel->call("harvester","is_task_finished",$task_uuid);
249
            }
250
        }
251
        elsif ($type eq 'import'){
252
            $kernel->call("harvester","is_task_finished",$task_uuid);
253
        }
254
    }
255
    $logger->debug("End deregistering $sender as $type task");
256
}
257
258
259
=head2 is_task_finished
260
261
    This event handler checks if the task has finished downloading and importing record.
262
    If it is finished downloading and importing, the task is deleted from the harvester.
263
264
    This only applies to non-repeating tasks.
265
266
=cut
267
268
sub is_task_finished {
269
    my ($self, $kernel, $heap, $session, $uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
270
    my $task = $kernel->call("harvester","get_task",$uuid);
271
    if ($task && (! $task->{downloading}) ){
272
        my $count = $self->get_import_count_for_task($uuid);
273
        if ( ! $count ) {
274
            #Clear this task out of the harvester as it's finished.
275
            $kernel->call("harvester","delete_task",$uuid);
276
            return 1;
277
        }
278
    }
279
    return 0;
280
}
281
282
=head2 register
283
284
    Internal method for registering a task on the harvester's scoreboard
285
286
=cut
287
288
sub register {
289
    my ($self, $kernel, $heap, $session, $sender, $type, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0,ARG1];
290
    my $logger = $self->{logger};
291
292
    my $scoreboard = $heap->{scoreboard};
293
294
295
    if ($type && $task_uuid){
296
        $logger->debug("Registering $sender as $type for $task_uuid");
297
298
        my $task = $heap->{tasks}->{$task_uuid};
299
        if ($task){
300
301
            if ($type){
302
                #Register the task uuid with the session id as a key for later recall
303
                $scoreboard->{session}->{$sender} = $task_uuid;
304
305
                #Register the session id as a certain type of session for a task
306
                $scoreboard->{task}->{$task_uuid}->{$type}->{$sender} = 1;
307
308
                if ($type eq "download"){
309
                    $task->{downloading} = 1;
310
311
                    my $task_repeats = $kernel->call("harvester","does_task_repeat",$task_uuid);
312
                    if ($task_repeats){
313
314
                        #NOTE: Set an effective until, so we know we're not getting records any newer than
315
                        #this moment.
316
                        my $dt = DateTime->now();
317
                        if ($dt){
318
                            #NOTE: Ideally, I'd like to make sure that we can use 'seconds' granularity, but
319
                            #it's valid for 'from' to be null, so it's impossible to know from the data whether
320
                            #or not the repository will support the seconds granularity.
321
                            #NOTE: Ideally, it would be good to use either 'day' granularity or 'seconds' granularity,
322
                            #but at the moment the interval is expressed as seconds only.
323
                            $dt->set_formatter($seconds_granularity);
324
                            $task->{effective_until} = "$dt";
325
                        }
326
                    }
327
328
                    $kernel->call("harvester","save_state");
329
                }
330
            }
331
        }
332
    }
333
}
334
335
=head2 does_task_repeat
336
337
    Internal method for checking if a task is supposed to repeat after it finishes
338
339
=cut
340
341
sub does_task_repeat {
342
    my ($self, $kernel, $heap, $session, $uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
343
    my $task = $kernel->call("harvester","get_task",$uuid);
344
    if ($task){
345
        my $interval = $task->{interval};
346
        my $parameters = $task->{parameters};
347
        if ($parameters){
348
            my $oai_pmh = $parameters->{oai_pmh};
349
            if ($oai_pmh){
350
                if ( $interval && ($oai_pmh->{verb} eq "ListRecords") && (! $oai_pmh->{until}) ){
351
                    return 1;
352
                }
353
            }
354
        }
355
    }
356
    return 0;
357
}
358
359
=head2 reset_imports_status
360
361
    Internal method that resets the import queue from wip to new
362
363
=cut
364
365
sub reset_imports_status {
366
    my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION];
367
368
    my $rs = Koha::OAI::Harvester::ImportQueues->new->search({
369
                status => "wip",
370
    });
371
    $rs->update({
372
        status => "new",
373
    });
374
}
375
376
=head2 restore_state
377
378
    Method to restore the harvester to a pre-existing state.
379
380
=cut
381
382
sub restore_state {
383
    my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION];
384
385
    my $state_file = $self->{state_file};
386
    if ($state_file){
387
        my $state_backup = "$state_file~";
388
389
        #NOTE: If there is a state backup, it means we crashed while saving the state. Otherwise,
390
        #let's try the regular state file if it exists.
391
        my $file_to_restore = ( -f $state_backup ) ? $state_backup : ( ( -f $state_file ) ? $state_file : undef );
392
        if ( $file_to_restore ){
393
            my $opened = open( my $fh, '<', $file_to_restore ) or die "Couldn't open state: $!";
394
            if ($opened){
395
                local $/;
396
                my $in = <$fh>;
397
                my $decoder = Sereal::Decoder->new;
398
                my $state = $decoder->decode($in);
399
                if ($state){
400
                    if ($state->{tasks}){
401
                        #Restore tasks from our saved state
402
                        $heap->{tasks} = $state->{tasks};
403
                        foreach my $uuid ( keys %{$heap->{tasks}} ){
404
                            my $task = $heap->{tasks}->{$uuid};
405
406
                            #If tasks were still downloading, restart the task
407
                            if ( ($task->{status} && $task->{status} eq "active") && $task->{downloading} ){
408
                                $task->{status} = "new";
409
                                $kernel->call("harvester","start_task",$task->{uuid});
410
                            }
411
                        }
412
                    }
413
                }
414
            }
415
        }
416
    }
417
}
418
419
=head2 save_state
420
421
    Method to save the existing state of the harvester
422
423
=cut
424
425
sub save_state {
426
    my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION];
427
    my $state_file = $self->{state_file};
428
    my $state_backup = "$state_file~";
429
430
    #Make a backup of existing state record
431
    my $moved = move($state_file,$state_backup);
432
433
    my $opened = open(my $fh, ">", $state_file) or die "Couldn't save state: $!";
434
    if ($opened){
435
        $fh->autoflush(1);
436
        my $tasks = $heap->{tasks};
437
        my $harvester_state = {
438
            tasks => $tasks,
439
        };
440
        my $encoder = Sereal::Encoder->new;
441
        my $out = $encoder->encode($harvester_state);
442
        local $\;
443
        my $printed = print $fh $out;
444
        if ($printed){
445
            close $fh;
446
            unlink($state_backup);
447
            return 1;
448
        }
449
    }
450
    return 0;
451
}
452
453
=head2 get_task
454
455
    This event handler returns a task from a harvester using the task's
456
    uuid as an argument.
457
458
=cut
459
460
sub get_task {
461
    my ($self, $kernel, $heap, $session, $uuid, $sender) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0, SENDER];
462
463
    if ( ! $uuid && $sender ){
464
        my $scoreboard = $heap->{scoreboard};
465
        my $uuid_by_session = $scoreboard->{session}->{$sender};
466
        if ($uuid_by_session){
467
            $uuid = $uuid_by_session;
468
        }
469
    }
470
471
    my $tasks = $heap->{tasks};
472
    if ($tasks && $uuid){
473
        my $task = $tasks->{$uuid};
474
        if ($task){
475
            return $task;
476
        }
477
    }
478
    return 0;
479
}
480
481
=head2 get_import_count_for_task
482
483
    This gets the count of the number of imports exist for a certain task
484
485
=cut
486
487
sub get_import_count_for_task {
488
    my ($self,$uuid) = @_;
489
    my $count = undef;
490
    if ($uuid){
491
        my $items = Koha::OAI::Harvester::ImportQueues->new->search({
492
            uuid => $uuid,
493
        });
494
        $count = $items->count;
495
    }
496
    return $count;
497
}
498
499
=head2 list_tasks
500
501
    This event handler returns a list of tasks that have been submitted
502
    to the harvester. It returns data like uuid, status, parameters,
503
    number of pending imports, etc.
504
505
=cut
506
507
sub list_tasks {
508
    my ($self, $kernel, $heap, $session, $status) = @_[OBJECT, KERNEL,HEAP,SESSION, ARG0];
509
    my @tasks = ();
510
    foreach my $uuid (sort keys %{$heap->{tasks}}){
511
        my $task = $heap->{tasks}->{$uuid};
512
        my $items = Koha::OAI::Harvester::ImportQueues->new->search({
513
            uuid => $uuid,
514
        });
515
        my $count = $items->count // 0;
516
        $task->{pending_imports} = $count;
517
        if ( ( ! $status ) || ( $status && $status eq $task->{status} ) ){
518
            push(@tasks, $task);
519
        }
520
521
    }
522
    return \@tasks;
523
}
524
525
=head2 create_task
526
527
    This event handler creates a spool directory for the task's imports.
528
    It also adds it to the harvester's memory and then saves memory to
529
    a persistent datastore.
530
531
    Newly created tasks have a status of "new".
532
533
=cut
534
535
sub create_task {
536
    my ($self, $kernel, $heap, $session, $incoming_task) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
537
    my $logger = $self->{logger};
538
    if ($incoming_task){
539
        my $uuid = $incoming_task->{uuid};
540
        if ( ! $heap->{tasks}->{$uuid} ){
541
542
            #Step One: assign a spool directory to this task
543
            my $spooldir = $self->{spooldir} // "/tmp";
544
            my $task_spooldir = "$spooldir/$uuid";
545
            if ( ! -d $task_spooldir ){
546
                my $made_spool_directory = make_path($task_spooldir);
547
                if ( ! $made_spool_directory ){
548
                    if ($logger){
549
                        $logger->warn("Unable to make task-specific spool directory at '$task_spooldir'");
550
                    }
551
                    return 0;
552
                }
553
            }
554
            $incoming_task->{spooldir} = $task_spooldir;
555
556
            #Step Two: assign new status
557
            $incoming_task->{status} = "new";
558
559
            #Step Three: add task to harvester's memory
560
            $heap->{tasks}->{ $uuid } = $incoming_task;
561
562
            #Step Four: save state
563
            $kernel->call($session,"save_state");
564
            return 1;
565
        }
566
    }
567
    return 0;
568
}
569
570
=head2 start_task
571
572
    This event handler marks a task as active in the harvester's memory,
573
    save the memory to a persistent datastore, then enqueues the task,
574
    so that it can be directed to the next available download worker.
575
576
    Newly started tasks have a status of "active".
577
578
=cut
579
580
sub start_task {
581
    my ($self, $session,$kernel,$heap,$uuid) = @_[OBJECT, SESSION,KERNEL,HEAP,ARG0];
582
    my $task = $heap->{tasks}->{$uuid};
583
    if ($task){
584
        if ( $task->{status} ne "active" ){
585
586
            #Clear any pre-existing error states
587
            delete $task->{error} if $task->{error};
588
589
            #Step One: mark task as active
590
            $task->{status} = "active";
591
592
            #Step Two: save state
593
            $kernel->call("harvester","save_state");
594
595
            #Step Three: enqueue task
596
            $kernel->post("oai-downloader",'enqueue','download_postback', $task);
597
598
            return 1;
599
        }
600
    }
601
    return 0;
602
}
603
604
=head2 repeat_task
605
606
    This method re-queues a task for downloading
607
608
=cut
609
610
sub repeat_task {
611
    my ($self, $session,$kernel,$heap,$uuid) = @_[OBJECT, SESSION,KERNEL,HEAP,ARG0];
612
    my $task = $heap->{tasks}->{$uuid};
613
    if ($task){
614
        my $interval = $task->{interval};
615
        if ($task->{downloading} && $interval){
616
            $kernel->post("oai-downloader",'enqueue','download_postback', $task);
617
        }
618
    }
619
}
620
621
=head2 stop_task
622
623
    This event handler prevents new workers from spawning, kills
624
    existing workers, and stops pending imports from being imported.
625
626
    Newly stopped tasks have a status of "stopped".
627
628
=cut
629
630
sub stop_task {
631
    my ($self, $kernel, $heap, $session, $sender, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0];
632
633
    my $task = $heap->{tasks}->{$task_uuid};
634
635
    if ($task && $task->{status} && $task->{status} ne "stopped" ){
636
637
        #Step One: deactivate task, so no new workers can be started
638
        $task->{status} = "stopped";
639
        #NOTE: You could also clear timers for new downloads, but that's probably unnecessary because of this step.
640
641
        #Step Two: kill workers
642
        my $scoreboard = $heap->{scoreboard};
643
        my $session_types = $scoreboard->{task}->{$task_uuid};
644
        if ($session_types){
645
            foreach my $type ( keys %$session_types ){
646
                my $sessions = $session_types->{$type};
647
                if ($sessions){
648
                    foreach my $session (keys %$sessions){
649
                        if ($session){
650
                            $kernel->signal($session, "cancel");
651
                        }
652
                    }
653
                }
654
            }
655
            #Remove the task uuid from the task key of the scoreboard
656
            delete $scoreboard->{task}->{$task_uuid};
657
            #NOTE: The task uuid will still exist under the session key,
658
            #but the sessions will deregister themselves and clean that up for you.
659
        }
660
661
        #Step Three: stop pending imports for this task
662
        my $items = Koha::OAI::Harvester::ImportQueues->new->search({
663
            uuid => $task_uuid,
664
        });
665
        my $rows_updated = $items->update({
666
            status => "stopped",
667
        });
668
669
        #Step Four: save state
670
        $kernel->call("harvester","save_state");
671
        return 1;
672
    }
673
    return 0;
674
}
675
676
=head2 delete_task
677
678
    Deleted tasks are stopped, pending imports are deleted from the
679
    database and file system, and then the task is removed from the harvester.
680
681
=cut
682
683
sub delete_task {
684
    my ($self, $kernel, $heap, $session, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
685
686
    my $task = $heap->{tasks}->{$task_uuid};
687
    if ($task){
688
        #Step One: stop task
689
        $kernel->call($session,"stop_task",$task_uuid);
690
691
        #Step Two: delete pending imports in database
692
        my $items = Koha::OAI::Harvester::ImportQueues->new->search({
693
            uuid => $task_uuid,
694
        });
695
        if ($items){
696
            my $rows_deleted = $items->delete;
697
            #NOTE: shows 0E0 instead of 0
698
        }
699
700
        #Step Three: remove task specific spool directory and files within it
701
        my $spooldir = $task->{spooldir};
702
        if ($spooldir){
703
            my $files_deleted = remove_tree($spooldir, { safe => 1 });
704
        }
705
706
        delete $heap->{tasks}->{$task_uuid};
707
708
        #Step Four: save state
709
        $kernel->call("harvester","save_state");
710
        return 1;
711
    }
712
    return 0;
713
}
714
715
1;
(-)a/Koha/OAI/Harvester/Biblio.pm (+49 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Biblio;
2
3
# Copyright Prosentient Systems 2018
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use Carp;
23
24
use base qw(Koha::Object);
25
26
=head1 NAME
27
28
Koha::OAI::Harvester::Biblio
29
30
=head1 SYNOPSIS
31
32
    use Koha::OAI::Harvester::Biblio;
33
    my $biblio = Koha::OAI::Harvester::Biblios->find($id);
34
35
=head1 METHODS
36
37
=cut
38
39
sub _type {
40
    return 'OaiHarvesterBiblio';
41
}
42
43
=head1 AUTHOR
44
45
David Cook <dcook@prosentient.com.au>
46
47
=cut
48
49
1;
(-)a/Koha/OAI/Harvester/Biblios.pm (+60 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Biblios;
2
3
# Copyright Prosentient Systems 2018
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use Carp;
23
24
use Koha::Database;
25
26
use Koha::OAI::Harvester::Biblio;
27
28
use base qw(Koha::Objects);
29
30
=head1 NAME
31
32
Koha::OAI::Harvester::Biblios
33
34
=head2 METHODS
35
36
=cut
37
38
=head3 _type
39
40
=cut
41
42
sub _type {
43
    return 'OaiHarvesterBiblio';
44
}
45
46
=head3 object_class
47
48
=cut
49
50
sub object_class {
51
    return 'Koha::OAI::Harvester::Biblio';
52
}
53
54
=head1 AUTHOR
55
56
David Cook <dcook@prosentient.com.au>
57
58
=cut
59
60
1;
(-)a/Koha/OAI/Harvester/Client.pm (+233 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Client;
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use URI;
22
use IO::Socket::UNIX;
23
use IO::Select;
24
use JSON;
25
use Encode;
26
27
=head1 NAME
28
29
Koha::OAI::Harvester::Client
30
31
=head1 SYNOPSIS
32
33
  use Koha::OAI::Harvester::Client;
34
  my $client = Koha::OAI::Harvester::Client->new($client_config);
35
36
=head1 METHODS
37
38
=head2 new
39
40
Create object
41
42
=cut
43
44
sub new {
45
    my ($class, $args) = @_;
46
    $args = {} unless defined $args;
47
    return bless ($args, $class);
48
}
49
50
=head2 connect
51
52
Connect to Koha::OAI::Harvester::Listener via a Unix socket
53
54
=cut
55
56
sub connect {
57
    my ($self) =  @_;
58
    my $socket_uri = $self->{socket_uri};
59
    if ($socket_uri){
60
        my $uri = URI->new($socket_uri);
61
        if ($uri && $uri->scheme eq 'unix'){
62
            my $socket_path = $uri->path;
63
            my $socket = IO::Socket::UNIX->new(
64
                Type => IO::Socket::UNIX::SOCK_STREAM(),
65
                Peer => $socket_path,
66
            );
67
            if ($socket){
68
                my $select = new IO::Select();
69
                $select->add($socket);
70
71
                $self->{_select} = $select;
72
                $self->{_socket} = $socket;
73
                my $message = $self->_read();
74
                if ($message){
75
                    if ($message eq 'HELLO'){
76
                        $self->{_connected} = 1;
77
                        return 1;
78
                    }
79
                }
80
            }
81
            else {
82
                warn "Failed to create socket."
83
            }
84
        }
85
    }
86
    return 0;
87
}
88
89
=head2 create
90
91
    Create a task on the harvester
92
93
=cut
94
95
sub create {
96
    my ($self,$task) = @_;
97
    my $message = {
98
        command => "create",
99
        body => {
100
            task => $task,
101
        }
102
    };
103
    my ($status) = $self->_exchange($message);
104
    return $status;
105
}
106
107
=head2 start
108
109
    Start a task on the harvester
110
111
=cut
112
113
sub start {
114
    my ($self,$uuid) = @_;
115
    my $message = {
116
        command => "start",
117
        body => {
118
            task => {
119
                uuid => $uuid,
120
            },
121
        }
122
    };
123
    my ($status) = $self->_exchange($message);
124
    return $status;
125
}
126
127
=head2 stop
128
129
    Stop a task on the harvester
130
131
=cut
132
133
sub stop {
134
    my ($self,$uuid) = @_;
135
    my $message = {
136
        command => "stop",
137
        body => {
138
            task => {
139
                uuid => $uuid,
140
            },
141
        }
142
    };
143
    my ($status) = $self->_exchange($message);
144
    return $status;
145
}
146
147
=head2 delete
148
149
    Delete a task on the harvester
150
151
=cut
152
153
sub delete {
154
    my ($self,$uuid) = @_;
155
    my $message = {
156
        command => "delete",
157
        body => {
158
            task => {
159
                uuid => $uuid,
160
            },
161
        }
162
    };
163
    my ($status) = $self->_exchange($message);
164
    return $status;
165
}
166
167
=head2 list
168
169
    List all tasks on the harvester
170
171
=cut
172
173
sub list {
174
    my ($self) = @_;
175
    my $message = {
176
        command => "list",
177
    };
178
    my ($status,$tasks) = $self->_exchange($message);
179
    return $tasks;
180
}
181
182
sub _exchange {
183
    my ($self,$message) = @_;
184
    my $status = 0;
185
    my $data;
186
    if ($message){
187
        my $output = to_json($message);
188
        if ($output){
189
            $self->_write($output);
190
            my $json_response = $self->_read();
191
            if ($json_response){
192
                my $response = from_json($json_response);
193
                $data = $response->{data} if $response->{data};
194
                $status = 1 if $response->{msg} && $response->{msg} eq "success";
195
            }
196
        }
197
    }
198
    return ($status,$data);
199
}
200
201
sub _write {
202
    my ($self, $output) = @_;
203
    if ($output){
204
        if (my $select = $self->{_select}){
205
            if (my @filehandles = $select->can_write(5)){
206
                foreach my $filehandle (@filehandles){
207
                    #Localize output record separator as null
208
                    local $\ = "\x00";
209
                    $output = Encode::encode("UTF-8",$output);
210
                    print $filehandle $output;
211
                }
212
            }
213
        }
214
    }
215
}
216
217
sub _read {
218
    my ($self) = @_;
219
    if (my $select = $self->{_select}){
220
        if (my @filehandles = $select->can_read(5)){
221
            foreach my $filehandle (@filehandles){
222
                #Localize input record separator as null
223
                local $/ = "\x00";
224
                my $message = <$filehandle>;
225
                chomp($message) if $message;
226
                $message = Encode::decode("UTF-8",$message);
227
                return $message;
228
            }
229
        }
230
    }
231
}
232
233
1;
(-)a/Koha/OAI/Harvester/Downloader.pm (+335 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Downloader;
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use URI;
22
use XML::LibXML::Reader;
23
use IO::Handle;
24
use JSON;
25
26
=head1 NAME
27
28
Koha::OAI::Harvester::Downloader
29
30
=head1 SYNOPSIS
31
32
    use Koha::OAI::Harvester::Downloader;
33
    my $oai_downloader = Koha::OAI::Harvester::Downloader->new();
34
35
    This class is used within a Koha::OAI::Harvester::Work::Download:: module.
36
37
=head1 METHODS
38
39
=head2 new
40
41
    Create object
42
43
=cut
44
45
sub new {
46
    my ($class, $args) = @_;
47
    $args = {} unless defined $args;
48
    return bless ($args, $class);
49
}
50
51
=head2 BuildURL
52
53
    Takes a baseURL and a mix of required and optional OAI-PMH arguments,
54
    and makes them into a suitable URL for an OAI-PMH request.
55
56
=cut
57
58
sub BuildURL {
59
    my ($self, $args) = @_;
60
    my $baseURL = $args->{baseURL};
61
    my $url = URI->new($baseURL);
62
    if ($url && $url->isa("URI")){
63
        my $verb = $args->{verb};
64
        if ($verb){
65
            my %parameters = (
66
                verb => $verb,
67
            );
68
            if ($verb eq "ListRecords"){
69
                my $resumptionToken = $args->{resumptionToken};
70
                my $metadataPrefix = $args->{metadataPrefix};
71
                if ($resumptionToken){
72
                    $parameters{resumptionToken} = $resumptionToken;
73
                }
74
                elsif ($metadataPrefix){
75
                    $parameters{metadataPrefix} = $metadataPrefix;
76
                    #Only add optional parameters if they're provided
77
                    foreach my $param ( qw( from until set ) ){
78
                        $parameters{$param} = $args->{$param} if $args->{$param};
79
                    }
80
                }
81
                else {
82
                    warn "BuildURL() requires an argument of either resumptionToken or metadataPrefix";
83
                    return;
84
                }
85
            }
86
            elsif ($verb eq "GetRecord"){
87
                my $metadataPrefix = $args->{metadataPrefix};
88
                my $identifier = $args->{identifier};
89
                if ($metadataPrefix && $identifier){
90
                    $parameters{metadataPrefix} = $metadataPrefix;
91
                    $parameters{identifier} = $identifier;
92
                }
93
                else {
94
                    warn "BuildURL() requires an argument of metadataPrefix and an argument of identifier";
95
                    return;
96
                }
97
            }
98
            $url->query_form(%parameters);
99
            return $url;
100
        }
101
        else {
102
            warn "BuildURL() requires a verb of GetRecord or ListRecords";
103
            return;
104
        }
105
    }
106
    else {
107
        warn "BuildURL() requires a base URL of type URI.";
108
        return;
109
    }
110
}
111
112
=head2 GetXMLStream
113
114
    Fork a child process to send the HTTP request, which sends chunks
115
    of XML via a pipe to the parent process.
116
117
    The parent process creates and returns a XML::LibXML::Reader object,
118
    which reads the XML stream coming through the pipe.
119
120
    Normally, using a DOM reader, you must wait to read the entire XML document
121
    into memory. However, using a stream reader, chunks are read into memory,
122
    processed, then discarded. It's faster and more efficient.
123
124
=cut
125
126
sub GetXMLStream {
127
    my ($self, $args) = @_;
128
    my $url = $args->{url};
129
    my $user_agent = $args->{user_agent};
130
    if ($url && $user_agent){
131
        pipe( CHILD, PARENT ) or die "Cannot created connected pipes: $!";
132
        CHILD->autoflush(1);
133
        PARENT->autoflush(1);
134
        if ( my $pid = fork ){
135
            #Parent process
136
            close PARENT;
137
            return \*CHILD;
138
        }
139
        else {
140
            #Child process
141
            close CHILD;
142
            my $response = $self->_request({
143
                url => $url,
144
                user_agent => $user_agent,
145
                file_handle => \*PARENT,
146
            });
147
            if ($response && $response->is_success){
148
                #HTTP request has successfully finished, so we close the file handle and exit the process
149
                close PARENT;
150
                CORE::exit(); #some modules like mod_perl redefine exit
151
            }
152
            else {
153
                warn "[child $$] OAI-PMH unsuccessful. Response status: ".$response->status_line."\n" if $response;
154
                CORE::exit();
155
            }
156
        }
157
    }
158
    else {
159
        warn "GetXMLStream() requires a 'url' argument and a 'user_agent' argument";
160
        return;
161
    }
162
}
163
164
sub _request {
165
    my ($self, $args) = @_;
166
    my $url = $args->{url};
167
    my $user_agent = $args->{user_agent};
168
    my $fh = $args->{file_handle};
169
170
    if ($url && $user_agent && $fh){
171
        my $request = HTTP::Request->new( GET => $url );
172
        my $response = $user_agent->request( $request, sub {
173
                my ($chunk_of_data, $ref_to_response, $ref_to_protocol) = @_;
174
                print $fh $chunk_of_data;
175
        });
176
        return $response;
177
    }
178
    else {
179
        warn "_request() requires a 'url' argument, 'user_agent' argument, and 'file_handle' argument.";
180
        return;
181
    }
182
}
183
184
=head2 ParseXMLStream
185
186
    Parse XML using XML::LibXML::Reader from a file handle (e.g. a stream)
187
188
=cut
189
190
191
sub ParseXMLStream {
192
    my ($self, $args) = @_;
193
194
    my $each_callback = $args->{each_callback};
195
    my $fh = $args->{file_handle};
196
    if ($fh){
197
        my $reader = XML::LibXML::Reader->new( FD => $fh, no_blanks => 1 );
198
        my $pattern = XML::LibXML::Pattern->new('oai:OAI-PMH|/oai:OAI-PMH/*', { 'oai' => "http://www.openarchives.org/OAI/2.0/" });
199
200
        my $repository;
201
202
        warn "Start parsing...";
203
        while (my $rv = $reader->nextPatternMatch($pattern)){
204
            #$rv == 1; successful
205
            #$rv == 0; end of document reached
206
            #$rv == -1; error
207
            if ($rv == -1){
208
                die "Parser error!";
209
            }
210
            #NOTE: We do this so we only get the opening tag of the element.
211
            next unless $reader->nodeType == XML_READER_TYPE_ELEMENT;
212
213
            my $localname = $reader->localName;
214
            if ( $localname eq "request" ){
215
                my $node = $reader->copyCurrentNode(1);
216
                $repository = $node->textContent;
217
            }
218
            elsif ( $localname eq "error" ){
219
                #See https://www.openarchives.org/OAI/openarchivesprotocol.html#ErrorConditions
220
                #We should probably die under all circumstances except "noRecordsMatch"
221
                my $node = $reader->copyCurrentNode(1);
222
                if ($node){
223
                    my $code = $node->getAttribute("code");
224
                    if ($code){
225
                        if ($code ne "noRecordsMatch"){
226
                            warn "Error code: $code";
227
                            die;
228
                        }
229
                    }
230
                }
231
            }
232
            elsif ( ($localname eq "ListRecords") || ($localname eq "GetRecord") ){
233
                my $each_pattern = XML::LibXML::Pattern->new('//oai:record|oai:resumptionToken', { 'oai' => "http://www.openarchives.org/OAI/2.0/" });
234
                while (my $each_rv =  $reader->nextPatternMatch($each_pattern)){
235
                    if ($rv == "-1"){
236
                        #NOTE: -1 denotes an error state
237
                        warn "Error getting pattern match";
238
                    }
239
                    next unless $reader->nodeType == XML_READER_TYPE_ELEMENT;
240
                    if ($reader->localName eq "record"){
241
                        my $node = $reader->copyCurrentNode(1);
242
                        #NOTE: Without the UTF-8 flag, UTF-8 data will be corrupted.
243
                        my $document = XML::LibXML::Document->new('1.0', 'UTF-8');
244
                        $document->setDocumentElement($node);
245
246
                       #Per record callback
247
                        if ($each_callback){
248
                            $each_callback->({
249
                                repository => $repository,
250
                                document => $document,
251
                            });
252
                        }
253
                    }
254
                    elsif ($reader->localName eq "resumptionToken"){
255
                        my $resumptionToken = $reader->readInnerXml;
256
                        return ($resumptionToken,$repository);
257
258
                    }
259
                }
260
            }
261
        } #/OAI-PMH document match
262
    }
263
    else {
264
        warn "ParseXMLStream() requires a 'file_handle' argument.";
265
    }
266
}
267
268
=head2 harvest
269
270
    Given a URL and a user-agent, start downloading OAI-PMH records from
271
    that URL.
272
273
=cut
274
275
sub harvest {
276
    my ($self,$args) = @_;
277
    my $url = $args->{url};
278
    my $ua = $args->{user_agent};
279
    my $callback = $args->{callback};
280
    my $complete_callback = $args->{complete_callback};
281
282
    if ($url && $ua){
283
284
        #NOTE: http://search.cpan.org/~shlomif/XML-LibXML-2.0128/lib/XML/LibXML/Parser.pod#ERROR_REPORTING
285
        while($url){
286
            warn "URL = $url";
287
            warn "Creating child process to download and feed parent process parser.";
288
            my $stream = $self->GetXMLStream({
289
                url => $url,
290
                user_agent => $ua,
291
            });
292
293
            warn "Creating parent process parser.";
294
            my ($resumptionToken) = $self->ParseXMLStream({
295
                file_handle => $stream,
296
                each_callback => $callback,
297
            });
298
            warn "Finished parsing current XML document.";
299
300
            if ($resumptionToken){
301
                #If there's a resumptionToken at the end of the stream,
302
                #we build a new URL and repeat this process again.
303
                $url->query_form({
304
                    verb => "ListRecords",
305
                    resumptionToken => $resumptionToken,
306
                });
307
            }
308
            else {
309
                warn "Finished harvest.";
310
                last;
311
            }
312
313
            warn "Reap child process downloader.";
314
            #Reap the dead child requester process before performing another request,
315
            #so we don't fill up the process table with zombie children.
316
            while ((my $child = waitpid(-1, 0)) > 0) {
317
                warn "Parent $$ reaped child process $child" . ($? ? " with exit code $?" : '') . ".\n";
318
            }
319
        }
320
321
        if ($complete_callback){
322
            warn "Run complete callback.";
323
324
            #Clear query string
325
            $url->query_form({});
326
327
            #Run complete callback using the actual URL from the request.
328
            $complete_callback->({
329
                repository => $url,
330
            });
331
        }
332
    }
333
}
334
335
1;
(-)a/Koha/OAI/Harvester/Histories.pm (+60 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Histories;
2
3
# Copyright Prosentient Systems 2018
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use Carp;
23
24
use Koha::Database;
25
26
use Koha::OAI::Harvester::History;
27
28
use base qw(Koha::Objects);
29
30
=head1 NAME
31
32
Koha::OAI::Harvester::Histories
33
34
=head2 METHODS
35
36
=cut
37
38
=head3 _type
39
40
=cut
41
42
sub _type {
43
    return 'OaiHarvesterHistory';
44
}
45
46
=head3 object_class
47
48
=cut
49
50
sub object_class {
51
    return 'Koha::OAI::Harvester::History';
52
}
53
54
=head1 AUTHOR
55
56
David Cook <dcook@prosentient.com.au>
57
58
=cut
59
60
1;
(-)a/Koha/OAI/Harvester/History.pm (+49 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::History;
2
3
# Copyright Prosentient Systems 2018
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use Carp;
23
24
use base qw(Koha::Object);
25
26
=head1 NAME
27
28
Koha::OAI::Harvester::History
29
30
=head1 SYNOPSIS
31
32
    use Koha::OAI::Harvester::History;
33
    my $request = Koha::OAI::Harvester::History->find($id);
34
35
=head1 METHODS
36
37
=cut
38
39
sub _type {
40
    return 'OaiHarvesterHistory';
41
}
42
43
=head1 AUTHOR
44
45
David Cook <dcook@prosentient.com.au>
46
47
=cut
48
49
1;
(-)a/Koha/OAI/Harvester/Import/MARCXML.pm (+163 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Import::MARCXML;
2
3
# Copyright 2016 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
use MARC::Record;
23
24
use C4::Context;
25
use C4::Biblio;
26
27
use constant MAX_MATCHES => 99999; #NOTE: This is an arbitrary value. We want to get all matches.
28
29
=head1 NAME
30
31
Koha::OAI::Harvester::Import::MARCXML
32
33
=head1 SYNOPSIS
34
35
    use Koha::OAI::Harvester::Import::MARCXML;
36
    my $marcxml = eval { Koha::OAI::Harvester::Import::MARCXML->new({ dom => $results, }) };
37
38
=head1 METHODS
39
40
=head2 new
41
42
    Create object
43
44
=cut
45
46
sub new {
47
    my ($class, $args) = @_;
48
    $args = {} unless defined $args;
49
    if ( (! $args->{dom}) && (! $args->{marc_record}) ){
50
        die "You must provide either a dom or marc_record argument to this constructor";
51
    }
52
    if ( $args->{dom} && ( ! $args->{marc_record} ) ){
53
        my $dom = $args->{dom};
54
        my $xml = $dom->toString(2);
55
        my $marcflavour = C4::Context->preference('marcflavour') || 'MARC21';
56
        my $marc_record = eval {MARC::Record::new_from_xml( $xml, "utf8", $marcflavour)};
57
        if ($@){
58
            die "Unable to create MARC::Record object";
59
        }
60
        if ($marc_record){
61
            $args->{marc_record} = $marc_record;
62
        }
63
    }
64
    return bless ($args, $class);
65
}
66
67
=head2 import_record
68
69
    Import a record into Koha
70
71
=cut
72
73
sub import_record {
74
    my ($self,$args) = @_;
75
    my $framework = $args->{framework};
76
    my $record_type = $args->{record_type};
77
    my $matcher = $args->{matcher};
78
    my $koha_id = $args->{koha_id};
79
80
    my $action = "error";
81
82
    #Try to find a matching Koha MARCXML record via Zebra
83
    if (! $koha_id && $matcher){
84
        my $matched_id = $self->_try_matcher({
85
            matcher => $matcher,
86
        });
87
        if ($matched_id){
88
            $koha_id = $matched_id;
89
        }
90
    }
91
92
    if ($koha_id){
93
        #Update
94
        ($action) = $self->_mod_koha_record({
95
            record_type => $record_type,
96
            framework => $framework,
97
            koha_id => $koha_id,
98
        });
99
    }
100
    else {
101
        #Add
102
        ($action,$koha_id) = $self->_add_koha_record({
103
            record_type => $record_type,
104
            framework => $framework,
105
        });
106
    }
107
108
    return ($action,$koha_id);
109
}
110
111
sub _try_matcher {
112
    my ($self, $args) = @_;
113
    my $marc_record = $self->{marc_record};
114
    my $matcher = $args->{matcher};
115
    my $matched_id;
116
    my @matches = $matcher->get_matches($marc_record, MAX_MATCHES);
117
    if (@matches){
118
        my $bestrecordmatch = shift @matches;
119
        if ($bestrecordmatch && $bestrecordmatch->{record_id}){
120
            $matched_id = $bestrecordmatch->{record_id};
121
        }
122
    }
123
    return $matched_id;
124
}
125
126
sub _add_koha_record {
127
    my ($self, $args) = @_;
128
    my $marc_record = $self->{marc_record};
129
    my $record_type = $args->{record_type} // "biblio";
130
    my $framework = $args->{framework};
131
    my $koha_id;
132
    my $action = "error";
133
    if ($record_type eq "biblio"){
134
        #NOTE: Strip item fields to prevent any accidentally getting through.
135
        C4::Biblio::_strip_item_fields($marc_record,$framework);
136
        my ($biblionumber,$biblioitemnumber) = C4::Biblio::AddBiblio($marc_record,$framework);
137
        if ($biblionumber){
138
            $action = "added";
139
            $koha_id = $biblionumber;
140
        }
141
    }
142
    return ($action,$koha_id);
143
}
144
145
sub _mod_koha_record {
146
    my ($self, $args) = @_;
147
    my $marc_record = $self->{marc_record};
148
    my $record_type = $args->{record_type} // "biblio";
149
    my $framework = $args->{framework};
150
    my $koha_id = $args->{koha_id};
151
    my $action = "error";
152
    if ($record_type eq "biblio"){
153
        #NOTE: Strip item fields to prevent any accidentally getting through.
154
        C4::Biblio::_strip_item_fields($marc_record,$framework);
155
        my $updated = C4::Biblio::ModBiblio($marc_record, $koha_id, $framework);
156
        if ($updated){
157
            $action = "updated";
158
        }
159
    }
160
    return ($action);
161
}
162
163
1;
(-)a/Koha/OAI/Harvester/Import/Record.pm (+348 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Import::Record;
2
3
# Copyright 2016 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
use XML::LibXML;
23
use XML::LibXSLT;
24
use URI;
25
use File::Basename;
26
27
use C4::Context;
28
use C4::Biblio;
29
30
use Koha::OAI::Harvester::Import::MARCXML;
31
use Koha::OAI::Harvester::Biblios;
32
use Koha::OAI::Harvester::History;
33
34
=head1 NAME
35
36
Koha::OAI::Harvester::Import::Record
37
38
=head1 SYNOPSIS
39
40
    use Koha::OAI::Harvester::Import::Record;
41
    my $oai_record = Koha::OAI::Harvester::Import::Record->new({
42
        doc => $dom,
43
        repository => $repository,
44
    });
45
46
=head1 METHODS
47
48
=head2 new
49
50
    Create object
51
52
=cut
53
54
sub new {
55
    my ($class, $args) = @_;
56
    $args = {} unless defined $args;
57
58
    die "You must provide a 'doc' argument to the constructor" unless $args->{doc};
59
    die "You must provide a 'repository' argument to the constructor" unless $args->{repository};
60
61
    if (my $doc = $args->{doc}){
62
63
        #Get the root element
64
        my $root = $doc->documentElement;
65
66
        #Register namespaces for searching purposes
67
        my $xpc = XML::LibXML::XPathContext->new();
68
        $xpc->registerNs('oai','http://www.openarchives.org/OAI/2.0/');
69
70
        my $xpath_identifier = XML::LibXML::XPathExpression->new("oai:header/oai:identifier");
71
        my $identifier = $xpc->findnodes($xpath_identifier,$root)->shift;
72
        $args->{header_identifier} = $identifier->textContent;
73
74
        my $xpath_datestamp = XML::LibXML::XPathExpression->new("oai:header/oai:datestamp");
75
        my $datestamp = $xpc->findnodes($xpath_datestamp,$root)->shift;
76
        $args->{header_datestamp} = $datestamp->textContent;
77
78
        my $xpath_status = XML::LibXML::XPathExpression->new(q{oai:header/@status});
79
        my $status_node = $xpc->findnodes($xpath_status,$root)->shift;
80
        $args->{header_status} = $status_node ? $status_node->textContent : "";
81
    }
82
83
    return bless ($args, $class);
84
}
85
86
=head2 is_deleted_upstream
87
88
    Returns true if OAI-PMH record is deleted upstream
89
90
=cut
91
92
sub is_deleted_upstream {
93
    my ($self, $args) = @_;
94
    if ($self->{header_status}){
95
        if ($self->{header_status} eq "deleted"){
96
            return 1;
97
        }
98
    }
99
    return 0;
100
}
101
102
=head2 set_filter
103
104
    $self->set_filter("/path/to/filter.xsl");
105
106
    Set a XSLT to use to filter records on import. This
107
    takes a full filepath as an argument.
108
109
=cut
110
111
sub set_filter {
112
    my ($self, $filter_definition) = @_;
113
114
    #Source a default XSLT to use for filtering
115
    my $htdocs  = C4::Context->config('intrahtdocs');
116
    my $theme   = C4::Context->preference("template");
117
    $self->{filter} = "$htdocs/$theme/en/xslt/StripOAIPMH.xsl";
118
    $self->{filter_type} = "xslt";
119
120
    if ($filter_definition && $filter_definition ne "default"){
121
        my ($filter_type, $filter) = $self->_parse_filter($filter_definition);
122
        if ($filter_type eq "xslt"){
123
            if (  -f $filter ){
124
                $self->{filter} = $filter;
125
                $self->{filter_type} = "xslt";
126
            }
127
        }
128
    }
129
}
130
131
sub _parse_filter {
132
    my ($self,$filter_definition) = @_;
133
    my ($type,$filter);
134
    my $filter_uri = URI->new($filter_definition);
135
    if ($filter_uri){
136
        my $scheme = $filter_uri->scheme;
137
        if ( ($scheme && $scheme eq "file") || ! $scheme ){
138
            my $path = $filter_uri->path;
139
            #Filters may theoretically be .xsl or .pm files
140
            my($filename, $dirs, $suffix) = fileparse($path,(".xsl",".pm"));
141
            if ($suffix){
142
                if ( $suffix eq ".xsl"){
143
                    $type = "xslt";
144
                    $filter = $path;
145
                }
146
            }
147
        }
148
    }
149
    return ($type,$filter);
150
}
151
152
=head2 filter
153
154
    Filters the OAI-PMH record using a filter
155
156
=cut
157
158
sub filter {
159
    my ($self) = @_;
160
    my $filtered = 0;
161
    my $doc = $self->{doc};
162
    my $filter = $self->{filter};
163
    my $filter_type = $self->{filter_type};
164
    if ($doc){
165
        if ($filter && -f $filter){
166
            if ($filter_type){
167
                if ( $filter_type eq 'xslt' ){
168
                    my $xslt = XML::LibXSLT->new();
169
                    my $style_doc = XML::LibXML->load_xml(location => $filter);
170
                    my $stylesheet = $xslt->parse_stylesheet($style_doc);
171
                    if ($stylesheet){
172
                        my $results = $stylesheet->transform($doc);
173
                        if ($results){
174
                            my $root = $results->documentElement;
175
                            if ($root){
176
                                my $namespace = $root->namespaceURI;
177
                                if ($namespace eq "http://www.loc.gov/MARC21/slim"){
178
                                    #NOTE: Both MARC21 and UNIMARC should be covered by this namespace
179
                                    my $marcxml = eval { Koha::OAI::Harvester::Import::MARCXML->new({ dom => $results, }) };
180
                                    if ($@){
181
                                        warn "Error Koha::OAI::Harvester::Import::MARCXML: $@";
182
                                        return;
183
                                    } else {
184
                                        return $marcxml;
185
                                    }
186
                                }
187
                            }
188
                        }
189
                    }
190
                }
191
            }
192
        }
193
    }
194
    return;
195
}
196
197
sub _find_koha_link {
198
    my ($self, $args) = @_;
199
    my $record_type = $args->{record_type} // "biblio";
200
    my $link_id;
201
    if ($record_type eq "biblio"){
202
        my $link = Koha::OAI::Harvester::Biblios->new->find(
203
            {
204
                oai_repository => $self->{repository},
205
                oai_identifier => $self->{header_identifier},
206
            },
207
            { key => "oai_record",}
208
        );
209
        if ($link && $link->biblionumber){
210
            $link_id = $link->biblionumber;
211
        }
212
    }
213
    return $link_id;
214
}
215
216
=head2 import_record
217
218
    my ($action,$record_id) = $oai_record->import_record({
219
        filter => $filter,
220
        framework => $framework,
221
        record_type => $record_type,
222
        matcher => $matcher,
223
    });
224
225
    $action eq "added" || "updated" || "deleted" || "not_found" || "error"
226
227
=cut
228
229
sub import_record {
230
    my ($self, $args) = @_;
231
    my $filter = $args->{filter} || 'default';
232
    my $framework = $args->{framework} || '';
233
    my $record_type = $args->{record_type} || 'biblio';
234
    my $matcher = $args->{matcher};
235
236
    my $action = "error";
237
238
    #Find linkage between OAI-PMH repository-identifier and Koha record id
239
    my $linked_id = $self->_find_koha_link({
240
        record_type => $record_type,
241
    });
242
243
    if ($self->is_deleted_upstream){
244
        #NOTE: If a record is deleted upstream, it will not contain a metadata element
245
        if ($linked_id){
246
            $action = $self->delete_koha_record({
247
                record_id => $linked_id,
248
                record_type => $record_type,
249
            });
250
        }
251
        else {
252
            $action = "not_found";
253
            #NOTE: If there's no OAI-PMH repository-identifier pair in the database,
254
            #then there's no perfect way to find a linked record to delete.
255
        }
256
    }
257
    else {
258
        $self->set_filter($filter);
259
260
261
        my $import_record = $self->filter();
262
263
        if ($import_record){
264
            ($action,$linked_id) = $import_record->import_record({
265
                framework => $framework,
266
                record_type => $record_type,
267
                matcher => $matcher,
268
                koha_id => $linked_id,
269
            });
270
271
            if ($linked_id){
272
                #Link Koha record ID to OAI-PMH details for this record type,
273
                #if linkage doesn't already exist.
274
                $self->link_koha_record({
275
                    record_type => $record_type,
276
                    koha_id => $linked_id,
277
                });
278
            }
279
        }
280
    }
281
282
    #Log record details to database
283
    Koha::OAI::Harvester::History->new({
284
        header_identifier => $self->{header_identifier},
285
        header_datestamp => $self->{header_datestamp},
286
        header_status => $self->{header_status},
287
        record => $self->{doc}->toString(1),
288
        repository => $self->{repository},
289
        status => $action,
290
        filter => $filter,
291
        framework => $framework,
292
        record_type => $record_type,
293
        matcher_code => $matcher ? $matcher->code : undef,
294
    })->store();
295
296
    return ($action,$linked_id);
297
}
298
299
=head2 link_koha_record
300
301
    Link an OAI-PMH record with a Koha record using
302
    the OAI-PMH repository and OAI-PMH identifier
303
304
=cut
305
306
sub link_koha_record {
307
    my ($self, $args) = @_;
308
    my $record_type = $args->{record_type} // "biblio";
309
    my $koha_id = $args->{koha_id};
310
    if ($koha_id){
311
        if ($record_type eq "biblio"){
312
            my $import_oai_biblio = Koha::OAI::Harvester::Biblios->new->find_or_create({
313
                oai_repository => $self->{repository},
314
                oai_identifier => $self->{header_identifier},
315
                biblionumber => $koha_id,
316
            });
317
            if ( ! $import_oai_biblio->in_storage ){
318
                $import_oai_biblio->insert;
319
            }
320
        }
321
    }
322
}
323
324
=head2 delete_koha_record
325
326
    Delete a Koha record
327
328
=cut
329
330
sub delete_koha_record {
331
    my ($self, $args) = @_;
332
    my $record_type = $args->{record_type} // "biblio";
333
    my $record_id = $args->{record_id};
334
335
    my $action = "error";
336
337
    if ($record_type eq "biblio"){
338
        my $error = C4::Biblio::DelBiblio($record_id);
339
        if (!$error){
340
            $action = "deleted";
341
            #NOTE: If there's no error, a cascading database delete should
342
            #automatically remove the link between the Koha biblionumber and OAI-PMH record too
343
        }
344
    }
345
    return $action;
346
}
347
348
1;
(-)a/Koha/OAI/Harvester/ImportQueue.pm (+49 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::ImportQueue;
2
3
# Copyright Prosentient Systems 2018
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use Carp;
23
24
use base qw(Koha::Object);
25
26
=head1 NAME
27
28
Koha::OAI::Harvester::ImportQueue
29
30
=head1 SYNOPSIS
31
32
    use Koha::OAI::Harvester::ImportQueue;
33
    my $request = Koha::OAI::Harvester::ImportQueue->find($id);
34
35
=head1 METHODS
36
37
=cut
38
39
sub _type {
40
    return 'OaiHarvesterImportQueue';
41
}
42
43
=head1 AUTHOR
44
45
David Cook <dcook@prosentient.com.au>
46
47
=cut
48
49
1;
(-)a/Koha/OAI/Harvester/ImportQueues.pm (+60 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::ImportQueues;
2
3
# Copyright Prosentient Systems 2018
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use Carp;
23
24
use Koha::Database;
25
26
use Koha::OAI::Harvester::ImportQueue;
27
28
use base qw(Koha::Objects);
29
30
=head1 NAME
31
32
Koha::OAI::Harvester::ImportQueues
33
34
=head2 METHODS
35
36
=cut
37
38
=head3 _type
39
40
=cut
41
42
sub _type {
43
    return 'OaiHarvesterImportQueue';
44
}
45
46
=head3 object_class
47
48
=cut
49
50
sub object_class {
51
    return 'Koha::OAI::Harvester::ImportQueue';
52
}
53
54
=head1 AUTHOR
55
56
David Cook <dcook@prosentient.com.au>
57
58
=cut
59
60
1;
(-)a/Koha/OAI/Harvester/Listener.pm (+246 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Listener;
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
use POE qw(Wheel::SocketFactory Wheel::ReadWrite);
23
use IO::Socket qw(AF_UNIX);
24
use JSON;
25
use URI;
26
use Encode;
27
28
=head1 NAME
29
30
Koha::OAI::Harvester::Listener
31
32
=head1 SYNOPSIS
33
34
    use Koha::OAI::Harvester::Listener;
35
    my $listener = Koha::OAI::Harvester::Listener->spawn({
36
        logger => $logger,
37
        socket => $socket_addr,
38
    });
39
40
=head1 METHODS
41
42
=head2 new
43
44
    Create object
45
46
=cut
47
48
sub new {
49
    my ($class, $args) = @_;
50
    $args = {} unless defined $args;
51
    return bless ($args, $class);
52
}
53
54
=head2 spawn
55
56
    Create POE session for listener
57
58
=cut
59
60
sub spawn {
61
    my ($class, $args) = @_;
62
    my $self = $class->new($args);
63
    my $socket = $args->{socket};
64
    POE::Session->create(
65
        args => [
66
            $socket,
67
        ],
68
        object_states => [
69
            $self => {
70
                _start => "on_start",
71
                "on_server_success" => "on_server_success",
72
                "on_server_error" => "on_server_error",
73
                "on_client_error" => "on_client_error",
74
                "on_client_input" => "on_client_input",
75
            },
76
        ],
77
    );
78
}
79
80
=head2 on_start
81
82
    Internal method for starting listener
83
84
=cut
85
86
sub on_start {
87
    my ($kernel,$heap,$socket_uri) = @_[KERNEL,HEAP,ARG0];
88
89
    my $uri = URI->new($socket_uri);
90
    if ($uri && $uri->scheme eq 'unix'){
91
        my $socket_path = $uri->path;
92
        unlink $socket_path if -S $socket_path;
93
        $heap->{server} = POE::Wheel::SocketFactory->new(
94
            SocketDomain => AF_UNIX,
95
            BindAddress => $socket_path,
96
            SuccessEvent => "on_server_success",
97
            FailureEvent => "on_server_error",
98
        );
99
100
        #Make the socket writeable to other users like Apache
101
        chmod 0666, $socket_path;
102
    }
103
}
104
105
=head2 on_server_success
106
107
    Internal event handler for successful connection to server
108
109
=cut
110
111
sub on_server_success {
112
    my ($self, $client_socket, $server_wheel_id, $heap, $session) = @_[OBJECT, ARG0, ARG3, HEAP,SESSION];
113
    my $logger = $self->{logger};
114
    my $null_filter = POE::Filter::Line->new(
115
         Literal => chr(0),
116
    );
117
    my $client_wheel = POE::Wheel::ReadWrite->new(
118
        Handle => $client_socket,
119
        InputEvent => "on_client_input",
120
        ErrorEvent => "on_client_error",
121
        InputFilter => $null_filter,
122
        OutputFilter => $null_filter,
123
    );
124
    $heap->{client}->{ $client_wheel->ID() } = $client_wheel;
125
    $logger->info("Connection ".$client_wheel->ID()." started.");
126
    #TODO: Add basic authentication here?
127
    $client_wheel->put("HELLO");
128
}
129
130
=head2 on_server_error
131
132
    Internal event handler for server errors
133
134
=cut
135
136
sub on_server_error {
137
    my ($self, $operation, $errnum, $errstr, $heap, $session) = @_[OBJECT, ARG0, ARG1, ARG2,HEAP, SESSION];
138
    my $logger = $self->{logger};
139
    $logger->error("Server $operation error $errnum: $errstr");
140
    delete $heap->{server};
141
}
142
143
=head2 on_client_error
144
145
    Internal event handler for errors relating to the client connection
146
147
=cut
148
149
sub on_client_error {
150
    my ($self, $wheel_id,$heap,$session) = @_[OBJECT, ARG3,HEAP,SESSION];
151
    my $logger = $self->{logger};
152
    $logger->info("Connection $wheel_id failed or ended.");
153
    delete $heap->{client}->{$wheel_id};
154
}
155
156
=head2 on_client_input
157
158
    Internal event handler for input from clients
159
160
=cut
161
162
sub on_client_input {
163
    my ($self, $input, $wheel_id, $session, $kernel, $heap) = @_[OBJECT, ARG0, ARG1, SESSION, KERNEL, HEAP];
164
    $input = Encode::decode("UTF-8",$input);
165
    my $logger = $self->{logger};
166
    $logger->debug("Server input: $input");
167
    my $server_response = { msg => "fail"};
168
    eval {
169
        my $json_input = from_json($input);
170
        my $command = $json_input->{command};
171
        my $body = $json_input->{body};
172
        if ($command){
173
            if ($command eq "create"){
174
                my $task = $body->{task};
175
                if ($task){
176
                    my $is_created = $kernel->call("harvester","create_task",$task);
177
                    if ($is_created){
178
                        $server_response->{msg} = "success";
179
                    }
180
                }
181
            }
182
            elsif ($command eq "start"){
183
                my $task = $body->{task};
184
                if ($task){
185
                    my $uuid = $task->{uuid};
186
                    #Fetch from memory now...
187
                    my $is_started = $kernel->call("harvester","start_task", $uuid);
188
                    if ($is_started){
189
                        $server_response->{msg} = "success";
190
                    }
191
                }
192
            }
193
            elsif ($command eq "stop"){
194
                my $task = $body->{task};
195
                if ($task){
196
                    if ($task->{uuid}){
197
                        my $is_stopped = $kernel->call("harvester","stop_task",$task->{uuid});
198
                        if ($is_stopped){
199
                            $server_response->{msg} = "success";
200
                        }
201
                    }
202
                }
203
            }
204
            elsif ($command eq "delete"){
205
                my $task = $body->{task};
206
                if ($task){
207
                    if ($task->{uuid}){
208
                        my $is_deleted = $kernel->call("harvester","delete_task",$task->{uuid});
209
                        if ($is_deleted){
210
                            $server_response->{msg} = "success";
211
                        }
212
                    }
213
                }
214
            }
215
            elsif ($command eq "list"){
216
                my $tasks = $kernel->call("harvester","list_tasks");
217
                if ($tasks){
218
                    $server_response->{msg} = "success";
219
                    $server_response->{data} = $tasks;
220
                }
221
            }
222
        }
223
    };
224
    if ($@){
225
        #NOTE: An error most likely means that something other than a valid JSON string was received
226
        $logger->error($@);
227
    }
228
229
    if ($server_response){
230
        eval {
231
            my $client = $heap->{client}->{$wheel_id};
232
            my $json_message = to_json($server_response, { pretty => 1 });
233
            if ($json_message){
234
                $logger->debug("Server output: $json_message");
235
                $json_message = Encode::encode("UTF-8",$json_message);
236
                $client->put($json_message);
237
            }
238
        };
239
        if ($@){
240
            #NOTE: An error means our response couldn't be serialised as JSON
241
            $logger->error($@);
242
        }
243
    }
244
}
245
246
1;
(-)a/Koha/OAI/Harvester/Request.pm (+187 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Request;
2
3
# Copyright Prosentient Systems 2017
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use Carp;
23
24
use base qw(Koha::Object);
25
26
#For validation
27
use URI;
28
use HTTP::OAI;
29
30
=head1 NAME
31
32
Koha::OAI::Harvester::Request
33
34
=head1 SYNOPSIS
35
36
    use Koha::OAI::Harvester::Request;
37
    my $request = Koha::OAI::Harvester::Requests->find($id);
38
39
=head1 METHODS
40
41
=cut
42
43
sub _type {
44
    return 'OaiHarvesterRequest';
45
}
46
47
=head2 validate
48
49
    Method to validate the parameters of an OAI-PMH request
50
51
=cut
52
53
sub validate {
54
    my ($self) = @_;
55
    my $errors = {};
56
57
    #Step one: validate URL
58
    my $uri = URI->new($self->http_url);
59
    if ( $uri && $uri->scheme && ($uri->scheme eq "http" || $uri->scheme eq "https") ){
60
61
        #Step two: validate access and authorization to URL
62
        my $harvester = $self->_harvester();
63
        my $identify = $harvester->Identify;
64
        if ($identify->is_success){
65
66
            #Step three: validate OAI-PMH parameters
67
68
            #Test Set
69
            my $set = $self->oai_set;
70
            if ($set){
71
                my $set_response = $harvester->ListSets();
72
                my @server_sets = $set_response->set;
73
                if ( ! grep {$_->setSpec eq $set} @server_sets ){
74
                    $errors->{oai_set}->{unavailable} = 1;
75
                }
76
            }
77
78
            #Test Metadata Prefix
79
            my $metadataPrefix = $self->oai_metadataPrefix;
80
            if ($metadataPrefix){
81
                my $metadata_response = $harvester->ListMetadataFormats();
82
                my @server_formats = $metadata_response->metadataFormat;
83
                if ( ! grep { $_->metadataPrefix eq $metadataPrefix } @server_formats ){
84
                    $errors->{oai_metadataPrefix}->{unavailable} = 1;
85
                }
86
            }
87
            else {
88
                $errors->{oai_metadataPrefix}->{missing} = 1;
89
            }
90
91
            #Test Granularity and Timestamps
92
            my $server_granularity = $identify->granularity;
93
            my $from = $self->oai_from;
94
            my $until = $self->oai_until;
95
            if ($from || $until){
96
                my ($from_granularity,$until_granularity);
97
                if ($from){
98
                    $from_granularity = _determine_granularity($from);
99
                    if ($from_granularity eq "YYYY-MM-DDThh:mm:ssZ"){
100
                        $errors->{oai_from}->{unavailable} = 1 if $server_granularity ne $from_granularity;
101
                    } elsif ($from_granularity eq "failed"){
102
                        $errors->{oai_from}->{malformed} = 1;
103
                    }
104
                }
105
                if ($until){
106
                    $until_granularity = _determine_granularity($until);
107
                    if ($until_granularity eq "YYYY-MM-DDThh:mm:ssZ"){
108
                        $errors->{oai_until}->{unavailable} = 1 if $server_granularity ne $until_granularity;
109
                    } elsif ($until_granularity eq "failed"){
110
                        $errors->{oai_until}->{malformed} = 1;
111
                    }
112
                }
113
                if ($from && $until){
114
                    if ($from_granularity ne $until_granularity){
115
                        $errors->{oai}->{granularity_mismatch} = 1;
116
                    }
117
                }
118
            }
119
120
            #Test if identifier is provided when using GetRecord
121
            my $verb = $self->oai_verb;
122
            if ($verb && $verb eq "GetRecord"){
123
                my $identifier = $self->oai_identifier;
124
                if (! $identifier){
125
                    $errors->{oai_identifier}->{missing} = 1;
126
                }
127
            }
128
        }
129
        elsif ($identify->is_error){
130
            foreach my $error ($identify->errors){
131
                if ($error->code =~ /^404$/){
132
                    $errors->{http}->{404} = 1;
133
                } elsif ($error->code =~ /^401$/){
134
                    $errors->{http}->{401} = 1;
135
                } else {
136
                    $errors->{http}->{generic} = 1;
137
                }
138
            }
139
        }
140
        else {
141
            $errors->{http}->{generic} = 1;
142
        }
143
    } else {
144
        $errors->{http_url}->{malformed} = 1;
145
    }
146
    return $errors;
147
}
148
149
sub _harvester {
150
    my ( $self ) = @_;
151
    my $harvester;
152
    if ($self->http_url){
153
        $harvester = new HTTP::OAI::Harvester( baseURL => $self->http_url );
154
        my $uri = URI->new($self->http_url);
155
        if ($uri->scheme && ($uri->scheme eq 'http' || $uri->scheme eq 'https') ){
156
            my $host = $uri->host;
157
            my $port = $uri->port;
158
            $harvester->credentials($host.":".$port, $self->http_realm, $self->http_username, $self->http_password);
159
        }
160
    }
161
    return $harvester;
162
}
163
164
sub _determine_granularity {
165
    my ($timestamp) = @_;
166
    my $granularity;
167
    if ($timestamp =~ /^(\d{4}-\d{2}-\d{2})(T\d{2}:\d{2}:\d{2}Z)?$/){
168
        if ($1 && $2){
169
            $granularity = "YYYY-MM-DDThh:mm:ssZ";
170
        } elsif ($1 && !$2){
171
            $granularity = "YYYY-MM-DD";
172
        } else {
173
            $granularity = "failed";
174
        }
175
    } else {
176
        $granularity = "failed";
177
    }
178
    return $granularity;
179
}
180
181
=head1 AUTHOR
182
183
David Cook <dcook@prosentient.com.au>
184
185
=cut
186
187
1;
(-)a/Koha/OAI/Harvester/Requests.pm (+62 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Requests;
2
3
# Copyright Prosentient Systems 2017
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use Carp;
23
24
use Koha::Database;
25
26
use Koha::OAI::Harvester::Request;
27
28
use base qw(Koha::Objects);
29
30
=head1 NAME
31
32
Koha::OAI::Harvester::Requests -
33
34
=head1 API
35
36
=head2 Class Methods
37
38
=cut
39
40
=head3 _type
41
42
=cut
43
44
sub _type {
45
    return 'OaiHarvesterRequest';
46
}
47
48
=head3 object_class
49
50
=cut
51
52
sub object_class {
53
    return 'Koha::OAI::Harvester::Request';
54
}
55
56
=head1 AUTHOR
57
58
David Cook <dcook@prosentient.com.au>
59
60
=cut
61
62
1;
(-)a/Koha/OAI/Harvester/Worker.pm (+215 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Worker;
2
3
# Copyright Prosentient Systems 2017
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use POE;
22
use DateTime;
23
use JSON;
24
25
=head1 NAME
26
27
Koha::OAI::Harvester::Worker
28
29
=head1 SYNOPSIS
30
31
    This is just a base class to use when writing workers for
32
    the OAI-PMH harvester.
33
34
=head1 METHODS
35
36
=head2 new
37
38
    Create object
39
40
=cut
41
42
sub new {
43
    my ($class, $args) = @_;
44
    $args = {} unless defined $args;
45
    $args->{type} = "worker" unless $args->{type};
46
    return bless ($args, $class);
47
}
48
49
=head2 run
50
51
    The entry point to getting a worker to process a task
52
53
=cut
54
55
sub run {
56
    my ($self,$args) = @_;
57
    my $postback = $args->{postback};
58
    my $task = $args->{task};
59
60
    POE::Session->create(
61
        object_states => [
62
            $self => {
63
                _start           => "on_start",
64
                got_child_stderr => "on_child_stderr",
65
                got_child_close  => "on_child_close",
66
                got_child_signal => "on_child_signal",
67
                got_child_stdout => "on_child_stdout",
68
                stop_worker      => "stop_worker",
69
                _stop            => "on_stop",
70
            },
71
        ],
72
        args => [
73
            $postback,
74
            $task,
75
        ],
76
    );
77
}
78
79
=head2 stop_worker
80
81
    Internal method for killing this worker's child processes
82
83
=cut
84
85
sub stop_worker {
86
    my ($self,$heap) = @_[OBJECT,HEAP];
87
    if (my $child_processes = $heap->{children_by_pid}){
88
        foreach my $child_pid (keys %$child_processes){
89
            my $child = $child_processes->{$child_pid};
90
            $child->kill();
91
        }
92
    }
93
}
94
95
=head2 on_stop
96
97
    Internal event handler for deregistering the worker session
98
    from the harvester's roster of workers
99
100
=cut
101
102
sub on_stop {
103
    my ($self,$kernel) = @_[OBJECT,KERNEL];
104
105
    #Deregister the worker session from the harvester's roster of workers
106
    $kernel->call("harvester","deregister",$self->{type});
107
}
108
109
=head2 on_child_stdout
110
111
    Internal event handler for reading output from a child process
112
113
=cut
114
115
# Wheel event, including the wheel's ID.
116
sub on_child_stdout {
117
    my ($self, $stdout_line, $wheel_id) = @_[OBJECT, ARG0, ARG1];
118
    my $type = $self->{type};
119
    my $child = $_[HEAP]{children_by_wid}{$wheel_id};
120
    my $logger = $self->{logger};
121
    if ($logger){
122
        $logger->debug("[$type][pid ".$child->PID."][STDOUT] $stdout_line");
123
    }
124
125
    my $postback = $_[HEAP]{postback};
126
    if ($postback){
127
        eval {
128
            my $message = from_json($stdout_line);
129
            if ($message){
130
                $postback->($message);
131
            }
132
        };
133
    }
134
}
135
136
=head2 on_child_stderr
137
138
    Internal event handler for reading errors from a child process
139
140
=cut
141
142
# Wheel event, including the wheel's ID.
143
sub on_child_stderr {
144
    my ($self,$stderr_line, $wheel_id) = @_[OBJECT, ARG0, ARG1];
145
    my $type = $self->{type};
146
    my $child = $_[HEAP]{children_by_wid}{$wheel_id};
147
    my $logger = $self->{logger};
148
    if ($logger){
149
        $logger->debug("[$type][pid ".$child->PID."][STDERR] $stderr_line");
150
    }
151
}
152
153
=head2 on_child_close
154
155
    Internal event handler for when a child closes its pipes
156
157
=cut
158
159
# Wheel event, including the wheel's ID.
160
sub on_child_close {
161
    my ($self,$heap,$wheel_id) = @_[OBJECT,HEAP,ARG0];
162
    my $type = $self->{type};
163
    my $logger = $self->{logger};
164
165
    my $child = delete $heap->{children_by_wid}->{$wheel_id};
166
167
    # May have been reaped by on_child_signal().
168
    unless (defined $child) {
169
        if ($logger){
170
            $logger->debug("[$type][wid $wheel_id] closed all pipes");
171
        }
172
        return;
173
    }
174
    if ($logger){
175
        $logger->debug("[$type][pid ".$child->PID."] closed all pipes");
176
    }
177
    delete $heap->{children_by_pid}->{$child->PID};
178
}
179
180
=head2 on_child_signal
181
182
    Internal event handler for when a child exits
183
184
=cut
185
186
sub on_child_signal {
187
    my ($self,$kernel,$pid,$status) = @_[OBJECT,KERNEL,ARG1,ARG2];
188
    my $type = $self->{type};
189
    my $logger = $self->{logger};
190
    if ($logger){
191
        $logger->debug("[$type][pid $pid] exited with status $status");
192
    }
193
194
    my $child = delete $_[HEAP]{children_by_pid}{$_[ARG1]};
195
196
    # May have been reaped by on_child_close().
197
    return unless defined $child;
198
199
    delete $_[HEAP]{children_by_wid}{$child->ID};
200
201
    #If the child doesn't complete successfully, we lodge an error
202
    #and stop the task.
203
    if ($status != 0){
204
        my $task = $kernel->call("harvester","get_task");
205
        if ($task){
206
            $task->{error} = 1;
207
            my $uuid = $task->{uuid};
208
            if ($uuid){
209
                $kernel->call("harvester","stop_task",$uuid);
210
            }
211
        }
212
    }
213
}
214
215
1;
(-)a/Koha/OAI/Harvester/Worker/Download/Stream.pm (+220 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Worker::Download::Stream;
2
3
# Copyright Prosentient Systems 2017
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use LWP::UserAgent;
22
use UUID;
23
use POE;
24
use JSON;
25
use File::Path qw/make_path/;
26
27
use C4::Context;
28
use Koha::OAI::Harvester::Downloader;
29
use parent 'Koha::OAI::Harvester::Worker';
30
31
=head1 NAME
32
33
Koha::OAI::Harvester::Worker::Download::Stream
34
35
=head1 SYNOPSIS
36
37
    This is a module used by the OAI-PMH harvester internally.
38
39
    As a bare minimum, it must define an "on_start" method.
40
41
=head1 METHODS
42
43
=head2 new
44
45
    Create object
46
47
=cut
48
49
sub new {
50
    my ($class, $args) = @_;
51
    $args = {} unless defined $args;
52
    $args->{type} = "download" unless $args->{type};
53
    return bless ($args, $class);
54
}
55
56
=head2 on_start
57
58
    Internal event handler for starting the processing of a task
59
60
=cut
61
62
sub on_start {
63
    my ($self, $kernel, $heap, $postback,$task,$session) = @_[OBJECT, KERNEL, HEAP, ARG0,ARG1,SESSION];
64
    #Save postback into heap so other event handlers can use it
65
    $heap->{postback} = $postback;
66
67
    my $task_uuid = $task->{uuid};
68
69
    $kernel->sig("cancel" => "stop_worker");
70
    $kernel->call("harvester","register",$self->{type},$task->{uuid});
71
72
    my $child = POE::Wheel::Run->new(
73
        ProgramArgs => [$task],
74
        Program => sub {
75
            my ($args) = @_;
76
            $self->do_work($args);
77
        },
78
        StdoutEvent  => "got_child_stdout",
79
        StderrEvent  => "got_child_stderr",
80
        CloseEvent   => "got_child_close",
81
        NoSetPgrp => 1, #Keep child processes in same group as parent. This is especially useful when using Ctrl+C to kill the whole group.
82
    );
83
84
     $_[KERNEL]->sig_child($child->PID, "got_child_signal");
85
86
    # Wheel events include the wheel's ID.
87
    $_[HEAP]{children_by_wid}{$child->ID} = $child;
88
89
    # Signal events include the process ID.
90
    $_[HEAP]{children_by_pid}{$child->PID} = $child;
91
92
    my $logger = $self->{logger};
93
    if ($logger){
94
        $logger->debug("Child pid ".$child->PID." started as wheel ".$child->ID);
95
    }
96
}
97
98
=head2 do_work
99
100
    Internal method for processing a download task
101
102
=cut
103
104
sub do_work {
105
    my ($self, $task) = @_;
106
    my $batch = ( $self->{batch} && int($self->{batch}) ) ? $self->{batch} : 100;
107
108
    #NOTE: Directory to spool files for processing
109
    my $spooldir = $task->{spooldir};
110
111
    my $task_uuid = $task->{uuid};
112
    my $task_parameters = $task->{parameters};
113
    my $interval = $task->{interval};
114
115
    my $oai_pmh_parameters = $task_parameters->{oai_pmh};
116
    my $import_parameters = $task_parameters->{import};
117
118
    #NOTE: Overwrite the 'from' and 'until' parameters for repeatable tasks
119
    if ( $interval && ! $oai_pmh_parameters->{until} ){
120
        if ($oai_pmh_parameters->{verb} eq "ListRecords"){
121
            #NOTE: 'effective_from' premiers on the first repetition (ie second request)
122
            $oai_pmh_parameters->{from} = $task->{effective_from} if $task->{effective_from};
123
            #NOTE: 'effective_until' appears on the first request
124
            $oai_pmh_parameters->{until} = $task->{effective_until} if $task->{effective_until};
125
        }
126
    }
127
128
    my $oai_downloader = Koha::OAI::Harvester::Downloader->new();
129
    my $url = $oai_downloader->BuildURL($oai_pmh_parameters);
130
131
    my $ua = LWP::UserAgent->new();
132
    #NOTE: setup HTTP Basic Authentication if parameters are supplied
133
    if($url && $url->host && $url->port){
134
        my $http_basic_auth = $task_parameters->{http_basic_auth};
135
        if ($http_basic_auth){
136
            my $username = $http_basic_auth->{username};
137
            my $password = $http_basic_auth->{password};
138
            my $realm = $http_basic_auth->{realm};
139
            $ua->credentials($url->host.":".$url->port, $realm, $username, $password);
140
        }
141
    }
142
143
    #NOTE: Prepare database statement handle
144
    my $dbh = C4::Context->dbh;
145
    my $sql = "insert into oai_harvester_import_queue (uuid,result) VALUES (?,?)";
146
    my $sth = $dbh->prepare($sql);
147
148
    if($url && $ua){
149
        #NOTE: You could define the callbacks as object methods instead... that might be nicer...
150
        #although I suppose it might be a much of a muchness.
151
        eval {
152
            my @filename_cache = ();
153
154
            $oai_downloader->harvest({
155
                user_agent => $ua,
156
                url => $url,
157
                callback => sub {
158
                    my ($args) = @_;
159
160
                    my $repository = $args->{repository};
161
                    my $document = $args->{document};
162
163
                    #If the spooldir has disappeared, re-create it.
164
                    if ( ! -d $spooldir ){
165
                        my $made_spool_directory = make_path($spooldir);
166
                    }
167
                    my ($uuid,$uuid_string);
168
                    UUID::generate($uuid);
169
                    UUID::unparse($uuid, $uuid_string);
170
                    my $file_uuid = $uuid_string;
171
                    my $filename = "$spooldir/$file_uuid";
172
                    my $state = $document->toFile($filename, 2);
173
                    if ($state){
174
                        push(@filename_cache,$filename);
175
                    }
176
177
                    if(scalar @filename_cache == $batch){
178
                        my $result = {
179
                            repository => $repository,
180
                            filenames => \@filename_cache,
181
                            filter => $import_parameters->{filter},
182
                            matcher_code => $import_parameters->{matcher_code},
183
                            frameworkcode => $import_parameters->{frameworkcode},
184
                            record_type => $import_parameters->{record_type},
185
                        };
186
                        eval {
187
                            my $json_result = to_json($result, { pretty => 1 });
188
                            $sth->execute($task_uuid,$json_result);
189
                        };
190
                        @filename_cache = ();
191
                    }
192
                },
193
                complete_callback => sub {
194
                    my ($args) = @_;
195
                    my $repository = $args->{repository};
196
                    if (@filename_cache){
197
                        my $result = {
198
                            repository => "$repository",
199
                            filenames => \@filename_cache,
200
                            filter => $import_parameters->{filter},
201
                            matcher_code => $import_parameters->{matcher_code},
202
                            frameworkcode => $import_parameters->{frameworkcode},
203
                            record_type => $import_parameters->{record_type},
204
                        };
205
                        eval {
206
                            my $json_result = to_json($result, { pretty => 1 });
207
                            $sth->execute($task_uuid,$json_result);
208
                        };
209
                    }
210
211
                },
212
            });
213
        };
214
        if ($@){
215
            die "Error during OAI-PMH download";
216
        }
217
    }
218
}
219
220
1;
(-)a/Koha/OAI/Harvester/Worker/Import.pm (+157 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Worker::Import;
2
3
# Copyright Prosentient Systems 2017
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use POE qw(Wheel::Run);
22
use JSON;
23
use XML::LibXML;
24
25
use C4::Context;
26
use C4::Matcher;
27
use Koha::OAI::Harvester::Import::Record;
28
29
use parent 'Koha::OAI::Harvester::Worker';
30
31
=head1 NAME
32
33
Koha::OAI::Harvester::Worker::Import
34
35
=head1 SYNOPSIS
36
37
    This is a module used by the OAI-PMH harvester internally.
38
39
    As a bare minimum, it must define an "on_start" method.
40
41
=head1 METHODS
42
43
=head2 new
44
45
     Create object
46
47
=cut
48
49
sub new {
50
    my ($class, $args) = @_;
51
    $args = {} unless defined $args;
52
    #NOTE: This type is used for logging and more importantly for registering with the harvester
53
    $args->{type} = "import" unless $args->{type};
54
    return bless ($args, $class);
55
}
56
57
=head2 on_start
58
59
    Internal event handler for starting the processing of a task
60
61
=cut
62
63
sub on_start {
64
    my ($self, $kernel, $heap, $postback,$task) = @_[OBJECT, KERNEL, HEAP, ARG0,ARG1];
65
66
    $kernel->call("harvester","register",$self->{type},$task->{uuid});
67
68
    $kernel->sig(cancel => "stop_worker");
69
70
    my $child = POE::Wheel::Run->new(
71
        ProgramArgs => [ $task ],
72
        Program => sub {
73
            my ($task,$args) = @_;
74
75
            my $debug = $args->{debug} // 0;
76
77
            if ($task){
78
                my $json_result = $task->{result};
79
                my $id = $task->{id};
80
                my $task_uuid = $task->{uuid};
81
                eval {
82
                    my $result = from_json($json_result);
83
                    if ($result){
84
                        my $repository = $result->{repository};
85
                        my $filenames = $result->{filenames};
86
                        my $filter = $result->{filter};
87
                        my $matcher_code = $result->{matcher_code};
88
                        my $frameworkcode = $result->{frameworkcode};
89
                        my $record_type = $result->{record_type};
90
91
                        my $matcher;
92
                        if ($matcher_code){
93
                            my $matcher_id = C4::Matcher::GetMatcherId($matcher_code);
94
                            $matcher = C4::Matcher->fetch($matcher_id);
95
                        }
96
97
                        foreach my $filename (@$filenames){
98
                            if ($filename){
99
                                if (-f $filename){
100
                                    my $dom = XML::LibXML->load_xml(location => $filename, { no_blanks => 1 });
101
                                    if ($dom){
102
                                        my $oai_record = Koha::OAI::Harvester::Import::Record->new({
103
                                            doc => $dom,
104
                                            repository => $repository,
105
                                        });
106
                                        if ($oai_record){
107
                                            my ($action,$linked_id) = $oai_record->import_record({
108
                                                filter => $filter,
109
                                                framework => $frameworkcode,
110
                                                record_type => $record_type,
111
                                                matcher => $matcher,
112
                                            });
113
                                            $debug && print STDOUT qq({ "import_result": { "task_uuid": "$task_uuid", "action": "$action", "filename": "$filename", "koha_id": "$linked_id" } }\n);
114
                                        }
115
                                    }
116
                                    my $unlinked = unlink $filename;
117
                                }
118
                            }
119
                        }
120
                    }
121
                };
122
                if ($@){
123
                    warn $@;
124
                }
125
                #NOTE: Even if the file doesn't exist, we still need to process the queue item.
126
127
                #NOTE: Don't do this via a postback in the parent process, as it's much faster to let the child process handle it.
128
129
                #NOTE: It's vital that files are unlinked before deleting from the database,
130
                #or you could get orphan files if the importer is interrupted.
131
                my $dbh = C4::Context->dbh;
132
                my $sql = "delete from oai_harvester_import_queue where id = ?";
133
                my $sth = $dbh->prepare($sql);
134
                $sth->execute($id);
135
            }
136
        },
137
        StdoutEvent  => "got_child_stdout",
138
        StderrEvent  => "got_child_stderr",
139
        CloseEvent   => "got_child_close",
140
        NoSetPgrp => 1, #Keep child processes in same group as parent. This is especially useful when using Ctrl+C to kill the whole group.
141
    );
142
143
    $_[KERNEL]->sig_child($child->PID, "got_child_signal");
144
145
    # Wheel events include the wheel's ID.
146
    $_[HEAP]{children_by_wid}{$child->ID} = $child;
147
148
    # Signal events include the process ID.
149
    $_[HEAP]{children_by_pid}{$child->PID} = $child;
150
151
    my $logger = $self->{logger};
152
    if ($logger){
153
        $logger->debug("Child pid ".$child->PID." started as wheel ".$child->ID);
154
    }
155
}
156
157
1;
(-)a/Makefile.PL (+14 lines)
Lines 360-365 my $target_map = { Link Here
360
  './skel/var/lib/koha/zebradb/biblios/tmp'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
360
  './skel/var/lib/koha/zebradb/biblios/tmp'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
361
  './skel/var/lock/koha/zebradb/rebuild' => { target => 'ZEBRA_LOCK_DIR', trimdir => 6 },
361
  './skel/var/lock/koha/zebradb/rebuild' => { target => 'ZEBRA_LOCK_DIR', trimdir => 6 },
362
  './skel/var/lib/koha/plugins' => { target => 'PLUGINS_DIR', trimdir => 6 },
362
  './skel/var/lib/koha/plugins' => { target => 'PLUGINS_DIR', trimdir => 6 },
363
  './skel/var/lib/koha/oai-pmh-harvester' => { target => 'OAI_LIB_DIR', trimdir => 6 },
364
  './skel/var/run/koha/oai-pmh-harvester' => { target => 'OAI_RUN_DIR', trimdir => 6 },
365
  './skel/var/spool/koha/oai-pmh-harvester' => { target => 'OAI_SPOOL_DIR', trimdir => 6 },
363
  './sms'                       => 'INTRANET_CGI_DIR',
366
  './sms'                       => 'INTRANET_CGI_DIR',
364
  './suggestion'                => 'INTRANET_CGI_DIR',
367
  './suggestion'                => 'INTRANET_CGI_DIR',
365
  './svc'                       => 'INTRANET_CGI_DIR',
368
  './svc'                       => 'INTRANET_CGI_DIR',
Lines 593-598 my $pl_files = { Link Here
593
         'blib/KOHA_CONF_DIR/koha-conf.xml',
596
         'blib/KOHA_CONF_DIR/koha-conf.xml',
594
         'blib/KOHA_CONF_DIR/koha-httpd.conf',
597
         'blib/KOHA_CONF_DIR/koha-httpd.conf',
595
         'blib/KOHA_CONF_DIR/log4perl.conf',
598
         'blib/KOHA_CONF_DIR/log4perl.conf',
599
         'blib/KOHA_CONF_DIR/oai-pmh-harvester.yaml',
596
         'blib/ZEBRA_CONF_DIR/etc/default.idx',
600
         'blib/ZEBRA_CONF_DIR/etc/default.idx',
597
         'blib/MISC_DIR/koha-install-log'
601
         'blib/MISC_DIR/koha-install-log'
598
         ],
602
         ],
Lines 1354-1359 sub get_target_directories { Link Here
1354
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'koha', 'plugins');
1358
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'koha', 'plugins');
1355
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1359
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1356
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
1360
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
1361
        $dirmap{'OAI_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'oai-pmh-harvester');
1362
        $dirmap{'OAI_LIB_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'oai-pmh-harvester');
1363
        $dirmap{'OAI_SPOOL_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'spool', 'oai-pmh-harvester');
1357
    } elsif ($mode eq 'dev') {
1364
    } elsif ($mode eq 'dev') {
1358
        my $curdir = File::Spec->rel2abs(File::Spec->curdir());
1365
        my $curdir = File::Spec->rel2abs(File::Spec->curdir());
1359
        $dirmap{'API_CGI_DIR'} = File::Spec->catdir($curdir, 'api');
1366
        $dirmap{'API_CGI_DIR'} = File::Spec->catdir($curdir, 'api');
Lines 1389-1394 sub get_target_directories { Link Here
1389
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'plugins');
1396
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'plugins');
1390
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1397
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1391
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
1398
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
1399
        $dirmap{'OAI_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'oai-pmh-harvester');
1400
        $dirmap{'OAI_LIB_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'oai-pmh-harvester');
1401
        $dirmap{'OAI_SPOOL_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'spool', 'oai-pmh-harvester');
1402
1392
    } else {
1403
    } else {
1393
        # mode is standard, i.e., 'fhs'
1404
        # mode is standard, i.e., 'fhs'
1394
        $dirmap{'API_CGI_DIR'} = File::Spec->catdir(@basedir, $package, 'api');
1405
        $dirmap{'API_CGI_DIR'} = File::Spec->catdir(@basedir, $package, 'api');
Lines 1413-1418 sub get_target_directories { Link Here
1413
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'plugins');
1424
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'plugins');
1414
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'zebradb');
1425
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'zebradb');
1415
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'run', $package, 'zebradb');
1426
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'run', $package, 'zebradb');
1427
        $dirmap{'OAI_RUN_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'run', $package, 'oai-pmh-harvester');
1428
        $dirmap{'OAI_LIB_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'oai-pmh-harvester');
1429
        $dirmap{'OAI_SPOOL_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'spool', $package, 'oai-pmh-harvester');
1416
    }
1430
    }
1417
1431
1418
    _get_env_overrides(\%dirmap);
1432
    _get_env_overrides(\%dirmap);
(-)a/admin/columns_settings.yml (+59 lines)
Lines 842-847 modules: Link Here
842
          columnname: spent
842
          columnname: spent
843
843
844
  tools:
844
  tools:
845
    oai-pmh-harvester-dashboard:
846
      saved-table:
847
        -
848
          columnname: name
849
        -
850
          columnname: url
851
        -
852
          columnname: set
853
        -
854
          columnname: from
855
        -
856
          columnname: until
857
        -
858
          columnname: interval
859
860
      submitted-table:
861
        -
862
          columnname: name
863
        -
864
          columnname: url
865
        -
866
          columnname: set
867
        -
868
          columnname: from
869
        -
870
          columnname: until
871
        -
872
          columnname: interval
873
        -
874
          columnname: "effective from"
875
        -
876
          columnname: "effective until"
877
        -
878
          columnname: "pending imports"
879
        -
880
          columnname: status
881
        -
882
          columnname: error
883
884
      history-table:
885
        -
886
          columnname: id
887
        -
888
          columnname: repository
889
        -
890
          columnname: identifier
891
        -
892
          columnname: datestamp
893
        -
894
          columnname: "upstream status"
895
        -
896
          columnname: "import status"
897
        -
898
          columnname: "import timestamp"
899
        -
900
          columnname: "imported record"
901
        -
902
          columnname: "downloaded record"
903
845
    notices:
904
    notices:
846
      lettert:
905
      lettert:
847
        -
906
        -
(-)a/debian/scripts/koha-create (+7 lines)
Lines 113-118 generate_config_file() { Link Here
113
        -e "s/__PLUGINS_DIR__/\/var\/lib\/koha\/$name\/plugins/g" \
113
        -e "s/__PLUGINS_DIR__/\/var\/lib\/koha\/$name\/plugins/g" \
114
        -e "s/__MEMCACHED_NAMESPACE__/$MEMCACHED_NAMESPACE/g" \
114
        -e "s/__MEMCACHED_NAMESPACE__/$MEMCACHED_NAMESPACE/g" \
115
        -e "s/__MEMCACHED_SERVERS__/$MEMCACHED_SERVERS/g" \
115
        -e "s/__MEMCACHED_SERVERS__/$MEMCACHED_SERVERS/g" \
116
        -e "s/__OAI_RUN_DIR__/\/var\/run\/koha\/$name\/oai-pmh-harvester/g" \
117
        -e "s/__OAI_LIB_DIR__/\/var\/lib\/koha\/$name\/oai-pmh-harvester/g" \
118
        -e "s/__OAI_SPOOL_DIR__/\/var\/spool\/koha\/$name\/oai-pmh-harvester/g" \
116
        "/etc/koha/$1" > "$2"
119
        "/etc/koha/$1" > "$2"
117
120
118
}
121
}
Lines 656-661 eof Link Here
656
    generate_config_file zebra.passwd.in \
659
    generate_config_file zebra.passwd.in \
657
        "/etc/koha/sites/$name/zebra.passwd"
660
        "/etc/koha/sites/$name/zebra.passwd"
658
661
662
    # Generate and install OAI-PMH harvester config file
663
    generate_config_file oai-pmh-harvester.yaml.in \
664
        "/etc/koha/sites/$name/oai-pmh-harvester.yaml"
665
659
    # Create a GPG-encrypted file for requesting a DB to be set up.
666
    # Create a GPG-encrypted file for requesting a DB to be set up.
660
    if [ "$op" = request ]
667
    if [ "$op" = request ]
661
    then
668
    then
(-)a/debian/scripts/koha-create-dirs (+3 lines)
Lines 56-66 do Link Here
56
    userdir "$name" "/var/lib/koha/$name/plugins"
56
    userdir "$name" "/var/lib/koha/$name/plugins"
57
    userdir "$name" "/var/lib/koha/$name/uploads"
57
    userdir "$name" "/var/lib/koha/$name/uploads"
58
    userdir "$name" "/var/lib/koha/$name/tmp"
58
    userdir "$name" "/var/lib/koha/$name/tmp"
59
    userdir "$name" "/var/lib/koha/$name/oai-pmh-harvester"
59
    userdir "$name" "/var/lock/koha/$name"
60
    userdir "$name" "/var/lock/koha/$name"
60
    userdir "$name" "/var/lock/koha/$name/authorities"
61
    userdir "$name" "/var/lock/koha/$name/authorities"
61
    userdir "$name" "/var/lock/koha/$name/biblios"
62
    userdir "$name" "/var/lock/koha/$name/biblios"
62
    userdir "$name" "/var/run/koha/$name"
63
    userdir "$name" "/var/run/koha/$name"
63
    userdir "$name" "/var/run/koha/$name/authorities"
64
    userdir "$name" "/var/run/koha/$name/authorities"
64
    userdir "$name" "/var/run/koha/$name/biblios"
65
    userdir "$name" "/var/run/koha/$name/biblios"
66
    userdir "$name" "/var/run/koha/$name/oai-pmh-harvester"
67
    userdir "$name" "/var/spool/koha/$name/oai-pmh-harvester"
65
done
68
done
66
69
(-)a/debian/templates/koha-conf-site.xml.in (+1 lines)
Lines 302-307 __END_SRU_PUBLICSERVER__ Link Here
302
 <zebra_max_record_size>1024</zebra_max_record_size>
302
 <zebra_max_record_size>1024</zebra_max_record_size>
303
 <queryparser_config>/etc/koha/searchengine/queryparser.yaml</queryparser_config>
303
 <queryparser_config>/etc/koha/searchengine/queryparser.yaml</queryparser_config>
304
 <log4perl_conf>__KOHA_CONF_DIR__/log4perl.conf</log4perl_conf>
304
 <log4perl_conf>__KOHA_CONF_DIR__/log4perl.conf</log4perl_conf>
305
 <oai_pmh_harvester_config>/etc/koha/sites/__KOHASITE__/oai-pmh-harvester.yaml</oai_pmh_harvester_config>
305
 <!-- Uncomment/edit next setting if you want to adjust zebra log levels.
306
 <!-- Uncomment/edit next setting if you want to adjust zebra log levels.
306
      Default is: none,fatal,warn.
307
      Default is: none,fatal,warn.
307
      You can also include: debug,log,malloc,all,request.
308
      You can also include: debug,log,malloc,all,request.
(-)a/debian/templates/oai-pmh-harvester-site.yaml.in (+22 lines)
Line 0 Link Here
1
---
2
#Harvester
3
socket: 'unix:__OAI_RUN_DIR__/harvesterd.sock'
4
5
logfile: '__LOG_DIR__/oai-pmh-harvester.log'
6
loglevel: 'WARN'
7
8
pidfile: '__OAI_RUN_DIR__/harvesterd.pid'
9
10
statefile: '__OAI_LIB_DIR__/harvesterd.state'
11
12
spooldir: '__OAI_SPOOL_DIR__'
13
14
#Downloader
15
download_workers: 1
16
download_module: 'Koha::OAI::Harvester::Worker::Download::Stream'
17
download_batch: 100
18
19
#Importer
20
import_workers: 1
21
import_poll: 5
22
import_module: 'Koha::OAI::Harvester::Worker::Import'
(-)a/etc/koha-conf.xml (+1 lines)
Lines 119-124 __PAZPAR2_TOGGLE_XML_POST__ Link Here
119
 <use_zebra_facets>1</use_zebra_facets>
119
 <use_zebra_facets>1</use_zebra_facets>
120
 <queryparser_config>__KOHA_CONF_DIR__/searchengine/queryparser.yaml</queryparser_config>
120
 <queryparser_config>__KOHA_CONF_DIR__/searchengine/queryparser.yaml</queryparser_config>
121
 <log4perl_conf>__KOHA_CONF_DIR__/log4perl.conf</log4perl_conf>
121
 <log4perl_conf>__KOHA_CONF_DIR__/log4perl.conf</log4perl_conf>
122
 <oai_pmh_harvester_config>__KOHA_CONF_DIR__/oai-pmh-harvester.yaml</oai_pmh_harvester_config>
122
 <memcached_servers>__MEMCACHED_SERVERS__</memcached_servers>
123
 <memcached_servers>__MEMCACHED_SERVERS__</memcached_servers>
123
 <memcached_namespace>__MEMCACHED_NAMESPACE__</memcached_namespace>
124
 <memcached_namespace>__MEMCACHED_NAMESPACE__</memcached_namespace>
124
 <template_cache_dir>__TEMPLATE_CACHE_DIR__</template_cache_dir>
125
 <template_cache_dir>__TEMPLATE_CACHE_DIR__</template_cache_dir>
(-)a/etc/oai-pmh-harvester.yaml (+22 lines)
Line 0 Link Here
1
---
2
#Harvester
3
socket: 'unix:__OAI_RUN_DIR__/harvesterd.sock'
4
5
logfile: '__LOG_DIR__/oai-pmh-harvester.log'
6
loglevel: 'WARN'
7
8
pidfile: '__OAI_RUN_DIR__/harvesterd.pid'
9
10
statefile: '__OAI_LIB_DIR__/harvesterd.state'
11
12
spooldir: '__OAI_SPOOL_DIR__'
13
14
#Downloader
15
download_workers: 1
16
download_module: 'Koha::OAI::Harvester::Worker::Download::Stream'
17
download_batch: 100
18
19
#Importer
20
import_workers: 1
21
import_poll: 5
22
import_module: 'Koha::OAI::Harvester::Worker::Import'
(-)a/installer/data/mysql/atomicupdate/bug_10662.sql (+73 lines)
Line 0 Link Here
1
--
2
-- Table structure for table 'oai_harvester_biblios'
3
--
4
5
CREATE TABLE IF NOT EXISTS `oai_harvester_biblios` (
6
  `import_oai_biblio_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
7
  `oai_repository` varchar(255) NOT NULL,
8
  `oai_identifier` varchar(255) DEFAULT NULL,
9
  `biblionumber` int(11) NOT NULL,
10
  PRIMARY KEY (`import_oai_biblio_id`),
11
  UNIQUE KEY `oai_record` (`oai_identifier`,`oai_repository`) USING BTREE,
12
  KEY `FK_import_oai_biblio_1` (`biblionumber`),
13
  CONSTRAINT `FK_import_oai_biblio_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE NO ACTION
14
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
15
16
--
17
-- Table structure for table 'oai_harvester_history'
18
--
19
20
CREATE TABLE IF NOT EXISTS  `oai_harvester_history` (
21
  `import_oai_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
22
  `repository` varchar(255) DEFAULT NULL,
23
  `header_identifier` varchar(255) DEFAULT NULL,
24
  `header_datestamp` datetime NOT NULL,
25
  `header_status` varchar(45) DEFAULT NULL,
26
  `record` longtext NOT NULL,
27
  `upload_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
28
  `status` varchar(45) NOT NULL,
29
  `filter` text NOT NULL,
30
  `framework` varchar(4) NOT NULL,
31
  `record_type` enum('biblio','auth','holdings') NOT NULL,
32
  `matcher_code` varchar(10) DEFAULT NULL,
33
  PRIMARY KEY (`import_oai_id`)
34
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
35
36
--
37
-- Table structure for table 'oai_harvester_import_queue'
38
--
39
40
CREATE TABLE IF NOT EXISTS `oai_harvester_import_queue` (
41
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
42
  `uuid` varchar(45) CHARACTER SET utf8 NOT NULL,
43
  `status` varchar(45) CHARACTER SET utf8 NOT NULL DEFAULT 'new',
44
  `result` text CHARACTER SET utf8 NOT NULL,
45
  `result_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
46
  PRIMARY KEY (`id`)
47
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
48
49
--
50
-- Table structure for table 'oai_harvester_requests'
51
--
52
53
CREATE TABLE IF NOT EXISTS `oai_harvester_requests` (
54
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
55
  `uuid` varchar(45) NOT NULL,
56
  `oai_verb` varchar(45) NOT NULL,
57
  `oai_metadataPrefix` varchar(255) NOT NULL,
58
  `oai_identifier` varchar(255) DEFAULT NULL,
59
  `oai_from` varchar(45) DEFAULT NULL,
60
  `oai_until` varchar(45) DEFAULT NULL,
61
  `oai_set` varchar(255) DEFAULT NULL,
62
  `http_url` varchar(255) DEFAULT NULL,
63
  `http_username` varchar(255) DEFAULT NULL,
64
  `http_password` varchar(255) DEFAULT NULL,
65
  `http_realm` varchar(255) DEFAULT NULL,
66
  `import_filter` varchar(255) NOT NULL,
67
  `import_framework_code` varchar(4) NOT NULL,
68
  `import_record_type` enum('biblio','auth','holdings') NOT NULL,
69
  `import_matcher_code` varchar(10) DEFAULT NULL,
70
  `interval` int(10) unsigned NOT NULL,
71
  `name` varchar(45) NOT NULL,
72
  PRIMARY KEY (`id`)
73
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
(-)a/installer/data/mysql/kohastructure.sql (+78 lines)
Lines 4342-4347 CREATE TABLE IF NOT EXISTS stockrotationitems ( Link Here
4342
      ON UPDATE CASCADE ON DELETE CASCADE
4342
      ON UPDATE CASCADE ON DELETE CASCADE
4343
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
4343
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
4344
4344
4345
--
4346
-- Table structure for table 'oai_harvester_biblios'
4347
--
4348
4349
DROP TABLE IF EXISTS `oai_harvester_biblios`;
4350
CREATE TABLE `oai_harvester_biblios` (
4351
  `import_oai_biblio_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
4352
  `oai_repository` varchar(191) NOT NULL,
4353
  `oai_identifier` varchar(191) DEFAULT NULL,
4354
  `biblionumber` int(11) NOT NULL,
4355
  PRIMARY KEY (`import_oai_biblio_id`),
4356
  UNIQUE KEY `oai_record` (`oai_identifier`,`oai_repository` (191)) USING BTREE,
4357
  KEY `FK_import_oai_biblio_1` (`biblionumber`),
4358
  CONSTRAINT `FK_import_oai_biblio_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE NO ACTION
4359
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
4360
4361
--
4362
-- Table structure for table 'oai_harvester_history'
4363
--
4364
4365
DROP TABLE IF EXISTS `oai_harvester_history`;
4366
CREATE TABLE `oai_harvester_history` (
4367
  `import_oai_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
4368
  `repository` varchar(255) DEFAULT NULL,
4369
  `header_identifier` varchar(255) DEFAULT NULL,
4370
  `header_datestamp` datetime NOT NULL,
4371
  `header_status` varchar(45) DEFAULT NULL,
4372
  `record` longtext NOT NULL,
4373
  `upload_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
4374
  `status` varchar(45) NOT NULL,
4375
  `filter` text NOT NULL,
4376
  `framework` varchar(4) NOT NULL,
4377
  `record_type` enum('biblio','auth','holdings') NOT NULL,
4378
  `matcher_code` varchar(10) DEFAULT NULL,
4379
  PRIMARY KEY (`import_oai_id`)
4380
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
4381
4382
--
4383
-- Table structure for table 'oai_harvester_import_queue'
4384
--
4385
4386
DROP TABLE IF EXISTS `oai_harvester_import_queue`;
4387
CREATE TABLE `oai_harvester_import_queue` (
4388
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
4389
  `uuid` varchar(45) CHARACTER SET utf8 NOT NULL,
4390
  `status` varchar(45) CHARACTER SET utf8 NOT NULL DEFAULT 'new',
4391
  `result` text CHARACTER SET utf8 NOT NULL,
4392
  `result_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
4393
  PRIMARY KEY (`id`)
4394
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
4395
4396
--
4397
-- Table structure for table 'oai_harvester_requests'
4398
--
4399
4400
DROP TABLE IF EXISTS `oai_harvester_requests`;
4401
CREATE TABLE `oai_harvester_requests` (
4402
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
4403
  `uuid` varchar(45) NOT NULL,
4404
  `oai_verb` varchar(45) NOT NULL,
4405
  `oai_metadataPrefix` varchar(255) NOT NULL,
4406
  `oai_identifier` varchar(255) DEFAULT NULL,
4407
  `oai_from` varchar(45) DEFAULT NULL,
4408
  `oai_until` varchar(45) DEFAULT NULL,
4409
  `oai_set` varchar(255) DEFAULT NULL,
4410
  `http_url` varchar(255) DEFAULT NULL,
4411
  `http_username` varchar(255) DEFAULT NULL,
4412
  `http_password` varchar(255) DEFAULT NULL,
4413
  `http_realm` varchar(255) DEFAULT NULL,
4414
  `import_filter` varchar(255) NOT NULL,
4415
  `import_framework_code` varchar(4) NOT NULL,
4416
  `import_record_type` enum('biblio','auth','holdings') NOT NULL,
4417
  `import_matcher_code` varchar(10) DEFAULT NULL,
4418
  `interval` int(10) unsigned NOT NULL,
4419
  `name` varchar(45) NOT NULL,
4420
  PRIMARY KEY (`id`)
4421
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
4422
4345
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
4423
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
4346
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
4424
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
4347
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
4425
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-menu.inc (-16 / +19 lines)
Lines 11-32 Link Here
11
<h5>Patrons and circulation</h5>
11
<h5>Patrons and circulation</h5>
12
<ul>
12
<ul>
13
    [% IF ( CAN_user_tools_manage_patron_lists ) %]
13
    [% IF ( CAN_user_tools_manage_patron_lists ) %]
14
	<li><a href="/cgi-bin/koha/patron_lists/lists.pl">Patron lists</a></li>
14
    <li><a href="/cgi-bin/koha/patron_lists/lists.pl">Patron lists</a></li>
15
    [% END %]
15
    [% END %]
16
    [% IF (CAN_user_clubs) %]
16
    [% IF (CAN_user_clubs) %]
17
        <li><a href="/cgi-bin/koha/clubs/clubs.pl">Patron clubs</a></li>
17
        <li><a href="/cgi-bin/koha/clubs/clubs.pl">Patron clubs</a></li>
18
    [% END %]
18
    [% END %]
19
    [% IF ( CAN_user_tools_moderate_comments ) %]
19
    [% IF ( CAN_user_tools_moderate_comments ) %]
20
	<li><a href="/cgi-bin/koha/reviews/reviewswaiting.pl">Comments</a></li>
20
    <li><a href="/cgi-bin/koha/reviews/reviewswaiting.pl">Comments</a></li>
21
    [% END %]
21
    [% END %]
22
    [% IF ( CAN_user_tools_import_patrons ) %]
22
    [% IF ( CAN_user_tools_import_patrons ) %]
23
	<li><a href="/cgi-bin/koha/tools/import_borrowers.pl">Import patrons</a></li>
23
    <li><a href="/cgi-bin/koha/tools/import_borrowers.pl">Import patrons</a></li>
24
    [% END %]
24
    [% END %]
25
    [% IF ( CAN_user_tools_edit_notices ) %]
25
    [% IF ( CAN_user_tools_edit_notices ) %]
26
    <li><a href="/cgi-bin/koha/tools/letter.pl">Notices &amp; slips</a></li>
26
    <li><a href="/cgi-bin/koha/tools/letter.pl">Notices &amp; slips</a></li>
27
    [% END %]
27
    [% END %]
28
    [% IF ( CAN_user_tools_edit_notice_status_triggers ) %]
28
    [% IF ( CAN_user_tools_edit_notice_status_triggers ) %]
29
	<li><a href="/cgi-bin/koha/tools/overduerules.pl">Overdue notice/status triggers</a></li>
29
    <li><a href="/cgi-bin/koha/tools/overduerules.pl">Overdue notice/status triggers</a></li>
30
    [% END %]
30
    [% END %]
31
    [% IF ( CAN_user_tools_label_creator ) %]
31
    [% IF ( CAN_user_tools_label_creator ) %]
32
    <li><a href="/cgi-bin/koha/patroncards/home.pl">Patron card creator</a></li>
32
    <li><a href="/cgi-bin/koha/patroncards/home.pl">Patron card creator</a></li>
Lines 41-47 Link Here
41
    <li><a href="/cgi-bin/koha/tags/review.pl">Tag moderation</a></li>
41
    <li><a href="/cgi-bin/koha/tags/review.pl">Tag moderation</a></li>
42
    [% END %]
42
    [% END %]
43
    [% IF ( CAN_user_tools_batch_upload_patron_images ) %]
43
    [% IF ( CAN_user_tools_batch_upload_patron_images ) %]
44
	<li><a href="/cgi-bin/koha/tools/picture-upload.pl">Upload patron images</a></li>
44
    <li><a href="/cgi-bin/koha/tools/picture-upload.pl">Upload patron images</a></li>
45
    [% END %]
45
    [% END %]
46
</ul>
46
</ul>
47
[% END %]
47
[% END %]
Lines 54-63 Link Here
54
<h5>Catalog</h5>
54
<h5>Catalog</h5>
55
<ul>
55
<ul>
56
    [% IF ( CAN_user_tools_items_batchdel ) %]
56
    [% IF ( CAN_user_tools_items_batchdel ) %]
57
	<li><a href="/cgi-bin/koha/tools/batchMod.pl?del=1">Batch item deletion</a></li>
57
    <li><a href="/cgi-bin/koha/tools/batchMod.pl?del=1">Batch item deletion</a></li>
58
    [% END %]
58
    [% END %]
59
    [% IF ( CAN_user_tools_items_batchmod ) %]
59
    [% IF ( CAN_user_tools_items_batchmod ) %]
60
	<li><a href="/cgi-bin/koha/tools/batchMod.pl">Batch item modification</a></li>
60
    <li><a href="/cgi-bin/koha/tools/batchMod.pl">Batch item modification</a></li>
61
    [% END %]
61
    [% END %]
62
    [% IF CAN_user_tools_records_batchdel %]
62
    [% IF CAN_user_tools_records_batchdel %]
63
      <li><a href="/cgi-bin/koha/tools/batch_delete_records.pl">Batch record deletion</a></li>
63
      <li><a href="/cgi-bin/koha/tools/batch_delete_records.pl">Batch record deletion</a></li>
Lines 75-82 Link Here
75
        <li><a href="/cgi-bin/koha/tools/inventory.pl">Inventory</a></li>
75
        <li><a href="/cgi-bin/koha/tools/inventory.pl">Inventory</a></li>
76
    [% END %]
76
    [% END %]
77
    [% IF ( CAN_user_tools_label_creator ) %]
77
    [% IF ( CAN_user_tools_label_creator ) %]
78
	<li><a href="/cgi-bin/koha/labels/label-home.pl">Label creator</a></li>
78
    <li><a href="/cgi-bin/koha/labels/label-home.pl">Label creator</a></li>
79
	<li><a href="/cgi-bin/koha/labels/spinelabel-home.pl">Quick spine label creator</a></li>
79
    <li><a href="/cgi-bin/koha/labels/spinelabel-home.pl">Quick spine label creator</a></li>
80
    [% END %]
80
    [% END %]
81
    [% IF ( CAN_user_tools_rotating_collections ) %]
81
    [% IF ( CAN_user_tools_rotating_collections ) %]
82
    <li><a href="/cgi-bin/koha/rotating_collections/rotatingCollections.pl">Rotating collections</a></li>
82
    <li><a href="/cgi-bin/koha/rotating_collections/rotatingCollections.pl">Rotating collections</a></li>
Lines 88-101 Link Here
88
        <li><a href="/cgi-bin/koha/tools/marc_modification_templates.pl">Manage MARC modification templates</a></li>
88
        <li><a href="/cgi-bin/koha/tools/marc_modification_templates.pl">Manage MARC modification templates</a></li>
89
    [% END %]
89
    [% END %]
90
    [% IF ( CAN_user_tools_stage_marc_import ) %]
90
    [% IF ( CAN_user_tools_stage_marc_import ) %]
91
	<li><a href="/cgi-bin/koha/tools/stage-marc-import.pl">Stage MARC for import</a></li>
91
    <li><a href="/cgi-bin/koha/tools/stage-marc-import.pl">Stage MARC for import</a></li>
92
    [% END %]
92
    [% END %]
93
    [% IF ( CAN_user_tools_manage_staged_marc ) %]
93
    [% IF ( CAN_user_tools_manage_staged_marc ) %]
94
	<li><a href="/cgi-bin/koha/tools/manage-marc-import.pl">Staged MARC management</a></li>
94
    <li><a href="/cgi-bin/koha/tools/manage-marc-import.pl">Staged MARC management</a></li>
95
    [% END %]
95
    [% END %]
96
    [% IF ( CAN_user_tools_upload_local_cover_images ) %]
96
    [% IF ( CAN_user_tools_upload_local_cover_images ) %]
97
    <li><a href="/cgi-bin/koha/tools/upload-cover-image.pl">Upload local cover image</a></li>
97
    <li><a href="/cgi-bin/koha/tools/upload-cover-image.pl">Upload local cover image</a></li>
98
    [% END %]
98
    [% END %]
99
    [% IF ( CAN_user_tools_manage_staged_marc ) %]
100
    <li><a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a></li>
101
    [% END %]
99
</ul>
102
</ul>
100
[% END %]
103
[% END %]
101
[% IF ( CAN_user_tools_edit_calendar || CAN_user_tools_manage_csv_profiles || CAN_user_tools_view_system_logs || CAN_user_tools_edit_news
104
[% IF ( CAN_user_tools_edit_calendar || CAN_user_tools_manage_csv_profiles || CAN_user_tools_view_system_logs || CAN_user_tools_edit_news
Lines 104-122 Link Here
104
<h5>Additional tools</h5>
107
<h5>Additional tools</h5>
105
<ul>
108
<ul>
106
    [% IF ( CAN_user_tools_edit_calendar ) %]
109
    [% IF ( CAN_user_tools_edit_calendar ) %]
107
	<li><a href="/cgi-bin/koha/tools/holidays.pl">Calendar</a></li>
110
    <li><a href="/cgi-bin/koha/tools/holidays.pl">Calendar</a></li>
108
    [% END %]
111
    [% END %]
109
    [% IF ( CAN_user_tools_manage_csv_profiles ) %]
112
    [% IF ( CAN_user_tools_manage_csv_profiles ) %]
110
	<li><a href="/cgi-bin/koha/tools/csv-profiles.pl">CSV profiles</a></li>
113
    <li><a href="/cgi-bin/koha/tools/csv-profiles.pl">CSV profiles</a></li>
111
    [% END %]
114
    [% END %]
112
    [% IF ( CAN_user_tools_view_system_logs ) %]
115
    [% IF ( CAN_user_tools_view_system_logs ) %]
113
	<li><a href="/cgi-bin/koha/tools/viewlog.pl">Log viewer</a></li>
116
    <li><a href="/cgi-bin/koha/tools/viewlog.pl">Log viewer</a></li>
114
    [% END %]
117
    [% END %]
115
    [% IF ( CAN_user_tools_edit_news ) %]
118
    [% IF ( CAN_user_tools_edit_news ) %]
116
	<li><a href="/cgi-bin/koha/tools/koha-news.pl">News</a></li>
119
    <li><a href="/cgi-bin/koha/tools/koha-news.pl">News</a></li>
117
    [% END %]
120
    [% END %]
118
    [% IF ( CAN_user_tools_schedule_tasks ) %]
121
    [% IF ( CAN_user_tools_schedule_tasks ) %]
119
	<li><a href="/cgi-bin/koha/tools/scheduler.pl">Task scheduler</a></li>
122
    <li><a href="/cgi-bin/koha/tools/scheduler.pl">Task scheduler</a></li>
120
    [% END %]
123
    [% END %]
121
    [% IF ( CAN_user_tools_edit_quotes ) %]
124
    [% IF ( CAN_user_tools_edit_quotes ) %]
122
       <li><a href="/cgi-bin/koha/tools/quotes.pl">Quote editor</a></li>
125
       <li><a href="/cgi-bin/koha/tools/quotes.pl">Quote editor</a></li>
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/dashboard.tt (+378 lines)
Line 0 Link Here
1
[% USE ColumnsSettings %]
2
[% SET footerjs = 1 %]
3
[% INCLUDE 'doc-head-open.inc' %]
4
<title>Koha &rsaquo; Tools &rsaquo; OAI-PMH harvester</title>
5
[% INCLUDE 'doc-head-close.inc' %]
6
[% dashboard_page = '/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl' %]
7
[% request_page = '/cgi-bin/koha/tools/oai-pmh-harvester/request.pl' %]
8
<style type="text/css">
9
    a.paginate_button {
10
        padding: 2px;
11
    }
12
</style>
13
</head>
14
<body id="tools_oai-pmh-harvester" class="tools">
15
[% INCLUDE 'header.inc' %]
16
[% INCLUDE 'cat-search.inc' %]
17
    <div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> &rsaquo; OAI-PMH harvester</div>
18
    <div id="doc3" class="yui-t2">
19
        <div id="bd">
20
            <div id="yui-main">
21
                <div class="yui-b">
22
                    <h1>OAI-PMH harvester</h1>
23
                    <div id="toolbar" class="btn-toolbar">
24
                        <a id="new-request" class="btn btn-default btn-sm" href="[% request_page | url %]?op=new"><i class="fa fa-plus"></i> New request</a>
25
                    </div>
26
                    [% IF ( harvester.offline ) %]
27
                        <div class="alert">
28
                            <span>OAI-PMH harvester offline</span>
29
                        </div>
30
                    [% END %]
31
                    <div id="dashboard-items" class="toptabs">
32
                        <ul>
33
                            <li>
34
                                <a href="#saved_requests">Saved requests <span id="saved_count">([% saved_requests.size | html %])</span></a>
35
                            </li>
36
                            <li>
37
                                <a href="#submitted_requests">Submitted requests <span id="submitted_count">([% submitted_requests.size | html %])</span></a>
38
                            </li>
39
                            <li>
40
                                <a href="#imports">Import history <span id="import_count">(0)</span></a>
41
                            </li>
42
                        </ul>
43
                        <div id="submitted_requests">
44
                            [% IF ( result.defined("start") ) %]
45
                                <div class="alert">
46
                                    [% IF ( result.start ) %]
47
                                        <span>Start succeeded</span>
48
                                    [% ELSE %]
49
                                        <span>Start failed</span>
50
                                    [% END %]
51
                                </div>
52
                            [% ELSIF ( result.defined("stop") ) %]
53
                                 <div class="alert">
54
                                    [% IF ( result.stop ) %]
55
                                        <span>Stop succeeded</span>
56
                                    [% ELSE %]
57
                                        <span>Stop failed</span>
58
                                    [% END %]
59
                                </div>
60
                            [% ELSIF ( result.defined("delete") ) %]
61
                                 <div class="alert">
62
                                    [% IF ( result.delete ) %]
63
                                        <span>Delete succeeded</span>
64
                                    [% ELSE %]
65
                                        <span>Delete failed</span>
66
                                    [% END %]
67
                                </div>
68
                            [% END %]
69
                            <table id="submitted-table">
70
                                <thead>
71
                                    <tr>
72
                                        <th>Name</th>
73
                                        <th>URL</th>
74
                                        <th>Set</th>
75
                                        <th>From</th>
76
                                        <th>Until</th>
77
                                        <th>Interval</th>
78
                                        <th>Effective from</th>
79
                                        <th>Effective until</th>
80
                                        <th>Pending imports</th>
81
                                        <th>Status</th>
82
                                        <th>Error</th>
83
                                        <th></th>
84
                                    </tr>
85
                                </thead>
86
                                <tbody>
87
                                [% IF ( submitted_requests ) %]
88
                                    [% FOREACH submitted_request IN submitted_requests %]
89
                                        <tr>
90
                                            <td>[% submitted_request.name | html %]</td>
91
                                            <td>[% submitted_request.parameters.oai_pmh.baseURL | html %]</td>
92
                                            <td>[% submitted_request.parameters.oai_pmh.set | html %]</td>
93
                                            <td>[% submitted_request.parameters.oai_pmh.from | html %]</td>
94
                                            <td>[% submitted_request.parameters.oai_pmh.until | html %]</td>
95
                                            <td>[% submitted_request.interval | html %]</td>
96
                                            <td>[% submitted_request.effective_from | html %]</td>
97
                                            <td>[% submitted_request.effective_until | html %]</td>
98
                                            <td>[% submitted_request.pending_imports |html %]</td>
99
                                            <td>
100
                                                [% IF ( submitted_status = submitted_request.status ) %]
101
                                                    [% IF ( submitted_status == "new" ) %]
102
                                                        <span>New</span>
103
                                                    [% ELSIF ( submitted_status == "active" ) %]
104
                                                        <span>Active</span>
105
                                                    [% ELSIF ( submitted_status == "stopped" ) %]
106
                                                        <span>Stopped</span>
107
                                                    [% END %]
108
                                                [% END %]
109
                                            </td>
110
                                            <td>
111
                                                [% IF ( submitted_request.error ) %]
112
                                                    <span>Harvest failure</span>
113
                                                [% END %]
114
                                            </td>
115
                                            <td>
116
                                                <div class="dropdown">
117
                                                    <a class="btn btn-default btn-xs dropdown-toggle" role="button" data-toggle="dropdown" href="#">Actions <span class="caret"></span></a>
118
                                                    <ul class="dropdown-menu pull-right" role="menu">
119
                                                          <li>
120
                                                              <a href="[% dashboard_page | url %]?op=start&uuid=[% submitted_request.uuid | url %]"><i class="fa fa-play"></i> Start</a>
121
                                                          </li>
122
                                                          <li>
123
                                                              <a href="[% dashboard_page | url %]?op=stop&uuid=[% submitted_request.uuid | url %]"><i class="fa fa-stop"></i> Stop</a>
124
                                                          </li>
125
                                                          <li>
126
                                                              <a href="[% dashboard_page | url %]?op=delete&uuid=[% submitted_request.uuid | url %]"><i class="fa fa-trash"></i> Delete</a>
127
                                                          </li>
128
                                                    </ul>
129
                                                </div>
130
                                            </td>
131
                                        </tr>
132
                                    [% END %]
133
                                [% END %]
134
                                </tbody>
135
                            </table>
136
                        </div>
137
                        <div id="saved_requests">
138
                            [% IF ( result.send.defined ) %]
139
                                <div class="alert">
140
                                    [% IF ( result.send ) %]
141
                                        <span>Submit succeeded</span>
142
                                    [% ELSE %]
143
                                        <span>Submit failed</span>
144
                                    [% END %]
145
                                </div>
146
                            [% END %]
147
                            <table id="saved-table">
148
                                <thead>
149
                                    <tr>
150
                                        <th>Name</th>
151
                                        <th>URL</th>
152
                                   <!-- <th>Verb</th>
153
                                        <th>Metadata prefix</th>
154
                                        <th>Identifier</th> -->
155
                                        <th>Set</th>
156
                                        <th>From</th>
157
                                        <th>Until</th>
158
                                        <th>Interval</th>
159
                                   <!-- <th>Filter</th>
160
                                        <th>Framework code</th>
161
                                        <th>Record type</th>
162
                                        <th>Matcher code</th> -->
163
                                        <th></th>
164
                                    </tr>
165
                                </thead>
166
                                <tbody>
167
                                [% IF ( saved_requests ) %]
168
                                    [% FOREACH saved_request IN saved_requests %]
169
                                        <tr>
170
                                            <td>[% saved_request.name | html %]</td>
171
                                            <td>[% saved_request.http_url | html %]</td>
172
                                       <!-- <td>[% saved_request.oai_verb | html %]</td>
173
                                            <td>[% saved_request.oai_metadataPrefix | html %]</td>
174
                                            <td>[% saved_request.oai_identifier | html %]</td> -->
175
                                            <td>[% saved_request.oai_set | html %]</td>
176
                                            <td>[% saved_request.oai_from | html %]</td>
177
                                            <td>[% saved_request.oai_until | html %]</td>
178
                                            <td>[% saved_request.interval | html %]</td>
179
                                       <!-- <td>
180
                                                [% IF ( saved_request.import_filter == "default" ) %]
181
                                                    <span>Default</span>
182
                                                [% ELSE %]
183
                                                    [% saved_request.import_filter | html %]
184
                                                [% END %]
185
                                            </td>
186
                                            <td>
187
                                                [% display_framework = "" %]
188
                                                [% FOREACH framework IN frameworks %]
189
                                                    [% IF ( framework.frameworkcode == saved_request.import_framework_code ) %]
190
                                                        [% display_framework = framework %]
191
                                                    [% END %]
192
                                                [% END %]
193
                                                [% IF ( display_framework ) %]
194
                                                    [% display_framework.frameworktext | html %]
195
                                                [% ELSE %]
196
                                                    [% saved_request.import_framework_code | html %]
197
                                                [% END %]
198
                                            </td>
199
                                            <td>
200
                                                [% IF ( saved_request.import_record_type == "biblio" ) %]
201
                                                    <span>Bibliographic</span>
202
                                                [% ELSE %]
203
                                                    [% saved_request.import_record_type | html %]
204
                                                [% END %]
205
                                            </td>
206
                                            <td>
207
                                                [% display_matcher = "" %]
208
                                                [% FOREACH matcher IN matchers %]
209
                                                    [% IF ( matcher.code == saved_request.import_matcher_code ) %]
210
                                                        [% display_matcher = matcher %]
211
                                                    [% END %]
212
                                                [% END %]
213
                                                [% IF ( display_matcher ) %]
214
                                                    [% display_matcher.description | html %]
215
                                                [% ELSE %]
216
                                                    [% saved_request.import_matcher_code | html %]
217
                                                [% END %]
218
                                            </td> -->
219
                                            <td>
220
                                                <div class="dropdown">
221
                                                    <a class="btn btn-default btn-xs dropdown-toggle" role="button" data-toggle="dropdown" href="#">Actions <span class="caret"></span></a>
222
                                                    <ul class="dropdown-menu pull-right" role="menu">
223
                                                          <li>
224
                                                              <a href="[% request_page | url %]?op=edit&id=[% saved_request.id | url %]"><i class="fa fa-pencil"></i> Edit</a>
225
                                                          </li>
226
                                                          <li>
227
                                                              <a href="[% request_page | url %]?op=new&id=[% saved_request.id | url %]"><i class="fa fa-copy"></i> Copy</a>
228
                                                          </li>
229
                                                          <li>
230
                                                              <a href="[% dashboard_page | url %]?op=send&id=[% saved_request.id | url %]"><i class="fa fa-send"></i> Submit</a>
231
                                                          </li>
232
                                                          <li>
233
                                                            <a href="[% request_page | url %]?op=delete&id=[% saved_request.id | url %]"><i class="fa fa-trash"></i> Delete</a>
234
                                                          </li>
235
                                                    </ul>
236
                                                </div>
237
                                            </td>
238
                                        </tr>
239
                                    [% END %]
240
                                [% END %]
241
                                </tbody>
242
                            </table>
243
                        </div>
244
                        <div id="imports">
245
                            <div class="btn-toolbar">
246
                                <button id="refresh-button" type="button" class="btn btn-default btn-sm">Refresh import history</button>
247
                            </div>
248
                            <table id="history-table">
249
                                <thead>
250
                                    <tr>
251
                                        <th>Id</th>
252
                                        <th>Repository</th>
253
                                        <th>Identifier</th>
254
                                        <th>Datestamp</th>
255
                                        <th>Upstream status</th>
256
                                        <th>Import status</th>
257
                                        <th>Import timestamp</th>
258
                                        <th>Imported record</th>
259
                                        <th>Downloaded record</th>
260
                                    </tr>
261
                                </thead>
262
                            </table>
263
264
                        </div>
265
                    </div>
266
                </div>
267
            </div>
268
            <div class="yui-b">
269
                [% INCLUDE 'tools-menu.inc' %]
270
            </div>
271
        </div>
272
273
[% MACRO jsinclude BLOCK %]
274
    [% INCLUDE 'datatables.inc' %]
275
    [% INCLUDE 'columns_settings.inc' %]
276
    <script type="text/javascript">
277
        $(document).ready(function() {
278
            $('#dashboard-items').tabs();
279
            [% IF ( result.start.defined || result.stop.defined || result.delete.defined ) %]
280
                $('#dashboard-items').tabs("option","active",1);
281
            [% END %]
282
283
            var saved_table_columns_settings = [% ColumnsSettings.GetColumns( 'tools','oai-pmh-harvester-dashboard', 'saved-table', 'json' ) | $raw %];
284
            var saved_table = KohaTable("saved-table",{},saved_table_columns_settings);
285
286
            var submitted_table_columns_settings = [% ColumnsSettings.GetColumns( 'tools','oai-pmh-harvester-dashboard', 'submitted-table', 'json' ) | $raw %];
287
            var submitted_table = KohaTable("submitted-table",{},submitted_table_columns_settings);
288
289
            var history_table_columns_settings = [% ColumnsSettings.GetColumns( 'tools','oai-pmh-harvester-dashboard', 'history-table', 'json' ) | $raw %];
290
            var history_table = KohaTable("history-table",{
291
                serverSide: true,
292
                searching: true,
293
                processing: true,
294
                ajax: {
295
                    "url": '/cgi-bin/koha/svc/oai-pmh-harvester/history',
296
                    contentType: 'application/json',
297
                    type: 'POST',
298
                    data: function ( d ) {
299
                        return JSON.stringify( d );
300
                    },
301
                    dataSrc: function (json){
302
                        var recordsTotal = json.recordsTotal;
303
                        if(recordsTotal){
304
                            $('#import_count').text( "("+recordsTotal+")" );
305
                        }
306
                        return json.data;
307
                    }
308
                },
309
                columns: [
310
                    { data: 'import_oai_id', },
311
                    { data: 'repository', },
312
                    { data: 'header_identifier', },
313
                    { data: 'header_datestamp', },
314
                    {
315
                        data: 'header_status', render: function (data, type, full, meta){
316
                            var display_string = _("Active");
317
                            if (data == "deleted"){
318
                                display_string = _("Deleted");
319
                            }
320
                            return display_string;
321
                        }
322
                    },
323
                    {
324
                        data: 'status', render: function (data, type, full, meta){
325
                            var display_string = data;
326
                            if (data == "added"){
327
                                display_string = _("Added");
328
                            }
329
                            else if (data == "error"){
330
                                display_string = _("Error");
331
                            }
332
                            else if (data == "not_found"){
333
                                display_string = _("Not found");
334
                            }
335
                            else if (data == "updated"){
336
                                display_string = _("Updated");
337
                            }
338
                            return display_string;
339
                        }
340
                    },
341
                    { data: 'upload_timestamp', },
342
                    {
343
                        data: 'imported_record', render: function (data, type, full, meta){
344
                            var display_string = data;
345
                            var record_type = (full.record_type) ? full.record_type : 'biblio';
346
                            if (data && record_type == 'biblio'){
347
                                var link_text = _("View biblio record");
348
                                var link = '<a href="/cgi-bin/koha/catalogue/detail.pl?biblionumber='+data+'">'+link_text+'</a>';
349
                                display_string = link;
350
                            }
351
                            return display_string;
352
                        }, searchable: false
353
354
                    },
355
                    {
356
                        data: 'import_oai_id', render: function (data, type, full, meta){
357
                            var display_string = data;
358
                            var link_text = _("View record");
359
                            var link = '<a href="/cgi-bin/koha/tools/oai-pmh-harvester/record.pl?import_oai_id='+data+'">'+link_text+'</a>';
360
                            display_string = link;
361
                            return display_string;
362
                        }, searchable: false
363
                    },
364
                ],
365
                //In theory, it would be nice to sort in descending order, but
366
                //it causes severe paging issues the data is frequently updated.
367
                //Caching would make the paging more stable, but the results would be stale.
368
                order: [
369
                    [ 0, 'asc' ],
370
                ]
371
            },history_table_columns_settings);
372
            $('#refresh-button').click(function(){
373
                history_table.dataTable().api().ajax.reload( null, false );
374
            });
375
        });
376
    </script>
377
[% END %]
378
[% INCLUDE 'intranet-bottom.inc' %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/record.tt (+23 lines)
Line 0 Link Here
1
[% INCLUDE 'doc-head-open.inc' %]
2
<title>Koha &rsaquo; Tools &rsaquo; OAI-PMH harvester &rsaquo; Downloaded record</title>
3
[% INCLUDE 'doc-head-close.inc' %]
4
</head>
5
<body id="tools_oai-pmh-harvester_request" class="tools">
6
[% INCLUDE 'header.inc' %]
7
[% INCLUDE 'cat-search.inc' %]
8
    <div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> &rsaquo; <a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a> &rsaquo; Downloaded record</div>
9
    <div id="doc3" class="yui-t2">
10
        <div id="bd">
11
            <div id="yui-main">
12
                <div class="yui-b">
13
                    <h1>Downloaded record</h1>
14
                    [% IF ( record ) %]
15
                        <div style="white-space:pre">[% record | xml %]</div>
16
                    [% END %]
17
                </div>
18
            </div>
19
            <div class="yui-b">
20
                [% INCLUDE 'tools-menu.inc' %]
21
            </div>
22
        </div>
23
[% INCLUDE 'intranet-bottom.inc' %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/request.tt (+244 lines)
Line 0 Link Here
1
[% USE Asset %]
2
[% USE raw %]
3
[% SET footerjs = 1 %]
4
[% INCLUDE 'doc-head-open.inc' %]
5
<title>Koha &rsaquo; Tools &rsaquo; OAI-PMH harvester &rsaquo; Request</title>
6
[% INCLUDE 'doc-head-close.inc' %]
7
<style type="text/css">
8
    /* Override staff-global.css which hides second, millisecond, and microsecond sliders */
9
    .ui_tpicker_second {
10
        display: block;
11
    }
12
</style>
13
</head>
14
<body id="tools_oai-pmh-harvester_request" class="tools">
15
[% INCLUDE 'header.inc' %]
16
[% INCLUDE 'cat-search.inc' %]
17
    <div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> &rsaquo; <a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a> &rsaquo; Request</div>
18
    <div id="doc3" class="yui-t2">
19
        <div id="bd">
20
            <div id="yui-main">
21
                <div class="yui-b">
22
                    [% IF ( op == "edit" ) %]
23
                        <h1>Edit OAI-PMH request</h1>
24
                    [% ELSE %]
25
                        <h1>New OAI-PMH request</h1>
26
                    [% END %]
27
                    [% IF ( test_parameters ) %]
28
                        [% IF ( errors.size ) %]
29
                            <div class="dialog message"><span class="text-danger">Tests failed!</span></div>
30
                        [% ELSE %]
31
                            <div class="dialog message"><span class="text-success">Tests succeeded!</span></div>
32
                        [% END %]
33
                    [% END %]
34
                    <form action="/cgi-bin/koha/tools/oai-pmh-harvester/request.pl" method="post" name="entry-form">
35
                        [% IF ( op == "new" ) %]
36
                            <input type="hidden" name="op" value="create" />
37
                        [% ELSIF ( op == "edit" ) %]
38
                             <input type="hidden" name="op" value="update" />
39
                        [% ELSE %]
40
                            <input type="hidden" name="op" value="[% op | html %]" />
41
                        [% END %]
42
                        [% IF ( id ) %]
43
                            <input type="hidden" name="id" value="[% id | html %]" />
44
                        [% END %]
45
                        <fieldset class="rows">
46
                            <ol>
47
                                <li>
48
                                    <label for="name">Name:</label>
49
                                    <input type="text" size="30" id="name" name="name" value="[% oai_pmh_request.name | html %]"/>
50
                                    <span class="help">This is just a short name to help in managing requests.</span>
51
                                </li>
52
                            </ol>
53
                        </fieldset>
54
                        <fieldset class="rows">
55
                            <legend>HTTP parameters:</legend>
56
                            <ol>
57
                                <li>
58
                                    <label for="http_url">URL:</label>
59
                                    <input type="text" size="30" id="http_url" name="http_url" value="[% oai_pmh_request.http_url | html %]">
60
                                    [% IF (errors.http_url.malformed) %]<span class="error">[This must be a properly formatted HTTP or HTTPS URL.]</span>[% END %]
61
                                    [% IF (errors.http.404) %]<span class="error">[Cannot find address specified by this URL.]</span>[% END %]
62
                                    [% IF (errors.http.401) %]<span class="error">[Permission denied to access this URL.]</span>[% END %]
63
                                    [% IF (errors.http.generic) %]<span class="error">[Unable to access this URL.]</span>[% END %]
64
                                </li>
65
                            </ol>
66
                            <span class="help">The following parameters are not required by all OAI-PMH repositories, so they may be optional for this task.</span>
67
                            <ol>
68
                                <li>
69
                                    <label for="http_username">Username:</label>
70
                                    <input type="text" size="30" id="http_username" name="http_username" value="[% oai_pmh_request.http_username | html %]">
71
                                </li>
72
                                <li>
73
                                    <label for="http_password">Password:</label>
74
                                    <input type="text" size="30" id="http_password" name="http_password" value="[% oai_pmh_request.http_password | html %]">
75
                                </li>
76
                                <li>
77
                                    <label for="http_realm">Realm:</label>
78
                                    <input type="text" size="30" id="http_realm" name="http_realm" value="[% oai_pmh_request.http_realm | html %]">
79
                                </li>
80
                            </ol>
81
                        </fieldset>
82
                        <fieldset class="rows">
83
                            <legend>OAI-PMH parameters:</legend>
84
                            <ol>
85
                                <li>
86
                                    <label for="oai_verb">Verb:</label>
87
                                    <select id="oai_verb" name="oai_verb">
88
                                        [% IF ( oai_pmh_request.oai_verb == "ListRecords" ) %]
89
                                        <option value="ListRecords" selected="selected">ListRecords</option>
90
                                        [% ELSE %]
91
                                        <option value="ListRecords">ListRecords</option>
92
                                        [% END %]
93
                                        [% IF ( oai_pmh_request.oai_verb == "GetRecord" ) %]
94
                                        <option value="GetRecord" selected="selected">GetRecord</option>
95
                                        [% ELSE %]
96
                                        <option value="GetRecord">GetRecord</option>
97
                                        [% END %]
98
                                    </select>
99
                                </li>
100
                                <li>
101
                                    <label for="oai_metadataPrefix">Metadata prefix:</label>
102
                                    <input type="text" size="30" id="oai_metadataPrefix" name="oai_metadataPrefix" value="[% oai_pmh_request.oai_metadataPrefix | html %]">
103
                                    [% IF (errors.oai_metadataPrefix.unavailable) %]<span class="error">[This metadataPrefix is unavailable from this OAI-PMH provider.]</span>[% END %]
104
                                    [% IF (errors.oai_metadataPrefix.missing) %]<span class="error">[metadataPrefix is a required field for an OAI-PMH request.]</span>[% END %]
105
                                </li>
106
                                <li>
107
                                    <label for="oai_identifier">Identifier:</label>
108
                                    <input type="text" size="30" id="oai_identifier" name="oai_identifier" value="[% oai_pmh_request.oai_identifier | html %]">
109
                                    [% IF (errors.oai_identifier.missing) %]<span class="error">[Identifier is a required field when using GetRecord for an OAI-PMH request.]</span>[% END %]
110
                                </li>
111
                                <li>
112
                                    <label for="oai_set">Set:</label>
113
                                    <input type="text" size="30" id="oai_set" name="oai_set" value="[% oai_pmh_request.oai_set | html %]">
114
                                    [% IF (errors.oai_set.unavailable) %]<span class="error">[This set is unavailable from this OAI-PMH provider.]</span>[% END %]
115
                                </li>
116
                                [% IF (errors.oai.granularity_mismatch) %]<span class="error">[You must specify the same granularity for both From and Until.]</span>[% END %]
117
                                <li>
118
                                    <label for="oai_from">From:</label>
119
                                    <input type="text" size="30" class="datetime_utc" id="oai_from" name="oai_from" value="[% oai_pmh_request.oai_from | html %]">
120
                                    <span class="help">This value will be treated as UTC time. Note that some repositories only support YYYY-MM-DD datestamps.</span>
121
                                    [% IF (errors.oai_from.malformed) %]<span class="error">[This must be in YYYY-MM-DD or YYYY-MM-DDThh:mm:ssZ format.]</span>[% END %]
122
                                    [% IF (errors.oai_from.unavailable) %]<span class="error">[This granularity is unsupported by this OAI-PMH provider.]</span>[% END %]
123
                                </li>
124
                                <li>
125
                                    <label for="oai_until">Until:</label>
126
                                    <input type="text" size="30" class="datetime_utc" id="oai_until" name="oai_until" value="[% oai_pmh_request.oai_until | html %]">
127
                                    <span class="help">This value will be treated as UTC time. Note that some repositories only support YYYY-MM-DD datestamps.</span>
128
                                    [% IF (errors.oai_until.malformed) %]<span class="error">[This must be in YYYY-MM-DD or YYYY-MM-DDThh:mm:ssZ format.]</span>[% END %]
129
                                    [% IF (errors.oai_until.unavailable) %]<span class="error">[This granularity is unsupported by this OAI-PMH provider.]</span>[% END %]
130
                                </li>
131
                            </ol>
132
                        </fieldset>
133
                        <fieldset class="rows">
134
                            <legend>Import parameters:</legend>
135
                            <ol>
136
                                <li>
137
                                    <label for="import_filter">Filter:</label>
138
                                    [% IF ( oai_pmh_request.import_filter == "default" ) %]
139
                                        <input type="text" size="30" id="import_filter" name="import_filter" value="default">
140
                                    [% ELSE %]
141
                                        <input type="text" size="30" id="import_filter" name="import_filter" value="[% oai_pmh_request.import_filter | html %]">
142
                                    [% END %]
143
                                    <span class="help">If no filter is entered, the default filter will be used.</span>
144
                                </li>
145
                                <li>
146
                                    <label for="import_framework_code">Framework code:</label>
147
                                    <select id="import_framework_code" name="import_framework_code">
148
                                        <option value="">Default framework</option>
149
                                        [% FOREACH framework IN frameworks %]
150
                                            [% IF ( oai_pmh_request.import_framework_code == framework.frameworkcode ) %]
151
                                                <option selected="selected" value="[% framework.frameworkcode | html %]">[% framework.frameworktext | html %]</option>
152
                                            [% ELSE %]
153
                                                <option value="[% framework.frameworkcode | html %]">[% framework.frameworktext | html %]</option>
154
                                            [% END %]
155
                                        [% END %]
156
                                    </select>
157
                                </li>
158
                                <li>
159
                                    <label for="import_record_type">Record type:</label>
160
                                    <select id="import_record_type" name="import_record_type">
161
                                        <option value="biblio">Bibliographic</option>
162
                                    </select>
163
                                </li>
164
                                <li>
165
                                    <label for="import_matcher_code">Record matcher:</label>
166
                                    <select id="import_matcher_code" name="import_matcher_code">
167
                                        <option value="">None</option>
168
                                        [% FOREACH matcher IN matchers %]
169
                                            [% IF ( oai_pmh_request.import_matcher_code == matcher.code ) %]
170
                                                <option value="[% matcher.code | html %]" selected="selected">[% matcher.description | html %]</option>
171
                                            [% ELSE %]
172
                                                <option value="[% matcher.code | html %]">[% matcher.description | html %]</option>
173
                                            [% END %]
174
                                        [% END %]
175
                                    </select>
176
                                    <span class="help">See <a href="/cgi-bin/koha/admin/matching-rules.pl">record matching rules</a> to add or edit rules.</span>
177
                                </li>
178
                            </ol>
179
                        </fieldset>
180
                        <fieldset class="rows">
181
                            <legend>Download parameters:</legend>
182
                            <ol>
183
                                <li>
184
                                    <label for="interval">Interval (seconds): </label>
185
                                    <input type="text" id="interval" name="interval" value="[% oai_pmh_request.interval | html %]" size="4">
186
                                    <span class="help">The download request will be repeated in intervals of this many seconds. Enter "0" if you want the task to only happen once.</span>
187
                                </li>
188
                            </ol>
189
                        </fieldset>
190
                        <fieldset class="action">
191
                            <input type="submit" name="test_parameters" value="Test parameters">
192
                            <input type="submit" name="save" value="Save">
193
                            <a class="cancel" href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">Cancel</a>
194
                        </fieldset>
195
                    </form>
196
                </div>
197
            </div>
198
            <div class="yui-b">
199
                [% INCLUDE 'tools-menu.inc' %]
200
            </div>
201
        </div>
202
[% MACRO jsinclude BLOCK %]
203
    [% INCLUDE 'calendar.inc' %]
204
    [% Asset.js("lib/jquery/plugins/jquery-ui-timepicker-addon.min.js") | $raw %]
205
    [% INCLUDE 'timepicker.inc' %]
206
    <script type="text/javascript">
207
        $(document).ready(function() {
208
            toggle_identifier();
209
            $("#oai_verb").on("click",toggle_identifier);
210
            $(".datetime_utc").datetimepicker({
211
                separator: "T",
212
                timeSuffix: 'Z',
213
                dateFormat: "yy-mm-dd",
214
                timeFormat: "HH:mm:ss",
215
                hour: 0,
216
                minute: 0,
217
                second: 0,
218
                showSecond: 1,
219
                // timezone doesn't work with the "Now" button in v1.4.3 although it appears to in v1.6.1
220
                // timezone: '+000'
221
            });
222
        });
223
        function toggle_identifier (){
224
            var verb = $("#oai_verb").find(":selected").val();
225
            var oai_identifier = $("#oai_identifier");
226
            var oai_set = $("#oai_set");
227
            var oai_from = $("#oai_from");
228
            var oai_until = $("#oai_until");
229
            if (verb == 'ListRecords'){
230
                oai_identifier.prop('disabled', true);
231
                oai_set.prop('disabled', false);
232
                oai_from.prop('disabled', false);
233
                oai_until.prop('disabled', false);
234
            }
235
            else if (verb == 'GetRecord'){
236
                oai_identifier.prop('disabled', false);
237
                oai_set.prop('disabled', true);
238
                oai_from.prop('disabled', true);
239
                oai_until.prop('disabled', true);
240
            }
241
        }
242
    </script>
243
[% END %]
244
[% INCLUDE 'intranet-bottom.inc' %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/tools-home.tt (+5 lines)
Lines 217-222 Link Here
217
    <dd>Utility to upload scanned cover images for display in OPAC</dd>
217
    <dd>Utility to upload scanned cover images for display in OPAC</dd>
218
    [% END %]
218
    [% END %]
219
219
220
    [% IF ( CAN_user_tools_manage_staged_marc ) %]
221
    <dt><a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a></dt>
222
    <dd>Harvest (ie download and import) records from remote sources using the OAI-PMH protocol</dd>
223
    [% END %]
224
220
</dl>
225
</dl>
221
</div>
226
</div>
222
227
(-)a/koha-tmpl/intranet-tmpl/prog/en/xslt/StripOAIPMH.xsl (+28 lines)
Line 0 Link Here
1
<?xml version="1.0" encoding="UTF-8"?>
2
<xsl:stylesheet version="1.0"
3
    xmlns:oai="http://www.openarchives.org/OAI/2.0/"
4
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
5
    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>
6
    <!-- NOTE: This XSLT strips the OAI-PMH wrapper from the metadata. -->
7
8
    <!-- Match the root oai:record element -->
9
    <xsl:template match="oai:record">
10
        <!-- Apply templates only to the child metadata element(s) -->
11
        <xsl:apply-templates select="oai:metadata" />
12
    </xsl:template>
13
14
    <!-- Matches an oai:metadata element -->
15
    <xsl:template match="oai:metadata">
16
        <!-- Create a copy of child attributes and nodes -->
17
        <xsl:apply-templates select="@* | node()" />
18
    </xsl:template>
19
20
    <!-- Identity transformation: this template copies attributes and nodes -->
21
    <xsl:template match="@* | node()">
22
        <!-- Create a copy of this attribute or node -->
23
        <xsl:copy>
24
            <!-- Recursively apply this template to the attributes and child nodes of this element -->
25
            <xsl:apply-templates select="@* | node()" />
26
        </xsl:copy>
27
    </xsl:template>
28
</xsl:stylesheet>
(-)a/misc/cronjobs/cleanup_database.pl (-1 / +14 lines)
Lines 26-31 use constant DEFAULT_LOGS_PURGEDAYS => 180; Link Here
26
use constant DEFAULT_SEARCHHISTORY_PURGEDAYS      => 30;
26
use constant DEFAULT_SEARCHHISTORY_PURGEDAYS      => 30;
27
use constant DEFAULT_SHARE_INVITATION_EXPIRY_DAYS => 14;
27
use constant DEFAULT_SHARE_INVITATION_EXPIRY_DAYS => 14;
28
use constant DEFAULT_DEBARMENTS_PURGEDAYS         => 30;
28
use constant DEFAULT_DEBARMENTS_PURGEDAYS         => 30;
29
use constant DEFAULT_IMPORTOAI_PURGEDAYS          => 30;
29
30
30
BEGIN {
31
BEGIN {
31
    # find Koha's Perl modules
32
    # find Koha's Perl modules
Lines 44-50 use Koha::UploadedFiles; Link Here
44
45
45
sub usage {
46
sub usage {
46
    print STDERR <<USAGE;
47
    print STDERR <<USAGE;
47
Usage: $0 [-h|--help] [--sessions] [--sessdays DAYS] [-v|--verbose] [--zebraqueue DAYS] [-m|--mail] [--merged] [--import DAYS] [--logs DAYS] [--searchhistory DAYS] [--restrictions DAYS] [--all-restrictions] [--fees DAYS] [--temp-uploads] [--temp-uploads-days DAYS] [--uploads-missing 0|1 ]
48
Usage: $0 [-h|--help] [--sessions] [--sessdays DAYS] [-v|--verbose] [--zebraqueue DAYS] [-m|--mail] [--merged] [--import DAYS] [--logs DAYS] [--searchhistory DAYS] [--restrictions DAYS] [--all-restrictions] [--fees DAYS] [--temp-uploads] [--temp-uploads-days DAYS] [--uploads-missing 0|1 ] [--importoai DAYS ]
48
49
49
   -h --help          prints this help message, and exits, ignoring all
50
   -h --help          prints this help message, and exits, ignoring all
50
                      other options
51
                      other options
Lines 82-87 Usage: $0 [-h|--help] [--sessions] [--sessdays DAYS] [-v|--verbose] [--zebraqueu Link Here
82
   --temp-uploads-days DAYS Override the corresponding preference value.
83
   --temp-uploads-days DAYS Override the corresponding preference value.
83
   --uploads-missing FLAG Delete upload records for missing files when FLAG is true, count them otherwise
84
   --uploads-missing FLAG Delete upload records for missing files when FLAG is true, count them otherwise
84
   --oauth-tokens     Delete expired OAuth2 tokens
85
   --oauth-tokens     Delete expired OAuth2 tokens
86
   --importoai DAYS    purge OAI-PMH records from import_oai table older than DAYS
87
                       days. Defaults to 30 days if no days specified.
85
USAGE
88
USAGE
86
    exit $_[0];
89
    exit $_[0];
87
}
90
}
Lines 108-113 my $temp_uploads; Link Here
108
my $temp_uploads_days;
111
my $temp_uploads_days;
109
my $uploads_missing;
112
my $uploads_missing;
110
my $oauth_tokens;
113
my $oauth_tokens;
114
my $importoai_days;
111
115
112
GetOptions(
116
GetOptions(
113
    'h|help'            => \$help,
117
    'h|help'            => \$help,
Lines 132-137 GetOptions( Link Here
132
    'temp-uploads-days:i' => \$temp_uploads_days,
136
    'temp-uploads-days:i' => \$temp_uploads_days,
133
    'uploads-missing:i' => \$uploads_missing,
137
    'uploads-missing:i' => \$uploads_missing,
134
    'oauth-tokens'      => \$oauth_tokens,
138
    'oauth-tokens'      => \$oauth_tokens,
139
    'importoai:i'       => \$importoai_days,
135
) || usage(1);
140
) || usage(1);
136
141
137
# Use default values
142
# Use default values
Lines 143-148 $mail = DEFAULT_MAIL_PURGEDAYS if defined($mail) Link Here
143
$pSearchhistory    = DEFAULT_SEARCHHISTORY_PURGEDAYS      if defined($pSearchhistory)    && $pSearchhistory == 0;
148
$pSearchhistory    = DEFAULT_SEARCHHISTORY_PURGEDAYS      if defined($pSearchhistory)    && $pSearchhistory == 0;
144
$pListShareInvites = DEFAULT_SHARE_INVITATION_EXPIRY_DAYS if defined($pListShareInvites) && $pListShareInvites == 0;
149
$pListShareInvites = DEFAULT_SHARE_INVITATION_EXPIRY_DAYS if defined($pListShareInvites) && $pListShareInvites == 0;
145
$pDebarments       = DEFAULT_DEBARMENTS_PURGEDAYS         if defined($pDebarments)       && $pDebarments == 0;
150
$pDebarments       = DEFAULT_DEBARMENTS_PURGEDAYS         if defined($pDebarments)       && $pDebarments == 0;
151
$importoai_days    = DEFAULT_IMPORTOAI_PURGEDAYS          if defined($importoai_days)    && $importoai_days == 0;
146
152
147
if ($help) {
153
if ($help) {
148
    usage(0);
154
    usage(0);
Lines 166-171 unless ( $sessions Link Here
166
    || $temp_uploads
172
    || $temp_uploads
167
    || defined $uploads_missing
173
    || defined $uploads_missing
168
    || $oauth_tokens
174
    || $oauth_tokens
175
    || $importoai_days
169
) {
176
) {
170
    print "You did not specify any cleanup work for the script to do.\n\n";
177
    print "You did not specify any cleanup work for the script to do.\n\n";
171
    usage(1);
178
    usage(1);
Lines 344-349 if ($oauth_tokens) { Link Here
344
    say "Removed $count expired OAuth2 tokens" if $verbose;
351
    say "Removed $count expired OAuth2 tokens" if $verbose;
345
}
352
}
346
353
354
if ($importoai_days){
355
    my $sql = "DELETE FROM import_oai WHERE date(upload_timestamp) < (date_sub(curdate(), INTERVAL ? DAY))";
356
    my $sth = $dbh->prepare($sql);
357
    $sth->execute($importoai_days) or die $dbh->errstr;
358
}
359
347
exit(0);
360
exit(0);
348
361
349
sub RemoveOldSessions {
362
sub RemoveOldSessions {
(-)a/misc/harvesterd.pl (+207 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
use Modern::Perl;
21
use Getopt::Long;
22
use Pod::Usage;
23
use Module::Load;
24
use Log::Log4perl qw(:easy);
25
use POE;
26
use YAML;
27
28
use C4::Context;
29
use Koha::Daemon;
30
use Koha::OAI::Harvester;
31
use Koha::OAI::Harvester::Listener;
32
33
binmode(STDOUT,':encoding(UTF-8)');
34
$|++;
35
36
my $help = 0;
37
my $daemonize = 0;
38
39
my ($socket_addr,$pidfile,$statefile,$spooldir);
40
my ($download_module,$import_module);
41
my ($batch,$download_workers,$import_workers,$import_poll);
42
my ($logfile,$log_level);
43
my $log_levels = {
44
    FATAL => $FATAL,
45
    ERROR => $ERROR,
46
    WARN => $WARN,
47
    INFO => $INFO,
48
    DEBUG => $DEBUG,
49
    TRACE => $TRACE,
50
};
51
52
my $context = C4::Context->new();
53
my $config_filename = $context->{config}->{oai_pmh_harvester_config};
54
if ($config_filename){
55
    my $config = YAML::LoadFile($config_filename);
56
    if ($config){
57
        $socket_addr = $config->{socket};
58
        $pidfile = $config->{pidfile};
59
        $statefile = $config->{statefile};
60
        $spooldir = $config->{spooldir};
61
        $logfile = $config->{logfile};
62
        $log_level = $config->{loglevel};
63
        $download_module = $config->{download_module};
64
        $import_module = $config->{import_module};
65
        $batch = $config->{download_batch};
66
        $download_workers = $config->{download_workers};
67
        $import_workers = $config->{import_workers};
68
        $import_poll = $config->{import_poll};
69
    }
70
}
71
72
GetOptions(
73
    "help|?"            => \$help,
74
    "daemon"            => \$daemonize,
75
    "socket-uri=s"          => \$socket_addr,
76
    "pid-file=s"             => \$pidfile,
77
    "state-file=s"      => \$statefile,
78
    "spool-dir=s"       => \$spooldir,
79
    "log-file=s"             => \$logfile,
80
    "log-level=s"       => \$log_level,
81
    "download-module=s" => \$download_module,
82
    "import-module=s"   => \$import_module,
83
) or pod2usage(2);
84
pod2usage(1) if $help;
85
86
my $level = ( $log_level && $log_levels->{$log_level} ) ? $log_levels->{$log_level} : $log_levels->{WARN};
87
Log::Log4perl->easy_init(
88
    {
89
        level => $level,
90
        file => "STDOUT",
91
        layout   => '[%d{yyyy-MM-dd HH:mm:ss}][%p] %m%n',
92
    }
93
);
94
my $logger = Log::Log4perl->get_logger();
95
96
unless($download_module){
97
    $download_module = "Koha::OAI::Harvester::Worker::Download::Stream";
98
}
99
unless($import_module){
100
    $import_module = "Koha::OAI::Harvester::Worker::Import";
101
}
102
103
foreach my $module ( $download_module, $import_module ){
104
    load $module;
105
}
106
my $downloader = $download_module->new({
107
    logger => $logger,
108
    batch => $batch,
109
});
110
my $importer = $import_module->new({
111
    logger => $logger,
112
});
113
114
my $daemon = Koha::Daemon->new({
115
    pidfile => $pidfile,
116
    logfile => $logfile,
117
    daemonize => $daemonize,
118
});
119
$daemon->run();
120
121
my $harvester = Koha::OAI::Harvester->spawn({
122
    Downloader => $downloader,
123
    DownloaderWorkers => $download_workers,
124
    Importer => $importer,
125
    ImporterWorkers => $import_workers,
126
    ImportQueuePoll => $import_poll,
127
    logger => $logger,
128
    state_file => $statefile,
129
    spooldir => $spooldir,
130
});
131
132
my $listener = Koha::OAI::Harvester::Listener->spawn({
133
    logger => $logger,
134
    socket => $socket_addr,
135
});
136
137
$logger->info("OAI-PMH harvester started.");
138
139
POE::Kernel->run();
140
141
exit;
142
143
=head1 NAME
144
145
harvesterd.pl - a daemon that asynchronously sends OAI-PMH requests and imports OAI-PMH records
146
147
=head1 SYNOPSIS
148
149
KOHA_CONF=/path/to/koha-conf.xml ./harvesterd.pl
150
151
=head1 OPTIONS
152
153
=over 8
154
155
=item B<--help>
156
157
Print a brief help message and exits.
158
159
=item B<--daemon>
160
161
Run program as a daemon (ie fork process, setsid, chdir to root, reset umask,
162
and close STDIN, STDOUT, and STDERR).
163
164
=item B<--log-file>
165
166
Specify a file to which to log STDOUT and STDERR.
167
168
=item B<--pid-file>
169
170
Specify a file to store the process id (this prevents multiple copies of the program
171
from running at the same time).
172
173
=item B<--socket-uri>
174
175
Specify a URI to use for the UNIX socket used to communicate with the daemon.
176
(e.g. unix:/path/to/socket.sock)
177
178
=item B<--state-file>
179
180
Specify a filename to use for storing the harvester's in-memory state.
181
182
In the event that the harvester crashes, it can resume from where it stopped.
183
184
=item B<--spool-dir>
185
186
Specify a directory to store downloaded OAI-PMH records prior to import.
187
188
=item B<--log-level>
189
190
Specify a log level for logging. The logger uses Log4Perl, which provides
191
FATAL, ERROR, WARN, INFO, DEBUG, and TRACE in order of descending priority.
192
193
Defaults to WARN level.
194
195
=item B<--download-module>
196
197
Specify a Perl module to use for downloading records. This is a specialty module,
198
which has particular requirements, so only advanced users should use this option.
199
200
=item B<--import-module>
201
202
Specify a Perl module to use for importing records. This is a specialty module,
203
which has particular requirements, so only advanced users should use this option.
204
205
=back
206
207
=cut
(-)a/rewrite-config.PL (-1 / +4 lines)
Lines 151-157 $prefix = $ENV{'INSTALL_BASE'} || "/usr"; Link Here
151
  "__MEMCACHED_SERVERS__" => "",
151
  "__MEMCACHED_SERVERS__" => "",
152
  "__MEMCACHED_NAMESPACE__" => "",
152
  "__MEMCACHED_NAMESPACE__" => "",
153
  "__FONT_DIR__" => "/usr/share/fonts/truetype/ttf-dejavu",
153
  "__FONT_DIR__" => "/usr/share/fonts/truetype/ttf-dejavu",
154
  "__TEMPLATE_CACHE_DIR__" => "/tmp/koha"
154
  "__TEMPLATE_CACHE_DIR__" => "/tmp/koha",
155
  "__OAI_RUN_DIR__" => "",
156
  "__OAI_LIB_DIR__" => "",
157
  "__OAI_SPOOL_DIR__" => "",
155
);
158
);
156
159
157
# Override configuration from the environment
160
# Override configuration from the environment
(-)a/svc/oai-pmh-harvester/history (+129 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use C4::Auth qw(check_cookie_auth haspermission get_session);
22
use JSON;
23
use Koha::OAI::Harvester::Biblios;
24
use Koha::OAI::Harvester::Histories;
25
26
my $input = new CGI;
27
28
my ( $auth_status, $sessionID ) =
29
  check_cookie_auth( $input->cookie('CGISESSID'));
30
31
if ( $auth_status ne "ok" ) {
32
    exit 0;
33
}
34
35
my $data = {
36
    data => [],
37
    recordsTotal => 0,
38
    recordsFiltered => 0,
39
    draw => undef,
40
};
41
42
my $length = 10;
43
my $start = 0;
44
my @order_by = ();
45
my @search = ();
46
47
if ($input->request_method eq "POST"){
48
    my $postdata = $input->param('POSTDATA');
49
    my $request = from_json($postdata);
50
    $data->{draw} = int( $request->{draw} ) if $request->{draw};
51
    $length = $request->{length} if $request->{length};
52
    $start = $request->{start} if $request->{start};
53
    if (my $search = $request->{search}){
54
        my $value = $search->{value};
55
        if ($value){
56
            foreach my $column (@{$request->{columns}}){
57
                if ($column->{data} && $column->{searchable}){
58
                    my $search_element = {
59
                        $column->{data} => { 'like', "%".$value."%" },
60
                    };
61
                    push(@search,$search_element);
62
                }
63
            }
64
        }
65
    }
66
    if (my $order = $request->{order}){
67
        foreach my $element (@$order){
68
            my $dir = $element->{dir};
69
            my $column_index = $element->{column};
70
            my $column = $request->{columns}->[$column_index];
71
            my $orderable = $column->{orderable};
72
            if ($orderable){
73
                my $column_name = $column->{data};
74
                my $direction;
75
                if ($dir){
76
                    if ($dir eq "asc" || $dir eq "desc"){
77
                        $direction = "-$dir";
78
                    }
79
                }
80
                if ($column_name && $direction){
81
                    my $single_order = {
82
                        $direction => $column_name,
83
                    };
84
                    push(@order_by,$single_order);
85
                }
86
            }
87
        }
88
    }
89
}
90
91
my $page = ( $start / $length ) + 1;
92
my $results = Koha::OAI::Harvester::Histories->new->search(
93
    \@search,
94
    {
95
        page => $page,
96
        rows => $length,
97
        order_by => \@order_by,
98
    },
99
);
100
my $count = Koha::OAI::Harvester::Histories->new->count;
101
my $filtered_count = $results->pager->total_entries;
102
my @rows = ();
103
while (my $obj = $results->next){
104
    my $row = $obj->unblessed;
105
    $row->{imported_record} = '';
106
    if ($row->{record_type} eq "biblio"){
107
        my $harvested_biblio = Koha::OAI::Harvester::Biblios->new->find(
108
                {
109
                    oai_repository => $row->{repository},
110
                    oai_identifier => $row->{header_identifier},
111
                },
112
                { key => "oai_record" },
113
        );
114
        $row->{imported_record} = $harvested_biblio->biblionumber if $harvested_biblio;
115
    }
116
    push(@rows,$row);
117
}
118
if ($count){
119
    $data->{recordsTotal} = $count;
120
    $data->{recordsFiltered} = $filtered_count;
121
    $data->{data} = \@rows if @rows;
122
}
123
124
binmode STDOUT, ":encoding(UTF-8)";
125
print $input->header(
126
    -type => 'application/json',
127
    -charset => 'UTF-8'
128
);
129
print to_json($data, { pretty => 1, });
(-)a/tools/oai-pmh-harvester/dashboard.pl (+134 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
use Modern::Perl;
21
use YAML;
22
23
use C4::Auth;
24
use C4::Context;
25
use C4::Output;
26
use Koha::OAI::Harvester::Client;
27
use Koha::OAI::Harvester::Requests;
28
use Koha::BiblioFrameworks;
29
use Koha::Database;
30
31
my $context = C4::Context->new();
32
my $config_filename = $context->{config}->{oai_pmh_harvester_config};
33
my $client_config = {};
34
if ($config_filename){
35
    my $config = YAML::LoadFile($config_filename);
36
    if ($config && $config->{socket}){
37
        $client_config->{socket_uri} = $config->{socket};
38
    }
39
}
40
41
my $input = new CGI;
42
43
my ($template, $loggedinuser, $cookie) =
44
    get_template_and_user({template_name => "tools/oai-pmh-harvester/dashboard.tt",
45
        query => $input,
46
        type => "intranet",
47
        authnotrequired => 0,
48
        flagsrequired => {tools => 'manage_staged_marc'},
49
    });
50
51
my $op = $input->param('op') // 'list';
52
my $id = $input->param('id');
53
my $uuid = $input->param('uuid');
54
55
my $client = Koha::OAI::Harvester::Client->new($client_config);
56
my $is_connected = $client->connect;
57
58
if ( ($op eq "send") && $id ){
59
    if ($is_connected){
60
        my $request = Koha::OAI::Harvester::Requests->find($id);
61
        if ($request){
62
            my $task = {
63
                name => $request->name,
64
                uuid => $request->uuid,
65
                interval => $request->interval,
66
                parameters => {
67
                    oai_pmh => {
68
                        baseURL => $request->http_url,
69
                        verb => $request->oai_verb,
70
                        metadataPrefix => $request->oai_metadataPrefix,
71
                        identifier => $request->oai_identifier,
72
                        set => $request->oai_set,
73
                        from => $request->oai_from,
74
                        until => $request->oai_until,
75
                    },
76
                    import => {
77
                        filter => $request->import_filter,
78
                        frameworkcode => $request->import_framework_code,
79
                        matcher_code => $request->import_matcher_code,
80
                        record_type => $request->import_record_type,
81
                    },
82
                },
83
            };
84
            if ($request->http_username && $request->http_password && $request->http_realm){
85
                $task->{parameters}->{http_basic_auth} = {
86
                    username => $request->http_username,
87
                    password => $request->http_password,
88
                    realm => $request->http_realm,
89
                };
90
            }
91
            my $is_created = $client->create($task);
92
            $template->{VARS}->{ result }->{ send } = $is_created;
93
        }
94
    }
95
}
96
elsif ( ($op eq "start") && ($uuid) ){
97
    if ($is_connected){
98
        my $is_started = $client->start($uuid);
99
        $template->{VARS}->{ result }->{ start } = $is_started;
100
    }
101
}
102
elsif ( ($op eq "stop") && ($uuid) ){
103
    if ($is_connected){
104
        my $is_stopped = $client->stop($uuid);
105
        $template->{VARS}->{ result }->{ stop } = $is_stopped;
106
    }
107
}
108
elsif ( ($op eq "delete") && ($uuid) ){
109
    if ($is_connected){
110
        my $is_deleted = $client->delete($uuid);
111
        $template->{VARS}->{ result }->{ delete } = $is_deleted;
112
    }
113
}
114
115
my $requests = Koha::OAI::Harvester::Requests->as_list;
116
$template->{VARS}->{ saved_requests } = $requests;
117
118
my $frameworks = Koha::BiblioFrameworks->as_list();
119
$template->{VARS}->{ frameworks } = $frameworks;
120
121
my $schema = Koha::Database->new()->schema();
122
my $matcher_rs = $schema->resultset("MarcMatcher");
123
my @matchers = $matcher_rs->all;
124
$template->{VARS}->{ matchers } = \@matchers;
125
126
if ($is_connected){
127
    my $submitted_requests = $client->list;
128
    $template->{VARS}->{ submitted_requests } = $submitted_requests;
129
}
130
else {
131
    $template->{VARS}->{ harvester }->{ offline } = 1;
132
}
133
134
output_html_with_http_headers($input, $cookie, $template->output);
(-)a/tools/oai-pmh-harvester/record.pl (+47 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
use Modern::Perl;
21
22
use C4::Auth;
23
use C4::Output;
24
use Koha::OAI::Harvester::Histories;
25
26
my $input = new CGI;
27
28
my ($template, $loggedinuser, $cookie) =
29
    get_template_and_user({template_name => "tools/oai-pmh-harvester/record.tt",
30
        query => $input,
31
        type => "intranet",
32
        authnotrequired => 0,
33
        flagsrequired => {tools => 'manage_staged_marc'},
34
    });
35
36
my $import_oai_id = $input->param('import_oai_id');
37
if ($import_oai_id){
38
    my $history = Koha::OAI::Harvester::Histories->new->find($import_oai_id);
39
    if ($history){
40
        my $record = $history->record;
41
        if ($record){
42
            $template->{VARS}->{ record } = $record;
43
        }
44
    }
45
}
46
47
output_html_with_http_headers($input, $cookie, $template->output);
(-)a/tools/oai-pmh-harvester/request.pl (-1 / +142 lines)
Line 0 Link Here
0
- 
1
#!/usr/bin/perl
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
use Modern::Perl;
21
use UUID;
22
23
use C4::Auth;
24
use C4::Output;
25
use Koha::OAI::Harvester::Requests;
26
use Koha::BiblioFrameworks;
27
use Koha::Database;
28
29
my $input = new CGI;
30
31
my ($template, $loggedinuser, $cookie) =
32
    get_template_and_user({template_name => "tools/oai-pmh-harvester/request.tt",
33
        query => $input,
34
        type => "intranet",
35
        authnotrequired => 0,
36
        flagsrequired => {tools => 'manage_staged_marc'},
37
    });
38
39
my $op = $input->param('op');
40
my $id = $input->param('id');
41
42
my @frameworks = Koha::BiblioFrameworks->as_list();
43
$template->{VARS}->{ frameworks } = \@frameworks;
44
45
my $schema = Koha::Database->new()->schema();
46
my $rs = $schema->resultset("MarcMatcher");
47
my @matchers = $rs->all;
48
$template->{VARS}->{ matchers } = \@matchers;
49
50
my $http_url = $input->param('http_url');
51
my $http_username = $input->param('http_username');
52
my $http_password = $input->param('http_password');
53
my $http_realm = $input->param('http_realm');
54
55
my $oai_verb = $input->param('oai_verb');
56
my $oai_metadataPrefix = $input->param('oai_metadataPrefix');
57
my $oai_identifier = $input->param('oai_identifier');
58
my $oai_from = $input->param('oai_from');
59
my $oai_until = $input->param('oai_until');
60
my $oai_set = $input->param('oai_set');
61
62
my $import_filter = $input->param('import_filter') // 'default';
63
my $import_framework_code = $input->param('import_framework_code');
64
my $import_record_type = $input->param('import_record_type');
65
my $import_matcher_code = $input->param('import_matcher_code');
66
67
my $interval = $input->param("interval") ? int ( $input->param("interval") ) : 0;
68
my $name = $input->param("name");
69
70
my $save = $input->param('save');
71
my $test_parameters = $input->param('test_parameters');
72
73
my $request = $id ? Koha::OAI::Harvester::Requests->find($id) : Koha::OAI::Harvester::Request->new();
74
if ($request){
75
    if ($op eq "create" || $op eq "update"){
76
        $request->set({
77
            name => $name,
78
            http_url => $http_url,
79
            http_username => $http_username,
80
            http_password => $http_password,
81
            http_realm => $http_realm,
82
            oai_verb => $oai_verb,
83
            oai_metadataPrefix => $oai_metadataPrefix,
84
            oai_identifier => $oai_identifier,
85
            oai_from => $oai_from,
86
            oai_until => $oai_until,
87
            oai_set => $oai_set,
88
            import_filter => $import_filter,
89
            import_framework_code => $import_framework_code,
90
            import_record_type => $import_record_type,
91
            import_matcher_code => $import_matcher_code,
92
            interval => $interval,
93
        });
94
    }
95
}
96
97
if ($test_parameters){
98
    my $errors = $request->validate();
99
    $template->{VARS}->{ errors } = $errors;
100
    $template->{VARS}->{ test_parameters } = 1;
101
}
102
103
if ($op eq "new"){
104
    #Empty form with some defaults
105
    $request->import_filter("default") unless $request->import_filter;
106
    $request->interval(0) unless $request->interval;
107
}
108
elsif ($op eq "create"){
109
    if ($save){
110
        my ($uuid,$uuid_string);
111
        UUID::generate($uuid);
112
        UUID::unparse($uuid, $uuid_string);
113
        $request->uuid($uuid_string);
114
        $request->store;
115
        print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
116
        exit;
117
    }
118
}
119
elsif ( $op eq "edit"){
120
    $template->{VARS}->{ id } = $id;
121
}
122
elsif ($op eq "update"){
123
    $template->{VARS}->{ id } = $id;
124
    if ($save){
125
        $request->store;
126
        print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
127
        exit;
128
    }
129
}
130
elsif ($op eq "delete"){
131
    if ($request){
132
        $request->delete;
133
        print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
134
    }
135
}
136
else {
137
    print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
138
}
139
$template->{VARS}->{ op } = $op;
140
$template->{VARS}->{ oai_pmh_request } = $request;
141
142
output_html_with_http_headers($input, $cookie, $template->output);

Return to bug 10662