View | Details | Raw Unified | Return to bug 10662
Collapse All | Expand All

(-)a/C4/Installer/PerlDependencies.pm (+10 lines)
Lines 898-903 our $PERL_DEPS = { Link Here
898
        'required' => '1',
898
        'required' => '1',
899
        'min_ver' => '1.017',
899
        'min_ver' => '1.017',
900
    },
900
    },
901
    'POE' => {
902
        'usage' => 'OAI-PMH harvester',
903
        'required' => 1,
904
        'min_ver' => '1.354',
905
    },
906
    'POE::Component::JobQueue' => {
907
        'usage' => 'OAI-PMH harvester',
908
        'required' => 1,
909
        'min_ver' => '0.570',
910
    },
901
};
911
};
902
912
903
1;
913
1;
(-)a/Koha/Daemon.pm (+151 lines)
Line 0 Link Here
1
package Koha::Daemon;
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
use POSIX; #For daemonizing
23
use Fcntl qw(:flock); #For pidfile
24
25
sub new {
26
    my ($class, $args) = @_;
27
    $args = {} unless defined $args;
28
    return bless ($args, $class);
29
}
30
31
#######################################################################
32
#NOTE: On Debian, you can use the daemon binary to make a process into a daemon,
33
# but the following can be used if you don't want to use that program.
34
35
sub daemonize {
36
    my ($self) = @_;
37
38
    my $pid = fork;
39
40
    die "Couldn't fork: $!" unless defined($pid);
41
    if ($pid){
42
        exit; #Parent exit
43
    }
44
45
    #Become a session leader (ie detach program from controlling terminal)
46
    POSIX::setsid() or die "Can't start a new session: $!";
47
48
    #Change to known system directory
49
    chdir('/');
50
51
    #Reset the file creation mask so only the daemon owner can read/write files it creates
52
    umask(066);
53
54
    #Close inherited file handles, so that you can truly run in the background.
55
    open STDIN,  '<', '/dev/null';
56
    open STDOUT, '>', '/dev/null';
57
    open STDERR, '>&STDOUT';
58
}
59
60
sub log_to_file {
61
    my ($self,$logfile) = @_;
62
63
    #Open a filehandle to append to a log file
64
    my $opened = open(LOG, '>>', $logfile);
65
    if ($opened){
66
        LOG->autoflush(1); #Make filehandle hot (ie don't buffer)
67
        *STDOUT = *LOG; #Re-assign STDOUT to LOG | --stdout
68
        *STDERR = *STDOUT; #Re-assign STDERR to STDOUT | --stderr
69
    }
70
    else {
71
        die "Unable to open a filehandle for $logfile: $!\n"; # --output
72
    }
73
}
74
75
sub make_pidfilehandle {
76
    my ($self,$pidfile) = @_;
77
    if ( ! -f $pidfile ){
78
        open(my $fh, '>', $pidfile) or die "Unable to write to $pidfile: $!\n";
79
    }
80
    open(my $pidfilehandle, '+<', $pidfile) or die "Unable to open a filehandle for $pidfile: $!\n";
81
    return $pidfilehandle;
82
}
83
84
sub get_pidfile {
85
    my ($self,$pidfile) = @_;
86
    #NOTE: We need to save the filehandle in the object, so any locks persist for the life of the object
87
    my $pidfilehandle = $self->{pidfilehandle} ||= $self->make_pidfilehandle($pidfile);
88
    return $pidfilehandle;
89
}
90
91
sub lock_pidfile {
92
    my ($self,$pidfilehandle) = @_;
93
    my $locked;
94
    if (flock($pidfilehandle, LOCK_EX|LOCK_NB)){
95
        $locked = 1;
96
97
    }
98
    return $locked;
99
}
100
101
sub write_pidfile {
102
    my ($self,$pidfilehandle) = @_;
103
    if ($pidfilehandle){
104
        truncate($pidfilehandle, 0);
105
        print $pidfilehandle $$."\n" or die $!;
106
        #Flush the filehandle so you're not suffering from buffering
107
        $pidfilehandle->flush();
108
        return 1;
109
    }
110
}
111
112
sub run {
113
    my ($self) = @_;
114
    my $pidfile = $self->{pidfile};
115
    my $logfile = $self->{logfile};
116
117
    if ($pidfile){
118
        my $pidfilehandle = $self->get_pidfile($pidfile);
119
        if ($pidfilehandle){
120
            my $locked = $self->lock_pidfile($pidfilehandle);
121
            if ( ! $locked ) {
122
                die "$0 is unable to lock pidfile...\n";
123
            }
124
        }
125
    }
126
127
    if (my $configure = $self->{configure}){
128
        $configure->($self);
129
    }
130
131
    if ($self->{daemonize}){
132
        $self->daemonize();
133
    }
134
135
    if ($pidfile){
136
        my $pidfilehandle = $self->get_pidfile($pidfile);
137
        if ($pidfilehandle){
138
            $self->write_pidfile($pidfilehandle);
139
        }
140
    }
141
142
    if ($logfile){
143
        $self->log_to_file($logfile);
144
    }
145
146
    if (my $loop = $self->{loop}){
147
        $loop->($self);
148
    }
149
}
150
151
1;
(-)a/Koha/OAI/Harvester.pm (+640 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester;
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
use POE qw(Component::JobQueue);
23
use JSON;
24
use Sereal::Encoder;
25
use Sereal::Decoder;
26
use IO::Handle;
27
use File::Copy;
28
use File::Path qw(make_path remove_tree);
29
use DateTime;
30
use DateTime::Format::Strptime;
31
32
use C4::Context;
33
use Koha::Database;
34
35
my $day_granularity = DateTime::Format::Strptime->new(
36
    pattern   => '%F',
37
);
38
my $seconds_granularity = DateTime::Format::Strptime->new(
39
    pattern   => '%FT%TZ',
40
);
41
42
sub new {
43
    my ($class, $args) = @_;
44
    $args = {} unless defined $args;
45
    return bless ($args, $class);
46
}
47
48
sub spawn {
49
    my ($class, $args) = @_;
50
    my $self = $class->new($args);
51
    my $downloader = $self->{Downloader};
52
    my $importer = $self->{Importer};
53
    my $download_worker_limit = ( $self->{DownloaderWorkers} && int($self->{DownloaderWorkers}) ) ? $self->{DownloaderWorkers} : 1;
54
    my $import_worker_limit = ( $self->{ImporterWorkers} && int($self->{ImporterWorkers}) ) ? $self->{ImporterWorkers} : 1;
55
    my $import_queue_poll = ( $self->{ImportQueuePoll} && int($self->{ImportQueuePoll}) ) ? $self->{ImportQueuePoll} : 5;
56
57
    #NOTE: This job queue should always be created before the
58
    #harvester so that you can start download jobs immediately
59
    #upon spawning the harvester.
60
    POE::Component::JobQueue->spawn(
61
        Alias         => 'oai-downloader',
62
        WorkerLimit   => $download_worker_limit,
63
        Worker        => sub {
64
            my ($postback, $task) = @_;
65
            if ($downloader){
66
                if ($task->{status} eq "active"){
67
                    $downloader->run({
68
                        postback => $postback,
69
                        task => $task,
70
                   });
71
                }
72
            }
73
        },
74
        Passive => {},
75
    );
76
77
    POE::Session->create(
78
        object_states => [
79
            $self => {
80
                _start => "on_start",
81
                get_task => "get_task",
82
                list_tasks => "list_tasks",
83
                create_task => "create_task",
84
                start_task => "start_task",
85
                stop_task => "stop_task",
86
                delete_task => "delete_task",
87
                repeat_task => "repeat_task",
88
                register => "register",
89
                deregister => "deregister",
90
                restore_state => "restore_state",
91
                save_state => "save_state",
92
                is_task_finished => "is_task_finished",
93
                does_task_repeat => "does_task_repeat",
94
                download_postback => "download_postback",
95
                reset_imports_status => "reset_imports_status",
96
            },
97
        ],
98
    );
99
100
    POE::Component::JobQueue->spawn(
101
        Alias         => 'oai-importer',
102
        WorkerLimit   => $import_worker_limit,
103
        Worker        => sub {
104
            my $meta_postback = shift;
105
106
            #NOTE: We need to only retrieve queue items for active tasks. Otherwise,
107
            #the importer will just spin its wheels on inactive tasks and do nothing.
108
            my $active_tasks = $poe_kernel->call("harvester","list_tasks","active");
109
            my @active_uuids = map { $_->{uuid} } @$active_tasks;
110
111
            my $schema = Koha::Database->new()->schema();
112
            my $rs = $schema->resultset('OaiHarvesterImportQueue')->search({
113
                uuid => \@active_uuids,
114
                status => "new"
115
            },{
116
                order_by => { -asc => 'id' },
117
                rows => 1,
118
            });
119
            my $result = $rs->first;
120
            if ($result){
121
                $result->status("wip");
122
                $result->update;
123
                my $task = {
124
                    id => $result->id,
125
                    uuid => $result->uuid,
126
                    result => $result->result,
127
                };
128
129
                my $postback = $meta_postback->($task);
130
                $importer->run({
131
                    postback => $postback,
132
                    task => $task,
133
                });
134
            }
135
        },
136
        Active => {
137
            PollInterval => $import_queue_poll,
138
            AckAlias => undef,
139
            AckState => undef,
140
        },
141
    );
142
143
    return;
144
}
145
146
sub on_start {
147
    my ($self, $kernel, $heap) = @_[OBJECT, KERNEL,HEAP];
148
    $kernel->alias_set('harvester');
149
    $heap->{scoreboard} = {};
150
151
    #Reset any 'wip' imports to 'new' so they can be re-tried.
152
    $kernel->call("harvester","reset_imports_status");
153
154
    #Restore state from state file
155
    $kernel->call("harvester","restore_state");
156
}
157
158
#NOTE: This isn't really implemented at the moment, as it's not really necessary.
159
sub download_postback {
160
    my ($kernel, $request_packet, $response_packet) = @_[KERNEL, ARG0, ARG1];
161
    my $message = $response_packet->[0];
162
}
163
164
=head3 deregister
165
166
    Remove the worker session from the harvester's in-memory scoreboard,
167
    unset the downloading flag if downloading is completed.
168
169
=cut
170
171
sub deregister {
172
    my ($self, $kernel, $heap, $session, $sender, $type) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0];
173
174
    my $scoreboard = $heap->{scoreboard};
175
176
    my $logger = $self->{logger};
177
    $logger->debug("Start deregistering $sender as $type task");
178
179
    my $task_uuid = delete $scoreboard->{session}->{$sender};
180
    #NOTE: If you don't check each step of the hashref, autovivication can lead to surprises.
181
    if ($task_uuid){
182
        if ($scoreboard->{task}->{$task_uuid}){
183
            if ($scoreboard->{task}->{$task_uuid}->{$type}){
184
                delete $scoreboard->{task}->{$task_uuid}->{$type}->{$sender};
185
            }
186
        }
187
    }
188
189
    my $task = $heap->{tasks}->{$task_uuid};
190
    if ($task && $task->{status} && ($task->{status} eq "active") ){
191
        #NOTE: Each task only has 1 download session, so we can now set/unset flags for the task.
192
        #NOTE: We should unset the downloading flag, if we're not going to repeat the task.
193
        if ($type eq "download"){
194
            my $task_repeats = $kernel->call("harvester","does_task_repeat",$task_uuid);
195
            if ($task_repeats){
196
                my $interval = $task->{interval};
197
198
                $task->{effective_from} = delete $task->{effective_until};
199
                $task->{download_timer} = $kernel->delay_set("repeat_task", $interval, $task_uuid);
200
            }
201
            else {
202
                $task->{downloading} = 0;
203
                $kernel->call("harvester","save_state");
204
                $kernel->call("harvester","is_task_finished",$task_uuid);
205
            }
206
        }
207
        elsif ($type eq 'import'){
208
            $kernel->call("harvester","is_task_finished",$task_uuid);
209
        }
210
    }
211
    $logger->debug("End deregistering $sender as $type task");
212
}
213
214
215
=head3 is_task_finished
216
217
    This event handler checks if the task has finished downloading and importing record.
218
    If it is finished downloading and importing, the task is deleted from the harvester.
219
220
    This only applies to non-repeating tasks.
221
222
=cut
223
224
sub is_task_finished {
225
    my ($self, $kernel, $heap, $session, $uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
226
    my $task = $kernel->call("harvester","get_task",$uuid);
227
    if ($task && (! $task->{downloading}) ){
228
        my $count = $self->get_import_count_for_task($uuid);
229
        if ( ! $count ) {
230
            #Clear this task out of the harvester as it's finished.
231
            $kernel->call("harvester","delete_task",$uuid);
232
            return 1;
233
        }
234
    }
235
    return 0;
236
}
237
238
sub register {
239
    my ($self, $kernel, $heap, $session, $sender, $type, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0,ARG1];
240
    my $logger = $self->{logger};
241
242
    my $scoreboard = $heap->{scoreboard};
243
244
245
    if ($type && $task_uuid){
246
        $logger->debug("Registering $sender as $type for $task_uuid");
247
248
        my $task = $heap->{tasks}->{$task_uuid};
249
        if ($task){
250
251
            if ($type){
252
                #Register the task uuid with the session id as a key for later recall
253
                $scoreboard->{session}->{$sender} = $task_uuid;
254
255
                #Register the session id as a certain type of session for a task
256
                $scoreboard->{task}->{$task_uuid}->{$type}->{$sender} = 1;
257
258
                if ($type eq "download"){
259
                    $task->{downloading} = 1;
260
261
                    my $task_repeats = $kernel->call("harvester","does_task_repeat",$task_uuid);
262
                    if ($task_repeats){
263
264
                        #NOTE: Set an effective until, so we know we're not getting records any newer than
265
                        #this moment.
266
                        my $dt = DateTime->now();
267
                        if ($dt){
268
                            #NOTE: Ideally, I'd like to make sure that we can use 'seconds' granularity, but
269
                            #it's valid for 'from' to be null, so it's impossible to know from the data whether
270
                            #or not the repository will support the seconds granularity.
271
                            #NOTE: Ideally, it would be good to use either 'day' granularity or 'seconds' granularity,
272
                            #but at the moment the interval is expressed as seconds only.
273
                            $dt->set_formatter($seconds_granularity);
274
                            $task->{effective_until} = "$dt";
275
                        }
276
                    }
277
278
                    $kernel->call("harvester","save_state");
279
                }
280
            }
281
        }
282
    }
283
}
284
285
sub does_task_repeat {
286
    my ($self, $kernel, $heap, $session, $uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
287
    my $task = $kernel->call("harvester","get_task",$uuid);
288
    if ($task){
289
        my $interval = $task->{interval};
290
        my $parameters = $task->{parameters};
291
        my $oai_pmh = $parameters->{oai_pmh} if $parameters->{oai_pmh};
292
        if ( $interval && ($oai_pmh->{verb} eq "ListRecords") && (! $oai_pmh->{until}) ){
293
            return 1;
294
        }
295
    }
296
    return 0;
297
}
298
299
300
301
sub reset_imports_status {
302
    my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION];
303
304
    my $schema = Koha::Database->new()->schema();
305
    my $rs = $schema->resultset('OaiHarvesterImportQueue')->search({
306
                status => "wip",
307
    });
308
    $rs->update({
309
        status => "new",
310
    });
311
}
312
313
sub restore_state {
314
    my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION];
315
316
    my $state_file = $self->{state_file};
317
    my $state_backup = "$state_file~";
318
319
    #NOTE: If there is a state backup, it means we crashed while saving the state. Otherwise,
320
    #let's try the regular state file if it exists.
321
    my $file_to_restore = ( -f $state_backup ) ? $state_backup : ( ( -f $state_file ) ? $state_file : undef );
322
    if ( $file_to_restore ){
323
        my $opened = open( my $fh, '<', $file_to_restore ) or die "Couldn't open state: $!";
324
        if ($opened){
325
            local $/;
326
            my $in = <$fh>;
327
            my $decoder = Sereal::Decoder->new;
328
            my $state = $decoder->decode($in);
329
            if ($state){
330
                if ($state->{tasks}){
331
                    #Restore tasks from our saved state
332
                    $heap->{tasks} = $state->{tasks};
333
                    foreach my $uuid ( keys %{$heap->{tasks}} ){
334
                        my $task = $heap->{tasks}->{$uuid};
335
336
                        #If tasks were still downloading, restart the task
337
                        if ( ($task->{status} && $task->{status} eq "active") && $task->{downloading} ){
338
                            $task->{status} = "new";
339
                            $kernel->call("harvester","start_task",$task->{uuid});
340
                        }
341
                    }
342
                }
343
            }
344
        }
345
    }
346
}
347
348
sub save_state {
349
    my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION];
350
    my $state_file = $self->{state_file};
351
    my $state_backup = "$state_file~";
352
353
    #Make a backup of existing state record
354
    my $moved = move($state_file,$state_backup);
355
356
    my $opened = open(my $fh, ">", $state_file) or die "Couldn't save state: $!";
357
    if ($opened){
358
        $fh->autoflush(1);
359
        my $tasks = $heap->{tasks};
360
        my $harvester_state = {
361
            tasks => $tasks,
362
        };
363
        my $encoder = Sereal::Encoder->new;
364
        my $out = $encoder->encode($harvester_state);
365
        local $\;
366
        my $printed = print $fh $out;
367
        if ($printed){
368
            close $fh;
369
            unlink($state_backup);
370
            return 1;
371
        }
372
    }
373
    return 0;
374
}
375
376
=head3 get_task
377
378
    This event handler returns a task from a harvester using the task's
379
    uuid as an argument.
380
381
=cut
382
383
sub get_task {
384
    my ($self, $kernel, $heap, $session, $uuid, $sender) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0, SENDER];
385
386
    if ( ! $uuid && $sender ){
387
        my $scoreboard = $heap->{scoreboard};
388
        my $uuid_by_session = $scoreboard->{session}->{$sender};
389
        if ($uuid_by_session){
390
            $uuid = $uuid_by_session;
391
        }
392
    }
393
394
    my $tasks = $heap->{tasks};
395
    if ($tasks && $uuid){
396
        my $task = $tasks->{$uuid};
397
        if ($task){
398
            return $task;
399
        }
400
    }
401
    return 0;
402
}
403
404
=head3 get_import_count_for_task
405
406
=cut
407
408
sub get_import_count_for_task {
409
    my ($self,$uuid) = @_;
410
    my $count = undef;
411
    if ($uuid){
412
        my $schema = Koha::Database->new()->schema();
413
        my $items = $schema->resultset('OaiHarvesterImportQueue')->search({
414
            uuid => $uuid,
415
        });
416
        $count = $items->count;
417
    }
418
    return $count;
419
}
420
421
=head3 list_tasks
422
423
    This event handler returns a list of tasks that have been submitted
424
    to the harvester. It returns data like uuid, status, parameters,
425
    number of pending imports, etc.
426
427
=cut
428
429
sub list_tasks {
430
    my ($self, $kernel, $heap, $session, $status) = @_[OBJECT, KERNEL,HEAP,SESSION, ARG0];
431
    my $schema = Koha::Database->new()->schema();
432
    my @tasks = ();
433
    foreach my $uuid (sort keys %{$heap->{tasks}}){
434
        my $task = $heap->{tasks}->{$uuid};
435
        my $items = $schema->resultset('OaiHarvesterImportQueue')->search({
436
            uuid => $uuid,
437
        });
438
        my $count = $items->count // 0;
439
        $task->{pending_imports} = $count;
440
        if ( ( ! $status ) || ( $status && $status eq $task->{status} ) ){
441
            push(@tasks, $task);
442
        }
443
444
    }
445
    return \@tasks;
446
}
447
448
=head3 create_task
449
450
    This event handler creates a spool directory for the task's imports.
451
    It also adds it to the harvester's memory and then saves memory to
452
    a persistent datastore.
453
454
    Newly created tasks have a status of "new".
455
456
=cut
457
458
sub create_task {
459
    my ($self, $kernel, $heap, $session, $incoming_task) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
460
    my $logger = $self->{logger};
461
    if ($incoming_task){
462
        my $uuid = $incoming_task->{uuid};
463
        if ( ! $heap->{tasks}->{$uuid} ){
464
465
            #Step One: assign a spool directory to this task
466
            my $spooldir = $self->{spooldir} // "/tmp";
467
            my $task_spooldir = "$spooldir/$uuid";
468
            if ( ! -d $task_spooldir ){
469
                my $made_spool_directory = make_path($task_spooldir);
470
                if ( ! $made_spool_directory ){
471
                    if ($logger){
472
                        $logger->warn("Unable to make task-specific spool directory at '$task_spooldir'");
473
                    }
474
                    return 0;
475
                }
476
            }
477
            $incoming_task->{spooldir} = $task_spooldir;
478
479
            #Step Two: assign new status
480
            $incoming_task->{status} = "new";
481
482
            #Step Three: add task to harvester's memory
483
            $heap->{tasks}->{ $uuid } = $incoming_task;
484
485
            #Step Four: save state
486
            $kernel->call($session,"save_state");
487
            return 1;
488
        }
489
    }
490
    return 0;
491
}
492
493
=head3 start_task
494
495
    This event handler marks a task as active in the harvester's memory,
496
    save the memory to a persistent datastore, then enqueues the task,
497
    so that it can be directed to the next available download worker.
498
499
    Newly started tasks have a status of "active".
500
501
=cut
502
503
sub start_task {
504
    my ($self, $session,$kernel,$heap,$uuid) = @_[OBJECT, SESSION,KERNEL,HEAP,ARG0];
505
    my $task = $heap->{tasks}->{$uuid};
506
    if ($task){
507
        if ( $task->{status} ne "active" ){
508
509
            #Clear any pre-existing error states
510
            delete $task->{error} if $task->{error};
511
512
            #Step One: mark task as active
513
            $task->{status} = "active";
514
515
            #Step Two: save state
516
            $kernel->call("harvester","save_state");
517
518
            #Step Three: enqueue task
519
            $kernel->post("oai-downloader",'enqueue','download_postback', $task);
520
521
            return 1;
522
        }
523
    }
524
    return 0;
525
}
526
527
=head3 repeat_task
528
529
530
531
=cut
532
533
sub repeat_task {
534
    my ($self, $session,$kernel,$heap,$uuid) = @_[OBJECT, SESSION,KERNEL,HEAP,ARG0];
535
    my $task = $heap->{tasks}->{$uuid};
536
    if ($task){
537
        my $interval = $task->{interval};
538
        if ($task->{downloading} && $interval){
539
            $kernel->post("oai-downloader",'enqueue','download_postback', $task);
540
        }
541
    }
542
}
543
544
=head3 stop_task
545
546
    This event handler prevents new workers from spawning, kills
547
    existing workers, and stops pending imports from being imported.
548
549
    Newly stopped tasks have a status of "stopped".
550
551
=cut
552
553
sub stop_task {
554
    my ($self, $kernel, $heap, $session, $sender, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0];
555
556
    my $task = $heap->{tasks}->{$task_uuid};
557
558
    if ($task && $task->{status} && $task->{status} ne "stopped" ){
559
560
        #Step One: deactivate task, so no new workers can be started
561
        $task->{status} = "stopped";
562
        #NOTE: You could also clear timers for new downloads, but that's probably unnecessary because of this step.
563
564
        #Step Two: kill workers
565
        my $scoreboard = $heap->{scoreboard};
566
        my $session_types = $scoreboard->{task}->{$task_uuid};
567
        if ($session_types){
568
            foreach my $type ( keys %$session_types ){
569
                my $sessions = $session_types->{$type};
570
                if ($sessions){
571
                    foreach my $session (keys %$sessions){
572
                        if ($session){
573
                            $kernel->signal($session, "cancel");
574
                        }
575
                    }
576
                }
577
            }
578
            #Remove the task uuid from the task key of the scoreboard
579
            delete $scoreboard->{task}->{$task_uuid};
580
            #NOTE: The task uuid will still exist under the session key,
581
            #but the sessions will deregister themselves and clean that up for you.
582
        }
583
584
        #Step Three: stop pending imports for this task
585
        my $schema = Koha::Database->new()->schema();
586
        my $items = $schema->resultset('OaiHarvesterImportQueue')->search({
587
            uuid => $task_uuid,
588
        });
589
        my $rows_updated = $items->update({
590
            status => "stopped",
591
        });
592
593
        #Step Four: save state
594
        $kernel->call("harvester","save_state");
595
        return 1;
596
    }
597
    return 0;
598
}
599
600
=head3 delete_task
601
602
    Deleted tasks are stopped, pending imports are deleted from the
603
    database and file system, and then the task is removed from the harvester.
604
605
=cut
606
607
sub delete_task {
608
    my ($self, $kernel, $heap, $session, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
609
610
    my $task = $heap->{tasks}->{$task_uuid};
611
    if ($task){
612
        #Step One: stop task
613
        $kernel->call($session,"stop_task",$task_uuid);
614
615
        #Step Two: delete pending imports in database
616
        my $schema = Koha::Database->new()->schema();
617
        my $items = $schema->resultset('OaiHarvesterImportQueue')->search({
618
            uuid => $task_uuid,
619
        });
620
        if ($items){
621
            my $rows_deleted = $items->delete;
622
            #NOTE: shows 0E0 instead of 0
623
        }
624
625
        #Step Three: remove task specific spool directory and files within it
626
        my $spooldir = $task->{spooldir};
627
        if ($spooldir){
628
            my $files_deleted = remove_tree($spooldir, { safe => 1 });
629
        }
630
631
        delete $heap->{tasks}->{$task_uuid};
632
633
        #Step Four: save state
634
        $kernel->call("harvester","save_state");
635
        return 1;
636
    }
637
    return 0;
638
}
639
640
1;
(-)a/Koha/OAI/Harvester/Client.pm (+177 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Client;
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use URI;
22
use IO::Socket::UNIX;
23
use IO::Select;
24
use JSON;
25
26
sub new {
27
    my ($class, $args) = @_;
28
    $args = {} unless defined $args;
29
    return bless ($args, $class);
30
}
31
32
sub connect {
33
    my ($self) =  @_;
34
    my $socket_uri = $self->{socket_uri};
35
    if ($socket_uri){
36
        my $uri = URI->new($socket_uri);
37
        if ($uri && $uri->scheme eq 'unix'){
38
            my $socket_path = $uri->path;
39
            my $socket = IO::Socket::UNIX->new(
40
                Type => IO::Socket::UNIX::SOCK_STREAM(),
41
                Peer => $socket_path,
42
            );
43
            if ($socket){
44
                my $select = new IO::Select();
45
                $select->add($socket);
46
47
                $self->{_select} = $select;
48
                $self->{_socket} = $socket;
49
                my $message = $self->_read();
50
                if ($message){
51
                    if ($message eq 'HELLO'){
52
                        $self->{_connected} = 1;
53
                        return 1;
54
                    }
55
                }
56
            }
57
            else {
58
                warn "Failed to create socket."
59
            }
60
        }
61
    }
62
    return 0;
63
}
64
65
sub create {
66
    my ($self,$task) = @_;
67
    my $message = {
68
        command => "create",
69
        body => {
70
            task => $task,
71
        }
72
    };
73
    my ($status) = $self->_exchange($message);
74
    return $status;
75
}
76
77
sub start {
78
    my ($self,$uuid) = @_;
79
    my $message = {
80
        command => "start",
81
        body => {
82
            task => {
83
                uuid => $uuid,
84
            },
85
        }
86
    };
87
    my ($status) = $self->_exchange($message);
88
    return $status;
89
}
90
91
sub stop {
92
    my ($self,$uuid) = @_;
93
    my $message = {
94
        command => "stop",
95
        body => {
96
            task => {
97
                uuid => $uuid,
98
            },
99
        }
100
    };
101
    my ($status) = $self->_exchange($message);
102
    return $status;
103
}
104
105
sub delete {
106
    my ($self,$uuid) = @_;
107
    my $message = {
108
        command => "delete",
109
        body => {
110
            task => {
111
                uuid => $uuid,
112
            },
113
        }
114
    };
115
    my ($status) = $self->_exchange($message);
116
    return $status;
117
}
118
119
sub list {
120
    my ($self) = @_;
121
    my $message = {
122
        command => "list",
123
    };
124
    my ($status,$tasks) = $self->_exchange($message);
125
    return $tasks;
126
}
127
128
sub _exchange {
129
    my ($self,$message) = @_;
130
    my $status = 0;
131
    my $data;
132
    if ($message){
133
        my $output = to_json($message);
134
        if ($output){
135
            $self->_write($output);
136
            my $json_response = $self->_read();
137
            if ($json_response){
138
                my $response = from_json($json_response);
139
                $data = $response->{data} if $response->{data};
140
                $status = 1 if $response->{msg} && $response->{msg} eq "success";
141
            }
142
        }
143
    }
144
    return ($status,$data);
145
}
146
147
sub _write {
148
    my ($self, $output) = @_;
149
    if ($output){
150
        if (my $select = $self->{_select}){
151
            if (my @filehandles = $select->can_write(5)){
152
                foreach my $filehandle (@filehandles){
153
                    #Localize output record separator as null
154
                    local $\ = "\x00";
155
                    print $filehandle $output;
156
                }
157
            }
158
        }
159
    }
160
}
161
162
sub _read {
163
    my ($self) = @_;
164
    if (my $select = $self->{_select}){
165
        if (my @filehandles = $select->can_read(5)){
166
            foreach my $filehandle (@filehandles){
167
                #Localize input record separator as null
168
                local $/ = "\x00";
169
                my $message = <$filehandle>;
170
                chomp($message) if $message;
171
                return $message;
172
            }
173
        }
174
    }
175
}
176
177
1;
(-)a/Koha/OAI/Harvester/Downloader.pm (+302 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Downloader;
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use URI;
22
use XML::LibXML::Reader;
23
use IO::Handle;
24
use JSON;
25
26
sub new {
27
    my ($class, $args) = @_;
28
    $args = {} unless defined $args;
29
    return bless ($args, $class);
30
}
31
32
=head2 BuildURL
33
34
    Takes a baseURL and a mix of required and optional OAI-PMH arguments,
35
    and makes them into a suitable URL for an OAI-PMH request.
36
37
=cut
38
39
sub BuildURL {
40
    my ($self, $args) = @_;
41
    my $baseURL = $args->{baseURL};
42
    my $url = URI->new($baseURL);
43
    if ($url && $url->isa("URI")){
44
        my $verb = $args->{verb};
45
        if ($verb){
46
            my %parameters = (
47
                verb => $verb,
48
            );
49
            if ($verb eq "ListRecords"){
50
                my $resumptionToken = $args->{resumptionToken};
51
                my $metadataPrefix = $args->{metadataPrefix};
52
                if ($resumptionToken){
53
                    $parameters{resumptionToken} = $resumptionToken;
54
                }
55
                elsif ($metadataPrefix){
56
                    $parameters{metadataPrefix} = $metadataPrefix;
57
                    #Only add optional parameters if they're provided
58
                    foreach my $param ( qw( from until set ) ){
59
                        $parameters{$param} = $args->{$param} if $args->{$param};
60
                    }
61
                }
62
                else {
63
                    warn "BuildURL() requires an argument of either resumptionToken or metadataPrefix";
64
                    return;
65
                }
66
            }
67
            elsif ($verb eq "GetRecord"){
68
                my $metadataPrefix = $args->{metadataPrefix};
69
                my $identifier = $args->{identifier};
70
                if ($metadataPrefix && $identifier){
71
                    $parameters{metadataPrefix} = $metadataPrefix;
72
                    $parameters{identifier} = $identifier;
73
                }
74
                else {
75
                    warn "BuildURL() requires an argument of metadataPrefix and an argument of identifier";
76
                    return;
77
                }
78
            }
79
            $url->query_form(%parameters);
80
            return $url;
81
        }
82
        else {
83
            warn "BuildURL() requires a verb of GetRecord or ListRecords";
84
            return;
85
        }
86
    }
87
    else {
88
        warn "BuildURL() requires a base URL of type URI.";
89
        return;
90
    }
91
}
92
93
=head2 OpenXMLStream
94
95
    Fork a child process to send the HTTP request, which sends chunks
96
    of XML via a pipe to the parent process.
97
98
    The parent process creates and returns a XML::LibXML::Reader object,
99
    which reads the XML stream coming through the pipe.
100
101
    Normally, using a DOM reader, you must wait to read the entire XML document
102
    into memory. However, using a stream reader, chunks are read into memory,
103
    processed, then discarded. It's faster and more efficient.
104
105
=cut
106
107
sub GetXMLStream {
108
    my ($self, $args) = @_;
109
    my $url = $args->{url};
110
    my $user_agent = $args->{user_agent};
111
    if ($url && $user_agent){
112
        pipe( CHILD, PARENT ) or die "Cannot created connected pipes: $!";
113
        CHILD->autoflush(1);
114
        PARENT->autoflush(1);
115
        if ( my $pid = fork ){
116
            #Parent process
117
            close PARENT;
118
            return \*CHILD;
119
        }
120
        else {
121
            #Child process
122
            close CHILD;
123
            my $response = $self->_request({
124
                url => $url,
125
                user_agent => $user_agent,
126
                file_handle => \*PARENT,
127
            });
128
            if ($response && $response->is_success){
129
                #HTTP request has successfully finished, so we close the file handle and exit the process
130
                close PARENT;
131
                CORE::exit(); #some modules like mod_perl redefine exit
132
            }
133
            else {
134
                warn "[child $$] OAI-PMH unsuccessful. Response status: ".$response->status_line."\n" if $response;
135
                CORE::exit();
136
            }
137
        }
138
    }
139
    else {
140
        warn "GetXMLStream() requires a 'url' argument and a 'user_agent' argument";
141
        return;
142
    }
143
}
144
145
sub _request {
146
    my ($self, $args) = @_;
147
    my $url = $args->{url};
148
    my $user_agent = $args->{user_agent};
149
    my $fh = $args->{file_handle};
150
151
    if ($url && $user_agent && $fh){
152
        my $request = HTTP::Request->new( GET => $url );
153
        my $response = $user_agent->request( $request, sub {
154
                my ($chunk_of_data, $ref_to_response, $ref_to_protocol) = @_;
155
                print $fh $chunk_of_data;
156
        });
157
        return $response;
158
    }
159
    else {
160
        warn "_request() requires a 'url' argument, 'user_agent' argument, and 'file_handle' argument.";
161
        return;
162
    }
163
}
164
165
sub ParseXMLStream {
166
    my ($self, $args) = @_;
167
168
    my $each_callback = $args->{each_callback};
169
    my $fh = $args->{file_handle};
170
    if ($fh){
171
        my $reader = XML::LibXML::Reader->new( FD => $fh, no_blanks => 1 );
172
        my $pattern = XML::LibXML::Pattern->new('oai:OAI-PMH|/oai:OAI-PMH/*', { 'oai' => "http://www.openarchives.org/OAI/2.0/" });
173
174
        my $repository;
175
176
        warn "Start parsing...";
177
        while (my $rv = $reader->nextPatternMatch($pattern)){
178
            #$rv == 1; successful
179
            #$rv == 0; end of document reached
180
            #$rv == -1; error
181
            if ($rv == -1){
182
                die "Parser error!";
183
            }
184
            #NOTE: We do this so we only get the opening tag of the element.
185
            next unless $reader->nodeType == XML_READER_TYPE_ELEMENT;
186
187
            my $localname = $reader->localName;
188
            if ( $localname eq "request" ){
189
                my $node = $reader->copyCurrentNode(1);
190
                $repository = $node->textContent;
191
            }
192
            elsif ( $localname eq "error" ){
193
                #See https://www.openarchives.org/OAI/openarchivesprotocol.html#ErrorConditions
194
                #We should probably die under all circumstances except "noRecordsMatch"
195
                my $node = $reader->copyCurrentNode(1);
196
                if ($node){
197
                    my $code = $node->getAttribute("code");
198
                    if ($code){
199
                        if ($code ne "noRecordsMatch"){
200
                            warn "Error code: $code";
201
                            die;
202
                        }
203
                    }
204
                }
205
            }
206
            elsif ( ($localname eq "ListRecords") || ($localname eq "GetRecord") ){
207
                my $each_pattern = XML::LibXML::Pattern->new('//oai:record|oai:resumptionToken', { 'oai' => "http://www.openarchives.org/OAI/2.0/" });
208
                while (my $each_rv =  $reader->nextPatternMatch($each_pattern)){
209
                    if ($rv == "-1"){
210
                        #NOTE: -1 denotes an error state
211
                        warn "Error getting pattern match";
212
                    }
213
                    next unless $reader->nodeType == XML_READER_TYPE_ELEMENT;
214
                    if ($reader->localName eq "record"){
215
                        my $node = $reader->copyCurrentNode(1);
216
                        #NOTE: Without the UTF-8 flag, UTF-8 data will be corrupted.
217
                        my $document = XML::LibXML::Document->new('1.0', 'UTF-8');
218
                        $document->setDocumentElement($node);
219
220
                       #Per record callback
221
                        if ($each_callback){
222
                            $each_callback->({
223
                                repository => $repository,
224
                                document => $document,
225
                            });
226
                        }
227
                    }
228
                    elsif ($reader->localName eq "resumptionToken"){
229
                        my $resumptionToken = $reader->readInnerXml;
230
                        return ($resumptionToken,$repository);
231
232
                    }
233
                }
234
            }
235
        } #/OAI-PMH document match
236
    }
237
    else {
238
        warn "ParseXMLStream() requires a 'file_handle' argument.";
239
    }
240
}
241
242
sub harvest {
243
    my ($self,$args) = @_;
244
    my $url = $args->{url};
245
    my $ua = $args->{user_agent};
246
    my $callback = $args->{callback};
247
    my $complete_callback = $args->{complete_callback};
248
249
    if ($url && $ua){
250
251
        #NOTE: http://search.cpan.org/~shlomif/XML-LibXML-2.0128/lib/XML/LibXML/Parser.pod#ERROR_REPORTING
252
        while($url){
253
            warn "URL = $url";
254
            warn "Creating child process to download and feed parent process parser.";
255
            my $stream = $self->GetXMLStream({
256
                url => $url,
257
                user_agent => $ua,
258
            });
259
260
            warn "Creating parent process parser.";
261
            my ($resumptionToken) = $self->ParseXMLStream({
262
                file_handle => $stream,
263
                each_callback => $callback,
264
            });
265
            warn "Finished parsing current XML document.";
266
267
            if ($resumptionToken){
268
                #If there's a resumptionToken at the end of the stream,
269
                #we build a new URL and repeat this process again.
270
                $url->query_form({
271
                    verb => "ListRecords",
272
                    resumptionToken => $resumptionToken,
273
                });
274
            }
275
            else {
276
                warn "Finished harvest.";
277
                last;
278
            }
279
280
            warn "Reap child process downloader.";
281
            #Reap the dead child requester process before performing another request,
282
            #so we don't fill up the process table with zombie children.
283
            while ((my $child = waitpid(-1, 0)) > 0) {
284
                warn "Parent $$ reaped child process $child" . ($? ? " with exit code $?" : '') . ".\n";
285
            }
286
        }
287
288
        if ($complete_callback){
289
            warn "Run complete callback.";
290
291
            #Clear query string
292
            $url->query_form({});
293
294
            #Run complete callback using the actual URL from the request.
295
            $complete_callback->({
296
                repository => $url,
297
            });
298
        }
299
    }
300
}
301
302
1;
(-)a/Koha/OAI/Harvester/Import/MARCXML.pm (+145 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Import::MARCXML;
2
3
# Copyright 2016 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
use MARC::Record;
23
24
use C4::Context;
25
use C4::Biblio;
26
27
use constant MAX_MATCHES => 99999; #NOTE: This is an arbitrary value. We want to get all matches.
28
29
sub new {
30
    my ($class, $args) = @_;
31
    $args = {} unless defined $args;
32
    if ( (! $args->{dom}) && (! $args->{marc_record}) ){
33
        die "You must provide either a dom or marc_record argument to this constructor";
34
    }
35
    if ( $args->{dom} && ( ! $args->{marc_record} ) ){
36
        my $dom = $args->{dom};
37
        my $xml = $dom->toString(2);
38
        my $marcflavour = C4::Context->preference('marcflavour') || 'MARC21';
39
        my $marc_record = eval {MARC::Record::new_from_xml( $xml, "utf8", $marcflavour)};
40
        if ($@){
41
            die "Unable to create MARC::Record object";
42
        }
43
        if ($marc_record){
44
            $args->{marc_record} = $marc_record;
45
        }
46
    }
47
    return bless ($args, $class);
48
}
49
50
sub import_record {
51
    my ($self,$args) = @_;
52
    my $framework = $args->{framework};
53
    my $record_type = $args->{record_type};
54
    my $matcher = $args->{matcher};
55
    my $koha_id = $args->{koha_id};
56
57
    my $action = "error";
58
59
    #Try to find a matching Koha MARCXML record via Zebra
60
    if (! $koha_id && $matcher){
61
        my $matched_id = $self->_try_matcher({
62
            matcher => $matcher,
63
        });
64
        if ($matched_id){
65
            $koha_id = $matched_id;
66
        }
67
    }
68
69
    if ($koha_id){
70
        #Update
71
        ($action) = $self->_mod_koha_record({
72
            record_type => $record_type,
73
            framework => $framework,
74
            koha_id => $koha_id,
75
        });
76
    }
77
    else {
78
        #Add
79
        ($action,$koha_id) = $self->_add_koha_record({
80
            record_type => $record_type,
81
            framework => $framework,
82
        });
83
    }
84
85
    return ($action,$koha_id);
86
}
87
88
sub _try_matcher {
89
    my ($self, $args) = @_;
90
    my $marc_record = $self->{marc_record};
91
    my $matcher = $args->{matcher};
92
    my $matched_id;
93
    my @matches = $matcher->get_matches($marc_record, MAX_MATCHES);
94
    if (@matches){
95
        #FIXME: You can remove this sort once Bug 17710 is pushed
96
        @matches = sort {
97
            $b->{'score'} cmp $a->{'score'} or
98
            $b->{'record_id'} cmp $a->{'record_id'}
99
        } @matches;
100
        my $bestrecordmatch = shift @matches;
101
        if ($bestrecordmatch && $bestrecordmatch->{record_id}){
102
            $matched_id = $bestrecordmatch->{record_id};
103
        }
104
    }
105
    return $matched_id;
106
}
107
108
sub _add_koha_record {
109
    my ($self, $args) = @_;
110
    my $marc_record = $self->{marc_record};
111
    my $record_type = $args->{record_type} // "biblio";
112
    my $framework = $args->{framework};
113
    my $koha_id;
114
    my $action = "error";
115
    if ($record_type eq "biblio"){
116
        #NOTE: Strip item fields to prevent any accidentally getting through.
117
        C4::Biblio::_strip_item_fields($marc_record,$framework);
118
        my ($biblionumber,$biblioitemnumber) = C4::Biblio::AddBiblio($marc_record,$framework);
119
        if ($biblionumber){
120
            $action = "added";
121
            $koha_id = $biblionumber;
122
        }
123
    }
124
    return ($action,$koha_id);
125
}
126
127
sub _mod_koha_record {
128
    my ($self, $args) = @_;
129
    my $marc_record = $self->{marc_record};
130
    my $record_type = $args->{record_type} // "biblio";
131
    my $framework = $args->{framework};
132
    my $koha_id = $args->{koha_id};
133
    my $action = "error";
134
    if ($record_type eq "biblio"){
135
        #NOTE: Strip item fields to prevent any accidentally getting through.
136
        C4::Biblio::_strip_item_fields($marc_record,$framework);
137
        my $updated = C4::Biblio::ModBiblio($marc_record, $koha_id, $framework);
138
        if ($updated){
139
            $action = "updated";
140
        }
141
    }
142
    return ($action);
143
}
144
145
1;
(-)a/Koha/OAI/Harvester/Import/RDFXML.pm (+47 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Import::RDFXML;
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
23
sub new {
24
    my ($class, $args) = @_;
25
    $args = {} unless defined $args;
26
	return bless ($args, $class);
27
}
28
29
sub import_record {
30
    my ($self,$args) = @_;
31
    my $framework = $args->{framework};
32
    my $record_type = $args->{record_type};
33
    my $matcher = $args->{matcher};
34
    my $koha_id = $args->{koha_id};
35
36
    my $action = "error";
37
38
    if (! $koha_id && $matcher){
39
        #Create minimal MARCXML record from RDFXML to be used for Zebra-based matching.
40
    }
41
42
43
    #TODO: add/update triplestore
44
    return ($action,$koha_id);
45
}
46
47
1;
(-)a/Koha/OAI/Harvester/Import/Record.pm (+304 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Import::Record;
2
3
# Copyright 2016 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
use XML::LibXML;
23
use XML::LibXSLT;
24
use URI;
25
use File::Basename;
26
27
use C4::Context;
28
use C4::Biblio;
29
30
use Koha::Database;
31
use Koha::OAI::Harvester::Import::MARCXML;
32
use Koha::OAI::Harvester::Import::RDFXML;
33
34
my $schema = Koha::Database->new()->schema();
35
36
sub new {
37
    my ($class, $args) = @_;
38
    $args = {} unless defined $args;
39
40
    die "You must provide a 'doc' argument to the constructor" unless $args->{doc};
41
    die "You must provide a 'repository' argument to the constructor" unless $args->{repository};
42
43
    if (my $doc = $args->{doc}){
44
45
        #Get the root element
46
        my $root = $doc->documentElement;
47
48
        #Register namespaces for searching purposes
49
        my $xpc = XML::LibXML::XPathContext->new();
50
        $xpc->registerNs('oai','http://www.openarchives.org/OAI/2.0/');
51
52
        my $xpath_identifier = XML::LibXML::XPathExpression->new("oai:header/oai:identifier");
53
        my $identifier = $xpc->findnodes($xpath_identifier,$root)->shift;
54
        $args->{header_identifier} = $identifier->textContent;
55
56
        my $xpath_datestamp = XML::LibXML::XPathExpression->new("oai:header/oai:datestamp");
57
        my $datestamp = $xpc->findnodes($xpath_datestamp,$root)->shift;
58
        $args->{header_datestamp} = $datestamp->textContent;
59
60
        my $xpath_status = XML::LibXML::XPathExpression->new(q{oai:header/@status});
61
        my $status_node = $xpc->findnodes($xpath_status,$root)->shift;
62
        $args->{header_status} = $status_node ? $status_node->textContent : "";
63
    }
64
65
    return bless ($args, $class);
66
}
67
68
sub is_deleted_upstream {
69
    my ($self, $args) = @_;
70
    if ($self->{header_status}){
71
        if ($self->{header_status} eq "deleted"){
72
            return 1;
73
        }
74
    }
75
    return 0;
76
}
77
78
sub set_filter {
79
    my ($self, $filter_definition) = @_;
80
81
    #Source a default XSLT to use for filtering
82
    my $htdocs  = C4::Context->config('intrahtdocs');
83
    my $theme   = C4::Context->preference("template");
84
    $self->{filter} = "$htdocs/$theme/en/xslt/StripOAIPMH.xsl";
85
    $self->{filter_type} = "xslt";
86
87
    if ($filter_definition && $filter_definition ne "default"){
88
        my ($filter_type, $filter) = $self->_parse_filter($filter_definition);
89
        if ($filter_type eq "xslt"){
90
            if (  -f $filter ){
91
                $self->{filter} = $filter;
92
                $self->{filter_type} = "xslt";
93
            }
94
        }
95
    }
96
}
97
98
sub _parse_filter {
99
    my ($self,$filter_definition) = @_;
100
    my ($type,$filter);
101
    my $filter_uri = URI->new($filter_definition);
102
    if ($filter_uri){
103
        my $scheme = $filter_uri->scheme;
104
        if ( ($scheme && $scheme eq "file") || ! $scheme ){
105
            my $path = $filter_uri->path;
106
            #Filters may theoretically be .xsl or .pm files
107
            my($filename, $dirs, $suffix) = fileparse($path,(".xsl",".pm"));
108
            if ($suffix){
109
                if ( $suffix eq ".xsl"){
110
                    $type = "xslt";
111
                    $filter = $path;
112
                }
113
            }
114
        }
115
    }
116
    return ($type,$filter);
117
}
118
119
sub filter {
120
    my ($self) = @_;
121
    my $filtered = 0;
122
    my $doc = $self->{doc};
123
    my $filter = $self->{filter};
124
    my $filter_type = $self->{filter_type};
125
    if ($doc){
126
        if ($filter && -f $filter){
127
            if ($filter_type){
128
                if ( $filter_type eq 'xslt' ){
129
                    my $xslt = XML::LibXSLT->new();
130
                    my $style_doc = XML::LibXML->load_xml(location => $filter);
131
                    my $stylesheet = $xslt->parse_stylesheet($style_doc);
132
                    if ($stylesheet){
133
                        my $results = $stylesheet->transform($doc);
134
						if ($results){
135
							my $root = $results->documentElement;
136
							if ($root){
137
								my $namespace = $root->namespaceURI;
138
								if ($namespace eq "http://www.loc.gov/MARC21/slim"){
139
									#NOTE: Both MARC21 and UNIMARC should be covered by this namespace
140
									my $marcxml = eval { Koha::OAI::Harvester::Import::MARCXML->new({ dom => $results, }) };
141
                                    if ($@){
142
                                        warn "Error Koha::OAI::Harvester::Import::MARCXML: $@";
143
                                        return undef;
144
                                    } else {
145
                                        return $marcxml;
146
                                    }
147
								} elsif ($namespace eq "http://www.w3.org/1999/02/22-rdf-syntax-ns#"){
148
									my $rdfxml = Koha::OAI::Harvester::Import::RDFXML->new({ dom => $results, });
149
									if ($rdfxml){
150
										return $rdfxml;
151
									}
152
								}
153
							}
154
						}
155
                    }
156
                }
157
            }
158
        }
159
    }
160
    return;
161
}
162
163
sub _find_koha_link {
164
    my ($self, $args) = @_;
165
    my $record_type = $args->{record_type} // "biblio";
166
    my $link_id;
167
    if ($record_type eq "biblio"){
168
        my $link = $schema->resultset('OaiHarvesterBiblio')->find(
169
            {
170
                oai_repository => $self->{repository},
171
                oai_identifier => $self->{header_identifier},
172
            },
173
            { key => "oai_record",}
174
        );
175
        if ($link && $link->biblionumber){
176
            $link_id = $link->biblionumber->id;
177
        }
178
    }
179
    return $link_id;
180
}
181
182
=head3
183
184
    my ($action,$record_id) = $oai_record->import_record({
185
        filter => $filter,
186
        framework => $framework,
187
        record_type => $record_type,
188
        matcher => $matcher,
189
    });
190
191
    $action eq "added" || "updated" || "deleted" || "not_found" || "error"
192
193
=cut
194
195
sub import_record {
196
    my ($self, $args) = @_;
197
    my $filter = $args->{filter} || 'default';
198
    my $framework = $args->{framework} || '';
199
    my $record_type = $args->{record_type} || 'biblio';
200
    my $matcher = $args->{matcher};
201
202
    my $action = "error";
203
204
    #Find linkage between OAI-PMH repository-identifier and Koha record id
205
    my $linked_id = $self->_find_koha_link({
206
        record_type => $record_type,
207
    });
208
209
    if ($self->is_deleted_upstream){
210
        #NOTE: If a record is deleted upstream, it will not contain a metadata element
211
        if ($linked_id){
212
            $action = $self->delete_koha_record({
213
                record_id => $linked_id,
214
                record_type => $record_type,
215
            });
216
            #TODO: How to handle RDF deletions?
217
            #NOTE: If we delete the Koha record, then we lose the link to our downloaded record, so I don't know if it matters...
218
            #...although surely we'd want some way of cleaning up records that aren't needed anymore? I suppose that could come up with a cronjob that scans for triples that aren't linked to...
219
        }
220
        else {
221
            $action = "not_found";
222
            #NOTE: If there's no OAI-PMH repository-identifier pair in the database,
223
            #then there's no perfect way to find a linked record to delete.
224
        }
225
    }
226
    else {
227
        $self->set_filter($filter);
228
229
230
        my $import_record = $self->filter();
231
232
        if ($import_record){
233
            ($action,$linked_id) = $import_record->import_record({
234
                framework => $framework,
235
                record_type => $record_type,
236
                matcher => $matcher,
237
                koha_id => $linked_id,
238
            });
239
240
            if ($linked_id){
241
                #Link Koha record ID to OAI-PMH details for this record type,
242
                #if linkage doesn't already exist.
243
                $self->link_koha_record({
244
                    record_type => $record_type,
245
                    koha_id => $linked_id,
246
                });
247
            }
248
        }
249
    }
250
251
    #Log record details to database
252
    my $importer = $schema->resultset('OaiHarvesterHistory')->create({
253
        header_identifier => $self->{header_identifier},
254
        header_datestamp => $self->{header_datestamp},
255
        header_status => $self->{header_status},
256
        record => $self->{doc}->toString(1),
257
        repository => $self->{repository},
258
        status => $action,
259
        filter => $filter,
260
        framework => $framework,
261
        record_type => $record_type,
262
        matcher_code => $matcher ? $matcher->code : undef,
263
    });
264
265
    return ($action,$linked_id);
266
}
267
268
sub link_koha_record {
269
    my ($self, $args) = @_;
270
    my $record_type = $args->{record_type} // "biblio";
271
    my $koha_id = $args->{koha_id};
272
    if ($koha_id){
273
        if ($record_type eq "biblio"){
274
            my $import_oai_biblio = $schema->resultset('OaiHarvesterBiblio')->find_or_create({
275
                oai_repository => $self->{repository},
276
                oai_identifier => $self->{header_identifier},
277
                biblionumber => $koha_id,
278
            });
279
            if ( ! $import_oai_biblio->in_storage ){
280
                $import_oai_biblio->insert;
281
            }
282
        }
283
    }
284
}
285
286
sub delete_koha_record {
287
    my ($self, $args) = @_;
288
    my $record_type = $args->{record_type} // "biblio";
289
	my $record_id = $args->{record_id};
290
291
    my $action = "error";
292
293
	if ($record_type eq "biblio"){
294
        my $error = C4::Biblio::DelBiblio($record_id);
295
        if (!$error){
296
            $action = "deleted";
297
            #NOTE: If there's no error, a cascading database delete should
298
            #automatically remove the link between the Koha biblionumber and OAI-PMH record too
299
        }
300
    }
301
    return $action;
302
}
303
304
1;
(-)a/Koha/OAI/Harvester/Listener.pm (+187 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Listener;
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
use POE qw(Wheel::SocketFactory Wheel::ReadWrite);
23
use IO::Socket qw(AF_UNIX);
24
use JSON;
25
use URI;
26
27
sub new {
28
    my ($class, $args) = @_;
29
    $args = {} unless defined $args;
30
    return bless ($args, $class);
31
}
32
33
sub spawn {
34
    my ($class, $args) = @_;
35
    my $self = $class->new($args);
36
    my $socket = $args->{socket};
37
    POE::Session->create(
38
        args => [
39
            $socket,
40
        ],
41
        object_states => [
42
            $self => {
43
                _start => "on_start",
44
                "on_server_success" => "on_server_success",
45
                "on_server_error" => "on_server_error",
46
                "on_client_error" => "on_client_error",
47
                "on_client_input" => "on_client_input",
48
            },
49
        ],
50
    );
51
}
52
53
sub on_start {
54
    my ($kernel,$heap,$socket_uri) = @_[KERNEL,HEAP,ARG0];
55
56
    my $uri = URI->new($socket_uri);
57
    if ($uri && $uri->scheme eq 'unix'){
58
        my $socket_path = $uri->path;
59
        unlink $socket_path if -S $socket_path;
60
        $heap->{server} = POE::Wheel::SocketFactory->new(
61
            SocketDomain => AF_UNIX,
62
            BindAddress => $socket_path,
63
            SuccessEvent => "on_server_success",
64
            FailureEvent => "on_server_error",
65
        );
66
67
        #Make the socket writeable to other users like Apache
68
        chmod 0666, $socket_path;
69
    }
70
}
71
72
sub on_server_success {
73
    my ($self, $client_socket, $server_wheel_id, $heap, $session) = @_[OBJECT, ARG0, ARG3, HEAP,SESSION];
74
    my $logger = $self->{logger};
75
    my $null_filter = POE::Filter::Line->new(
76
         Literal => chr(0),
77
    );
78
    my $client_wheel = POE::Wheel::ReadWrite->new(
79
        Handle => $client_socket,
80
        InputEvent => "on_client_input",
81
        ErrorEvent => "on_client_error",
82
        InputFilter => $null_filter,
83
        OutputFilter => $null_filter,
84
    );
85
    $heap->{client}->{ $client_wheel->ID() } = $client_wheel;
86
    $logger->info("Connection ".$client_wheel->ID()." started.");
87
    #TODO: Add basic authentication here?
88
    $client_wheel->put("HELLO");
89
}
90
91
sub on_server_error {
92
    my ($self, $operation, $errnum, $errstr, $heap, $session) = @_[OBJECT, ARG0, ARG1, ARG2,HEAP, SESSION];
93
    my $logger = $self->{logger};
94
    $logger->error("Server $operation error $errnum: $errstr");
95
    delete $heap->{server};
96
}
97
98
sub on_client_error {
99
    my ($self, $wheel_id,$heap,$session) = @_[OBJECT, ARG3,HEAP,SESSION];
100
    my $logger = $self->{logger};
101
    $logger->info("Connection $wheel_id failed or ended.");
102
    delete $heap->{client}->{$wheel_id};
103
}
104
105
sub on_client_input {
106
    my ($self, $input, $wheel_id, $session, $kernel, $heap) = @_[OBJECT, ARG0, ARG1, SESSION, KERNEL, HEAP];
107
    my $logger = $self->{logger};
108
    $logger->debug("Server input: $input");
109
    my $server_response = { msg => "fail"};
110
    eval {
111
        my $json_input = from_json($input);
112
        my $command = $json_input->{command};
113
        my $body = $json_input->{body};
114
        if ($command){
115
            if ($command eq "create"){
116
                my $task = $body->{task};
117
                if ($task){
118
                    my $is_created = $kernel->call("harvester","create_task",$task);
119
                    if ($is_created){
120
                        $server_response->{msg} = "success";
121
                    }
122
                }
123
            }
124
            elsif ($command eq "start"){
125
                my $task = $body->{task};
126
                if ($task){
127
                    my $uuid = $task->{uuid};
128
                    #Fetch from memory now...
129
                    my $is_started = $kernel->call("harvester","start_task", $uuid);
130
                    if ($is_started){
131
                        $server_response->{msg} = "success";
132
                    }
133
                }
134
            }
135
            elsif ($command eq "stop"){
136
                my $task = $body->{task};
137
                if ($task){
138
                    if ($task->{uuid}){
139
                        my $is_stopped = $kernel->call("harvester","stop_task",$task->{uuid});
140
                        if ($is_stopped){
141
                            $server_response->{msg} = "success";
142
                        }
143
                    }
144
                }
145
            }
146
            elsif ($command eq "delete"){
147
                my $task = $body->{task};
148
                if ($task){
149
                    if ($task->{uuid}){
150
                        my $is_deleted = $kernel->call("harvester","delete_task",$task->{uuid});
151
                        if ($is_deleted){
152
                            $server_response->{msg} = "success";
153
                        }
154
                    }
155
                }
156
            }
157
            elsif ($command eq "list"){
158
                my $tasks = $kernel->call("harvester","list_tasks");
159
                if ($tasks){
160
                    $server_response->{msg} = "success";
161
                    $server_response->{data} = $tasks;
162
                }
163
            }
164
        }
165
    };
166
    if ($@){
167
        #NOTE: An error most likely means that something other than a valid JSON string was received
168
        $logger->error($@);
169
    }
170
171
    if ($server_response){
172
        eval {
173
            my $client = $heap->{client}->{$wheel_id};
174
            my $json_message = to_json($server_response, { pretty => 1 });
175
            if ($json_message){
176
                $logger->debug("Server output: $json_message");
177
                $client->put($json_message);
178
            }
179
        };
180
        if ($@){
181
            #NOTE: An error means our response couldn't be serialised as JSON
182
            $logger->error($@);
183
        }
184
    }
185
}
186
187
1;
(-)a/Koha/OAI/Harvester/Request.pm (+184 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Request;
2
3
# Copyright Prosentient Systems 2017
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use Carp;
23
24
use base qw(Koha::Object);
25
26
#For validation
27
use URI;
28
use HTTP::OAI;
29
30
=head1 NAME
31
32
Koha::OAI::Harvester::Request -
33
34
=head1 API
35
36
=head2 Class Methods
37
38
=cut
39
40
41
42
=head3 _type
43
44
=cut
45
46
sub _type {
47
    return 'OaiHarvesterRequest';
48
}
49
50
sub validate {
51
    my ($self) = @_;
52
    my $errors = {};
53
54
    #Step one: validate URL
55
    my $uri = URI->new($self->http_url);
56
    if ( $uri && $uri->scheme && ($uri->scheme eq "http" || $uri->scheme eq "https") ){
57
58
        #Step two: validate access and authorization to URL
59
        my $harvester = $self->_harvester();
60
        my $identify = $harvester->Identify;
61
        if ($identify->is_success){
62
63
            #Step three: validate OAI-PMH parameters
64
65
            #Test Set
66
			my $set = $self->oai_set;
67
            if ($set){
68
                my $set_response = $harvester->ListSets();
69
				my @server_sets = $set_response->set;
70
				if ( ! grep {$_->setSpec eq $set} @server_sets ){
71
                    $errors->{oai_set}->{unavailable} = 1;
72
                }
73
            }
74
75
            #Test Metadata Prefix
76
			my $metadataPrefix = $self->oai_metadataPrefix;
77
			if ($metadataPrefix){
78
				my $metadata_response = $harvester->ListMetadataFormats();
79
				my @server_formats = $metadata_response->metadataFormat;
80
				if ( ! grep { $_->metadataPrefix eq $metadataPrefix } @server_formats ){
81
                    $errors->{oai_metadataPrefix}->{unavailable} = 1;
82
                }
83
			}
84
            else {
85
                $errors->{oai_metadataPrefix}->{missing} = 1;
86
            }
87
88
            #Test Granularity and Timestamps
89
			my $server_granularity = $identify->granularity;
90
			my $from = $self->oai_from;
91
			my $until = $self->oai_until;
92
			if ($from || $until){
93
				my ($from_granularity,$until_granularity);
94
				if ($from){
95
					$from_granularity = _determine_granularity($from);
96
					if ($from_granularity eq "YYYY-MM-DDThh:mm:ssZ"){
97
						$errors->{oai_from}->{unavailable} = 1 if $server_granularity ne $from_granularity;
98
					} elsif ($from_granularity eq "failed"){
99
						$errors->{oai_from}->{malformed} = 1;
100
					}
101
				}
102
				if ($until){
103
					$until_granularity = _determine_granularity($until);
104
					if ($until_granularity eq "YYYY-MM-DDThh:mm:ssZ"){
105
						$errors->{oai_until}->{unavailable} = 1 if $server_granularity ne $until_granularity;
106
					} elsif ($until_granularity eq "failed"){
107
						$errors->{oai_until}->{malformed} = 1;
108
					}
109
				}
110
				if ($from && $until){
111
					if ($from_granularity ne $until_granularity){
112
						$errors->{oai}->{granularity_mismatch} = 1;
113
					}
114
				}
115
			}
116
117
            #Test if identifier is provided when using GetRecord
118
            my $verb = $self->oai_verb;
119
            if ($verb && $verb eq "GetRecord"){
120
                my $identifier = $self->oai_identifier;
121
                if (! $identifier){
122
                    $errors->{oai_identifier}->{missing} = 1;
123
                }
124
            }
125
        }
126
        elsif ($identify->is_error){
127
            foreach my $error ($identify->errors){
128
                if ($error->code =~ /^404$/){
129
                    $errors->{http}->{404} = 1;
130
                } elsif ($error->code =~ /^401$/){
131
                    $errors->{http}->{401} = 1;
132
                } else {
133
                    $errors->{http}->{generic} = 1;
134
                }
135
            }
136
        }
137
        else {
138
            $errors->{http}->{generic} = 1;
139
        }
140
    } else {
141
        $errors->{http_url}->{malformed} = 1;
142
    }
143
    return $errors;
144
}
145
146
sub _harvester {
147
    my ( $self ) = @_;
148
    my $harvester;
149
    if ($self->http_url){
150
        $harvester = new HTTP::OAI::Harvester( baseURL => $self->http_url );
151
		my $uri = URI->new($self->http_url);
152
        if ($uri->scheme && ($uri->scheme eq 'http' || $uri->scheme eq 'https') ){
153
            my $host = $uri->host;
154
            my $port = $uri->port;
155
            $harvester->credentials($host.":".$port, $self->http_realm, $self->http_username, $self->http_password);
156
        }
157
    }
158
    return $harvester;
159
}
160
161
sub _determine_granularity {
162
	my ($timestamp) = @_;
163
	my $granularity;
164
	if ($timestamp =~ /^(\d{4}-\d{2}-\d{2})(T\d{2}:\d{2}:\d{2}Z)?$/){
165
		if ($1 && $2){
166
			$granularity = "YYYY-MM-DDThh:mm:ssZ";
167
		} elsif ($1 && !$2){
168
			$granularity = "YYYY-MM-DD";
169
		} else {
170
			$granularity = "failed";
171
		}
172
	} else {
173
		$granularity = "failed";
174
	}
175
	return $granularity;
176
}
177
178
=head1 AUTHOR
179
180
David Cook <dcook@prosentient.com.au>
181
182
=cut
183
184
1;
(-)a/Koha/OAI/Harvester/Requests.pm (+62 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Requests;
2
3
# Copyright Prosentient Systems 2017
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use Carp;
23
24
use Koha::Database;
25
26
use Koha::OAI::Harvester::Request;
27
28
use base qw(Koha::Objects);
29
30
=head1 NAME
31
32
Koha::OAI::Harvester::Requests -
33
34
=head1 API
35
36
=head2 Class Methods
37
38
=cut
39
40
=head3 _type
41
42
=cut
43
44
sub _type {
45
    return 'OaiHarvesterRequest';
46
}
47
48
=head3 object_class
49
50
=cut
51
52
sub object_class {
53
    return 'Koha::OAI::Harvester::Request';
54
}
55
56
=head1 AUTHOR
57
58
David Cook <dcook@prosentient.com.au>
59
60
=cut
61
62
1;
(-)a/Koha/OAI/Harvester/Worker.pm (+156 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Worker;
2
3
# Copyright Prosentient Systems 2017
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use POE;
22
use DateTime;
23
use JSON;
24
25
sub new {
26
    my ($class, $args) = @_;
27
    $args = {} unless defined $args;
28
    $args->{type} = "worker" unless $args->{type};
29
    return bless ($args, $class);
30
}
31
32
sub run {
33
    my ($self,$args) = @_;
34
    my $postback = $args->{postback};
35
    my $task = $args->{task};
36
37
    POE::Session->create(
38
        object_states => [
39
            $self => {
40
                _start           => "on_start",
41
                got_child_stderr => "on_child_stderr",
42
                got_child_close  => "on_child_close",
43
                got_child_signal => "on_child_signal",
44
                got_child_stdout => "on_child_stdout",
45
                stop_worker      => "stop_worker",
46
                _stop            => "on_stop",
47
            },
48
        ],
49
        args => [
50
            $postback,
51
            $task,
52
        ],
53
    );
54
}
55
56
sub stop_worker {
57
    my ($self,$heap) = @_[OBJECT,HEAP];
58
    if (my $child_processes = $heap->{children_by_pid}){
59
        foreach my $child_pid (keys %$child_processes){
60
            my $child = $child_processes->{$child_pid};
61
            $child->kill();
62
        }
63
    }
64
}
65
66
67
sub on_stop {
68
    my ($self,$kernel) = @_[OBJECT,KERNEL];
69
70
    #Deregister the worker session from the harvester's roster of workers
71
    $kernel->call("harvester","deregister",$self->{type});
72
}
73
74
# Wheel event, including the wheel's ID.
75
sub on_child_stdout {
76
    my ($self, $stdout_line, $wheel_id) = @_[OBJECT, ARG0, ARG1];
77
    my $type = $self->{type};
78
    my $child = $_[HEAP]{children_by_wid}{$wheel_id};
79
    my $logger = $self->{logger};
80
    if ($logger){
81
        $logger->debug("[$type][pid ".$child->PID."][STDOUT] $stdout_line");
82
    }
83
84
    my $postback = $_[HEAP]{postback};
85
    if ($postback){
86
        eval {
87
            my $message = from_json($stdout_line);
88
            if ($message){
89
                $postback->($message);
90
            }
91
        };
92
    }
93
}
94
95
# Wheel event, including the wheel's ID.
96
sub on_child_stderr {
97
    my ($self,$stderr_line, $wheel_id) = @_[OBJECT, ARG0, ARG1];
98
    my $type = $self->{type};
99
    my $child = $_[HEAP]{children_by_wid}{$wheel_id};
100
    my $logger = $self->{logger};
101
    if ($logger){
102
        $logger->debug("[$type][pid ".$child->PID."][STDERR] $stderr_line");
103
    }
104
}
105
106
# Wheel event, including the wheel's ID.
107
sub on_child_close {
108
    my ($self,$heap,$wheel_id) = @_[OBJECT,HEAP,ARG0];
109
    my $type = $self->{type};
110
    my $logger = $self->{logger};
111
112
    my $child = delete $heap->{children_by_wid}->{$wheel_id};
113
114
    # May have been reaped by on_child_signal().
115
    unless (defined $child) {
116
        if ($logger){
117
            $logger->debug("[$type][wid $wheel_id] closed all pipes");
118
        }
119
        return;
120
    }
121
    if ($logger){
122
        $logger->debug("[$type][pid ".$child->PID."] closed all pipes");
123
    }
124
    delete $heap->{children_by_pid}->{$child->PID};
125
}
126
127
sub on_child_signal {
128
    my ($self,$kernel,$pid,$status) = @_[OBJECT,KERNEL,ARG1,ARG2];
129
    my $type = $self->{type};
130
    my $logger = $self->{logger};
131
    if ($logger){
132
        $logger->debug("[$type][pid $pid] exited with status $status");
133
    }
134
135
    my $child = delete $_[HEAP]{children_by_pid}{$_[ARG1]};
136
137
    # May have been reaped by on_child_close().
138
    return unless defined $child;
139
140
    delete $_[HEAP]{children_by_wid}{$child->ID};
141
142
    #If the child doesn't complete successfully, we lodge an error
143
    #and stop the task.
144
    if ($status != 0){
145
        my $task = $kernel->call("harvester","get_task");
146
        if ($task){
147
            $task->{error} = 1;
148
            my $uuid = $task->{uuid};
149
            if ($uuid){
150
                $kernel->call("harvester","stop_task",$uuid);
151
            }
152
        }
153
    }
154
}
155
156
1;
(-)a/Koha/OAI/Harvester/Worker/Download/Stream.pm (+187 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Worker::Download::Stream;
2
3
# Copyright Prosentient Systems 2017
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use LWP::UserAgent;
22
use UUID qw/uuid/;
23
use POE;
24
use JSON;
25
use File::Path qw/make_path/;
26
27
use C4::Context;
28
use Koha::OAI::Harvester::Downloader;
29
use parent 'Koha::OAI::Harvester::Worker';
30
31
sub new {
32
    my ($class, $args) = @_;
33
    $args = {} unless defined $args;
34
    $args->{type} = "download" unless $args->{type};
35
    return bless ($args, $class);
36
}
37
38
sub on_start {
39
    my ($self, $kernel, $heap, $postback,$task,$session) = @_[OBJECT, KERNEL, HEAP, ARG0,ARG1,SESSION];
40
    #Save postback into heap so other event handlers can use it
41
    $heap->{postback} = $postback;
42
43
    my $task_uuid = $task->{uuid};
44
45
    $kernel->sig("cancel" => "stop_worker");
46
    $kernel->call("harvester","register",$self->{type},$task->{uuid});
47
48
    my $child = POE::Wheel::Run->new(
49
        ProgramArgs => [$task],
50
        Program => sub {
51
            my ($args) = @_;
52
            $self->do_work($args);
53
        },
54
        StdoutEvent  => "got_child_stdout",
55
        StderrEvent  => "got_child_stderr",
56
        CloseEvent   => "got_child_close",
57
        NoSetPgrp => 1, #Keep child processes in same group as parent. This is especially useful when using Ctrl+C to kill the whole group.
58
    );
59
60
     $_[KERNEL]->sig_child($child->PID, "got_child_signal");
61
62
    # Wheel events include the wheel's ID.
63
    $_[HEAP]{children_by_wid}{$child->ID} = $child;
64
65
    # Signal events include the process ID.
66
    $_[HEAP]{children_by_pid}{$child->PID} = $child;
67
68
    my $logger = $self->{logger};
69
    if ($logger){
70
        $logger->debug("Child pid ".$child->PID." started as wheel ".$child->ID);
71
    }
72
}
73
74
sub do_work {
75
    my ($self, $task) = @_;
76
    my $batch = ( $self->{batch} && int($self->{batch}) ) ? $self->{batch} : 100;
77
78
    #NOTE: Directory to spool files for processing
79
    my $spooldir = $task->{spooldir};
80
81
    my $task_uuid = $task->{uuid};
82
    my $task_parameters = $task->{parameters};
83
    my $interval = $task->{interval};
84
85
    my $oai_pmh_parameters = $task_parameters->{oai_pmh};
86
    my $import_parameters = $task_parameters->{import};
87
88
    #NOTE: Overwrite the 'from' and 'until' parameters for repeatable tasks
89
    if ( $interval && ! $oai_pmh_parameters->{until} ){
90
        if ($oai_pmh_parameters->{verb} eq "ListRecords"){
91
            #NOTE: 'effective_from' premiers on the first repetition (ie second request)
92
            $oai_pmh_parameters->{from} = $task->{effective_from} if $task->{effective_from};
93
            #NOTE: 'effective_until' appears on the first request
94
            $oai_pmh_parameters->{until} = $task->{effective_until} if $task->{effective_until};
95
        }
96
    }
97
98
    my $oai_downloader = Koha::OAI::Harvester::Downloader->new();
99
    my $url = $oai_downloader->BuildURL($oai_pmh_parameters);
100
101
    my $ua = LWP::UserAgent->new();
102
    #NOTE: setup HTTP Basic Authentication if parameters are supplied
103
    if($url && $url->host && $url->port){
104
        my $http_basic_auth = $task_parameters->{http_basic_auth};
105
        if ($http_basic_auth){
106
            my $username = $http_basic_auth->{username};
107
            my $password = $http_basic_auth->{password};
108
            my $realm = $http_basic_auth->{realm};
109
            $ua->credentials($url->host.":".$url->port, $realm, $username, $password);
110
        }
111
    }
112
113
    #NOTE: Prepare database statement handle
114
    my $dbh = C4::Context->dbh;
115
    my $sql = "insert into oai_harvester_import_queue (uuid,result) VALUES (?,?)";
116
    my $sth = $dbh->prepare($sql);
117
118
    if($url && $ua){
119
        #NOTE: You could define the callbacks as object methods instead... that might be nicer...
120
        #although I suppose it might be a much of a muchness.
121
        eval {
122
            my @filename_cache = ();
123
124
            $oai_downloader->harvest({
125
                user_agent => $ua,
126
                url => $url,
127
                callback => sub {
128
                    my ($args) = @_;
129
130
                    my $repository = $args->{repository};
131
                    my $document = $args->{document};
132
133
                    #If the spooldir has disappeared, re-create it.
134
                    if ( ! -d $spooldir ){
135
                        my $made_spool_directory = make_path($spooldir);
136
                    }
137
                    my $file_uuid = uuid();
138
                    my $filename = "$spooldir/$file_uuid";
139
                    my $state = $document->toFile($filename, 2);
140
                    if ($state){
141
                        push(@filename_cache,$filename);
142
                    }
143
144
                    if(scalar @filename_cache == $batch){
145
                        my $result = {
146
                            repository => $repository,
147
                            filenames => \@filename_cache,
148
                            filter => $import_parameters->{filter},
149
                            matcher_code => $import_parameters->{matcher_code},
150
                            frameworkcode => $import_parameters->{frameworkcode},
151
                            record_type => $import_parameters->{record_type},
152
                        };
153
                        eval {
154
                            my $json_result = to_json($result, { pretty => 1 });
155
                            $sth->execute($task_uuid,$json_result);
156
                        };
157
                        @filename_cache = ();
158
                    }
159
                },
160
                complete_callback => sub {
161
                    my ($args) = @_;
162
                    my $repository = $args->{repository};
163
                    if (@filename_cache){
164
                        my $result = {
165
                            repository => "$repository",
166
                            filenames => \@filename_cache,
167
                            filter => $import_parameters->{filter},
168
                            matcher_code => $import_parameters->{matcher_code},
169
                            frameworkcode => $import_parameters->{frameworkcode},
170
                            record_type => $import_parameters->{record_type},
171
                        };
172
                        eval {
173
                            my $json_result = to_json($result, { pretty => 1 });
174
                            $sth->execute($task_uuid,$json_result);
175
                        };
176
                    }
177
178
                },
179
            });
180
        };
181
        if ($@){
182
            die "Error during OAI-PMH download";
183
        }
184
    }
185
}
186
187
1;
(-)a/Koha/OAI/Harvester/Worker/Import.pm (+133 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Worker::Import;
2
3
# Copyright Prosentient Systems 2017
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use POE qw(Wheel::Run);
22
use JSON;
23
use XML::LibXML;
24
25
use C4::Context;
26
use C4::Matcher;
27
use Koha::OAI::Harvester::Import::Record;
28
29
use parent 'Koha::OAI::Harvester::Worker';
30
31
sub new {
32
    my ($class, $args) = @_;
33
    $args = {} unless defined $args;
34
    #NOTE: This type is used for logging and more importantly for registering with the harvester
35
    $args->{type} = "import" unless $args->{type};
36
    return bless ($args, $class);
37
}
38
39
sub on_start {
40
    my ($self, $kernel, $heap, $postback,$task) = @_[OBJECT, KERNEL, HEAP, ARG0,ARG1];
41
42
    $kernel->call("harvester","register",$self->{type},$task->{uuid});
43
44
    $kernel->sig(cancel => "stop_worker");
45
46
    my $child = POE::Wheel::Run->new(
47
        ProgramArgs => [ $task ],
48
        Program => sub {
49
            my ($task,$args) = @_;
50
51
            my $debug = $args->{debug} // 0;
52
53
            if ($task){
54
                my $json_result = $task->{result};
55
                my $id = $task->{id};
56
                my $task_uuid = $task->{uuid};
57
                eval {
58
                    my $result = from_json($json_result);
59
                    if ($result){
60
                        my $repository = $result->{repository};
61
                        my $filenames = $result->{filenames};
62
                        my $filter = $result->{filter};
63
                        my $matcher_code = $result->{matcher_code};
64
                        my $frameworkcode = $result->{frameworkcode};
65
                        my $record_type = $result->{record_type};
66
67
                        my $matcher;
68
                        if ($matcher_code){
69
                            my $matcher_id = C4::Matcher::GetMatcherId($matcher_code);
70
                            $matcher = C4::Matcher->fetch($matcher_id);
71
                        }
72
73
                        foreach my $filename (@$filenames){
74
                            if ($filename){
75
                                if (-f $filename){
76
                                    my $dom = XML::LibXML->load_xml(location => $filename, { no_blanks => 1 });
77
                                    if ($dom){
78
                                        my $oai_record = Koha::OAI::Harvester::Import::Record->new({
79
                                            doc => $dom,
80
                                            repository => $repository,
81
                                        });
82
                                        if ($oai_record){
83
                                            my ($action,$linked_id) = $oai_record->import_record({
84
                                                filter => $filter,
85
                                                framework => $frameworkcode,
86
                                                record_type => $record_type,
87
                                                matcher => $matcher,
88
                                            });
89
                                            $debug && print STDOUT qq({ "import_result": { "task_uuid": "$task_uuid", "action": "$action", "filename": "$filename", "koha_id": "$linked_id" } }\n);
90
                                        }
91
                                    }
92
                                    my $unlinked = unlink $filename;
93
                                }
94
                            }
95
                        }
96
                    }
97
                };
98
                if ($@){
99
                    warn $@;
100
                }
101
                #NOTE: Even if the file doesn't exist, we still need to process the queue item.
102
103
                #NOTE: Don't do this via a postback in the parent process, as it's much faster to let the child process handle it.
104
105
                #NOTE: It's vital that files are unlinked before deleting from the database,
106
                #or you could get orphan files if the importer is interrupted.
107
                my $dbh = C4::Context->dbh;
108
                my $sql = "delete from oai_harvester_import_queue where id = ?";
109
                my $sth = $dbh->prepare($sql);
110
                $sth->execute($id);
111
            }
112
        },
113
        StdoutEvent  => "got_child_stdout",
114
        StderrEvent  => "got_child_stderr",
115
        CloseEvent   => "got_child_close",
116
        NoSetPgrp => 1, #Keep child processes in same group as parent. This is especially useful when using Ctrl+C to kill the whole group.
117
    );
118
119
    $_[KERNEL]->sig_child($child->PID, "got_child_signal");
120
121
    # Wheel events include the wheel's ID.
122
    $_[HEAP]{children_by_wid}{$child->ID} = $child;
123
124
    # Signal events include the process ID.
125
    $_[HEAP]{children_by_pid}{$child->PID} = $child;
126
127
    my $logger = $self->{logger};
128
    if ($logger){
129
        $logger->debug("Child pid ".$child->PID." started as wheel ".$child->ID);
130
    }
131
}
132
133
1;
(-)a/Koha/Schema/Result/OaiHarvesterBiblio.pm (+120 lines)
Line 0 Link Here
1
use utf8;
2
package Koha::Schema::Result::OaiHarvesterBiblio;
3
4
# Created by DBIx::Class::Schema::Loader
5
# DO NOT MODIFY THE FIRST PART OF THIS FILE
6
7
=head1 NAME
8
9
Koha::Schema::Result::OaiHarvesterBiblio
10
11
=cut
12
13
use strict;
14
use warnings;
15
16
use base 'DBIx::Class::Core';
17
18
=head1 TABLE: C<oai_harvester_biblios>
19
20
=cut
21
22
__PACKAGE__->table("oai_harvester_biblios");
23
24
=head1 ACCESSORS
25
26
=head2 import_oai_biblio_id
27
28
  data_type: 'integer'
29
  extra: {unsigned => 1}
30
  is_auto_increment: 1
31
  is_nullable: 0
32
33
=head2 oai_repository
34
35
  data_type: 'varchar'
36
  is_nullable: 0
37
  size: 255
38
39
=head2 oai_identifier
40
41
  data_type: 'varchar'
42
  is_nullable: 1
43
  size: 255
44
45
=head2 biblionumber
46
47
  data_type: 'integer'
48
  is_foreign_key: 1
49
  is_nullable: 0
50
51
=cut
52
53
__PACKAGE__->add_columns(
54
  "import_oai_biblio_id",
55
  {
56
    data_type => "integer",
57
    extra => { unsigned => 1 },
58
    is_auto_increment => 1,
59
    is_nullable => 0,
60
  },
61
  "oai_repository",
62
  { data_type => "varchar", is_nullable => 0, size => 255 },
63
  "oai_identifier",
64
  { data_type => "varchar", is_nullable => 1, size => 255 },
65
  "biblionumber",
66
  { data_type => "integer", is_foreign_key => 1, is_nullable => 0 },
67
);
68
69
=head1 PRIMARY KEY
70
71
=over 4
72
73
=item * L</import_oai_biblio_id>
74
75
=back
76
77
=cut
78
79
__PACKAGE__->set_primary_key("import_oai_biblio_id");
80
81
=head1 UNIQUE CONSTRAINTS
82
83
=head2 C<oai_record>
84
85
=over 4
86
87
=item * L</oai_identifier>
88
89
=item * L</oai_repository>
90
91
=back
92
93
=cut
94
95
__PACKAGE__->add_unique_constraint("oai_record", ["oai_identifier", "oai_repository"]);
96
97
=head1 RELATIONS
98
99
=head2 biblionumber
100
101
Type: belongs_to
102
103
Related object: L<Koha::Schema::Result::Biblio>
104
105
=cut
106
107
__PACKAGE__->belongs_to(
108
  "biblionumber",
109
  "Koha::Schema::Result::Biblio",
110
  { biblionumber => "biblionumber" },
111
  { is_deferrable => 1, on_delete => "CASCADE", on_update => "NO ACTION" },
112
);
113
114
115
# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-03-29 12:23:43
116
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:2URn8tPABMKC+JuIMfGeYw
117
118
119
# You can replace this text with custom code or comments, and it will be preserved on regeneration
120
1;
(-)a/Koha/Schema/Result/OaiHarvesterHistory.pm (+163 lines)
Line 0 Link Here
1
use utf8;
2
package Koha::Schema::Result::OaiHarvesterHistory;
3
4
# Created by DBIx::Class::Schema::Loader
5
# DO NOT MODIFY THE FIRST PART OF THIS FILE
6
7
=head1 NAME
8
9
Koha::Schema::Result::OaiHarvesterHistory
10
11
=cut
12
13
use strict;
14
use warnings;
15
16
use base 'DBIx::Class::Core';
17
18
=head1 TABLE: C<oai_harvester_history>
19
20
=cut
21
22
__PACKAGE__->table("oai_harvester_history");
23
24
=head1 ACCESSORS
25
26
=head2 import_oai_id
27
28
  data_type: 'integer'
29
  extra: {unsigned => 1}
30
  is_auto_increment: 1
31
  is_nullable: 0
32
33
=head2 repository
34
35
  data_type: 'varchar'
36
  is_nullable: 1
37
  size: 255
38
39
=head2 header_identifier
40
41
  data_type: 'varchar'
42
  is_nullable: 1
43
  size: 255
44
45
=head2 header_datestamp
46
47
  data_type: 'datetime'
48
  datetime_undef_if_invalid: 1
49
  is_nullable: 0
50
51
=head2 header_status
52
53
  data_type: 'varchar'
54
  is_nullable: 1
55
  size: 45
56
57
=head2 record
58
59
  data_type: 'longtext'
60
  is_nullable: 0
61
62
=head2 upload_timestamp
63
64
  data_type: 'timestamp'
65
  datetime_undef_if_invalid: 1
66
  default_value: current_timestamp
67
  is_nullable: 0
68
69
=head2 status
70
71
  data_type: 'varchar'
72
  is_nullable: 0
73
  size: 45
74
75
=head2 filter
76
77
  data_type: 'text'
78
  is_nullable: 0
79
80
=head2 framework
81
82
  data_type: 'varchar'
83
  is_nullable: 0
84
  size: 4
85
86
=head2 record_type
87
88
  data_type: 'enum'
89
  extra: {list => ["biblio","auth","holdings"]}
90
  is_nullable: 0
91
92
=head2 matcher_code
93
94
  data_type: 'varchar'
95
  is_nullable: 1
96
  size: 10
97
98
=cut
99
100
__PACKAGE__->add_columns(
101
  "import_oai_id",
102
  {
103
    data_type => "integer",
104
    extra => { unsigned => 1 },
105
    is_auto_increment => 1,
106
    is_nullable => 0,
107
  },
108
  "repository",
109
  { data_type => "varchar", is_nullable => 1, size => 255 },
110
  "header_identifier",
111
  { data_type => "varchar", is_nullable => 1, size => 255 },
112
  "header_datestamp",
113
  {
114
    data_type => "datetime",
115
    datetime_undef_if_invalid => 1,
116
    is_nullable => 0,
117
  },
118
  "header_status",
119
  { data_type => "varchar", is_nullable => 1, size => 45 },
120
  "record",
121
  { data_type => "longtext", is_nullable => 0 },
122
  "upload_timestamp",
123
  {
124
    data_type => "timestamp",
125
    datetime_undef_if_invalid => 1,
126
    default_value => \"current_timestamp",
127
    is_nullable => 0,
128
  },
129
  "status",
130
  { data_type => "varchar", is_nullable => 0, size => 45 },
131
  "filter",
132
  { data_type => "text", is_nullable => 0 },
133
  "framework",
134
  { data_type => "varchar", is_nullable => 0, size => 4 },
135
  "record_type",
136
  {
137
    data_type => "enum",
138
    extra => { list => ["biblio", "auth", "holdings"] },
139
    is_nullable => 0,
140
  },
141
  "matcher_code",
142
  { data_type => "varchar", is_nullable => 1, size => 10 },
143
);
144
145
=head1 PRIMARY KEY
146
147
=over 4
148
149
=item * L</import_oai_id>
150
151
=back
152
153
=cut
154
155
__PACKAGE__->set_primary_key("import_oai_id");
156
157
158
# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-03-29 12:23:43
159
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Rp/ZEZsKVlLo2vaM3M37ow
160
161
162
# You can replace this text with custom code or comments, and it will be preserved on regeneration
163
1;
(-)a/Koha/Schema/Result/OaiHarvesterImportQueue.pm (+106 lines)
Line 0 Link Here
1
use utf8;
2
package Koha::Schema::Result::OaiHarvesterImportQueue;
3
4
# Created by DBIx::Class::Schema::Loader
5
# DO NOT MODIFY THE FIRST PART OF THIS FILE
6
7
=head1 NAME
8
9
Koha::Schema::Result::OaiHarvesterImportQueue
10
11
=cut
12
13
use strict;
14
use warnings;
15
16
use base 'DBIx::Class::Core';
17
18
=head1 TABLE: C<oai_harvester_import_queue>
19
20
=cut
21
22
__PACKAGE__->table("oai_harvester_import_queue");
23
24
=head1 ACCESSORS
25
26
=head2 id
27
28
  data_type: 'integer'
29
  extra: {unsigned => 1}
30
  is_auto_increment: 1
31
  is_nullable: 0
32
33
=head2 uuid
34
35
  data_type: 'varchar'
36
  is_nullable: 0
37
  size: 45
38
39
=head2 status
40
41
  data_type: 'varchar'
42
  default_value: 'new'
43
  is_nullable: 0
44
  size: 45
45
46
=head2 result
47
48
  data_type: 'text'
49
  is_nullable: 0
50
51
=head2 result_timestamp
52
53
  data_type: 'timestamp'
54
  datetime_undef_if_invalid: 1
55
  default_value: current_timestamp
56
  is_nullable: 0
57
58
=cut
59
60
__PACKAGE__->add_columns(
61
  "id",
62
  {
63
    data_type => "integer",
64
    extra => { unsigned => 1 },
65
    is_auto_increment => 1,
66
    is_nullable => 0,
67
  },
68
  "uuid",
69
  { data_type => "varchar", is_nullable => 0, size => 45 },
70
  "status",
71
  {
72
    data_type => "varchar",
73
    default_value => "new",
74
    is_nullable => 0,
75
    size => 45,
76
  },
77
  "result",
78
  { data_type => "text", is_nullable => 0 },
79
  "result_timestamp",
80
  {
81
    data_type => "timestamp",
82
    datetime_undef_if_invalid => 1,
83
    default_value => \"current_timestamp",
84
    is_nullable => 0,
85
  },
86
);
87
88
=head1 PRIMARY KEY
89
90
=over 4
91
92
=item * L</id>
93
94
=back
95
96
=cut
97
98
__PACKAGE__->set_primary_key("id");
99
100
101
# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-03-29 12:23:43
102
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zBD+hMawbvu7sonuLRHnCA
103
104
105
# You can replace this text with custom code or comments, and it will be preserved on regeneration
106
1;
(-)a/Koha/Schema/Result/OaiHarvesterRequest.pm (+209 lines)
Line 0 Link Here
1
use utf8;
2
package Koha::Schema::Result::OaiHarvesterRequest;
3
4
# Created by DBIx::Class::Schema::Loader
5
# DO NOT MODIFY THE FIRST PART OF THIS FILE
6
7
=head1 NAME
8
9
Koha::Schema::Result::OaiHarvesterRequest
10
11
=cut
12
13
use strict;
14
use warnings;
15
16
use base 'DBIx::Class::Core';
17
18
=head1 TABLE: C<oai_harvester_requests>
19
20
=cut
21
22
__PACKAGE__->table("oai_harvester_requests");
23
24
=head1 ACCESSORS
25
26
=head2 id
27
28
  data_type: 'integer'
29
  extra: {unsigned => 1}
30
  is_auto_increment: 1
31
  is_nullable: 0
32
33
=head2 uuid
34
35
  data_type: 'varchar'
36
  is_nullable: 0
37
  size: 45
38
39
=head2 oai_verb
40
41
  data_type: 'varchar'
42
  is_nullable: 0
43
  size: 45
44
45
=head2 oai_metadataPrefix
46
47
  accessor: 'oai_metadata_prefix'
48
  data_type: 'varchar'
49
  is_nullable: 0
50
  size: 255
51
52
=head2 oai_identifier
53
54
  data_type: 'varchar'
55
  is_nullable: 1
56
  size: 255
57
58
=head2 oai_from
59
60
  data_type: 'varchar'
61
  is_nullable: 1
62
  size: 45
63
64
=head2 oai_until
65
66
  data_type: 'varchar'
67
  is_nullable: 1
68
  size: 45
69
70
=head2 oai_set
71
72
  data_type: 'varchar'
73
  is_nullable: 1
74
  size: 255
75
76
=head2 http_url
77
78
  data_type: 'varchar'
79
  is_nullable: 1
80
  size: 255
81
82
=head2 http_username
83
84
  data_type: 'varchar'
85
  is_nullable: 1
86
  size: 255
87
88
=head2 http_password
89
90
  data_type: 'varchar'
91
  is_nullable: 1
92
  size: 255
93
94
=head2 http_realm
95
96
  data_type: 'varchar'
97
  is_nullable: 1
98
  size: 255
99
100
=head2 import_filter
101
102
  data_type: 'varchar'
103
  is_nullable: 0
104
  size: 255
105
106
=head2 import_framework_code
107
108
  data_type: 'varchar'
109
  is_nullable: 0
110
  size: 4
111
112
=head2 import_record_type
113
114
  data_type: 'enum'
115
  extra: {list => ["biblio","auth","holdings"]}
116
  is_nullable: 0
117
118
=head2 import_matcher_code
119
120
  data_type: 'varchar'
121
  is_nullable: 1
122
  size: 10
123
124
=head2 interval
125
126
  data_type: 'integer'
127
  extra: {unsigned => 1}
128
  is_nullable: 0
129
130
=head2 name
131
132
  data_type: 'varchar'
133
  is_nullable: 0
134
  size: 45
135
136
=cut
137
138
__PACKAGE__->add_columns(
139
  "id",
140
  {
141
    data_type => "integer",
142
    extra => { unsigned => 1 },
143
    is_auto_increment => 1,
144
    is_nullable => 0,
145
  },
146
  "uuid",
147
  { data_type => "varchar", is_nullable => 0, size => 45 },
148
  "oai_verb",
149
  { data_type => "varchar", is_nullable => 0, size => 45 },
150
  "oai_metadataPrefix",
151
  {
152
    accessor => "oai_metadata_prefix",
153
    data_type => "varchar",
154
    is_nullable => 0,
155
    size => 255,
156
  },
157
  "oai_identifier",
158
  { data_type => "varchar", is_nullable => 1, size => 255 },
159
  "oai_from",
160
  { data_type => "varchar", is_nullable => 1, size => 45 },
161
  "oai_until",
162
  { data_type => "varchar", is_nullable => 1, size => 45 },
163
  "oai_set",
164
  { data_type => "varchar", is_nullable => 1, size => 255 },
165
  "http_url",
166
  { data_type => "varchar", is_nullable => 1, size => 255 },
167
  "http_username",
168
  { data_type => "varchar", is_nullable => 1, size => 255 },
169
  "http_password",
170
  { data_type => "varchar", is_nullable => 1, size => 255 },
171
  "http_realm",
172
  { data_type => "varchar", is_nullable => 1, size => 255 },
173
  "import_filter",
174
  { data_type => "varchar", is_nullable => 0, size => 255 },
175
  "import_framework_code",
176
  { data_type => "varchar", is_nullable => 0, size => 4 },
177
  "import_record_type",
178
  {
179
    data_type => "enum",
180
    extra => { list => ["biblio", "auth", "holdings"] },
181
    is_nullable => 0,
182
  },
183
  "import_matcher_code",
184
  { data_type => "varchar", is_nullable => 1, size => 10 },
185
  "interval",
186
  { data_type => "integer", extra => { unsigned => 1 }, is_nullable => 0 },
187
  "name",
188
  { data_type => "varchar", is_nullable => 0, size => 45 },
189
);
190
191
=head1 PRIMARY KEY
192
193
=over 4
194
195
=item * L</id>
196
197
=back
198
199
=cut
200
201
__PACKAGE__->set_primary_key("id");
202
203
204
# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-04-07 11:26:24
205
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Vm/b4yurmr8WF7z+Fo6KEw
206
207
208
# You can replace this text with custom code or comments, and it will be preserved on regeneration
209
1;
(-)a/Makefile.PL (+14 lines)
Lines 360-365 my $target_map = { Link Here
360
  './skel/var/lib/koha/zebradb/biblios/tmp'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
360
  './skel/var/lib/koha/zebradb/biblios/tmp'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
361
  './skel/var/lock/koha/zebradb/rebuild' => { target => 'ZEBRA_LOCK_DIR', trimdir => 6 },
361
  './skel/var/lock/koha/zebradb/rebuild' => { target => 'ZEBRA_LOCK_DIR', trimdir => 6 },
362
  './skel/var/lib/koha/plugins' => { target => 'PLUGINS_DIR', trimdir => 6 },
362
  './skel/var/lib/koha/plugins' => { target => 'PLUGINS_DIR', trimdir => 6 },
363
  './skel/var/lib/koha/oai-pmh-harvester' => { target => 'OAI_LIB_DIR', trimdir => 6 },
364
  './skel/var/run/koha/oai-pmh-harvester' => { target => 'OAI_RUN_DIR', trimdir => 6 },
365
  './skel/var/spool/koha/oai-pmh-harvester' => { target => 'OAI_SPOOL_DIR', trimdir => 6 },
363
  './sms'                       => 'INTRANET_CGI_DIR',
366
  './sms'                       => 'INTRANET_CGI_DIR',
364
  './suggestion'                => 'INTRANET_CGI_DIR',
367
  './suggestion'                => 'INTRANET_CGI_DIR',
365
  './svc'                       => 'INTRANET_CGI_DIR',
368
  './svc'                       => 'INTRANET_CGI_DIR',
Lines 594-599 my $pl_files = { Link Here
594
         'blib/KOHA_CONF_DIR/koha-conf.xml',
597
         'blib/KOHA_CONF_DIR/koha-conf.xml',
595
         'blib/KOHA_CONF_DIR/koha-httpd.conf',
598
         'blib/KOHA_CONF_DIR/koha-httpd.conf',
596
         'blib/KOHA_CONF_DIR/log4perl.conf',
599
         'blib/KOHA_CONF_DIR/log4perl.conf',
600
         'blib/KOHA_CONF_DIR/oai-pmh-harvester.yaml',
597
         'blib/ZEBRA_CONF_DIR/etc/default.idx',
601
         'blib/ZEBRA_CONF_DIR/etc/default.idx',
598
         'blib/MISC_DIR/koha-install-log'
602
         'blib/MISC_DIR/koha-install-log'
599
         ],
603
         ],
Lines 1355-1360 sub get_target_directories { Link Here
1355
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'koha', 'plugins');
1359
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'koha', 'plugins');
1356
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1360
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1357
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
1361
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
1362
        $dirmap{'OAI_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'oai-pmh-harvester');
1363
        $dirmap{'OAI_LIB_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'oai-pmh-harvester');
1364
        $dirmap{'OAI_SPOOL_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'spool', 'oai-pmh-harvester');
1358
    } elsif ($mode eq 'dev') {
1365
    } elsif ($mode eq 'dev') {
1359
        my $curdir = File::Spec->rel2abs(File::Spec->curdir());
1366
        my $curdir = File::Spec->rel2abs(File::Spec->curdir());
1360
        $dirmap{'API_CGI_DIR'} = File::Spec->catdir($curdir, 'api');
1367
        $dirmap{'API_CGI_DIR'} = File::Spec->catdir($curdir, 'api');
Lines 1390-1395 sub get_target_directories { Link Here
1390
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'plugins');
1397
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'plugins');
1391
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1398
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1392
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
1399
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
1400
        $dirmap{'OAI_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'oai-pmh-harvester');
1401
        $dirmap{'OAI_LIB_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'oai-pmh-harvester');
1402
        $dirmap{'OAI_SPOOL_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'spool', 'oai-pmh-harvester');
1403
1393
    } else {
1404
    } else {
1394
        # mode is standard, i.e., 'fhs'
1405
        # mode is standard, i.e., 'fhs'
1395
        $dirmap{'API_CGI_DIR'} = File::Spec->catdir(@basedir, $package, 'api');
1406
        $dirmap{'API_CGI_DIR'} = File::Spec->catdir(@basedir, $package, 'api');
Lines 1414-1419 sub get_target_directories { Link Here
1414
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'plugins');
1425
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'plugins');
1415
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'zebradb');
1426
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'zebradb');
1416
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'run', $package, 'zebradb');
1427
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'run', $package, 'zebradb');
1428
        $dirmap{'OAI_RUN_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'run', $package, 'oai-pmh-harvester');
1429
        $dirmap{'OAI_LIB_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'oai-pmh-harvester');
1430
        $dirmap{'OAI_SPOOL_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'spool', $package, 'oai-pmh-harvester');
1417
    }
1431
    }
1418
1432
1419
    _get_env_overrides(\%dirmap);
1433
    _get_env_overrides(\%dirmap);
(-)a/debian/scripts/koha-create (+7 lines)
Lines 113-118 generate_config_file() { Link Here
113
        -e "s/__PLUGINS_DIR__/\/var\/lib\/koha\/$name\/plugins/g" \
113
        -e "s/__PLUGINS_DIR__/\/var\/lib\/koha\/$name\/plugins/g" \
114
        -e "s/__MEMCACHED_NAMESPACE__/$MEMCACHED_NAMESPACE/g" \
114
        -e "s/__MEMCACHED_NAMESPACE__/$MEMCACHED_NAMESPACE/g" \
115
        -e "s/__MEMCACHED_SERVERS__/$MEMCACHED_SERVERS/g" \
115
        -e "s/__MEMCACHED_SERVERS__/$MEMCACHED_SERVERS/g" \
116
        -e "s/__OAI_RUN_DIR__/\/var\/run\/koha\/$name\/oai-pmh-harvester/g" \
117
        -e "s/__OAI_LIB_DIR__/\/var\/lib\/koha\/$name\/oai-pmh-harvester/g" \
118
        -e "s/__OAI_SPOOL_DIR__/\/var\/spool\/koha\/$name\/oai-pmh-harvester/g" \
116
        "/etc/koha/$1" > "$2"
119
        "/etc/koha/$1" > "$2"
117
120
118
}
121
}
Lines 656-661 eof Link Here
656
    generate_config_file triplestore.yaml.in \
659
    generate_config_file triplestore.yaml.in \
657
        "/etc/koha/sites/$name/triplestore.yaml"
660
        "/etc/koha/sites/$name/triplestore.yaml"
658
661
662
    # Generate and install OAI-PMH harvester config file
663
    generate_config_file oai-pmh-harvester.yaml.in \
664
        "/etc/koha/sites/$name/oai-pmh-harvester.yaml"
665
659
    # Create a GPG-encrypted file for requesting a DB to be set up.
666
    # Create a GPG-encrypted file for requesting a DB to be set up.
660
    if [ "$op" = request ]
667
    if [ "$op" = request ]
661
    then
668
    then
(-)a/debian/scripts/koha-create-dirs (+3 lines)
Lines 56-66 do Link Here
56
    userdir "$name" "/var/lib/koha/$name/plugins"
56
    userdir "$name" "/var/lib/koha/$name/plugins"
57
    userdir "$name" "/var/lib/koha/$name/uploads"
57
    userdir "$name" "/var/lib/koha/$name/uploads"
58
    userdir "$name" "/var/lib/koha/$name/tmp"
58
    userdir "$name" "/var/lib/koha/$name/tmp"
59
    userdir "$name" "/var/lib/koha/$name/oai-pmh-harvester"
59
    userdir "$name" "/var/lock/koha/$name"
60
    userdir "$name" "/var/lock/koha/$name"
60
    userdir "$name" "/var/lock/koha/$name/authorities"
61
    userdir "$name" "/var/lock/koha/$name/authorities"
61
    userdir "$name" "/var/lock/koha/$name/biblios"
62
    userdir "$name" "/var/lock/koha/$name/biblios"
62
    userdir "$name" "/var/run/koha/$name"
63
    userdir "$name" "/var/run/koha/$name"
63
    userdir "$name" "/var/run/koha/$name/authorities"
64
    userdir "$name" "/var/run/koha/$name/authorities"
64
    userdir "$name" "/var/run/koha/$name/biblios"
65
    userdir "$name" "/var/run/koha/$name/biblios"
66
    userdir "$name" "/var/run/koha/$name/oai-pmh-harvester"
67
    userdir "$name" "/var/spool/koha/$name/oai-pmh-harvester"
65
done
68
done
66
69
(-)a/debian/templates/koha-conf-site.xml.in (+1 lines)
Lines 298-303 __END_SRU_PUBLICSERVER__ Link Here
298
 <use_zebra_facets>1</use_zebra_facets>
298
 <use_zebra_facets>1</use_zebra_facets>
299
 <queryparser_config>/etc/koha/searchengine/queryparser.yaml</queryparser_config>
299
 <queryparser_config>/etc/koha/searchengine/queryparser.yaml</queryparser_config>
300
 <log4perl_conf>__KOHA_CONF_DIR__/log4perl.conf</log4perl_conf>
300
 <log4perl_conf>__KOHA_CONF_DIR__/log4perl.conf</log4perl_conf>
301
 <oai_pmh_harvester_config>/etc/koha/sites/__KOHASITE__/oai_pmh_harvester.yaml</oai_pmh_harvester_config>
301
 <!-- Uncomment/edit next setting if you want to adjust zebra log levels.
302
 <!-- Uncomment/edit next setting if you want to adjust zebra log levels.
302
      Default is: none,fatal,warn.
303
      Default is: none,fatal,warn.
303
      You can also include: debug,log,malloc,all,request.
304
      You can also include: debug,log,malloc,all,request.
(-)a/debian/templates/oai-pmh-harvester-site.yaml.in (+22 lines)
Line 0 Link Here
1
---
2
#Harvester
3
socket: 'unix:__OAI_RUN_DIR__/harvesterd.sock'
4
5
logfile: '__LOG_DIR__/oai-pmh-harvester.log'
6
loglevel: 'WARN'
7
8
pidfile: '__OAI_RUN_DIR__/harvesterd.pid'
9
10
statefile: '__OAI_LIB_DIR__/harvesterd.state'
11
12
spooldir: '__OAI_SPOOL_DIR__'
13
14
#Downloader
15
download_workers: 1
16
download_module: 'Koha::OAI::Harvester::Worker::Download::Stream'
17
download_batch: 100
18
19
#Importer
20
import_workers: 1
21
import_poll: 5
22
import_module: 'Koha::OAI::Harvester::Worker::Import'
(-)a/etc/koha-conf.xml (+1 lines)
Lines 119-124 __PAZPAR2_TOGGLE_XML_POST__ Link Here
119
 <use_zebra_facets>1</use_zebra_facets>
119
 <use_zebra_facets>1</use_zebra_facets>
120
 <queryparser_config>__KOHA_CONF_DIR__/searchengine/queryparser.yaml</queryparser_config>
120
 <queryparser_config>__KOHA_CONF_DIR__/searchengine/queryparser.yaml</queryparser_config>
121
 <log4perl_conf>__KOHA_CONF_DIR__/log4perl.conf</log4perl_conf>
121
 <log4perl_conf>__KOHA_CONF_DIR__/log4perl.conf</log4perl_conf>
122
 <oai_pmh_harvester_config>__KOHA_CONF_DIR__/oai-pmh-harvester.yaml</oai_pmh_harvester_config>
122
 <memcached_servers>__MEMCACHED_SERVERS__</memcached_servers>
123
 <memcached_servers>__MEMCACHED_SERVERS__</memcached_servers>
123
 <memcached_namespace>__MEMCACHED_NAMESPACE__</memcached_namespace>
124
 <memcached_namespace>__MEMCACHED_NAMESPACE__</memcached_namespace>
124
 <template_cache_dir>__TEMPLATE_CACHE_DIR__</template_cache_dir>
125
 <template_cache_dir>__TEMPLATE_CACHE_DIR__</template_cache_dir>
(-)a/etc/oai-pmh-harvester.yaml (+22 lines)
Line 0 Link Here
1
---
2
#Harvester
3
socket: 'unix:__OAI_RUN_DIR__/harvesterd.sock'
4
5
logfile: '__LOG_DIR__/oai-pmh-harvester.log'
6
loglevel: 'WARN'
7
8
pidfile: '__OAI_RUN_DIR__/harvesterd.pid'
9
10
statefile: '__OAI_LIB_DIR__/harvesterd.state'
11
12
spooldir: '__OAI_SPOOL_DIR__'
13
14
#Downloader
15
download_workers: 1
16
download_module: 'Koha::OAI::Harvester::Worker::Download::Stream'
17
download_batch: 100
18
19
#Importer
20
import_workers: 1
21
import_poll: 5
22
import_module: 'Koha::OAI::Harvester::Worker::Import'
(-)a/installer/data/mysql/atomicupdate/bug_10662.sql (+73 lines)
Line 0 Link Here
1
--
2
-- Table structure for table 'oai_harvester_biblios'
3
--
4
5
CREATE TABLE IF NOT EXISTS `oai_harvester_biblios` (
6
  `import_oai_biblio_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
7
  `oai_repository` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
8
  `oai_identifier` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
9
  `biblionumber` int(11) NOT NULL,
10
  PRIMARY KEY (`import_oai_biblio_id`),
11
  UNIQUE KEY `oai_record` (`oai_identifier`,`oai_repository`) USING BTREE,
12
  KEY `FK_import_oai_biblio_1` (`biblionumber`),
13
  CONSTRAINT `FK_import_oai_biblio_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE NO ACTION
14
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
15
16
--
17
-- Table structure for table 'oai_harvester_history'
18
--
19
20
CREATE TABLE IF NOT EXISTS  `oai_harvester_history` (
21
  `import_oai_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
22
  `repository` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
23
  `header_identifier` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
24
  `header_datestamp` datetime NOT NULL,
25
  `header_status` varchar(45) COLLATE utf8_unicode_ci DEFAULT NULL,
26
  `record` longtext COLLATE utf8_unicode_ci NOT NULL,
27
  `upload_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
28
  `status` varchar(45) COLLATE utf8_unicode_ci NOT NULL,
29
  `filter` text COLLATE utf8_unicode_ci NOT NULL,
30
  `framework` varchar(4) COLLATE utf8_unicode_ci NOT NULL,
31
  `record_type` enum('biblio','auth','holdings') COLLATE utf8_unicode_ci NOT NULL,
32
  `matcher_code` varchar(10) COLLATE utf8_unicode_ci DEFAULT NULL,
33
  PRIMARY KEY (`import_oai_id`)
34
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
35
36
--
37
-- Table structure for table 'oai_harvester_import_queue'
38
--
39
40
CREATE TABLE IF NOT EXISTS `oai_harvester_import_queue` (
41
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
42
  `uuid` varchar(45) CHARACTER SET utf8 NOT NULL,
43
  `status` varchar(45) CHARACTER SET utf8 NOT NULL DEFAULT 'new',
44
  `result` text CHARACTER SET utf8 NOT NULL,
45
  `result_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
46
  PRIMARY KEY (`id`)
47
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
48
49
--
50
-- Table structure for table 'oai_harvester_requests'
51
--
52
53
CREATE TABLE IF NOT EXISTS `oai_harvester_requests` (
54
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
55
  `uuid` varchar(45) NOT NULL,
56
  `oai_verb` varchar(45) NOT NULL,
57
  `oai_metadataPrefix` varchar(255) NOT NULL,
58
  `oai_identifier` varchar(255) DEFAULT NULL,
59
  `oai_from` varchar(45) DEFAULT NULL,
60
  `oai_until` varchar(45) DEFAULT NULL,
61
  `oai_set` varchar(255) DEFAULT NULL,
62
  `http_url` varchar(255) DEFAULT NULL,
63
  `http_username` varchar(255) DEFAULT NULL,
64
  `http_password` varchar(255) DEFAULT NULL,
65
  `http_realm` varchar(255) DEFAULT NULL,
66
  `import_filter` varchar(255) NOT NULL,
67
  `import_framework_code` varchar(4) NOT NULL,
68
  `import_record_type` enum('biblio','auth','holdings') NOT NULL,
69
  `import_matcher_code` varchar(10) DEFAULT NULL,
70
  `interval` int(10) unsigned NOT NULL,
71
  `name` varchar(45) NOT NULL,
72
  PRIMARY KEY (`id`)
73
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
(-)a/installer/data/mysql/kohastructure.sql (+74 lines)
Lines 4192-4197 CREATE TABLE `oauth_access_tokens` ( Link Here
4192
    PRIMARY KEY (`access_token`)
4192
    PRIMARY KEY (`access_token`)
4193
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
4193
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
4194
4194
4195
--
4196
-- Table structure for table 'oai_harvester_biblios'
4197
--
4198
4199
CREATE TABLE IF NOT EXISTS `oai_harvester_biblios` (
4200
  `import_oai_biblio_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
4201
  `oai_repository` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
4202
  `oai_identifier` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
4203
  `biblionumber` int(11) NOT NULL,
4204
  PRIMARY KEY (`import_oai_biblio_id`),
4205
  UNIQUE KEY `oai_record` (`oai_identifier`,`oai_repository`) USING BTREE,
4206
  KEY `FK_import_oai_biblio_1` (`biblionumber`),
4207
  CONSTRAINT `FK_import_oai_biblio_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE NO ACTION
4208
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
4209
4210
--
4211
-- Table structure for table 'oai_harvester_history'
4212
--
4213
4214
CREATE TABLE IF NOT EXISTS  `oai_harvester_history` (
4215
  `import_oai_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
4216
  `repository` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
4217
  `header_identifier` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
4218
  `header_datestamp` datetime NOT NULL,
4219
  `header_status` varchar(45) COLLATE utf8_unicode_ci DEFAULT NULL,
4220
  `record` longtext COLLATE utf8_unicode_ci NOT NULL,
4221
  `upload_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
4222
  `status` varchar(45) COLLATE utf8_unicode_ci NOT NULL,
4223
  `filter` text COLLATE utf8_unicode_ci NOT NULL,
4224
  `framework` varchar(4) COLLATE utf8_unicode_ci NOT NULL,
4225
  `record_type` enum('biblio','auth','holdings') COLLATE utf8_unicode_ci NOT NULL,
4226
  `matcher_code` varchar(10) COLLATE utf8_unicode_ci DEFAULT NULL,
4227
  PRIMARY KEY (`import_oai_id`)
4228
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
4229
4230
--
4231
-- Table structure for table 'oai_harvester_import_queue'
4232
--
4233
4234
CREATE TABLE IF NOT EXISTS `oai_harvester_import_queue` (
4235
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
4236
  `uuid` varchar(45) CHARACTER SET utf8 NOT NULL,
4237
  `status` varchar(45) CHARACTER SET utf8 NOT NULL DEFAULT 'new',
4238
  `result` text CHARACTER SET utf8 NOT NULL,
4239
  `result_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
4240
  PRIMARY KEY (`id`)
4241
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
4242
4243
--
4244
-- Table structure for table 'oai_harvester_requests'
4245
--
4246
4247
CREATE TABLE IF NOT EXISTS `oai_harvester_requests` (
4248
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
4249
  `uuid` varchar(45) NOT NULL,
4250
  `oai_verb` varchar(45) NOT NULL,
4251
  `oai_metadataPrefix` varchar(255) NOT NULL,
4252
  `oai_identifier` varchar(255) DEFAULT NULL,
4253
  `oai_from` varchar(45) DEFAULT NULL,
4254
  `oai_until` varchar(45) DEFAULT NULL,
4255
  `oai_set` varchar(255) DEFAULT NULL,
4256
  `http_url` varchar(255) DEFAULT NULL,
4257
  `http_username` varchar(255) DEFAULT NULL,
4258
  `http_password` varchar(255) DEFAULT NULL,
4259
  `http_realm` varchar(255) DEFAULT NULL,
4260
  `import_filter` varchar(255) NOT NULL,
4261
  `import_framework_code` varchar(4) NOT NULL,
4262
  `import_record_type` enum('biblio','auth','holdings') NOT NULL,
4263
  `import_matcher_code` varchar(10) DEFAULT NULL,
4264
  `interval` int(10) unsigned NOT NULL,
4265
  `name` varchar(45) NOT NULL,
4266
  PRIMARY KEY (`id`)
4267
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
4268
4195
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
4269
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
4196
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
4270
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
4197
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
4271
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-menu.inc (+3 lines)
Lines 81-86 Link Here
81
    [% IF ( CAN_user_tools_upload_local_cover_images ) %]
81
    [% IF ( CAN_user_tools_upload_local_cover_images ) %]
82
    <li><a href="/cgi-bin/koha/tools/upload-cover-image.pl">Upload local cover image</a></li>
82
    <li><a href="/cgi-bin/koha/tools/upload-cover-image.pl">Upload local cover image</a></li>
83
    [% END %]
83
    [% END %]
84
    [% IF ( CAN_user_tools_manage_staged_marc ) %]
85
	<li><a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a></li>
86
    [% END %]
84
</ul>
87
</ul>
85
<h5>Additional tools</h5>
88
<h5>Additional tools</h5>
86
<ul>
89
<ul>
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/dashboard.tt (+371 lines)
Line 0 Link Here
1
[% INCLUDE 'doc-head-open.inc' %]
2
<title>Koha &rsaquo; Tools &rsaquo; OAI-PMH harvester</title>
3
[% INCLUDE 'doc-head-close.inc' %]
4
[% INCLUDE 'datatables.inc' %]
5
[% dashboard_page = '/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl' %]
6
[% request_page = '/cgi-bin/koha/tools/oai-pmh-harvester/request.pl' %]
7
<script type="text/javascript">
8
//<![CDATA[
9
    $(document).ready(function() {
10
        $('#dashboard-items').tabs();
11
        [% IF ( result.start.defined || result.stop.defined || result.delete.defined ) %]
12
            $('#dashboard-items').tabs("option","active",1);
13
        [% END %]
14
15
        var saved_table = $('#saved-table').DataTable({});
16
        var submitted_table = $('#submitted-table').DataTable({});
17
        var history_table = $('#history-table').DataTable({
18
            serverSide: true,
19
            searching: true,
20
            processing: true,
21
            ajax: {
22
                "url": '/cgi-bin/koha/svc/oai-pmh-harvester/history',
23
                contentType: 'application/json',
24
                type: 'POST',
25
                data: function ( d ) {
26
                    return JSON.stringify( d );
27
                },
28
                dataSrc: function (json){
29
                    var recordsTotal = json.recordsTotal;
30
                    if(recordsTotal){
31
                        $('#import_count').text( "("+recordsTotal+")" );
32
                    }
33
                    return json.data;
34
                }
35
            },
36
            columns: [
37
                { data: 'import_oai_id', },
38
                { data: 'repository', },
39
                { data: 'header_identifier', },
40
                { data: 'header_datestamp', },
41
                {
42
                    data: 'header_status', render: function (data, type, full, meta){
43
                        var display_string = _("Active");
44
                        if (data == "deleted"){
45
                            display_string = _("Deleted");
46
                        }
47
                        return display_string;
48
                    }
49
                },
50
                {
51
                    data: 'status', render: function (data, type, full, meta){
52
                        var display_string = data;
53
                        if (data == "added"){
54
                            display_string = _("Added");
55
                        }
56
                        else if (data == "error"){
57
                            display_string = _("Error");
58
                        }
59
                        else if (data == "not_found"){
60
                            display_string = _("Not found");
61
                        }
62
                        else if (data == "updated"){
63
                            display_string = _("Updated");
64
                        }
65
                        return display_string;
66
                    }
67
                },
68
                { data: 'upload_timestamp', },
69
                {
70
                    data: 'imported_record', render: function (data, type, full, meta){
71
                        var display_string = data;
72
                        var record_type = (full.record_type) ? full.record_type : 'biblio';
73
                        if (data && record_type == 'biblio'){
74
                            var link_text = _("View biblio record");
75
                            var link = '<a href="/cgi-bin/koha/catalogue/detail.pl?biblionumber='+data+'">'+link_text+'</a>';
76
                            display_string = link;
77
                        }
78
                        return display_string;
79
                    }
80
81
                },
82
                {
83
                    data: 'import_oai_id', render: function (data, type, full, meta){
84
                        var display_string = data;
85
                        var link_text = _("View record");
86
                        var link = '<a href="/cgi-bin/koha/tools/oai-pmh-harvester/record.pl?import_oai_id='+data+'">'+link_text+'</a>';
87
                        display_string = link;
88
                        return display_string;
89
                    }
90
                },
91
            ],
92
            //In theory, it would be nice to sort in descending order, but
93
            //it causes severe paging issues the data is frequently updated.
94
            //Caching would make the paging more stable, but the results would be stale.
95
            order: [
96
                [ 0, 'asc' ],
97
            ]
98
        });
99
        $('#refresh-button').click(function(){
100
            history_table.ajax.reload( null, false );
101
        });
102
    });
103
//]]>
104
</script>
105
<style type="text/css">
106
    a.paginate_button {
107
        padding: 2px;
108
    }
109
</style>
110
</head>
111
<body id="tools_oai-pmh-harvester" class="tools">
112
[% INCLUDE 'header.inc' %]
113
[% INCLUDE 'cat-search.inc' %]
114
    <div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> &rsaquo; OAI-PMH harvester</div>
115
    <div id="doc3" class="yui-t2">
116
        <div id="bd">
117
            <div id="yui-main">
118
                <div class="yui-b">
119
                    <h1>OAI-PMH harvester</h1>
120
                    <div id="toolbar" class="btn-toolbar">
121
                        <a id="new-request" class="btn btn-default btn-sm" href="[% request_page %]?op=new"><i class="fa fa-plus"></i> New request</a>
122
                    </div>
123
                    [% IF ( harvester.offline ) %]
124
                        <div class="alert">
125
                            <span>OAI-PMH harvester offline</span>
126
                        </div>
127
                    [% END %]
128
                    <div id="dashboard-items" class="toptabs">
129
                        <ul>
130
                            <li>
131
                                <a href="#saved_requests">Saved requests <span id="saved_count">([% saved_requests.size %])</span></a>
132
                            </li>
133
                            <li>
134
                                <a href="#submitted_requests">Submitted requests <span id="submitted_count">([% submitted_requests.size %])</span></a>
135
                            </li>
136
                            <li>
137
                                <a href="#imports">Import history <span id="import_count">(0)</span></a>
138
                            </li>
139
                        </ul>
140
                        <div id="submitted_requests">
141
                            [% IF ( result.start.defined ) %]
142
                                <div class="alert">
143
                                    [% IF ( result.start ) %]
144
                                        <span>Start succeeded</span>
145
                                    [% ELSE %]
146
                                        <span>Start failed</span>
147
                                    [% END %]
148
                                </div>
149
                            [% ELSIF ( result.stop.defined ) %]
150
                                 <div class="alert">
151
                                    [% IF ( result.stop ) %]
152
                                        <span>Stop succeeded</span>
153
                                    [% ELSE %]
154
                                        <span>Stop failed</span>
155
                                    [% END %]
156
                                </div>
157
                            [% ELSIF ( result.delete.defined ) %]
158
                                 <div class="alert">
159
                                    [% IF ( result.delete ) %]
160
                                        <span>Delete succeeded</span>
161
                                    [% ELSE %]
162
                                        <span>Delete failed</span>
163
                                    [% END %]
164
                                </div>
165
                            [% END %]
166
                            <table id="submitted-table">
167
                                <thead>
168
                                    <tr>
169
                                        <th>Name</th>
170
                                        <th>URL</th>
171
                                        <th>Set</th>
172
                                        <th>From</th>
173
                                        <th>Until</th>
174
                                        <th>Interval</th>
175
                                        <th>Effective from</th>
176
                                        <th>Effective until</th>
177
                                        <th>Pending imports</th>
178
                                        <th>Status</th>
179
                                        <th>Error</th>
180
                                        <th></th>
181
                                    </tr>
182
                                </thead>
183
                                <tbody>
184
                                [% IF ( submitted_requests ) %]
185
                                    [% FOREACH submitted_request IN submitted_requests %]
186
                                        <tr>
187
                                            <td>[% submitted_request.name %]</td>
188
                                            <td>[% submitted_request.parameters.oai_pmh.baseURL %]</td>
189
                                            <td>[% submitted_request.parameters.oai_pmh.set %]</td>
190
                                            <td>[% submitted_request.parameters.oai_pmh.from %]</td>
191
                                            <td>[% submitted_request.parameters.oai_pmh.until %]</td>
192
                                            <td>[% submitted_request.interval %]</td>
193
                                            <td>[% submitted_request.effective_from %]</td>
194
                                            <td>[% submitted_request.effective_until %]</td>
195
                                            <td>[% submitted_request.pending_imports %]</td>
196
                                            <td>
197
                                                [% IF ( submitted_status = submitted_request.status ) %]
198
                                                    [% IF ( submitted_status == "new" ) %]
199
                                                        <span>New</span>
200
                                                    [% ELSIF ( submitted_status == "active" ) %]
201
                                                        <span>Active</span>
202
                                                    [% ELSIF ( submitted_status == "stopped" ) %]
203
                                                        <span>Stopped</span>
204
                                                    [% END %]
205
                                                [% END %]
206
                                            </td>
207
                                            <td>
208
                                                [% IF ( submitted_request.error ) %]
209
                                                    <span>Harvest failure</span>
210
                                                [% END %]
211
                                            </td>
212
                                            <td>
213
                                                <div class="dropdown">
214
                                                    <a class="btn btn-default btn-xs dropdown-toggle" role="button" data-toggle="dropdown" href="#">Actions <span class="caret"></span></a>
215
                                                    <ul class="dropdown-menu pull-right" role="menu">
216
                                                          <li>
217
                                                              <a href="[% dashboard_page %]?op=start&uuid=[% submitted_request.uuid %]"><i class="fa fa-play"></i> Start</a>
218
                                                          </li>
219
                                                          <li>
220
                                                              <a href="[% dashboard_page %]?op=stop&uuid=[% submitted_request.uuid %]"><i class="fa fa-stop"></i> Stop</a>
221
                                                          </li>
222
                                                          <li>
223
                                                              <a href="[% dashboard_page %]?op=delete&uuid=[% submitted_request.uuid %]"><i class="fa fa-trash"></i> Delete</a>
224
                                                          </li>
225
                                                    </ul>
226
                                                </div>
227
                                            </td>
228
                                        </tr>
229
                                    [% END %]
230
                                [% END %]
231
                                </tbody>
232
                            </table>
233
                        </div>
234
                        <div id="saved_requests">
235
                            [% IF ( result.send.defined ) %]
236
                                <div class="alert">
237
                                    [% IF ( result.send ) %]
238
                                        <span>Send succeeded</span>
239
                                    [% ELSE %]
240
                                        <span>Send failed</span>
241
                                    [% END %]
242
                                </div>
243
                            [% END %]
244
                            <table id="saved-table">
245
                                <thead>
246
                                    <tr>
247
                                        <th>Name</th>
248
                                        <th>URL</th>
249
                                   <!-- <th>Verb</th>
250
                                        <th>Metadata prefix</th>
251
                                        <th>Identifier</th> -->
252
                                        <th>Set</th>
253
                                        <th>From</th>
254
                                        <th>Until</th>
255
                                        <th>Interval</th>
256
                                   <!-- <th>Filter</th>
257
                                        <th>Framework code</th>
258
                                        <th>Record type</th>
259
                                        <th>Matcher code</th> -->
260
                                        <th></th>
261
                                    </tr>
262
                                </thead>
263
                                <tbody>
264
                                [% IF ( saved_requests ) %]
265
                                    [% FOREACH saved_request IN saved_requests %]
266
                                        <tr>
267
                                            <td>[% saved_request.name %]</td>
268
                                            <td>[% saved_request.http_url %]</td>
269
                                       <!-- <td>[% saved_request.oai_verb %]</td>
270
                                            <td>[% saved_request.oai_metadataPrefix %]</td>
271
                                            <td>[% saved_request.oai_identifier %]</td> -->
272
                                            <td>[% saved_request.oai_set %]</td>
273
                                            <td>[% saved_request.oai_from %]</td>
274
                                            <td>[% saved_request.oai_until %]</td>
275
                                            <td>[% saved_request.interval %]</td>
276
                                       <!-- <td>
277
                                                [% IF ( saved_request.import_filter == "default" ) %]
278
                                                    <span>Default</span>
279
                                                [% ELSE %]
280
                                                    [% saved_request.import_filter %]
281
                                                [% END %]
282
                                            </td>
283
                                            <td>
284
                                                [%  display_framework = "";
285
                                                    FOREACH framework IN frameworks;
286
                                                        IF ( framework.frameworkcode == saved_request.import_framework_code );
287
                                                            display_framework = framework;
288
                                                        END;
289
                                                    END;
290
                                                %]
291
                                                [% IF ( display_framework ) %]
292
                                                    [% display_framework.frameworktext %]
293
                                                [% ELSE %]
294
                                                    [% saved_request.import_framework_code %]
295
                                                [% END %]
296
                                            </td>
297
                                            <td>
298
                                                [% IF ( saved_request.import_record_type == "biblio" ) %]
299
                                                    <span>Bibliographic</span>
300
                                                [% ELSE %]
301
                                                    [% saved_request.import_record_type %]
302
                                                [% END %]
303
                                            </td>
304
                                            <td>
305
                                                [%  display_matcher = "";
306
                                                    FOREACH matcher IN matchers;
307
                                                        IF ( matcher.code == saved_request.import_matcher_code );
308
                                                            display_matcher = matcher;
309
                                                        END;
310
                                                    END;
311
                                                %]
312
                                                [% IF ( display_matcher ) %]
313
                                                    [% display_matcher.description %]
314
                                                [% ELSE %]
315
                                                    [% saved_request.import_matcher_code %]
316
                                                [% END %]
317
                                            </td> -->
318
                                            <td>
319
                                                <div class="dropdown">
320
                                                    <a class="btn btn-default btn-xs dropdown-toggle" role="button" data-toggle="dropdown" href="#">Actions <span class="caret"></span></a>
321
                                                    <ul class="dropdown-menu pull-right" role="menu">
322
                                                          <li>
323
                                                              <a href="[% request_page %]?op=edit&id=[% saved_request.id %]"><i class="fa fa-pencil"></i> Edit</a>
324
                                                          </li>
325
                                                          <li>
326
                                                              <a href="[% request_page %]?op=new&id=[% saved_request.id %]"><i class="fa fa-copy"></i> Copy</a>
327
                                                          </li>
328
                                                          <li>
329
                                                              <a href="[% dashboard_page %]?op=send&id=[% saved_request.id %]"><i class="fa fa-send"></i> Send</a>
330
                                                          </li>
331
                                                          <li>
332
                                                            <a href="[% request_page %]?op=delete&id=[% saved_request.id %]"><i class="fa fa-trash"></i> Delete</a>
333
                                                          </li>
334
                                                    </ul>
335
                                                </div>
336
                                            </td>
337
                                        </tr>
338
                                    [% END %]
339
                                [% END %]
340
                                </tbody>
341
                            </table>
342
                        </div>
343
                        <div id="imports">
344
                            <div class="btn-toolbar">
345
                                <button id="refresh-button" type="button" class="btn btn-default btn-sm">Refresh import history</button>
346
                            </div>
347
                            <table id="history-table">
348
                                <thead>
349
                                    <tr>
350
                                        <th>Id</th>
351
                                        <th>Repository</th>
352
                                        <th>Identifier</th>
353
                                        <th>Datestamp</th>
354
                                        <th>Upstream status</th>
355
                                        <th>Import status</th>
356
                                        <th>Import timestamp</th>
357
                                        <th>Imported record</th>
358
                                        <th>Downloaded record</th>
359
                                    </tr>
360
                                </thead>
361
                            </table>
362
363
                        </div>
364
                    </div>
365
                </div>
366
            </div>
367
            <div class="yui-b">
368
                [% INCLUDE 'tools-menu.inc' %]
369
            </div>
370
        </div>
371
[% INCLUDE 'intranet-bottom.inc' %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/record.tt (+23 lines)
Line 0 Link Here
1
[% INCLUDE 'doc-head-open.inc' %]
2
<title>Koha &rsaquo; Tools &rsaquo; OAI-PMH harvester &rsaquo; Downloaded record</title>
3
[% INCLUDE 'doc-head-close.inc' %]
4
</head>
5
<body id="tools_oai-pmh-harvester_request" class="tools">
6
[% INCLUDE 'header.inc' %]
7
[% INCLUDE 'cat-search.inc' %]
8
    <div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> &rsaquo; <a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a> &rsaquo; Downloaded record</div>
9
    <div id="doc3" class="yui-t2">
10
        <div id="bd">
11
            <div id="yui-main">
12
                <div class="yui-b">
13
                    <h1>Downloaded record</h1>
14
                    [% IF ( record ) %]
15
                        <div style="white-space:pre">[% record | xml %]</div>
16
                    [% END %]
17
                </div>
18
            </div>
19
            <div class="yui-b">
20
                [% INCLUDE 'tools-menu.inc' %]
21
            </div>
22
        </div>
23
[% INCLUDE 'intranet-bottom.inc' %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/request.tt (+241 lines)
Line 0 Link Here
1
[% INCLUDE 'doc-head-open.inc' %]
2
<title>Koha &rsaquo; Tools &rsaquo; OAI-PMH harvester &rsaquo; Request</title>
3
[% INCLUDE 'doc-head-close.inc' %]
4
[% INCLUDE 'calendar.inc' %]
5
<script type="text/javascript" src="[% interface %]/lib/jquery/plugins/jquery-ui-timepicker-addon.min.js"></script>
6
[% INCLUDE 'timepicker.inc' %]
7
<script type="text/javascript">
8
//<![CDATA[
9
    $(document).ready(function() {
10
        toggle_identifier();
11
        $("#oai_verb").on("click",toggle_identifier);
12
        $(".datetime_utc").datetimepicker({
13
            separator: "T",
14
            timeSuffix: 'Z',
15
            dateFormat: "yy-mm-dd",
16
            timeFormat: "HH:mm:ss",
17
            hour: 0,
18
            minute: 0,
19
            second: 0,
20
            showSecond: 1,
21
            // timezone doesn't work with the "Now" button in v1.4.3 although it appears to in v1.6.1
22
            // timezone: '+000'
23
        });
24
    });
25
    function toggle_identifier (){
26
        var verb = $("#oai_verb").find(":selected").val();
27
        var oai_identifier = $("#oai_identifier");
28
        var oai_set = $("#oai_set");
29
        var oai_from = $("#oai_from");
30
        var oai_until = $("#oai_until");
31
        if (verb == 'ListRecords'){
32
            oai_identifier.prop('disabled', true);
33
            oai_set.prop('disabled', false);
34
            oai_from.prop('disabled', false);
35
            oai_until.prop('disabled', false);
36
        }
37
        else if (verb == 'GetRecord'){
38
            oai_identifier.prop('disabled', false);
39
            oai_set.prop('disabled', true);
40
            oai_from.prop('disabled', true);
41
            oai_until.prop('disabled', true);
42
        }
43
    }
44
//]]>
45
</script>
46
<style type="text/css">
47
    /* Override staff-global.css which hides second, millisecond, and microsecond sliders */
48
    .ui_tpicker_second {
49
        display: block;
50
    }
51
</style>
52
</head>
53
<body id="tools_oai-pmh-harvester_request" class="tools">
54
[% INCLUDE 'header.inc' %]
55
[% INCLUDE 'cat-search.inc' %]
56
    <div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> &rsaquo; <a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a> &rsaquo; Request</div>
57
    <div id="doc3" class="yui-t2">
58
        <div id="bd">
59
            <div id="yui-main">
60
                <div class="yui-b">
61
                    [% IF ( op == "edit" ) %]
62
                        <h1>Edit OAI-PMH request</h1>
63
                    [% ELSE %]
64
                        <h1>New OAI-PMH request</h1>
65
                    [% END %]
66
                    [% IF ( test_parameters ) %]
67
                        [% IF ( errors.size ) %]
68
                            <span class="text-danger">Tests failed!</span>
69
                        [% ELSE %]
70
                            <span class="text-success">Tests successful!</span>
71
                        [% END %]
72
                    [% END %]
73
                    <form action="/cgi-bin/koha/tools/oai-pmh-harvester/request.pl" method="post" name="entry-form">
74
                        [% IF ( op == "new" ) %]
75
                            <input type="hidden" name="op" value="create" />
76
                        [% ELSIF ( op == "edit" ) %]
77
                             <input type="hidden" name="op" value="update" />
78
                        [% ELSE %]
79
                            <input type="hidden" name="op" value="[% op %]" />
80
                        [% END %]
81
                        [% IF ( id ) %]
82
                            <input type="hidden" name="id" value="[% id %]" />
83
                        [% END %]
84
                        <fieldset class="rows">
85
                            <ol>
86
                                <li>
87
                                    <label for="name">Name:</label>
88
                                    <input type="text" size="30" id="name" name="name" value="[% oai_pmh_request.name %]"/>
89
                                    <span class="help">This is just a short name to help in managing requests.</span>
90
                                </li>
91
                            </ol>
92
                        </fieldset>
93
                        <fieldset class="rows">
94
                            <legend>HTTP parameters:</legend>
95
                            <ol>
96
                                <li>
97
                                    <label for="http_url">URL:</label>
98
                                    <input type="text" size="30" id="http_url" name="http_url" value="[% oai_pmh_request.http_url %]">
99
                                    [% IF (errors.http_url.malformed) %]<span class="error">[This must be a properly formatted HTTP or HTTPS URL.]</span>[% END %]
100
                                    [% IF (errors.http.404) %]<span class="error">[Cannot find address specified by this URL.]</span>[% END %]
101
                                    [% IF (errors.http.401) %]<span class="error">[Permission denied to access this URL.]</span>[% END %]
102
                                    [% IF (errors.http.generic) %]<span class="error">[Unable to access this URL.]</span>[% END %]
103
                                </li>
104
                            </ol>
105
                            <span class="help">The following parameters are not required by all OAI-PMH repositories, so they may be optional for this task.</span>
106
                            <ol>
107
                                <li>
108
                                    <label for="http_username">Username:</label>
109
                                    <input type="text" size="30" id="http_username" name="http_username" value="[% oai_pmh_request.http_username %]">
110
                                </li>
111
                                <li>
112
                                    <label for="http_password">Password:</label>
113
                                    <input type="text" size="30" id="http_password" name="http_password" value="[% oai_pmh_request.http_password %]">
114
                                </li>
115
                                <li>
116
                                    <label for="http_realm">Realm:</label>
117
                                    <input type="text" size="30" id="http_realm" name="http_realm" value="[% oai_pmh_request.http_realm %]">
118
                                </li>
119
                            </ol>
120
                        </fieldset>
121
                        <fieldset class="rows">
122
                            <legend>OAI-PMH parameters:</legend>
123
                            <ol>
124
                                <li>
125
                                    <label for="oai_verb">Verb:</label>
126
                                    <select id="oai_verb" name="oai_verb">
127
                                        [% IF ( oai_pmh_request.oai_verb == "ListRecords" ) %]
128
                                        <option value="ListRecords" selected="selected">ListRecords</option>
129
                                        [% ELSE %]
130
                                        <option value="ListRecords">ListRecords</option>
131
                                        [% END %]
132
                                        [% IF ( oai_pmh_request.oai_verb == "GetRecord" ) %]
133
                                        <option value="GetRecord" selected="selected">GetRecord</option>
134
                                        [% ELSE %]
135
                                        <option value="GetRecord">GetRecord</option>
136
                                        [% END %]
137
                                    </select>
138
                                </li>
139
                                <li>
140
                                    <label for="oai_metadataPrefix">Metadata prefix:</label>
141
                                    <input type="text" size="30" id="oai_metadataPrefix" name="oai_metadataPrefix" value="[% oai_pmh_request.oai_metadataPrefix %]">
142
                                    [% IF (errors.oai_metadataPrefix.unavailable) %]<span class="error">[This metadataPrefix is unavailable from this OAI-PMH provider.]</span>[% END %]
143
                                    [% IF (errors.oai_metadataPrefix.missing) %]<span class="error">[metadataPrefix is a required field for an OAI-PMH request.]</span>[% END %]
144
                                </li>
145
                                <li>
146
                                    <label for="oai_identifier">Identifier:</label>
147
                                    <input type="text" size="30" id="oai_identifier" name="oai_identifier" value="[% oai_pmh_request.oai_identifier %]">
148
                                    [% IF (errors.oai_identifier.missing) %]<span class="error">[Identifier is a required field when using GetRecord for an OAI-PMH request.]</span>[% END %]
149
                                </li>
150
                                <li>
151
                                    <label for="oai_set">Set:</label>
152
                                    <input type="text" size="30" id="oai_set" name="oai_set" value="[% oai_pmh_request.oai_set %]">
153
                                    [% IF (errors.oai_set.unavailable) %]<span class="error">[This set is unavailable from this OAI-PMH provider.]</span>[% END %]
154
                                </li>
155
                                [% IF (errors.oai.granularity_mismatch) %]<span class="error">[You must specify the same granularity for both From and Until.]</span>[% END %]
156
                                <li>
157
                                    <label for="oai_from">From:</label>
158
                                    <input type="text" size="30" class="datetime_utc" id="oai_from" name="oai_from" value="[% oai_pmh_request.oai_from %]">
159
                                    <span class="help">This value will be treated as UTC time. Note that some repositories only support YYYY-MM-DD datestamps.</span>
160
                                    [% IF (errors.oai_from.malformed) %]<span class="error">[This must be in YYYY-MM-DD or YYYY-MM-DDThh:mm:ssZ format.]</span>[% END %]
161
                                    [% IF (errors.oai_from.unavailable) %]<span class="error">[This granularity is unsupported by this OAI-PMH provider.]</span>[% END %]
162
                                </li>
163
                                <li>
164
                                    <label for="oai_until">Until:</label>
165
                                    <input type="text" size="30" class="datetime_utc" id="oai_until" name="oai_until" value="[% oai_pmh_request.oai_until %]">
166
                                    <span class="help">This value will be treated as UTC time. Note that some repositories only support YYYY-MM-DD datestamps.</span>
167
                                    [% IF (errors.oai_until.malformed) %]<span class="error">[This must be in YYYY-MM-DD or YYYY-MM-DDThh:mm:ssZ format.]</span>[% END %]
168
                                    [% IF (errors.oai_until.unavailable) %]<span class="error">[This granularity is unsupported by this OAI-PMH provider.]</span>[% END %]
169
                                </li>
170
                            </ol>
171
                        </fieldset>
172
                        <fieldset class="rows">
173
                            <legend>Import parameters:</legend>
174
                            <ol>
175
                                <li>
176
                                    <label for="import_filter">Filter:</label>
177
                                    [% IF ( oai_pmh_request.import_filter == "default" ) %]
178
                                        <input type="text" size="30" id="import_filter" name="import_filter" value="default">
179
                                    [% ELSE %]
180
                                        <input type="text" size="30" id="import_filter" name="import_filter" value="[% oai_pmh_request.import_filter %]">
181
                                    [% END %]
182
                                    <span class="help">If no filter is entered, the default filter will be used.</span>
183
                                </li>
184
                                <li>
185
                                    <label for="import_framework_code">Framework code:</label>
186
                                    <select id="import_framework_code" name="import_framework_code">
187
                                        <option value="">Default framework</option>
188
                                        [% FOREACH framework IN frameworks %]
189
                                            [% IF ( oai_pmh_request.import_framework_code == framework.frameworkcode ) %]
190
                                                <option selected="selected" value="[% framework.frameworkcode %]">[% framework.frameworktext %]</option>
191
                                            [% ELSE %]
192
                                                <option value="[% framework.frameworkcode %]">[% framework.frameworktext %]</option>
193
                                            [% END %]
194
                                        [% END %]
195
                                    </select>
196
                                </li>
197
                                <li>
198
                                    <label for="import_record_type">Record type:</label>
199
                                    <select id="import_record_type" name="import_record_type">
200
                                        <option value="biblio">Bibliographic</option>
201
                                    </select>
202
                                </li>
203
                                <li>
204
                                    <label for="import_matcher_code">Record matcher:</label>
205
                                    <select id="import_matcher_code" name="import_matcher_code">
206
                                        <option value="">None</option>
207
                                        [% FOREACH matcher IN matchers %]
208
                                            [% IF ( oai_pmh_request.import_matcher_code == matcher.code ) %]
209
                                                <option value="[% matcher.code %]" selected="selected">[% matcher.description %]</option>
210
                                            [% ELSE %]
211
                                                <option value="[% matcher.code %]">[% matcher.description %]</option>
212
                                            [% END %]
213
                                        [% END %]
214
                                    </select>
215
                                    <span class="help">See <a href="/cgi-bin/koha/admin/matching-rules.pl">record matching rules</a> to add or edit rules.</span>
216
                                </li>
217
                            </ol>
218
                        </fieldset>
219
                        <fieldset class="rows">
220
                            <legend>Download parameters:</legend>
221
                            <ol>
222
                                <li>
223
                                    <label for="interval">Interval (seconds): </label>
224
                                    <input type="text" id="interval" name="interval" value="[% oai_pmh_request.interval %]" size="4">
225
                                    <span class="help">The download request will be repeated in intervals of this many seconds. Enter "0" if you want the task to only happen once.</span>
226
                                </li>
227
                            </ol>
228
                        </fieldset>
229
                        <fieldset class="action">
230
                            <input type="submit" name="test_parameters" value="Test parameters">
231
                            <input type="submit" name="save" value="Save">
232
                            <a class="cancel" href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">Cancel</a>
233
                        </fieldset>
234
                    </form>
235
                </div>
236
            </div>
237
            <div class="yui-b">
238
                [% INCLUDE 'tools-menu.inc' %]
239
            </div>
240
        </div>
241
[% INCLUDE 'intranet-bottom.inc' %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/tools-home.tt (+5 lines)
Lines 195-200 Link Here
195
    <dd>Utility to upload scanned cover images for display in OPAC</dd>
195
    <dd>Utility to upload scanned cover images for display in OPAC</dd>
196
    [% END %]
196
    [% END %]
197
197
198
    [% IF ( CAN_user_tools_manage_staged_marc ) %]
199
    <dt><a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a></dt>
200
    <dd>Harvest (ie download and import) records from remote sources using the OAI-PMH protocol</dd>
201
    [% END %]
202
198
</dl>
203
</dl>
199
</div>
204
</div>
200
205
(-)a/koha-tmpl/intranet-tmpl/prog/en/xslt/StripOAIPMH.xsl (+28 lines)
Line 0 Link Here
1
<?xml version="1.0" encoding="UTF-8"?>
2
<xsl:stylesheet version="1.0"
3
    xmlns:oai="http://www.openarchives.org/OAI/2.0/"
4
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
5
    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>
6
    <!-- NOTE: This XSLT strips the OAI-PMH wrapper from the metadata. -->
7
8
    <!-- Match the root oai:record element -->
9
    <xsl:template match="oai:record">
10
        <!-- Apply templates only to the child metadata element(s) -->
11
        <xsl:apply-templates select="oai:metadata" />
12
    </xsl:template>
13
14
    <!-- Matches an oai:metadata element -->
15
    <xsl:template match="oai:metadata">
16
        <!-- Create a copy of child attributes and nodes -->
17
        <xsl:apply-templates select="@* | node()" />
18
    </xsl:template>
19
20
    <!-- Identity transformation: this template copies attributes and nodes -->
21
    <xsl:template match="@* | node()">
22
        <!-- Create a copy of this attribute or node -->
23
        <xsl:copy>
24
            <!-- Recursively apply this template to the attributes and child nodes of this element -->
25
            <xsl:apply-templates select="@* | node()" />
26
        </xsl:copy>
27
    </xsl:template>
28
</xsl:stylesheet>
(-)a/misc/cronjobs/cleanup_database.pl (-1 / +14 lines)
Lines 26-31 use constant DEFAULT_LOGS_PURGEDAYS => 180; Link Here
26
use constant DEFAULT_SEARCHHISTORY_PURGEDAYS      => 30;
26
use constant DEFAULT_SEARCHHISTORY_PURGEDAYS      => 30;
27
use constant DEFAULT_SHARE_INVITATION_EXPIRY_DAYS => 14;
27
use constant DEFAULT_SHARE_INVITATION_EXPIRY_DAYS => 14;
28
use constant DEFAULT_DEBARMENTS_PURGEDAYS         => 30;
28
use constant DEFAULT_DEBARMENTS_PURGEDAYS         => 30;
29
use constant DEFAULT_IMPORTOAI_PURGEDAYS          => 30;
29
30
30
BEGIN {
31
BEGIN {
31
    # find Koha's Perl modules
32
    # find Koha's Perl modules
Lines 44-50 use Koha::UploadedFiles; Link Here
44
45
45
sub usage {
46
sub usage {
46
    print STDERR <<USAGE;
47
    print STDERR <<USAGE;
47
Usage: $0 [-h|--help] [--sessions] [--sessdays DAYS] [-v|--verbose] [--zebraqueue DAYS] [-m|--mail] [--merged] [--import DAYS] [--logs DAYS] [--searchhistory DAYS] [--restrictions DAYS] [--all-restrictions] [--fees DAYS] [--temp-uploads] [--temp-uploads-days DAYS] [--uploads-missing 0|1 ]
48
Usage: $0 [-h|--help] [--sessions] [--sessdays DAYS] [-v|--verbose] [--zebraqueue DAYS] [-m|--mail] [--merged] [--import DAYS] [--logs DAYS] [--searchhistory DAYS] [--restrictions DAYS] [--all-restrictions] [--fees DAYS] [--temp-uploads] [--temp-uploads-days DAYS] [--uploads-missing 0|1 ] [--importoai DAYS ]
48
49
49
   -h --help          prints this help message, and exits, ignoring all
50
   -h --help          prints this help message, and exits, ignoring all
50
                      other options
51
                      other options
Lines 82-87 Usage: $0 [-h|--help] [--sessions] [--sessdays DAYS] [-v|--verbose] [--zebraqueu Link Here
82
   --temp-uploads-days DAYS Override the corresponding preference value.
83
   --temp-uploads-days DAYS Override the corresponding preference value.
83
   --uploads-missing FLAG Delete upload records for missing files when FLAG is true, count them otherwise
84
   --uploads-missing FLAG Delete upload records for missing files when FLAG is true, count them otherwise
84
   --oauth-tokens     Delete expired OAuth2 tokens
85
   --oauth-tokens     Delete expired OAuth2 tokens
86
   --importoai DAYS    purge OAI-PMH records from import_oai table older than DAYS
87
                       days. Defaults to 30 days if no days specified.
85
USAGE
88
USAGE
86
    exit $_[0];
89
    exit $_[0];
87
}
90
}
Lines 108-113 my $temp_uploads; Link Here
108
my $temp_uploads_days;
111
my $temp_uploads_days;
109
my $uploads_missing;
112
my $uploads_missing;
110
my $oauth_tokens;
113
my $oauth_tokens;
114
my $importoai_days;
111
115
112
GetOptions(
116
GetOptions(
113
    'h|help'            => \$help,
117
    'h|help'            => \$help,
Lines 132-137 GetOptions( Link Here
132
    'temp-uploads-days:i' => \$temp_uploads_days,
136
    'temp-uploads-days:i' => \$temp_uploads_days,
133
    'uploads-missing:i' => \$uploads_missing,
137
    'uploads-missing:i' => \$uploads_missing,
134
    'oauth-tokens'      => \$oauth_tokens,
138
    'oauth-tokens'      => \$oauth_tokens,
139
    'importoai:i'       => \$importoai_days,
135
) || usage(1);
140
) || usage(1);
136
141
137
# Use default values
142
# Use default values
Lines 143-148 $mail = DEFAULT_MAIL_PURGEDAYS if defined($mail) Link Here
143
$pSearchhistory    = DEFAULT_SEARCHHISTORY_PURGEDAYS      if defined($pSearchhistory)    && $pSearchhistory == 0;
148
$pSearchhistory    = DEFAULT_SEARCHHISTORY_PURGEDAYS      if defined($pSearchhistory)    && $pSearchhistory == 0;
144
$pListShareInvites = DEFAULT_SHARE_INVITATION_EXPIRY_DAYS if defined($pListShareInvites) && $pListShareInvites == 0;
149
$pListShareInvites = DEFAULT_SHARE_INVITATION_EXPIRY_DAYS if defined($pListShareInvites) && $pListShareInvites == 0;
145
$pDebarments       = DEFAULT_DEBARMENTS_PURGEDAYS         if defined($pDebarments)       && $pDebarments == 0;
150
$pDebarments       = DEFAULT_DEBARMENTS_PURGEDAYS         if defined($pDebarments)       && $pDebarments == 0;
151
$importoai_days    = DEFAULT_IMPORTOAI_PURGEDAYS          if defined($importoai_days)    && $importoai_days == 0;
146
152
147
if ($help) {
153
if ($help) {
148
    usage(0);
154
    usage(0);
Lines 166-171 unless ( $sessions Link Here
166
    || $temp_uploads
172
    || $temp_uploads
167
    || defined $uploads_missing
173
    || defined $uploads_missing
168
    || $oauth_tokens
174
    || $oauth_tokens
175
    || $importoai_days
169
) {
176
) {
170
    print "You did not specify any cleanup work for the script to do.\n\n";
177
    print "You did not specify any cleanup work for the script to do.\n\n";
171
    usage(1);
178
    usage(1);
Lines 344-349 if ($oauth_tokens) { Link Here
344
    say "Removed $count expired OAuth2 tokens" if $verbose;
351
    say "Removed $count expired OAuth2 tokens" if $verbose;
345
}
352
}
346
353
354
if ($importoai_days){
355
    my $sql = "DELETE FROM import_oai WHERE date(upload_timestamp) < (date_sub(curdate(), INTERVAL ? DAY))";
356
    my $sth = $dbh->prepare($sql);
357
    $sth->execute($importoai_days) or die $dbh->errstr;
358
}
359
347
exit(0);
360
exit(0);
348
361
349
sub RemoveOldSessions {
362
sub RemoveOldSessions {
(-)a/misc/harvesterd.pl (+207 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
use Modern::Perl;
21
use Getopt::Long;
22
use Pod::Usage;
23
use Module::Load;
24
use Log::Log4perl qw(:easy);
25
use POE;
26
use YAML;
27
28
use C4::Context;
29
use Koha::Daemon;
30
use Koha::OAI::Harvester;
31
use Koha::OAI::Harvester::Listener;
32
33
binmode(STDOUT,':encoding(UTF-8)');
34
$|++;
35
36
my $help = 0;
37
my $daemonize = 0;
38
39
my ($socket_addr,$pidfile,$statefile,$spooldir);
40
my ($download_module,$import_module);
41
my ($batch,$download_workers,$import_workers,$import_poll);
42
my ($logfile,$log_level);
43
my $log_levels = {
44
    FATAL => $FATAL,
45
    ERROR => $ERROR,
46
    WARN => $WARN,
47
    INFO => $INFO,
48
    DEBUG => $DEBUG,
49
    TRACE => $TRACE,
50
};
51
52
my $context = C4::Context->new();
53
my $config_filename = $context->{config}->{oai_pmh_harvester_config};
54
if ($config_filename){
55
    my $config = YAML::LoadFile($config_filename);
56
    if ($config){
57
        $socket_addr = $config->{socket};
58
        $pidfile = $config->{pidfile};
59
        $statefile = $config->{statefile};
60
        $spooldir = $config->{spooldir};
61
        $logfile = $config->{logfile};
62
        $log_level = $config->{loglevel};
63
        $download_module = $config->{download_module};
64
        $import_module = $config->{import_module};
65
        $batch = $config->{download_batch};
66
        $download_workers = $config->{download_workers};
67
        $import_workers = $config->{import_workers};
68
        $import_poll = $config->{import_poll};
69
    }
70
}
71
72
GetOptions(
73
    "help|?"            => \$help,
74
    "daemon"            => \$daemonize,
75
    "socket-uri=s"          => \$socket_addr,
76
    "pid-file=s"             => \$pidfile,
77
    "state-file=s"      => \$statefile,
78
    "spool-dir=s"       => \$spooldir,
79
    "log-file=s"             => \$logfile,
80
    "log-level=s"       => \$log_level,
81
    "download-module=s" => \$download_module,
82
    "import-module=s"   => \$import_module,
83
) or pod2usage(2);
84
pod2usage(1) if $help;
85
86
my $level = ( $log_level && $log_levels->{$log_level} ) ? $log_levels->{$log_level} : $log_levels->{WARN};
87
Log::Log4perl->easy_init(
88
    {
89
        level => $level,
90
        file => "STDOUT",
91
        layout   => '[%d{yyyy-MM-dd HH:mm:ss}][%p] %m%n',
92
    }
93
);
94
my $logger = Log::Log4perl->get_logger();
95
96
unless($download_module){
97
    $download_module = "Koha::OAI::Harvester::Worker::Download::Stream";
98
}
99
unless($import_module){
100
    $import_module = "Koha::OAI::Harvester::Worker::Import";
101
}
102
103
foreach my $module ( $download_module, $import_module ){
104
    load $module;
105
}
106
my $downloader = $download_module->new({
107
    logger => $logger,
108
    batch => $batch,
109
});
110
my $importer = $import_module->new({
111
    logger => $logger,
112
});
113
114
my $daemon = Koha::Daemon->new({
115
    pidfile => $pidfile,
116
    logfile => $logfile,
117
    daemonize => $daemonize,
118
});
119
$daemon->run();
120
121
my $harvester = Koha::OAI::Harvester->spawn({
122
    Downloader => $downloader,
123
    DownloaderWorkers => $download_workers,
124
    Importer => $importer,
125
    ImporterWorkers => $import_workers,
126
    ImportQueuePoll => $import_poll,
127
    logger => $logger,
128
    state_file => $statefile,
129
    spooldir => $spooldir,
130
});
131
132
my $listener = Koha::OAI::Harvester::Listener->spawn({
133
    logger => $logger,
134
    socket => $socket_addr,
135
});
136
137
$logger->info("OAI-PMH harvester started.");
138
139
POE::Kernel->run();
140
141
exit;
142
143
=head1 NAME
144
145
harvesterd.pl - a daemon that asynchronously sends OAI-PMH requests and imports OAI-PMH records
146
147
=head1 SYNOPSIS
148
149
KOHA_CONF=/path/to/koha-conf.xml ./harvesterd.pl
150
151
=head1 OPTIONS
152
153
=over 8
154
155
=item B<--help>
156
157
Print a brief help message and exits.
158
159
=item B<--daemon>
160
161
Run program as a daemon (ie fork process, setsid, chdir to root, reset umask,
162
and close STDIN, STDOUT, and STDERR).
163
164
=item B<--log-file>
165
166
Specify a file to which to log STDOUT and STDERR.
167
168
=item B<--pid-file>
169
170
Specify a file to store the process id (this prevents multiple copies of the program
171
from running at the same time).
172
173
=item B<--socket-uri>
174
175
Specify a URI to use for the UNIX socket used to communicate with the daemon.
176
(e.g. unix:/path/to/socket.sock)
177
178
=item B<--state-file>
179
180
Specify a filename to use for storing the harvester's in-memory state.
181
182
In the event that the harvester crashes, it can resume from where it stopped.
183
184
=item B<--spool-dir>
185
186
Specify a directory to store downloaded OAI-PMH records prior to import.
187
188
=item B<--log-level>
189
190
Specify a log level for logging. The logger uses Log4Perl, which provides
191
FATAL, ERROR, WARN, INFO, DEBUG, and TRACE in order of descending priority.
192
193
Defaults to WARN level.
194
195
=item B<--download-module>
196
197
Specify a Perl module to use for downloading records. This is a specialty module,
198
which has particular requirements, so only advanced users should use this option.
199
200
=item B<--import-module>
201
202
Specify a Perl module to use for importing records. This is a specialty module,
203
which has particular requirements, so only advanced users should use this option.
204
205
=back
206
207
=cut
(-)a/rewrite-config.PL (+3 lines)
Lines 152-157 $prefix = $ENV{'INSTALL_BASE'} || "/usr"; Link Here
152
  "__MEMCACHED_NAMESPACE__" => "",
152
  "__MEMCACHED_NAMESPACE__" => "",
153
  "__FONT_DIR__" => "/usr/share/fonts/truetype/ttf-dejavu",
153
  "__FONT_DIR__" => "/usr/share/fonts/truetype/ttf-dejavu",
154
  "__TEMPLATE_CACHE_DIR__" => "/tmp/koha"
154
  "__TEMPLATE_CACHE_DIR__" => "/tmp/koha"
155
  "__OAI_RUN_DIR__" => "",
156
  "__OAI_LIB_DIR__" => "",
157
  "__OAI_SPOOL_DIR__" => "",
155
);
158
);
156
159
157
# Override configuration from the environment
160
# Override configuration from the environment
(-)a/svc/oai-pmh-harvester/history (+132 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use C4::Auth qw(check_cookie_auth haspermission get_session);
22
use JSON;
23
use Koha::Database;
24
25
my $input = new CGI;
26
27
my ( $auth_status, $sessionID ) =
28
  check_cookie_auth( $input->cookie('CGISESSID'));
29
30
if ( $auth_status ne "ok" ) {
31
    exit 0;
32
}
33
34
my $data = {
35
    data => [],
36
    recordsTotal => 0,
37
    recordsFiltered => 0,
38
    draw => undef,
39
};
40
41
my $length = 10;
42
my $start = 0;
43
my @order_by = ();
44
my @search = ();
45
46
if ($input->request_method eq "POST"){
47
    my $postdata = $input->param('POSTDATA');
48
    my $request = from_json($postdata);
49
    $data->{draw} = int( $request->{draw} ) if $request->{draw};
50
    $length = $request->{length} if $request->{length};
51
    $start = $request->{start} if $request->{start};
52
    if (my $search = $request->{search}){
53
        my $value = $search->{value};
54
        if ($value){
55
            foreach my $column (@{$request->{columns}}){
56
                if ($column->{data} && $column->{searchable}){
57
                    my $search_element = {
58
                        $column->{data} => { 'like', "%".$value."%" },
59
                    };
60
                    push(@search,$search_element);
61
                }
62
            }
63
        }
64
    }
65
    if (my $order = $request->{order}){
66
        foreach my $element (@$order){
67
            my $dir = $element->{dir};
68
            my $column_index = $element->{column};
69
            my $column = $request->{columns}->[$column_index];
70
            my $orderable = $column->{orderable};
71
            if ($orderable){
72
                my $column_name = $column->{data};
73
                my $direction;
74
                if ($dir){
75
                    if ($dir eq "asc" || $dir eq "desc"){
76
                        $direction = "-$dir";
77
                    }
78
                }
79
                if ($column_name && $direction){
80
                    my $single_order = {
81
                        $direction => $column_name,
82
                    };
83
                    push(@order_by,$single_order);
84
                }
85
            }
86
        }
87
    }
88
}
89
90
my $page = ( $start / $length ) + 1;
91
my $schema = Koha::Database->new()->schema();
92
if ($schema){
93
    my $rs = $schema->resultset("OaiHarvesterHistory");
94
    my $results = $rs->search(
95
        \@search,
96
        {
97
            result_class => 'DBIx::Class::ResultClass::HashRefInflator',
98
            page => $page,
99
            rows => $length,
100
            order_by => \@order_by,
101
        },
102
    );
103
    my $count = $rs->count;
104
    my $filtered_count = $results->pager->total_entries;
105
    my @rows = ();
106
    while (my $row = $results->next){
107
        $row->{imported_record} = '';
108
        if ($row->{record_type} eq "biblio"){
109
            my $harvested_biblio = $schema->resultset("OaiHarvesterBiblio")->find(
110
                    {
111
                        oai_repository => $row->{repository},
112
                        oai_identifier => $row->{header_identifier},
113
                    },
114
                    { key => "oai_record" },
115
            );
116
            $row->{imported_record} = $harvested_biblio->biblionumber->id if $harvested_biblio;
117
        }
118
        push(@rows,$row);
119
    }
120
    if ($count){
121
        $data->{recordsTotal} = $count;
122
        $data->{recordsFiltered} = $filtered_count;
123
        $data->{data} = \@rows if @rows;
124
    }
125
}
126
127
binmode STDOUT, ":encoding(UTF-8)";
128
print $input->header(
129
    -type => 'application/json',
130
    -charset => 'UTF-8'
131
);
132
print to_json($data, { pretty => 1, });
(-)a/tools/oai-pmh-harvester/dashboard.pl (+134 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
use Modern::Perl;
21
use YAML;
22
23
use C4::Auth;
24
use C4::Context;
25
use C4::Output;
26
use Koha::OAI::Harvester::Client;
27
use Koha::OAI::Harvester::Requests;
28
use Koha::BiblioFrameworks;
29
use Koha::Database;
30
31
my $context = C4::Context->new();
32
my $config_filename = $context->{config}->{oai_pmh_harvester_config};
33
my $client_config = {};
34
if ($config_filename){
35
    my $config = YAML::LoadFile($config_filename);
36
    if ($config && $config->{socket}){
37
        $client_config->{socket_uri} = $config->{socket};
38
    }
39
}
40
41
my $input = new CGI;
42
43
my ($template, $loggedinuser, $cookie) =
44
    get_template_and_user({template_name => "tools/oai-pmh-harvester/dashboard.tt",
45
        query => $input,
46
        type => "intranet",
47
        authnotrequired => 0,
48
        flagsrequired => {tools => 'manage_staged_marc'},
49
    });
50
51
my $op = $input->param('op') // 'list';
52
my $id = $input->param('id');
53
my $uuid = $input->param('uuid');
54
55
my $client = Koha::OAI::Harvester::Client->new($client_config);
56
my $is_connected = $client->connect;
57
58
if ( ($op eq "send") && $id ){
59
    if ($is_connected){
60
        my $request = Koha::OAI::Harvester::Requests->find($id);
61
        if ($request){
62
            my $task = {
63
                name => $request->name,
64
                uuid => $request->uuid,
65
                interval => $request->interval,
66
                parameters => {
67
                    oai_pmh => {
68
                        baseURL => $request->http_url,
69
                        verb => $request->oai_verb,
70
                        metadataPrefix => $request->oai_metadataPrefix,
71
                        identifier => $request->oai_identifier,
72
                        set => $request->oai_set,
73
                        from => $request->oai_from,
74
                        until => $request->oai_until,
75
                    },
76
                    import => {
77
                        filter => $request->import_filter,
78
                        frameworkcode => $request->import_framework_code,
79
                        matcher_code => $request->import_matcher_code,
80
                        record_type => $request->import_record_type,
81
                    },
82
                },
83
            };
84
            if ($request->http_username && $request->http_password && $request->http_realm){
85
                $task->{parameters}->{http_basic_auth} = {
86
                    username => $request->http_username,
87
                    password => $request->http_password,
88
                    realm => $request->http_realm,
89
                };
90
            }
91
            my $is_created = $client->create($task);
92
            $template->{VARS}->{ result }->{ send } = $is_created;
93
        }
94
    }
95
}
96
elsif ( ($op eq "start") && ($uuid) ){
97
    if ($is_connected){
98
        my $is_started = $client->start($uuid);
99
        $template->{VARS}->{ result }->{ start } = $is_started;
100
    }
101
}
102
elsif ( ($op eq "stop") && ($uuid) ){
103
    if ($is_connected){
104
        my $is_stopped = $client->stop($uuid);
105
        $template->{VARS}->{ result }->{ stop } = $is_stopped;
106
    }
107
}
108
elsif ( ($op eq "delete") && ($uuid) ){
109
    if ($is_connected){
110
        my $is_deleted = $client->delete($uuid);
111
        $template->{VARS}->{ result }->{ delete } = $is_deleted;
112
    }
113
}
114
115
my $requests = Koha::OAI::Harvester::Requests->as_list;
116
$template->{VARS}->{ saved_requests } = $requests;
117
118
my $frameworks = Koha::BiblioFrameworks->as_list();
119
$template->{VARS}->{ frameworks } = $frameworks;
120
121
my $schema = Koha::Database->new()->schema();
122
my $matcher_rs = $schema->resultset("MarcMatcher");
123
my @matchers = $matcher_rs->all;
124
$template->{VARS}->{ matchers } = \@matchers;
125
126
if ($is_connected){
127
    my $submitted_requests = $client->list;
128
    $template->{VARS}->{ submitted_requests } = $submitted_requests;
129
}
130
else {
131
    $template->{VARS}->{ harvester }->{ offline } = 1;
132
}
133
134
output_html_with_http_headers($input, $cookie, $template->output);
(-)a/tools/oai-pmh-harvester/record.pl (+53 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
use Modern::Perl;
21
22
use C4::Auth;
23
use C4::Output;
24
use Koha::Database;
25
26
my $input = new CGI;
27
28
my ($template, $loggedinuser, $cookie) =
29
    get_template_and_user({template_name => "tools/oai-pmh-harvester/record.tt",
30
        query => $input,
31
        type => "intranet",
32
        authnotrequired => 0,
33
        flagsrequired => {tools => 'manage_staged_marc'},
34
    });
35
36
my $import_oai_id = $input->param('import_oai_id');
37
if ($import_oai_id){
38
    my $schema = Koha::Database->new()->schema();
39
    if ($schema){
40
        my $rs = $schema->resultset("OaiHarvesterHistory");
41
        if ($rs){
42
            my $row = $rs->find($import_oai_id);
43
            if ($row){
44
                my $record = $row->record;
45
                if ($record){
46
                    $template->{VARS}->{ record } = $record;
47
                }
48
            }
49
        }
50
    }
51
}
52
53
output_html_with_http_headers($input, $cookie, $template->output);
(-)a/tools/oai-pmh-harvester/request.pl (-1 / +140 lines)
Line 0 Link Here
0
- 
1
#!/usr/bin/perl
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
use Modern::Perl;
21
use UUID qw/uuid/;
22
23
use C4::Auth;
24
use C4::Output;
25
use Koha::OAI::Harvester::Requests;
26
use Koha::BiblioFrameworks;
27
use Koha::Database;
28
29
my $input = new CGI;
30
31
my ($template, $loggedinuser, $cookie) =
32
    get_template_and_user({template_name => "tools/oai-pmh-harvester/request.tt",
33
        query => $input,
34
        type => "intranet",
35
        authnotrequired => 0,
36
        flagsrequired => {tools => 'manage_staged_marc'},
37
    });
38
39
my $op = $input->param('op');
40
my $id = $input->param('id');
41
42
my @frameworks = Koha::BiblioFrameworks->as_list();
43
$template->{VARS}->{ frameworks } = \@frameworks;
44
45
my $schema = Koha::Database->new()->schema();
46
my $rs = $schema->resultset("MarcMatcher");
47
my @matchers = $rs->all;
48
$template->{VARS}->{ matchers } = \@matchers;
49
50
my $http_url = $input->param('http_url');
51
my $http_username = $input->param('http_username');
52
my $http_password = $input->param('http_password');
53
my $http_realm = $input->param('http_realm');
54
55
my $oai_verb = $input->param('oai_verb');
56
my $oai_metadataPrefix = $input->param('oai_metadataPrefix');
57
my $oai_identifier = $input->param('oai_identifier');
58
my $oai_from = $input->param('oai_from');
59
my $oai_until = $input->param('oai_until');
60
my $oai_set = $input->param('oai_set');
61
62
my $import_filter = $input->param('import_filter') // 'default';
63
my $import_framework_code = $input->param('import_framework_code');
64
my $import_record_type = $input->param('import_record_type');
65
my $import_matcher_code = $input->param('import_matcher_code');
66
67
my $interval = $input->param("interval") ? int ( $input->param("interval") ) : 0;
68
my $name = $input->param("name");
69
70
my $save = $input->param('save');
71
my $test_parameters = $input->param('test_parameters');
72
73
my $request = $id ? Koha::OAI::Harvester::Requests->find($id) : Koha::OAI::Harvester::Request->new();
74
if ($request){
75
    if ($op eq "create" || $op eq "update"){
76
        $request->set({
77
            name => $name,
78
            http_url => $http_url,
79
            http_username => $http_username,
80
            http_password => $http_password,
81
            http_realm => $http_realm,
82
            oai_verb => $oai_verb,
83
            oai_metadataPrefix => $oai_metadataPrefix,
84
            oai_identifier => $oai_identifier,
85
            oai_from => $oai_from,
86
            oai_until => $oai_until,
87
            oai_set => $oai_set,
88
            import_filter => $import_filter,
89
            import_framework_code => $import_framework_code,
90
            import_record_type => $import_record_type,
91
            import_matcher_code => $import_matcher_code,
92
            interval => $interval,
93
        });
94
    }
95
}
96
97
if ($test_parameters){
98
    my $errors = $request->validate();
99
    $template->{VARS}->{ errors } = $errors;
100
    $template->{VARS}->{ test_parameters } = 1;
101
}
102
103
if ($op eq "new"){
104
    #Empty form with some defaults
105
    $request->import_filter("default") unless $request->import_filter;
106
    $request->interval(0) unless $request->interval;
107
}
108
elsif ($op eq "create"){
109
    if ($save){
110
        my $uuid = uuid;
111
        $request->uuid($uuid);
112
        $request->store;
113
        print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
114
        exit;
115
    }
116
}
117
elsif ( $op eq "edit"){
118
    $template->{VARS}->{ id } = $id;
119
}
120
elsif ($op eq "update"){
121
    $template->{VARS}->{ id } = $id;
122
    if ($save){
123
        $request->store;
124
        print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
125
        exit;
126
    }
127
}
128
elsif ($op eq "delete"){
129
    if ($request){
130
        $request->delete;
131
        print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
132
    }
133
}
134
else {
135
    print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
136
}
137
$template->{VARS}->{ op } = $op;
138
$template->{VARS}->{ oai_pmh_request } = $request;
139
140
output_html_with_http_headers($input, $cookie, $template->output);

Return to bug 10662