View | Details | Raw Unified | Return to bug 10662
Collapse All | Expand All

(-)a/C4/Installer/PerlDependencies.pm (+10 lines)
Lines 893-898 our $PERL_DEPS = { Link Here
893
        required => '1',
893
        required => '1',
894
        min_ver  => '0.37',
894
        min_ver  => '0.37',
895
    },
895
    },
896
    'POE' => {
897
        'usage' => 'OAI-PMH harvester',
898
        'required' => 1,
899
        'min_ver' => '1.354',
900
    },
901
    'POE::Component::JobQueue' => {
902
        'usage' => 'OAI-PMH harvester',
903
        'required' => 1,
904
        'min_ver' => '0.570',
905
    },
896
};
906
};
897
907
898
1;
908
1;
(-)a/Koha/Daemon.pm (+152 lines)
Line 0 Link Here
1
package Koha::Daemon;
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
use POSIX; #For daemonizing
23
use Fcntl qw(:flock); #For pidfile
24
25
sub new {
26
    my ($class, $args) = @_;
27
    $args = {} unless defined $args;
28
    return bless ($args, $class);
29
}
30
31
#######################################################################
32
#NOTE: On Debian, you can use the daemon binary to make a process into a daemon,
33
# but the following can be used if you don't want to use that program.
34
35
sub daemonize {
36
    my ($self) = @_;
37
38
    my $pid = fork;
39
40
    die "Couldn't fork: $!" unless defined($pid);
41
    if ($pid){
42
        exit; #Parent exit
43
    }
44
45
    #Become a session leader (ie detach program from controlling terminal)
46
    POSIX::setsid() or die "Can't start a new session: $!";
47
48
    #Change to known system directory
49
    chdir('/');
50
51
    #Reset the file creation mask so only the daemon owner can read/write files it creates
52
    umask(066);
53
54
    #Close inherited file handles, so that you can truly run in the background.
55
    open STDIN,  '<', '/dev/null';
56
    open STDOUT, '>', '/dev/null';
57
    open STDERR, '>&STDOUT';
58
}
59
60
sub log_to_file {
61
    my ($self,$logfile) = @_;
62
63
    #Open a filehandle to append to a log file
64
    my $opened = open(my $fh, '>>', $logfile);
65
    if ($opened){
66
        $fh->autoflush(1); #Make filehandle hot (ie don't buffer)
67
        *STDOUT = *$fh; #Re-assign STDOUT to LOG | --stdout
68
        *STDERR = *STDOUT; #Re-assign STDERR to STDOUT | --stderr
69
    }
70
    else {
71
        die "Unable to open a filehandle for $logfile: $!\n"; # --output
72
    }
73
}
74
75
sub make_pidfilehandle {
76
    my ($self,$pidfile) = @_;
77
    if ( ! -f $pidfile ){
78
        open(my $fh, '>', $pidfile) or die "Unable to write to $pidfile: $!\n";
79
        close($fh);
80
    }
81
    open(my $pidfilehandle, '+<', $pidfile) or die "Unable to open a filehandle for $pidfile: $!\n";
82
    return $pidfilehandle;
83
}
84
85
sub get_pidfile {
86
    my ($self,$pidfile) = @_;
87
    #NOTE: We need to save the filehandle in the object, so any locks persist for the life of the object
88
    my $pidfilehandle = $self->{pidfilehandle} ||= $self->make_pidfilehandle($pidfile);
89
    return $pidfilehandle;
90
}
91
92
sub lock_pidfile {
93
    my ($self,$pidfilehandle) = @_;
94
    my $locked;
95
    if (flock($pidfilehandle, LOCK_EX|LOCK_NB)){
96
        $locked = 1;
97
98
    }
99
    return $locked;
100
}
101
102
sub write_pidfile {
103
    my ($self,$pidfilehandle) = @_;
104
    if ($pidfilehandle){
105
        truncate($pidfilehandle, 0);
106
        print $pidfilehandle $$."\n" or die $!;
107
        #Flush the filehandle so you're not suffering from buffering
108
        $pidfilehandle->flush();
109
        return 1;
110
    }
111
}
112
113
sub run {
114
    my ($self) = @_;
115
    my $pidfile = $self->{pidfile};
116
    my $logfile = $self->{logfile};
117
118
    if ($pidfile){
119
        my $pidfilehandle = $self->get_pidfile($pidfile);
120
        if ($pidfilehandle){
121
            my $locked = $self->lock_pidfile($pidfilehandle);
122
            if ( ! $locked ) {
123
                die "$0 is unable to lock pidfile...\n";
124
            }
125
        }
126
    }
127
128
    if (my $configure = $self->{configure}){
129
        $configure->($self);
130
    }
131
132
    if ($self->{daemonize}){
133
        $self->daemonize();
134
    }
135
136
    if ($pidfile){
137
        my $pidfilehandle = $self->get_pidfile($pidfile);
138
        if ($pidfilehandle){
139
            $self->write_pidfile($pidfilehandle);
140
        }
141
    }
142
143
    if ($logfile){
144
        $self->log_to_file($logfile);
145
    }
146
147
    if (my $loop = $self->{loop}){
148
        $loop->($self);
149
    }
150
}
151
152
1;
(-)a/Koha/OAI/Harvester.pm (+652 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester;
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
use POE qw(Component::JobQueue);
23
use JSON;
24
use Sereal::Encoder;
25
use Sereal::Decoder;
26
use IO::Handle;
27
use File::Copy;
28
use File::Path qw(make_path remove_tree);
29
use DateTime;
30
use DateTime::Format::Strptime;
31
32
use C4::Context;
33
use Koha::Database;
34
35
=head1 API
36
37
=head2 Class Methods
38
39
=cut
40
41
my $day_granularity = DateTime::Format::Strptime->new(
42
    pattern   => '%F',
43
);
44
my $seconds_granularity = DateTime::Format::Strptime->new(
45
    pattern   => '%FT%TZ',
46
);
47
48
sub new {
49
    my ($class, $args) = @_;
50
    $args = {} unless defined $args;
51
    return bless ($args, $class);
52
}
53
54
sub spawn {
55
    my ($class, $args) = @_;
56
    my $self = $class->new($args);
57
    my $downloader = $self->{Downloader};
58
    my $importer = $self->{Importer};
59
    my $download_worker_limit = ( $self->{DownloaderWorkers} && int($self->{DownloaderWorkers}) ) ? $self->{DownloaderWorkers} : 1;
60
    my $import_worker_limit = ( $self->{ImporterWorkers} && int($self->{ImporterWorkers}) ) ? $self->{ImporterWorkers} : 1;
61
    my $import_queue_poll = ( $self->{ImportQueuePoll} && int($self->{ImportQueuePoll}) ) ? $self->{ImportQueuePoll} : 5;
62
63
    #NOTE: This job queue should always be created before the
64
    #harvester so that you can start download jobs immediately
65
    #upon spawning the harvester.
66
    POE::Component::JobQueue->spawn(
67
        Alias         => 'oai-downloader',
68
        WorkerLimit   => $download_worker_limit,
69
        Worker        => sub {
70
            my ($postback, $task) = @_;
71
            if ($downloader){
72
                if ($task->{status} eq "active"){
73
                    $downloader->run({
74
                        postback => $postback,
75
                        task => $task,
76
                   });
77
                }
78
            }
79
        },
80
        Passive => {},
81
    );
82
83
    POE::Session->create(
84
        object_states => [
85
            $self => {
86
                _start => "on_start",
87
                get_task => "get_task",
88
                list_tasks => "list_tasks",
89
                create_task => "create_task",
90
                start_task => "start_task",
91
                stop_task => "stop_task",
92
                delete_task => "delete_task",
93
                repeat_task => "repeat_task",
94
                register => "register",
95
                deregister => "deregister",
96
                restore_state => "restore_state",
97
                save_state => "save_state",
98
                is_task_finished => "is_task_finished",
99
                does_task_repeat => "does_task_repeat",
100
                download_postback => "download_postback",
101
                reset_imports_status => "reset_imports_status",
102
            },
103
        ],
104
    );
105
106
    POE::Component::JobQueue->spawn(
107
        Alias         => 'oai-importer',
108
        WorkerLimit   => $import_worker_limit,
109
        Worker        => sub {
110
            my $meta_postback = shift;
111
112
            #NOTE: We need to only retrieve queue items for active tasks. Otherwise,
113
            #the importer will just spin its wheels on inactive tasks and do nothing.
114
            my $active_tasks = $poe_kernel->call("harvester","list_tasks","active");
115
            my @active_uuids = map { $_->{uuid} } @$active_tasks;
116
117
            my $schema = Koha::Database->new()->schema();
118
            my $rs = $schema->resultset('OaiHarvesterImportQueue')->search({
119
                uuid => \@active_uuids,
120
                status => "new"
121
            },{
122
                order_by => { -asc => 'id' },
123
                rows => 1,
124
            });
125
            my $result = $rs->first;
126
            if ($result){
127
                $result->status("wip");
128
                $result->update;
129
                my $task = {
130
                    id => $result->id,
131
                    uuid => $result->uuid,
132
                    result => $result->result,
133
                };
134
135
                my $postback = $meta_postback->($task);
136
                $importer->run({
137
                    postback => $postback,
138
                    task => $task,
139
                });
140
            }
141
        },
142
        Active => {
143
            PollInterval => $import_queue_poll,
144
            AckAlias => undef,
145
            AckState => undef,
146
        },
147
    );
148
149
    return;
150
}
151
152
sub on_start {
153
    my ($self, $kernel, $heap) = @_[OBJECT, KERNEL,HEAP];
154
    $kernel->alias_set('harvester');
155
    $heap->{scoreboard} = {};
156
157
    #Reset any 'wip' imports to 'new' so they can be re-tried.
158
    $kernel->call("harvester","reset_imports_status");
159
160
    #Restore state from state file
161
    $kernel->call("harvester","restore_state");
162
}
163
164
#NOTE: This isn't really implemented at the moment, as it's not really necessary.
165
sub download_postback {
166
    my ($kernel, $request_packet, $response_packet) = @_[KERNEL, ARG0, ARG1];
167
    my $message = $response_packet->[0];
168
}
169
170
=head3 deregister
171
172
    Remove the worker session from the harvester's in-memory scoreboard,
173
    unset the downloading flag if downloading is completed.
174
175
=cut
176
177
sub deregister {
178
    my ($self, $kernel, $heap, $session, $sender, $type) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0];
179
180
    my $scoreboard = $heap->{scoreboard};
181
182
    my $logger = $self->{logger};
183
    $logger->debug("Start deregistering $sender as $type task");
184
185
    my $task_uuid = delete $scoreboard->{session}->{$sender};
186
    #NOTE: If you don't check each step of the hashref, autovivication can lead to surprises.
187
    if ($task_uuid){
188
        if ($scoreboard->{task}->{$task_uuid}){
189
            if ($scoreboard->{task}->{$task_uuid}->{$type}){
190
                delete $scoreboard->{task}->{$task_uuid}->{$type}->{$sender};
191
            }
192
        }
193
    }
194
195
    my $task = $heap->{tasks}->{$task_uuid};
196
    if ($task && $task->{status} && ($task->{status} eq "active") ){
197
        #NOTE: Each task only has 1 download session, so we can now set/unset flags for the task.
198
        #NOTE: We should unset the downloading flag, if we're not going to repeat the task.
199
        if ($type eq "download"){
200
            my $task_repeats = $kernel->call("harvester","does_task_repeat",$task_uuid);
201
            if ($task_repeats){
202
                my $interval = $task->{interval};
203
204
                $task->{effective_from} = delete $task->{effective_until};
205
                $task->{download_timer} = $kernel->delay_set("repeat_task", $interval, $task_uuid);
206
            }
207
            else {
208
                $task->{downloading} = 0;
209
                $kernel->call("harvester","save_state");
210
                $kernel->call("harvester","is_task_finished",$task_uuid);
211
            }
212
        }
213
        elsif ($type eq 'import'){
214
            $kernel->call("harvester","is_task_finished",$task_uuid);
215
        }
216
    }
217
    $logger->debug("End deregistering $sender as $type task");
218
}
219
220
221
=head3 is_task_finished
222
223
    This event handler checks if the task has finished downloading and importing record.
224
    If it is finished downloading and importing, the task is deleted from the harvester.
225
226
    This only applies to non-repeating tasks.
227
228
=cut
229
230
sub is_task_finished {
231
    my ($self, $kernel, $heap, $session, $uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
232
    my $task = $kernel->call("harvester","get_task",$uuid);
233
    if ($task && (! $task->{downloading}) ){
234
        my $count = $self->get_import_count_for_task($uuid);
235
        if ( ! $count ) {
236
            #Clear this task out of the harvester as it's finished.
237
            $kernel->call("harvester","delete_task",$uuid);
238
            return 1;
239
        }
240
    }
241
    return 0;
242
}
243
244
sub register {
245
    my ($self, $kernel, $heap, $session, $sender, $type, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0,ARG1];
246
    my $logger = $self->{logger};
247
248
    my $scoreboard = $heap->{scoreboard};
249
250
251
    if ($type && $task_uuid){
252
        $logger->debug("Registering $sender as $type for $task_uuid");
253
254
        my $task = $heap->{tasks}->{$task_uuid};
255
        if ($task){
256
257
            if ($type){
258
                #Register the task uuid with the session id as a key for later recall
259
                $scoreboard->{session}->{$sender} = $task_uuid;
260
261
                #Register the session id as a certain type of session for a task
262
                $scoreboard->{task}->{$task_uuid}->{$type}->{$sender} = 1;
263
264
                if ($type eq "download"){
265
                    $task->{downloading} = 1;
266
267
                    my $task_repeats = $kernel->call("harvester","does_task_repeat",$task_uuid);
268
                    if ($task_repeats){
269
270
                        #NOTE: Set an effective until, so we know we're not getting records any newer than
271
                        #this moment.
272
                        my $dt = DateTime->now();
273
                        if ($dt){
274
                            #NOTE: Ideally, I'd like to make sure that we can use 'seconds' granularity, but
275
                            #it's valid for 'from' to be null, so it's impossible to know from the data whether
276
                            #or not the repository will support the seconds granularity.
277
                            #NOTE: Ideally, it would be good to use either 'day' granularity or 'seconds' granularity,
278
                            #but at the moment the interval is expressed as seconds only.
279
                            $dt->set_formatter($seconds_granularity);
280
                            $task->{effective_until} = "$dt";
281
                        }
282
                    }
283
284
                    $kernel->call("harvester","save_state");
285
                }
286
            }
287
        }
288
    }
289
}
290
291
sub does_task_repeat {
292
    my ($self, $kernel, $heap, $session, $uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
293
    my $task = $kernel->call("harvester","get_task",$uuid);
294
    if ($task){
295
        my $interval = $task->{interval};
296
        my $parameters = $task->{parameters};
297
        if ($parameters){
298
            my $oai_pmh = $parameters->{oai_pmh};
299
            if ($oai_pmh){
300
                if ( $interval && ($oai_pmh->{verb} eq "ListRecords") && (! $oai_pmh->{until}) ){
301
                    return 1;
302
                }
303
            }
304
        }
305
    }
306
    return 0;
307
}
308
309
310
311
sub reset_imports_status {
312
    my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION];
313
314
    my $schema = Koha::Database->new()->schema();
315
    my $rs = $schema->resultset('OaiHarvesterImportQueue')->search({
316
                status => "wip",
317
    });
318
    $rs->update({
319
        status => "new",
320
    });
321
}
322
323
sub restore_state {
324
    my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION];
325
326
    my $state_file = $self->{state_file};
327
    if ($state_file){
328
        my $state_backup = "$state_file~";
329
330
        #NOTE: If there is a state backup, it means we crashed while saving the state. Otherwise,
331
        #let's try the regular state file if it exists.
332
        my $file_to_restore = ( -f $state_backup ) ? $state_backup : ( ( -f $state_file ) ? $state_file : undef );
333
        if ( $file_to_restore ){
334
            my $opened = open( my $fh, '<', $file_to_restore ) or die "Couldn't open state: $!";
335
            if ($opened){
336
                local $/;
337
                my $in = <$fh>;
338
                my $decoder = Sereal::Decoder->new;
339
                my $state = $decoder->decode($in);
340
                if ($state){
341
                    if ($state->{tasks}){
342
                        #Restore tasks from our saved state
343
                        $heap->{tasks} = $state->{tasks};
344
                        foreach my $uuid ( keys %{$heap->{tasks}} ){
345
                            my $task = $heap->{tasks}->{$uuid};
346
347
                            #If tasks were still downloading, restart the task
348
                            if ( ($task->{status} && $task->{status} eq "active") && $task->{downloading} ){
349
                                $task->{status} = "new";
350
                                $kernel->call("harvester","start_task",$task->{uuid});
351
                            }
352
                        }
353
                    }
354
                }
355
            }
356
        }
357
    }
358
}
359
360
sub save_state {
361
    my ($self, $kernel, $heap, $session) = @_[OBJECT, KERNEL,HEAP,SESSION];
362
    my $state_file = $self->{state_file};
363
    my $state_backup = "$state_file~";
364
365
    #Make a backup of existing state record
366
    my $moved = move($state_file,$state_backup);
367
368
    my $opened = open(my $fh, ">", $state_file) or die "Couldn't save state: $!";
369
    if ($opened){
370
        $fh->autoflush(1);
371
        my $tasks = $heap->{tasks};
372
        my $harvester_state = {
373
            tasks => $tasks,
374
        };
375
        my $encoder = Sereal::Encoder->new;
376
        my $out = $encoder->encode($harvester_state);
377
        local $\;
378
        my $printed = print $fh $out;
379
        if ($printed){
380
            close $fh;
381
            unlink($state_backup);
382
            return 1;
383
        }
384
    }
385
    return 0;
386
}
387
388
=head3 get_task
389
390
    This event handler returns a task from a harvester using the task's
391
    uuid as an argument.
392
393
=cut
394
395
sub get_task {
396
    my ($self, $kernel, $heap, $session, $uuid, $sender) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0, SENDER];
397
398
    if ( ! $uuid && $sender ){
399
        my $scoreboard = $heap->{scoreboard};
400
        my $uuid_by_session = $scoreboard->{session}->{$sender};
401
        if ($uuid_by_session){
402
            $uuid = $uuid_by_session;
403
        }
404
    }
405
406
    my $tasks = $heap->{tasks};
407
    if ($tasks && $uuid){
408
        my $task = $tasks->{$uuid};
409
        if ($task){
410
            return $task;
411
        }
412
    }
413
    return 0;
414
}
415
416
=head3 get_import_count_for_task
417
418
=cut
419
420
sub get_import_count_for_task {
421
    my ($self,$uuid) = @_;
422
    my $count = undef;
423
    if ($uuid){
424
        my $schema = Koha::Database->new()->schema();
425
        my $items = $schema->resultset('OaiHarvesterImportQueue')->search({
426
            uuid => $uuid,
427
        });
428
        $count = $items->count;
429
    }
430
    return $count;
431
}
432
433
=head3 list_tasks
434
435
    This event handler returns a list of tasks that have been submitted
436
    to the harvester. It returns data like uuid, status, parameters,
437
    number of pending imports, etc.
438
439
=cut
440
441
sub list_tasks {
442
    my ($self, $kernel, $heap, $session, $status) = @_[OBJECT, KERNEL,HEAP,SESSION, ARG0];
443
    my $schema = Koha::Database->new()->schema();
444
    my @tasks = ();
445
    foreach my $uuid (sort keys %{$heap->{tasks}}){
446
        my $task = $heap->{tasks}->{$uuid};
447
        my $items = $schema->resultset('OaiHarvesterImportQueue')->search({
448
            uuid => $uuid,
449
        });
450
        my $count = $items->count // 0;
451
        $task->{pending_imports} = $count;
452
        if ( ( ! $status ) || ( $status && $status eq $task->{status} ) ){
453
            push(@tasks, $task);
454
        }
455
456
    }
457
    return \@tasks;
458
}
459
460
=head3 create_task
461
462
    This event handler creates a spool directory for the task's imports.
463
    It also adds it to the harvester's memory and then saves memory to
464
    a persistent datastore.
465
466
    Newly created tasks have a status of "new".
467
468
=cut
469
470
sub create_task {
471
    my ($self, $kernel, $heap, $session, $incoming_task) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
472
    my $logger = $self->{logger};
473
    if ($incoming_task){
474
        my $uuid = $incoming_task->{uuid};
475
        if ( ! $heap->{tasks}->{$uuid} ){
476
477
            #Step One: assign a spool directory to this task
478
            my $spooldir = $self->{spooldir} // "/tmp";
479
            my $task_spooldir = "$spooldir/$uuid";
480
            if ( ! -d $task_spooldir ){
481
                my $made_spool_directory = make_path($task_spooldir);
482
                if ( ! $made_spool_directory ){
483
                    if ($logger){
484
                        $logger->warn("Unable to make task-specific spool directory at '$task_spooldir'");
485
                    }
486
                    return 0;
487
                }
488
            }
489
            $incoming_task->{spooldir} = $task_spooldir;
490
491
            #Step Two: assign new status
492
            $incoming_task->{status} = "new";
493
494
            #Step Three: add task to harvester's memory
495
            $heap->{tasks}->{ $uuid } = $incoming_task;
496
497
            #Step Four: save state
498
            $kernel->call($session,"save_state");
499
            return 1;
500
        }
501
    }
502
    return 0;
503
}
504
505
=head3 start_task
506
507
    This event handler marks a task as active in the harvester's memory,
508
    save the memory to a persistent datastore, then enqueues the task,
509
    so that it can be directed to the next available download worker.
510
511
    Newly started tasks have a status of "active".
512
513
=cut
514
515
sub start_task {
516
    my ($self, $session,$kernel,$heap,$uuid) = @_[OBJECT, SESSION,KERNEL,HEAP,ARG0];
517
    my $task = $heap->{tasks}->{$uuid};
518
    if ($task){
519
        if ( $task->{status} ne "active" ){
520
521
            #Clear any pre-existing error states
522
            delete $task->{error} if $task->{error};
523
524
            #Step One: mark task as active
525
            $task->{status} = "active";
526
527
            #Step Two: save state
528
            $kernel->call("harvester","save_state");
529
530
            #Step Three: enqueue task
531
            $kernel->post("oai-downloader",'enqueue','download_postback', $task);
532
533
            return 1;
534
        }
535
    }
536
    return 0;
537
}
538
539
=head3 repeat_task
540
541
542
543
=cut
544
545
sub repeat_task {
546
    my ($self, $session,$kernel,$heap,$uuid) = @_[OBJECT, SESSION,KERNEL,HEAP,ARG0];
547
    my $task = $heap->{tasks}->{$uuid};
548
    if ($task){
549
        my $interval = $task->{interval};
550
        if ($task->{downloading} && $interval){
551
            $kernel->post("oai-downloader",'enqueue','download_postback', $task);
552
        }
553
    }
554
}
555
556
=head3 stop_task
557
558
    This event handler prevents new workers from spawning, kills
559
    existing workers, and stops pending imports from being imported.
560
561
    Newly stopped tasks have a status of "stopped".
562
563
=cut
564
565
sub stop_task {
566
    my ($self, $kernel, $heap, $session, $sender, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,SENDER,ARG0];
567
568
    my $task = $heap->{tasks}->{$task_uuid};
569
570
    if ($task && $task->{status} && $task->{status} ne "stopped" ){
571
572
        #Step One: deactivate task, so no new workers can be started
573
        $task->{status} = "stopped";
574
        #NOTE: You could also clear timers for new downloads, but that's probably unnecessary because of this step.
575
576
        #Step Two: kill workers
577
        my $scoreboard = $heap->{scoreboard};
578
        my $session_types = $scoreboard->{task}->{$task_uuid};
579
        if ($session_types){
580
            foreach my $type ( keys %$session_types ){
581
                my $sessions = $session_types->{$type};
582
                if ($sessions){
583
                    foreach my $session (keys %$sessions){
584
                        if ($session){
585
                            $kernel->signal($session, "cancel");
586
                        }
587
                    }
588
                }
589
            }
590
            #Remove the task uuid from the task key of the scoreboard
591
            delete $scoreboard->{task}->{$task_uuid};
592
            #NOTE: The task uuid will still exist under the session key,
593
            #but the sessions will deregister themselves and clean that up for you.
594
        }
595
596
        #Step Three: stop pending imports for this task
597
        my $schema = Koha::Database->new()->schema();
598
        my $items = $schema->resultset('OaiHarvesterImportQueue')->search({
599
            uuid => $task_uuid,
600
        });
601
        my $rows_updated = $items->update({
602
            status => "stopped",
603
        });
604
605
        #Step Four: save state
606
        $kernel->call("harvester","save_state");
607
        return 1;
608
    }
609
    return 0;
610
}
611
612
=head3 delete_task
613
614
    Deleted tasks are stopped, pending imports are deleted from the
615
    database and file system, and then the task is removed from the harvester.
616
617
=cut
618
619
sub delete_task {
620
    my ($self, $kernel, $heap, $session, $task_uuid) = @_[OBJECT, KERNEL,HEAP,SESSION,ARG0];
621
622
    my $task = $heap->{tasks}->{$task_uuid};
623
    if ($task){
624
        #Step One: stop task
625
        $kernel->call($session,"stop_task",$task_uuid);
626
627
        #Step Two: delete pending imports in database
628
        my $schema = Koha::Database->new()->schema();
629
        my $items = $schema->resultset('OaiHarvesterImportQueue')->search({
630
            uuid => $task_uuid,
631
        });
632
        if ($items){
633
            my $rows_deleted = $items->delete;
634
            #NOTE: shows 0E0 instead of 0
635
        }
636
637
        #Step Three: remove task specific spool directory and files within it
638
        my $spooldir = $task->{spooldir};
639
        if ($spooldir){
640
            my $files_deleted = remove_tree($spooldir, { safe => 1 });
641
        }
642
643
        delete $heap->{tasks}->{$task_uuid};
644
645
        #Step Four: save state
646
        $kernel->call("harvester","save_state");
647
        return 1;
648
    }
649
    return 0;
650
}
651
652
1;
(-)a/Koha/OAI/Harvester/Client.pm (+177 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Client;
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use URI;
22
use IO::Socket::UNIX;
23
use IO::Select;
24
use JSON;
25
26
sub new {
27
    my ($class, $args) = @_;
28
    $args = {} unless defined $args;
29
    return bless ($args, $class);
30
}
31
32
sub connect {
33
    my ($self) =  @_;
34
    my $socket_uri = $self->{socket_uri};
35
    if ($socket_uri){
36
        my $uri = URI->new($socket_uri);
37
        if ($uri && $uri->scheme eq 'unix'){
38
            my $socket_path = $uri->path;
39
            my $socket = IO::Socket::UNIX->new(
40
                Type => IO::Socket::UNIX::SOCK_STREAM(),
41
                Peer => $socket_path,
42
            );
43
            if ($socket){
44
                my $select = new IO::Select();
45
                $select->add($socket);
46
47
                $self->{_select} = $select;
48
                $self->{_socket} = $socket;
49
                my $message = $self->_read();
50
                if ($message){
51
                    if ($message eq 'HELLO'){
52
                        $self->{_connected} = 1;
53
                        return 1;
54
                    }
55
                }
56
            }
57
            else {
58
                warn "Failed to create socket."
59
            }
60
        }
61
    }
62
    return 0;
63
}
64
65
sub create {
66
    my ($self,$task) = @_;
67
    my $message = {
68
        command => "create",
69
        body => {
70
            task => $task,
71
        }
72
    };
73
    my ($status) = $self->_exchange($message);
74
    return $status;
75
}
76
77
sub start {
78
    my ($self,$uuid) = @_;
79
    my $message = {
80
        command => "start",
81
        body => {
82
            task => {
83
                uuid => $uuid,
84
            },
85
        }
86
    };
87
    my ($status) = $self->_exchange($message);
88
    return $status;
89
}
90
91
sub stop {
92
    my ($self,$uuid) = @_;
93
    my $message = {
94
        command => "stop",
95
        body => {
96
            task => {
97
                uuid => $uuid,
98
            },
99
        }
100
    };
101
    my ($status) = $self->_exchange($message);
102
    return $status;
103
}
104
105
sub delete {
106
    my ($self,$uuid) = @_;
107
    my $message = {
108
        command => "delete",
109
        body => {
110
            task => {
111
                uuid => $uuid,
112
            },
113
        }
114
    };
115
    my ($status) = $self->_exchange($message);
116
    return $status;
117
}
118
119
sub list {
120
    my ($self) = @_;
121
    my $message = {
122
        command => "list",
123
    };
124
    my ($status,$tasks) = $self->_exchange($message);
125
    return $tasks;
126
}
127
128
sub _exchange {
129
    my ($self,$message) = @_;
130
    my $status = 0;
131
    my $data;
132
    if ($message){
133
        my $output = to_json($message);
134
        if ($output){
135
            $self->_write($output);
136
            my $json_response = $self->_read();
137
            if ($json_response){
138
                my $response = from_json($json_response);
139
                $data = $response->{data} if $response->{data};
140
                $status = 1 if $response->{msg} && $response->{msg} eq "success";
141
            }
142
        }
143
    }
144
    return ($status,$data);
145
}
146
147
sub _write {
148
    my ($self, $output) = @_;
149
    if ($output){
150
        if (my $select = $self->{_select}){
151
            if (my @filehandles = $select->can_write(5)){
152
                foreach my $filehandle (@filehandles){
153
                    #Localize output record separator as null
154
                    local $\ = "\x00";
155
                    print $filehandle $output;
156
                }
157
            }
158
        }
159
    }
160
}
161
162
sub _read {
163
    my ($self) = @_;
164
    if (my $select = $self->{_select}){
165
        if (my @filehandles = $select->can_read(5)){
166
            foreach my $filehandle (@filehandles){
167
                #Localize input record separator as null
168
                local $/ = "\x00";
169
                my $message = <$filehandle>;
170
                chomp($message) if $message;
171
                return $message;
172
            }
173
        }
174
    }
175
}
176
177
1;
(-)a/Koha/OAI/Harvester/Downloader.pm (+308 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Downloader;
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use URI;
22
use XML::LibXML::Reader;
23
use IO::Handle;
24
use JSON;
25
26
=head1 API
27
28
=head2 Class Methods
29
30
=cut
31
32
sub new {
33
    my ($class, $args) = @_;
34
    $args = {} unless defined $args;
35
    return bless ($args, $class);
36
}
37
38
=head2 BuildURL
39
40
    Takes a baseURL and a mix of required and optional OAI-PMH arguments,
41
    and makes them into a suitable URL for an OAI-PMH request.
42
43
=cut
44
45
sub BuildURL {
46
    my ($self, $args) = @_;
47
    my $baseURL = $args->{baseURL};
48
    my $url = URI->new($baseURL);
49
    if ($url && $url->isa("URI")){
50
        my $verb = $args->{verb};
51
        if ($verb){
52
            my %parameters = (
53
                verb => $verb,
54
            );
55
            if ($verb eq "ListRecords"){
56
                my $resumptionToken = $args->{resumptionToken};
57
                my $metadataPrefix = $args->{metadataPrefix};
58
                if ($resumptionToken){
59
                    $parameters{resumptionToken} = $resumptionToken;
60
                }
61
                elsif ($metadataPrefix){
62
                    $parameters{metadataPrefix} = $metadataPrefix;
63
                    #Only add optional parameters if they're provided
64
                    foreach my $param ( qw( from until set ) ){
65
                        $parameters{$param} = $args->{$param} if $args->{$param};
66
                    }
67
                }
68
                else {
69
                    warn "BuildURL() requires an argument of either resumptionToken or metadataPrefix";
70
                    return;
71
                }
72
            }
73
            elsif ($verb eq "GetRecord"){
74
                my $metadataPrefix = $args->{metadataPrefix};
75
                my $identifier = $args->{identifier};
76
                if ($metadataPrefix && $identifier){
77
                    $parameters{metadataPrefix} = $metadataPrefix;
78
                    $parameters{identifier} = $identifier;
79
                }
80
                else {
81
                    warn "BuildURL() requires an argument of metadataPrefix and an argument of identifier";
82
                    return;
83
                }
84
            }
85
            $url->query_form(%parameters);
86
            return $url;
87
        }
88
        else {
89
            warn "BuildURL() requires a verb of GetRecord or ListRecords";
90
            return;
91
        }
92
    }
93
    else {
94
        warn "BuildURL() requires a base URL of type URI.";
95
        return;
96
    }
97
}
98
99
=head2 OpenXMLStream
100
101
    Fork a child process to send the HTTP request, which sends chunks
102
    of XML via a pipe to the parent process.
103
104
    The parent process creates and returns a XML::LibXML::Reader object,
105
    which reads the XML stream coming through the pipe.
106
107
    Normally, using a DOM reader, you must wait to read the entire XML document
108
    into memory. However, using a stream reader, chunks are read into memory,
109
    processed, then discarded. It's faster and more efficient.
110
111
=cut
112
113
sub GetXMLStream {
114
    my ($self, $args) = @_;
115
    my $url = $args->{url};
116
    my $user_agent = $args->{user_agent};
117
    if ($url && $user_agent){
118
        pipe( CHILD, PARENT ) or die "Cannot created connected pipes: $!";
119
        CHILD->autoflush(1);
120
        PARENT->autoflush(1);
121
        if ( my $pid = fork ){
122
            #Parent process
123
            close PARENT;
124
            return \*CHILD;
125
        }
126
        else {
127
            #Child process
128
            close CHILD;
129
            my $response = $self->_request({
130
                url => $url,
131
                user_agent => $user_agent,
132
                file_handle => \*PARENT,
133
            });
134
            if ($response && $response->is_success){
135
                #HTTP request has successfully finished, so we close the file handle and exit the process
136
                close PARENT;
137
                CORE::exit(); #some modules like mod_perl redefine exit
138
            }
139
            else {
140
                warn "[child $$] OAI-PMH unsuccessful. Response status: ".$response->status_line."\n" if $response;
141
                CORE::exit();
142
            }
143
        }
144
    }
145
    else {
146
        warn "GetXMLStream() requires a 'url' argument and a 'user_agent' argument";
147
        return;
148
    }
149
}
150
151
sub _request {
152
    my ($self, $args) = @_;
153
    my $url = $args->{url};
154
    my $user_agent = $args->{user_agent};
155
    my $fh = $args->{file_handle};
156
157
    if ($url && $user_agent && $fh){
158
        my $request = HTTP::Request->new( GET => $url );
159
        my $response = $user_agent->request( $request, sub {
160
                my ($chunk_of_data, $ref_to_response, $ref_to_protocol) = @_;
161
                print $fh $chunk_of_data;
162
        });
163
        return $response;
164
    }
165
    else {
166
        warn "_request() requires a 'url' argument, 'user_agent' argument, and 'file_handle' argument.";
167
        return;
168
    }
169
}
170
171
sub ParseXMLStream {
172
    my ($self, $args) = @_;
173
174
    my $each_callback = $args->{each_callback};
175
    my $fh = $args->{file_handle};
176
    if ($fh){
177
        my $reader = XML::LibXML::Reader->new( FD => $fh, no_blanks => 1 );
178
        my $pattern = XML::LibXML::Pattern->new('oai:OAI-PMH|/oai:OAI-PMH/*', { 'oai' => "http://www.openarchives.org/OAI/2.0/" });
179
180
        my $repository;
181
182
        warn "Start parsing...";
183
        while (my $rv = $reader->nextPatternMatch($pattern)){
184
            #$rv == 1; successful
185
            #$rv == 0; end of document reached
186
            #$rv == -1; error
187
            if ($rv == -1){
188
                die "Parser error!";
189
            }
190
            #NOTE: We do this so we only get the opening tag of the element.
191
            next unless $reader->nodeType == XML_READER_TYPE_ELEMENT;
192
193
            my $localname = $reader->localName;
194
            if ( $localname eq "request" ){
195
                my $node = $reader->copyCurrentNode(1);
196
                $repository = $node->textContent;
197
            }
198
            elsif ( $localname eq "error" ){
199
                #See https://www.openarchives.org/OAI/openarchivesprotocol.html#ErrorConditions
200
                #We should probably die under all circumstances except "noRecordsMatch"
201
                my $node = $reader->copyCurrentNode(1);
202
                if ($node){
203
                    my $code = $node->getAttribute("code");
204
                    if ($code){
205
                        if ($code ne "noRecordsMatch"){
206
                            warn "Error code: $code";
207
                            die;
208
                        }
209
                    }
210
                }
211
            }
212
            elsif ( ($localname eq "ListRecords") || ($localname eq "GetRecord") ){
213
                my $each_pattern = XML::LibXML::Pattern->new('//oai:record|oai:resumptionToken', { 'oai' => "http://www.openarchives.org/OAI/2.0/" });
214
                while (my $each_rv =  $reader->nextPatternMatch($each_pattern)){
215
                    if ($rv == "-1"){
216
                        #NOTE: -1 denotes an error state
217
                        warn "Error getting pattern match";
218
                    }
219
                    next unless $reader->nodeType == XML_READER_TYPE_ELEMENT;
220
                    if ($reader->localName eq "record"){
221
                        my $node = $reader->copyCurrentNode(1);
222
                        #NOTE: Without the UTF-8 flag, UTF-8 data will be corrupted.
223
                        my $document = XML::LibXML::Document->new('1.0', 'UTF-8');
224
                        $document->setDocumentElement($node);
225
226
                       #Per record callback
227
                        if ($each_callback){
228
                            $each_callback->({
229
                                repository => $repository,
230
                                document => $document,
231
                            });
232
                        }
233
                    }
234
                    elsif ($reader->localName eq "resumptionToken"){
235
                        my $resumptionToken = $reader->readInnerXml;
236
                        return ($resumptionToken,$repository);
237
238
                    }
239
                }
240
            }
241
        } #/OAI-PMH document match
242
    }
243
    else {
244
        warn "ParseXMLStream() requires a 'file_handle' argument.";
245
    }
246
}
247
248
sub harvest {
249
    my ($self,$args) = @_;
250
    my $url = $args->{url};
251
    my $ua = $args->{user_agent};
252
    my $callback = $args->{callback};
253
    my $complete_callback = $args->{complete_callback};
254
255
    if ($url && $ua){
256
257
        #NOTE: http://search.cpan.org/~shlomif/XML-LibXML-2.0128/lib/XML/LibXML/Parser.pod#ERROR_REPORTING
258
        while($url){
259
            warn "URL = $url";
260
            warn "Creating child process to download and feed parent process parser.";
261
            my $stream = $self->GetXMLStream({
262
                url => $url,
263
                user_agent => $ua,
264
            });
265
266
            warn "Creating parent process parser.";
267
            my ($resumptionToken) = $self->ParseXMLStream({
268
                file_handle => $stream,
269
                each_callback => $callback,
270
            });
271
            warn "Finished parsing current XML document.";
272
273
            if ($resumptionToken){
274
                #If there's a resumptionToken at the end of the stream,
275
                #we build a new URL and repeat this process again.
276
                $url->query_form({
277
                    verb => "ListRecords",
278
                    resumptionToken => $resumptionToken,
279
                });
280
            }
281
            else {
282
                warn "Finished harvest.";
283
                last;
284
            }
285
286
            warn "Reap child process downloader.";
287
            #Reap the dead child requester process before performing another request,
288
            #so we don't fill up the process table with zombie children.
289
            while ((my $child = waitpid(-1, 0)) > 0) {
290
                warn "Parent $$ reaped child process $child" . ($? ? " with exit code $?" : '') . ".\n";
291
            }
292
        }
293
294
        if ($complete_callback){
295
            warn "Run complete callback.";
296
297
            #Clear query string
298
            $url->query_form({});
299
300
            #Run complete callback using the actual URL from the request.
301
            $complete_callback->({
302
                repository => $url,
303
            });
304
        }
305
    }
306
}
307
308
1;
(-)a/Koha/OAI/Harvester/Import/MARCXML.pm (+140 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Import::MARCXML;
2
3
# Copyright 2016 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
use MARC::Record;
23
24
use C4::Context;
25
use C4::Biblio;
26
27
use constant MAX_MATCHES => 99999; #NOTE: This is an arbitrary value. We want to get all matches.
28
29
sub new {
30
    my ($class, $args) = @_;
31
    $args = {} unless defined $args;
32
    if ( (! $args->{dom}) && (! $args->{marc_record}) ){
33
        die "You must provide either a dom or marc_record argument to this constructor";
34
    }
35
    if ( $args->{dom} && ( ! $args->{marc_record} ) ){
36
        my $dom = $args->{dom};
37
        my $xml = $dom->toString(2);
38
        my $marcflavour = C4::Context->preference('marcflavour') || 'MARC21';
39
        my $marc_record = eval {MARC::Record::new_from_xml( $xml, "utf8", $marcflavour)};
40
        if ($@){
41
            die "Unable to create MARC::Record object";
42
        }
43
        if ($marc_record){
44
            $args->{marc_record} = $marc_record;
45
        }
46
    }
47
    return bless ($args, $class);
48
}
49
50
sub import_record {
51
    my ($self,$args) = @_;
52
    my $framework = $args->{framework};
53
    my $record_type = $args->{record_type};
54
    my $matcher = $args->{matcher};
55
    my $koha_id = $args->{koha_id};
56
57
    my $action = "error";
58
59
    #Try to find a matching Koha MARCXML record via Zebra
60
    if (! $koha_id && $matcher){
61
        my $matched_id = $self->_try_matcher({
62
            matcher => $matcher,
63
        });
64
        if ($matched_id){
65
            $koha_id = $matched_id;
66
        }
67
    }
68
69
    if ($koha_id){
70
        #Update
71
        ($action) = $self->_mod_koha_record({
72
            record_type => $record_type,
73
            framework => $framework,
74
            koha_id => $koha_id,
75
        });
76
    }
77
    else {
78
        #Add
79
        ($action,$koha_id) = $self->_add_koha_record({
80
            record_type => $record_type,
81
            framework => $framework,
82
        });
83
    }
84
85
    return ($action,$koha_id);
86
}
87
88
sub _try_matcher {
89
    my ($self, $args) = @_;
90
    my $marc_record = $self->{marc_record};
91
    my $matcher = $args->{matcher};
92
    my $matched_id;
93
    my @matches = $matcher->get_matches($marc_record, MAX_MATCHES);
94
    if (@matches){
95
        my $bestrecordmatch = shift @matches;
96
        if ($bestrecordmatch && $bestrecordmatch->{record_id}){
97
            $matched_id = $bestrecordmatch->{record_id};
98
        }
99
    }
100
    return $matched_id;
101
}
102
103
sub _add_koha_record {
104
    my ($self, $args) = @_;
105
    my $marc_record = $self->{marc_record};
106
    my $record_type = $args->{record_type} // "biblio";
107
    my $framework = $args->{framework};
108
    my $koha_id;
109
    my $action = "error";
110
    if ($record_type eq "biblio"){
111
        #NOTE: Strip item fields to prevent any accidentally getting through.
112
        C4::Biblio::_strip_item_fields($marc_record,$framework);
113
        my ($biblionumber,$biblioitemnumber) = C4::Biblio::AddBiblio($marc_record,$framework);
114
        if ($biblionumber){
115
            $action = "added";
116
            $koha_id = $biblionumber;
117
        }
118
    }
119
    return ($action,$koha_id);
120
}
121
122
sub _mod_koha_record {
123
    my ($self, $args) = @_;
124
    my $marc_record = $self->{marc_record};
125
    my $record_type = $args->{record_type} // "biblio";
126
    my $framework = $args->{framework};
127
    my $koha_id = $args->{koha_id};
128
    my $action = "error";
129
    if ($record_type eq "biblio"){
130
        #NOTE: Strip item fields to prevent any accidentally getting through.
131
        C4::Biblio::_strip_item_fields($marc_record,$framework);
132
        my $updated = C4::Biblio::ModBiblio($marc_record, $koha_id, $framework);
133
        if ($updated){
134
            $action = "updated";
135
        }
136
    }
137
    return ($action);
138
}
139
140
1;
(-)a/Koha/OAI/Harvester/Import/Record.pm (+301 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Import::Record;
2
3
# Copyright 2016 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
use XML::LibXML;
23
use XML::LibXSLT;
24
use URI;
25
use File::Basename;
26
27
use C4::Context;
28
use C4::Biblio;
29
30
use Koha::Database;
31
use Koha::OAI::Harvester::Import::MARCXML;
32
33
=head1 API
34
35
=head2 Class Methods
36
37
=cut
38
39
my $schema = Koha::Database->new()->schema();
40
41
sub new {
42
    my ($class, $args) = @_;
43
    $args = {} unless defined $args;
44
45
    die "You must provide a 'doc' argument to the constructor" unless $args->{doc};
46
    die "You must provide a 'repository' argument to the constructor" unless $args->{repository};
47
48
    if (my $doc = $args->{doc}){
49
50
        #Get the root element
51
        my $root = $doc->documentElement;
52
53
        #Register namespaces for searching purposes
54
        my $xpc = XML::LibXML::XPathContext->new();
55
        $xpc->registerNs('oai','http://www.openarchives.org/OAI/2.0/');
56
57
        my $xpath_identifier = XML::LibXML::XPathExpression->new("oai:header/oai:identifier");
58
        my $identifier = $xpc->findnodes($xpath_identifier,$root)->shift;
59
        $args->{header_identifier} = $identifier->textContent;
60
61
        my $xpath_datestamp = XML::LibXML::XPathExpression->new("oai:header/oai:datestamp");
62
        my $datestamp = $xpc->findnodes($xpath_datestamp,$root)->shift;
63
        $args->{header_datestamp} = $datestamp->textContent;
64
65
        my $xpath_status = XML::LibXML::XPathExpression->new(q{oai:header/@status});
66
        my $status_node = $xpc->findnodes($xpath_status,$root)->shift;
67
        $args->{header_status} = $status_node ? $status_node->textContent : "";
68
    }
69
70
    return bless ($args, $class);
71
}
72
73
sub is_deleted_upstream {
74
    my ($self, $args) = @_;
75
    if ($self->{header_status}){
76
        if ($self->{header_status} eq "deleted"){
77
            return 1;
78
        }
79
    }
80
    return 0;
81
}
82
83
sub set_filter {
84
    my ($self, $filter_definition) = @_;
85
86
    #Source a default XSLT to use for filtering
87
    my $htdocs  = C4::Context->config('intrahtdocs');
88
    my $theme   = C4::Context->preference("template");
89
    $self->{filter} = "$htdocs/$theme/en/xslt/StripOAIPMH.xsl";
90
    $self->{filter_type} = "xslt";
91
92
    if ($filter_definition && $filter_definition ne "default"){
93
        my ($filter_type, $filter) = $self->_parse_filter($filter_definition);
94
        if ($filter_type eq "xslt"){
95
            if (  -f $filter ){
96
                $self->{filter} = $filter;
97
                $self->{filter_type} = "xslt";
98
            }
99
        }
100
    }
101
}
102
103
sub _parse_filter {
104
    my ($self,$filter_definition) = @_;
105
    my ($type,$filter);
106
    my $filter_uri = URI->new($filter_definition);
107
    if ($filter_uri){
108
        my $scheme = $filter_uri->scheme;
109
        if ( ($scheme && $scheme eq "file") || ! $scheme ){
110
            my $path = $filter_uri->path;
111
            #Filters may theoretically be .xsl or .pm files
112
            my($filename, $dirs, $suffix) = fileparse($path,(".xsl",".pm"));
113
            if ($suffix){
114
                if ( $suffix eq ".xsl"){
115
                    $type = "xslt";
116
                    $filter = $path;
117
                }
118
            }
119
        }
120
    }
121
    return ($type,$filter);
122
}
123
124
sub filter {
125
    my ($self) = @_;
126
    my $filtered = 0;
127
    my $doc = $self->{doc};
128
    my $filter = $self->{filter};
129
    my $filter_type = $self->{filter_type};
130
    if ($doc){
131
        if ($filter && -f $filter){
132
            if ($filter_type){
133
                if ( $filter_type eq 'xslt' ){
134
                    my $xslt = XML::LibXSLT->new();
135
                    my $style_doc = XML::LibXML->load_xml(location => $filter);
136
                    my $stylesheet = $xslt->parse_stylesheet($style_doc);
137
                    if ($stylesheet){
138
                        my $results = $stylesheet->transform($doc);
139
                        if ($results){
140
                            my $root = $results->documentElement;
141
                            if ($root){
142
                                my $namespace = $root->namespaceURI;
143
                                if ($namespace eq "http://www.loc.gov/MARC21/slim"){
144
                                    #NOTE: Both MARC21 and UNIMARC should be covered by this namespace
145
                                    my $marcxml = eval { Koha::OAI::Harvester::Import::MARCXML->new({ dom => $results, }) };
146
                                    if ($@){
147
                                        warn "Error Koha::OAI::Harvester::Import::MARCXML: $@";
148
                                        return;
149
                                    } else {
150
                                        return $marcxml;
151
                                    }
152
                                }
153
                            }
154
                        }
155
                    }
156
                }
157
            }
158
        }
159
    }
160
    return;
161
}
162
163
sub _find_koha_link {
164
    my ($self, $args) = @_;
165
    my $record_type = $args->{record_type} // "biblio";
166
    my $link_id;
167
    if ($record_type eq "biblio"){
168
        my $link = $schema->resultset('OaiHarvesterBiblio')->find(
169
            {
170
                oai_repository => $self->{repository},
171
                oai_identifier => $self->{header_identifier},
172
            },
173
            { key => "oai_record",}
174
        );
175
        if ($link && $link->biblionumber){
176
            $link_id = $link->biblionumber->id;
177
        }
178
    }
179
    return $link_id;
180
}
181
182
=head3 import_record
183
184
    my ($action,$record_id) = $oai_record->import_record({
185
        filter => $filter,
186
        framework => $framework,
187
        record_type => $record_type,
188
        matcher => $matcher,
189
    });
190
191
    $action eq "added" || "updated" || "deleted" || "not_found" || "error"
192
193
=cut
194
195
sub import_record {
196
    my ($self, $args) = @_;
197
    my $filter = $args->{filter} || 'default';
198
    my $framework = $args->{framework} || '';
199
    my $record_type = $args->{record_type} || 'biblio';
200
    my $matcher = $args->{matcher};
201
202
    my $action = "error";
203
204
    #Find linkage between OAI-PMH repository-identifier and Koha record id
205
    my $linked_id = $self->_find_koha_link({
206
        record_type => $record_type,
207
    });
208
209
    if ($self->is_deleted_upstream){
210
        #NOTE: If a record is deleted upstream, it will not contain a metadata element
211
        if ($linked_id){
212
            $action = $self->delete_koha_record({
213
                record_id => $linked_id,
214
                record_type => $record_type,
215
            });
216
        }
217
        else {
218
            $action = "not_found";
219
            #NOTE: If there's no OAI-PMH repository-identifier pair in the database,
220
            #then there's no perfect way to find a linked record to delete.
221
        }
222
    }
223
    else {
224
        $self->set_filter($filter);
225
226
227
        my $import_record = $self->filter();
228
229
        if ($import_record){
230
            ($action,$linked_id) = $import_record->import_record({
231
                framework => $framework,
232
                record_type => $record_type,
233
                matcher => $matcher,
234
                koha_id => $linked_id,
235
            });
236
237
            if ($linked_id){
238
                #Link Koha record ID to OAI-PMH details for this record type,
239
                #if linkage doesn't already exist.
240
                $self->link_koha_record({
241
                    record_type => $record_type,
242
                    koha_id => $linked_id,
243
                });
244
            }
245
        }
246
    }
247
248
    #Log record details to database
249
    my $importer = $schema->resultset('OaiHarvesterHistory')->create({
250
        header_identifier => $self->{header_identifier},
251
        header_datestamp => $self->{header_datestamp},
252
        header_status => $self->{header_status},
253
        record => $self->{doc}->toString(1),
254
        repository => $self->{repository},
255
        status => $action,
256
        filter => $filter,
257
        framework => $framework,
258
        record_type => $record_type,
259
        matcher_code => $matcher ? $matcher->code : undef,
260
    });
261
262
    return ($action,$linked_id);
263
}
264
265
sub link_koha_record {
266
    my ($self, $args) = @_;
267
    my $record_type = $args->{record_type} // "biblio";
268
    my $koha_id = $args->{koha_id};
269
    if ($koha_id){
270
        if ($record_type eq "biblio"){
271
            my $import_oai_biblio = $schema->resultset('OaiHarvesterBiblio')->find_or_create({
272
                oai_repository => $self->{repository},
273
                oai_identifier => $self->{header_identifier},
274
                biblionumber => $koha_id,
275
            });
276
            if ( ! $import_oai_biblio->in_storage ){
277
                $import_oai_biblio->insert;
278
            }
279
        }
280
    }
281
}
282
283
sub delete_koha_record {
284
    my ($self, $args) = @_;
285
    my $record_type = $args->{record_type} // "biblio";
286
    my $record_id = $args->{record_id};
287
288
    my $action = "error";
289
290
    if ($record_type eq "biblio"){
291
        my $error = C4::Biblio::DelBiblio($record_id);
292
        if (!$error){
293
            $action = "deleted";
294
            #NOTE: If there's no error, a cascading database delete should
295
            #automatically remove the link between the Koha biblionumber and OAI-PMH record too
296
        }
297
    }
298
    return $action;
299
}
300
301
1;
(-)a/Koha/OAI/Harvester/Listener.pm (+187 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Listener;
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
use POE qw(Wheel::SocketFactory Wheel::ReadWrite);
23
use IO::Socket qw(AF_UNIX);
24
use JSON;
25
use URI;
26
27
sub new {
28
    my ($class, $args) = @_;
29
    $args = {} unless defined $args;
30
    return bless ($args, $class);
31
}
32
33
sub spawn {
34
    my ($class, $args) = @_;
35
    my $self = $class->new($args);
36
    my $socket = $args->{socket};
37
    POE::Session->create(
38
        args => [
39
            $socket,
40
        ],
41
        object_states => [
42
            $self => {
43
                _start => "on_start",
44
                "on_server_success" => "on_server_success",
45
                "on_server_error" => "on_server_error",
46
                "on_client_error" => "on_client_error",
47
                "on_client_input" => "on_client_input",
48
            },
49
        ],
50
    );
51
}
52
53
sub on_start {
54
    my ($kernel,$heap,$socket_uri) = @_[KERNEL,HEAP,ARG0];
55
56
    my $uri = URI->new($socket_uri);
57
    if ($uri && $uri->scheme eq 'unix'){
58
        my $socket_path = $uri->path;
59
        unlink $socket_path if -S $socket_path;
60
        $heap->{server} = POE::Wheel::SocketFactory->new(
61
            SocketDomain => AF_UNIX,
62
            BindAddress => $socket_path,
63
            SuccessEvent => "on_server_success",
64
            FailureEvent => "on_server_error",
65
        );
66
67
        #Make the socket writeable to other users like Apache
68
        chmod 0666, $socket_path;
69
    }
70
}
71
72
sub on_server_success {
73
    my ($self, $client_socket, $server_wheel_id, $heap, $session) = @_[OBJECT, ARG0, ARG3, HEAP,SESSION];
74
    my $logger = $self->{logger};
75
    my $null_filter = POE::Filter::Line->new(
76
         Literal => chr(0),
77
    );
78
    my $client_wheel = POE::Wheel::ReadWrite->new(
79
        Handle => $client_socket,
80
        InputEvent => "on_client_input",
81
        ErrorEvent => "on_client_error",
82
        InputFilter => $null_filter,
83
        OutputFilter => $null_filter,
84
    );
85
    $heap->{client}->{ $client_wheel->ID() } = $client_wheel;
86
    $logger->info("Connection ".$client_wheel->ID()." started.");
87
    #TODO: Add basic authentication here?
88
    $client_wheel->put("HELLO");
89
}
90
91
sub on_server_error {
92
    my ($self, $operation, $errnum, $errstr, $heap, $session) = @_[OBJECT, ARG0, ARG1, ARG2,HEAP, SESSION];
93
    my $logger = $self->{logger};
94
    $logger->error("Server $operation error $errnum: $errstr");
95
    delete $heap->{server};
96
}
97
98
sub on_client_error {
99
    my ($self, $wheel_id,$heap,$session) = @_[OBJECT, ARG3,HEAP,SESSION];
100
    my $logger = $self->{logger};
101
    $logger->info("Connection $wheel_id failed or ended.");
102
    delete $heap->{client}->{$wheel_id};
103
}
104
105
sub on_client_input {
106
    my ($self, $input, $wheel_id, $session, $kernel, $heap) = @_[OBJECT, ARG0, ARG1, SESSION, KERNEL, HEAP];
107
    my $logger = $self->{logger};
108
    $logger->debug("Server input: $input");
109
    my $server_response = { msg => "fail"};
110
    eval {
111
        my $json_input = from_json($input);
112
        my $command = $json_input->{command};
113
        my $body = $json_input->{body};
114
        if ($command){
115
            if ($command eq "create"){
116
                my $task = $body->{task};
117
                if ($task){
118
                    my $is_created = $kernel->call("harvester","create_task",$task);
119
                    if ($is_created){
120
                        $server_response->{msg} = "success";
121
                    }
122
                }
123
            }
124
            elsif ($command eq "start"){
125
                my $task = $body->{task};
126
                if ($task){
127
                    my $uuid = $task->{uuid};
128
                    #Fetch from memory now...
129
                    my $is_started = $kernel->call("harvester","start_task", $uuid);
130
                    if ($is_started){
131
                        $server_response->{msg} = "success";
132
                    }
133
                }
134
            }
135
            elsif ($command eq "stop"){
136
                my $task = $body->{task};
137
                if ($task){
138
                    if ($task->{uuid}){
139
                        my $is_stopped = $kernel->call("harvester","stop_task",$task->{uuid});
140
                        if ($is_stopped){
141
                            $server_response->{msg} = "success";
142
                        }
143
                    }
144
                }
145
            }
146
            elsif ($command eq "delete"){
147
                my $task = $body->{task};
148
                if ($task){
149
                    if ($task->{uuid}){
150
                        my $is_deleted = $kernel->call("harvester","delete_task",$task->{uuid});
151
                        if ($is_deleted){
152
                            $server_response->{msg} = "success";
153
                        }
154
                    }
155
                }
156
            }
157
            elsif ($command eq "list"){
158
                my $tasks = $kernel->call("harvester","list_tasks");
159
                if ($tasks){
160
                    $server_response->{msg} = "success";
161
                    $server_response->{data} = $tasks;
162
                }
163
            }
164
        }
165
    };
166
    if ($@){
167
        #NOTE: An error most likely means that something other than a valid JSON string was received
168
        $logger->error($@);
169
    }
170
171
    if ($server_response){
172
        eval {
173
            my $client = $heap->{client}->{$wheel_id};
174
            my $json_message = to_json($server_response, { pretty => 1 });
175
            if ($json_message){
176
                $logger->debug("Server output: $json_message");
177
                $client->put($json_message);
178
            }
179
        };
180
        if ($@){
181
            #NOTE: An error means our response couldn't be serialised as JSON
182
            $logger->error($@);
183
        }
184
    }
185
}
186
187
1;
(-)a/Koha/OAI/Harvester/Request.pm (+184 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Request;
2
3
# Copyright Prosentient Systems 2017
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use Carp;
23
24
use base qw(Koha::Object);
25
26
#For validation
27
use URI;
28
use HTTP::OAI;
29
30
=head1 NAME
31
32
Koha::OAI::Harvester::Request -
33
34
=head1 API
35
36
=head2 Class Methods
37
38
=cut
39
40
41
42
=head3 _type
43
44
=cut
45
46
sub _type {
47
    return 'OaiHarvesterRequest';
48
}
49
50
sub validate {
51
    my ($self) = @_;
52
    my $errors = {};
53
54
    #Step one: validate URL
55
    my $uri = URI->new($self->http_url);
56
    if ( $uri && $uri->scheme && ($uri->scheme eq "http" || $uri->scheme eq "https") ){
57
58
        #Step two: validate access and authorization to URL
59
        my $harvester = $self->_harvester();
60
        my $identify = $harvester->Identify;
61
        if ($identify->is_success){
62
63
            #Step three: validate OAI-PMH parameters
64
65
            #Test Set
66
            my $set = $self->oai_set;
67
            if ($set){
68
                my $set_response = $harvester->ListSets();
69
                my @server_sets = $set_response->set;
70
                if ( ! grep {$_->setSpec eq $set} @server_sets ){
71
                    $errors->{oai_set}->{unavailable} = 1;
72
                }
73
            }
74
75
            #Test Metadata Prefix
76
            my $metadataPrefix = $self->oai_metadataPrefix;
77
            if ($metadataPrefix){
78
                my $metadata_response = $harvester->ListMetadataFormats();
79
                my @server_formats = $metadata_response->metadataFormat;
80
                if ( ! grep { $_->metadataPrefix eq $metadataPrefix } @server_formats ){
81
                    $errors->{oai_metadataPrefix}->{unavailable} = 1;
82
                }
83
            }
84
            else {
85
                $errors->{oai_metadataPrefix}->{missing} = 1;
86
            }
87
88
            #Test Granularity and Timestamps
89
            my $server_granularity = $identify->granularity;
90
            my $from = $self->oai_from;
91
            my $until = $self->oai_until;
92
            if ($from || $until){
93
                my ($from_granularity,$until_granularity);
94
                if ($from){
95
                    $from_granularity = _determine_granularity($from);
96
                    if ($from_granularity eq "YYYY-MM-DDThh:mm:ssZ"){
97
                        $errors->{oai_from}->{unavailable} = 1 if $server_granularity ne $from_granularity;
98
                    } elsif ($from_granularity eq "failed"){
99
                        $errors->{oai_from}->{malformed} = 1;
100
                    }
101
                }
102
                if ($until){
103
                    $until_granularity = _determine_granularity($until);
104
                    if ($until_granularity eq "YYYY-MM-DDThh:mm:ssZ"){
105
                        $errors->{oai_until}->{unavailable} = 1 if $server_granularity ne $until_granularity;
106
                    } elsif ($until_granularity eq "failed"){
107
                        $errors->{oai_until}->{malformed} = 1;
108
                    }
109
                }
110
                if ($from && $until){
111
                    if ($from_granularity ne $until_granularity){
112
                        $errors->{oai}->{granularity_mismatch} = 1;
113
                    }
114
                }
115
            }
116
117
            #Test if identifier is provided when using GetRecord
118
            my $verb = $self->oai_verb;
119
            if ($verb && $verb eq "GetRecord"){
120
                my $identifier = $self->oai_identifier;
121
                if (! $identifier){
122
                    $errors->{oai_identifier}->{missing} = 1;
123
                }
124
            }
125
        }
126
        elsif ($identify->is_error){
127
            foreach my $error ($identify->errors){
128
                if ($error->code =~ /^404$/){
129
                    $errors->{http}->{404} = 1;
130
                } elsif ($error->code =~ /^401$/){
131
                    $errors->{http}->{401} = 1;
132
                } else {
133
                    $errors->{http}->{generic} = 1;
134
                }
135
            }
136
        }
137
        else {
138
            $errors->{http}->{generic} = 1;
139
        }
140
    } else {
141
        $errors->{http_url}->{malformed} = 1;
142
    }
143
    return $errors;
144
}
145
146
sub _harvester {
147
    my ( $self ) = @_;
148
    my $harvester;
149
    if ($self->http_url){
150
        $harvester = new HTTP::OAI::Harvester( baseURL => $self->http_url );
151
        my $uri = URI->new($self->http_url);
152
        if ($uri->scheme && ($uri->scheme eq 'http' || $uri->scheme eq 'https') ){
153
            my $host = $uri->host;
154
            my $port = $uri->port;
155
            $harvester->credentials($host.":".$port, $self->http_realm, $self->http_username, $self->http_password);
156
        }
157
    }
158
    return $harvester;
159
}
160
161
sub _determine_granularity {
162
    my ($timestamp) = @_;
163
    my $granularity;
164
    if ($timestamp =~ /^(\d{4}-\d{2}-\d{2})(T\d{2}:\d{2}:\d{2}Z)?$/){
165
        if ($1 && $2){
166
            $granularity = "YYYY-MM-DDThh:mm:ssZ";
167
        } elsif ($1 && !$2){
168
            $granularity = "YYYY-MM-DD";
169
        } else {
170
            $granularity = "failed";
171
        }
172
    } else {
173
        $granularity = "failed";
174
    }
175
    return $granularity;
176
}
177
178
=head1 AUTHOR
179
180
David Cook <dcook@prosentient.com.au>
181
182
=cut
183
184
1;
(-)a/Koha/OAI/Harvester/Requests.pm (+62 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Requests;
2
3
# Copyright Prosentient Systems 2017
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use Carp;
23
24
use Koha::Database;
25
26
use Koha::OAI::Harvester::Request;
27
28
use base qw(Koha::Objects);
29
30
=head1 NAME
31
32
Koha::OAI::Harvester::Requests -
33
34
=head1 API
35
36
=head2 Class Methods
37
38
=cut
39
40
=head3 _type
41
42
=cut
43
44
sub _type {
45
    return 'OaiHarvesterRequest';
46
}
47
48
=head3 object_class
49
50
=cut
51
52
sub object_class {
53
    return 'Koha::OAI::Harvester::Request';
54
}
55
56
=head1 AUTHOR
57
58
David Cook <dcook@prosentient.com.au>
59
60
=cut
61
62
1;
(-)a/Koha/OAI/Harvester/Worker.pm (+156 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Worker;
2
3
# Copyright Prosentient Systems 2017
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use POE;
22
use DateTime;
23
use JSON;
24
25
sub new {
26
    my ($class, $args) = @_;
27
    $args = {} unless defined $args;
28
    $args->{type} = "worker" unless $args->{type};
29
    return bless ($args, $class);
30
}
31
32
sub run {
33
    my ($self,$args) = @_;
34
    my $postback = $args->{postback};
35
    my $task = $args->{task};
36
37
    POE::Session->create(
38
        object_states => [
39
            $self => {
40
                _start           => "on_start",
41
                got_child_stderr => "on_child_stderr",
42
                got_child_close  => "on_child_close",
43
                got_child_signal => "on_child_signal",
44
                got_child_stdout => "on_child_stdout",
45
                stop_worker      => "stop_worker",
46
                _stop            => "on_stop",
47
            },
48
        ],
49
        args => [
50
            $postback,
51
            $task,
52
        ],
53
    );
54
}
55
56
sub stop_worker {
57
    my ($self,$heap) = @_[OBJECT,HEAP];
58
    if (my $child_processes = $heap->{children_by_pid}){
59
        foreach my $child_pid (keys %$child_processes){
60
            my $child = $child_processes->{$child_pid};
61
            $child->kill();
62
        }
63
    }
64
}
65
66
67
sub on_stop {
68
    my ($self,$kernel) = @_[OBJECT,KERNEL];
69
70
    #Deregister the worker session from the harvester's roster of workers
71
    $kernel->call("harvester","deregister",$self->{type});
72
}
73
74
# Wheel event, including the wheel's ID.
75
sub on_child_stdout {
76
    my ($self, $stdout_line, $wheel_id) = @_[OBJECT, ARG0, ARG1];
77
    my $type = $self->{type};
78
    my $child = $_[HEAP]{children_by_wid}{$wheel_id};
79
    my $logger = $self->{logger};
80
    if ($logger){
81
        $logger->debug("[$type][pid ".$child->PID."][STDOUT] $stdout_line");
82
    }
83
84
    my $postback = $_[HEAP]{postback};
85
    if ($postback){
86
        eval {
87
            my $message = from_json($stdout_line);
88
            if ($message){
89
                $postback->($message);
90
            }
91
        };
92
    }
93
}
94
95
# Wheel event, including the wheel's ID.
96
sub on_child_stderr {
97
    my ($self,$stderr_line, $wheel_id) = @_[OBJECT, ARG0, ARG1];
98
    my $type = $self->{type};
99
    my $child = $_[HEAP]{children_by_wid}{$wheel_id};
100
    my $logger = $self->{logger};
101
    if ($logger){
102
        $logger->debug("[$type][pid ".$child->PID."][STDERR] $stderr_line");
103
    }
104
}
105
106
# Wheel event, including the wheel's ID.
107
sub on_child_close {
108
    my ($self,$heap,$wheel_id) = @_[OBJECT,HEAP,ARG0];
109
    my $type = $self->{type};
110
    my $logger = $self->{logger};
111
112
    my $child = delete $heap->{children_by_wid}->{$wheel_id};
113
114
    # May have been reaped by on_child_signal().
115
    unless (defined $child) {
116
        if ($logger){
117
            $logger->debug("[$type][wid $wheel_id] closed all pipes");
118
        }
119
        return;
120
    }
121
    if ($logger){
122
        $logger->debug("[$type][pid ".$child->PID."] closed all pipes");
123
    }
124
    delete $heap->{children_by_pid}->{$child->PID};
125
}
126
127
sub on_child_signal {
128
    my ($self,$kernel,$pid,$status) = @_[OBJECT,KERNEL,ARG1,ARG2];
129
    my $type = $self->{type};
130
    my $logger = $self->{logger};
131
    if ($logger){
132
        $logger->debug("[$type][pid $pid] exited with status $status");
133
    }
134
135
    my $child = delete $_[HEAP]{children_by_pid}{$_[ARG1]};
136
137
    # May have been reaped by on_child_close().
138
    return unless defined $child;
139
140
    delete $_[HEAP]{children_by_wid}{$child->ID};
141
142
    #If the child doesn't complete successfully, we lodge an error
143
    #and stop the task.
144
    if ($status != 0){
145
        my $task = $kernel->call("harvester","get_task");
146
        if ($task){
147
            $task->{error} = 1;
148
            my $uuid = $task->{uuid};
149
            if ($uuid){
150
                $kernel->call("harvester","stop_task",$uuid);
151
            }
152
        }
153
    }
154
}
155
156
1;
(-)a/Koha/OAI/Harvester/Worker/Download/Stream.pm (+190 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Worker::Download::Stream;
2
3
# Copyright Prosentient Systems 2017
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use LWP::UserAgent;
22
use UUID;
23
use POE;
24
use JSON;
25
use File::Path qw/make_path/;
26
27
use C4::Context;
28
use Koha::OAI::Harvester::Downloader;
29
use parent 'Koha::OAI::Harvester::Worker';
30
31
sub new {
32
    my ($class, $args) = @_;
33
    $args = {} unless defined $args;
34
    $args->{type} = "download" unless $args->{type};
35
    return bless ($args, $class);
36
}
37
38
sub on_start {
39
    my ($self, $kernel, $heap, $postback,$task,$session) = @_[OBJECT, KERNEL, HEAP, ARG0,ARG1,SESSION];
40
    #Save postback into heap so other event handlers can use it
41
    $heap->{postback} = $postback;
42
43
    my $task_uuid = $task->{uuid};
44
45
    $kernel->sig("cancel" => "stop_worker");
46
    $kernel->call("harvester","register",$self->{type},$task->{uuid});
47
48
    my $child = POE::Wheel::Run->new(
49
        ProgramArgs => [$task],
50
        Program => sub {
51
            my ($args) = @_;
52
            $self->do_work($args);
53
        },
54
        StdoutEvent  => "got_child_stdout",
55
        StderrEvent  => "got_child_stderr",
56
        CloseEvent   => "got_child_close",
57
        NoSetPgrp => 1, #Keep child processes in same group as parent. This is especially useful when using Ctrl+C to kill the whole group.
58
    );
59
60
     $_[KERNEL]->sig_child($child->PID, "got_child_signal");
61
62
    # Wheel events include the wheel's ID.
63
    $_[HEAP]{children_by_wid}{$child->ID} = $child;
64
65
    # Signal events include the process ID.
66
    $_[HEAP]{children_by_pid}{$child->PID} = $child;
67
68
    my $logger = $self->{logger};
69
    if ($logger){
70
        $logger->debug("Child pid ".$child->PID." started as wheel ".$child->ID);
71
    }
72
}
73
74
sub do_work {
75
    my ($self, $task) = @_;
76
    my $batch = ( $self->{batch} && int($self->{batch}) ) ? $self->{batch} : 100;
77
78
    #NOTE: Directory to spool files for processing
79
    my $spooldir = $task->{spooldir};
80
81
    my $task_uuid = $task->{uuid};
82
    my $task_parameters = $task->{parameters};
83
    my $interval = $task->{interval};
84
85
    my $oai_pmh_parameters = $task_parameters->{oai_pmh};
86
    my $import_parameters = $task_parameters->{import};
87
88
    #NOTE: Overwrite the 'from' and 'until' parameters for repeatable tasks
89
    if ( $interval && ! $oai_pmh_parameters->{until} ){
90
        if ($oai_pmh_parameters->{verb} eq "ListRecords"){
91
            #NOTE: 'effective_from' premiers on the first repetition (ie second request)
92
            $oai_pmh_parameters->{from} = $task->{effective_from} if $task->{effective_from};
93
            #NOTE: 'effective_until' appears on the first request
94
            $oai_pmh_parameters->{until} = $task->{effective_until} if $task->{effective_until};
95
        }
96
    }
97
98
    my $oai_downloader = Koha::OAI::Harvester::Downloader->new();
99
    my $url = $oai_downloader->BuildURL($oai_pmh_parameters);
100
101
    my $ua = LWP::UserAgent->new();
102
    #NOTE: setup HTTP Basic Authentication if parameters are supplied
103
    if($url && $url->host && $url->port){
104
        my $http_basic_auth = $task_parameters->{http_basic_auth};
105
        if ($http_basic_auth){
106
            my $username = $http_basic_auth->{username};
107
            my $password = $http_basic_auth->{password};
108
            my $realm = $http_basic_auth->{realm};
109
            $ua->credentials($url->host.":".$url->port, $realm, $username, $password);
110
        }
111
    }
112
113
    #NOTE: Prepare database statement handle
114
    my $dbh = C4::Context->dbh;
115
    my $sql = "insert into oai_harvester_import_queue (uuid,result) VALUES (?,?)";
116
    my $sth = $dbh->prepare($sql);
117
118
    if($url && $ua){
119
        #NOTE: You could define the callbacks as object methods instead... that might be nicer...
120
        #although I suppose it might be a much of a muchness.
121
        eval {
122
            my @filename_cache = ();
123
124
            $oai_downloader->harvest({
125
                user_agent => $ua,
126
                url => $url,
127
                callback => sub {
128
                    my ($args) = @_;
129
130
                    my $repository = $args->{repository};
131
                    my $document = $args->{document};
132
133
                    #If the spooldir has disappeared, re-create it.
134
                    if ( ! -d $spooldir ){
135
                        my $made_spool_directory = make_path($spooldir);
136
                    }
137
                    my ($uuid,$uuid_string);
138
                    UUID::generate($uuid);
139
                    UUID::unparse($uuid, $uuid_string);
140
                    my $file_uuid = $uuid_string;
141
                    my $filename = "$spooldir/$file_uuid";
142
                    my $state = $document->toFile($filename, 2);
143
                    if ($state){
144
                        push(@filename_cache,$filename);
145
                    }
146
147
                    if(scalar @filename_cache == $batch){
148
                        my $result = {
149
                            repository => $repository,
150
                            filenames => \@filename_cache,
151
                            filter => $import_parameters->{filter},
152
                            matcher_code => $import_parameters->{matcher_code},
153
                            frameworkcode => $import_parameters->{frameworkcode},
154
                            record_type => $import_parameters->{record_type},
155
                        };
156
                        eval {
157
                            my $json_result = to_json($result, { pretty => 1 });
158
                            $sth->execute($task_uuid,$json_result);
159
                        };
160
                        @filename_cache = ();
161
                    }
162
                },
163
                complete_callback => sub {
164
                    my ($args) = @_;
165
                    my $repository = $args->{repository};
166
                    if (@filename_cache){
167
                        my $result = {
168
                            repository => "$repository",
169
                            filenames => \@filename_cache,
170
                            filter => $import_parameters->{filter},
171
                            matcher_code => $import_parameters->{matcher_code},
172
                            frameworkcode => $import_parameters->{frameworkcode},
173
                            record_type => $import_parameters->{record_type},
174
                        };
175
                        eval {
176
                            my $json_result = to_json($result, { pretty => 1 });
177
                            $sth->execute($task_uuid,$json_result);
178
                        };
179
                    }
180
181
                },
182
            });
183
        };
184
        if ($@){
185
            die "Error during OAI-PMH download";
186
        }
187
    }
188
}
189
190
1;
(-)a/Koha/OAI/Harvester/Worker/Import.pm (+133 lines)
Line 0 Link Here
1
package Koha::OAI::Harvester::Worker::Import;
2
3
# Copyright Prosentient Systems 2017
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use POE qw(Wheel::Run);
22
use JSON;
23
use XML::LibXML;
24
25
use C4::Context;
26
use C4::Matcher;
27
use Koha::OAI::Harvester::Import::Record;
28
29
use parent 'Koha::OAI::Harvester::Worker';
30
31
sub new {
32
    my ($class, $args) = @_;
33
    $args = {} unless defined $args;
34
    #NOTE: This type is used for logging and more importantly for registering with the harvester
35
    $args->{type} = "import" unless $args->{type};
36
    return bless ($args, $class);
37
}
38
39
sub on_start {
40
    my ($self, $kernel, $heap, $postback,$task) = @_[OBJECT, KERNEL, HEAP, ARG0,ARG1];
41
42
    $kernel->call("harvester","register",$self->{type},$task->{uuid});
43
44
    $kernel->sig(cancel => "stop_worker");
45
46
    my $child = POE::Wheel::Run->new(
47
        ProgramArgs => [ $task ],
48
        Program => sub {
49
            my ($task,$args) = @_;
50
51
            my $debug = $args->{debug} // 0;
52
53
            if ($task){
54
                my $json_result = $task->{result};
55
                my $id = $task->{id};
56
                my $task_uuid = $task->{uuid};
57
                eval {
58
                    my $result = from_json($json_result);
59
                    if ($result){
60
                        my $repository = $result->{repository};
61
                        my $filenames = $result->{filenames};
62
                        my $filter = $result->{filter};
63
                        my $matcher_code = $result->{matcher_code};
64
                        my $frameworkcode = $result->{frameworkcode};
65
                        my $record_type = $result->{record_type};
66
67
                        my $matcher;
68
                        if ($matcher_code){
69
                            my $matcher_id = C4::Matcher::GetMatcherId($matcher_code);
70
                            $matcher = C4::Matcher->fetch($matcher_id);
71
                        }
72
73
                        foreach my $filename (@$filenames){
74
                            if ($filename){
75
                                if (-f $filename){
76
                                    my $dom = XML::LibXML->load_xml(location => $filename, { no_blanks => 1 });
77
                                    if ($dom){
78
                                        my $oai_record = Koha::OAI::Harvester::Import::Record->new({
79
                                            doc => $dom,
80
                                            repository => $repository,
81
                                        });
82
                                        if ($oai_record){
83
                                            my ($action,$linked_id) = $oai_record->import_record({
84
                                                filter => $filter,
85
                                                framework => $frameworkcode,
86
                                                record_type => $record_type,
87
                                                matcher => $matcher,
88
                                            });
89
                                            $debug && print STDOUT qq({ "import_result": { "task_uuid": "$task_uuid", "action": "$action", "filename": "$filename", "koha_id": "$linked_id" } }\n);
90
                                        }
91
                                    }
92
                                    my $unlinked = unlink $filename;
93
                                }
94
                            }
95
                        }
96
                    }
97
                };
98
                if ($@){
99
                    warn $@;
100
                }
101
                #NOTE: Even if the file doesn't exist, we still need to process the queue item.
102
103
                #NOTE: Don't do this via a postback in the parent process, as it's much faster to let the child process handle it.
104
105
                #NOTE: It's vital that files are unlinked before deleting from the database,
106
                #or you could get orphan files if the importer is interrupted.
107
                my $dbh = C4::Context->dbh;
108
                my $sql = "delete from oai_harvester_import_queue where id = ?";
109
                my $sth = $dbh->prepare($sql);
110
                $sth->execute($id);
111
            }
112
        },
113
        StdoutEvent  => "got_child_stdout",
114
        StderrEvent  => "got_child_stderr",
115
        CloseEvent   => "got_child_close",
116
        NoSetPgrp => 1, #Keep child processes in same group as parent. This is especially useful when using Ctrl+C to kill the whole group.
117
    );
118
119
    $_[KERNEL]->sig_child($child->PID, "got_child_signal");
120
121
    # Wheel events include the wheel's ID.
122
    $_[HEAP]{children_by_wid}{$child->ID} = $child;
123
124
    # Signal events include the process ID.
125
    $_[HEAP]{children_by_pid}{$child->PID} = $child;
126
127
    my $logger = $self->{logger};
128
    if ($logger){
129
        $logger->debug("Child pid ".$child->PID." started as wheel ".$child->ID);
130
    }
131
}
132
133
1;
(-)a/Koha/Schema/Result/OaiHarvesterBiblio.pm (+120 lines)
Line 0 Link Here
1
use utf8;
2
package Koha::Schema::Result::OaiHarvesterBiblio;
3
4
# Created by DBIx::Class::Schema::Loader
5
# DO NOT MODIFY THE FIRST PART OF THIS FILE
6
7
=head1 NAME
8
9
Koha::Schema::Result::OaiHarvesterBiblio
10
11
=cut
12
13
use strict;
14
use warnings;
15
16
use base 'DBIx::Class::Core';
17
18
=head1 TABLE: C<oai_harvester_biblios>
19
20
=cut
21
22
__PACKAGE__->table("oai_harvester_biblios");
23
24
=head1 ACCESSORS
25
26
=head2 import_oai_biblio_id
27
28
  data_type: 'integer'
29
  extra: {unsigned => 1}
30
  is_auto_increment: 1
31
  is_nullable: 0
32
33
=head2 oai_repository
34
35
  data_type: 'varchar'
36
  is_nullable: 0
37
  size: 255
38
39
=head2 oai_identifier
40
41
  data_type: 'varchar'
42
  is_nullable: 1
43
  size: 255
44
45
=head2 biblionumber
46
47
  data_type: 'integer'
48
  is_foreign_key: 1
49
  is_nullable: 0
50
51
=cut
52
53
__PACKAGE__->add_columns(
54
  "import_oai_biblio_id",
55
  {
56
    data_type => "integer",
57
    extra => { unsigned => 1 },
58
    is_auto_increment => 1,
59
    is_nullable => 0,
60
  },
61
  "oai_repository",
62
  { data_type => "varchar", is_nullable => 0, size => 255 },
63
  "oai_identifier",
64
  { data_type => "varchar", is_nullable => 1, size => 255 },
65
  "biblionumber",
66
  { data_type => "integer", is_foreign_key => 1, is_nullable => 0 },
67
);
68
69
=head1 PRIMARY KEY
70
71
=over 4
72
73
=item * L</import_oai_biblio_id>
74
75
=back
76
77
=cut
78
79
__PACKAGE__->set_primary_key("import_oai_biblio_id");
80
81
=head1 UNIQUE CONSTRAINTS
82
83
=head2 C<oai_record>
84
85
=over 4
86
87
=item * L</oai_identifier>
88
89
=item * L</oai_repository>
90
91
=back
92
93
=cut
94
95
__PACKAGE__->add_unique_constraint("oai_record", ["oai_identifier", "oai_repository"]);
96
97
=head1 RELATIONS
98
99
=head2 biblionumber
100
101
Type: belongs_to
102
103
Related object: L<Koha::Schema::Result::Biblio>
104
105
=cut
106
107
__PACKAGE__->belongs_to(
108
  "biblionumber",
109
  "Koha::Schema::Result::Biblio",
110
  { biblionumber => "biblionumber" },
111
  { is_deferrable => 1, on_delete => "CASCADE", on_update => "NO ACTION" },
112
);
113
114
115
# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-03-29 12:23:43
116
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:2URn8tPABMKC+JuIMfGeYw
117
118
119
# You can replace this text with custom code or comments, and it will be preserved on regeneration
120
1;
(-)a/Koha/Schema/Result/OaiHarvesterHistory.pm (+163 lines)
Line 0 Link Here
1
use utf8;
2
package Koha::Schema::Result::OaiHarvesterHistory;
3
4
# Created by DBIx::Class::Schema::Loader
5
# DO NOT MODIFY THE FIRST PART OF THIS FILE
6
7
=head1 NAME
8
9
Koha::Schema::Result::OaiHarvesterHistory
10
11
=cut
12
13
use strict;
14
use warnings;
15
16
use base 'DBIx::Class::Core';
17
18
=head1 TABLE: C<oai_harvester_history>
19
20
=cut
21
22
__PACKAGE__->table("oai_harvester_history");
23
24
=head1 ACCESSORS
25
26
=head2 import_oai_id
27
28
  data_type: 'integer'
29
  extra: {unsigned => 1}
30
  is_auto_increment: 1
31
  is_nullable: 0
32
33
=head2 repository
34
35
  data_type: 'varchar'
36
  is_nullable: 1
37
  size: 255
38
39
=head2 header_identifier
40
41
  data_type: 'varchar'
42
  is_nullable: 1
43
  size: 255
44
45
=head2 header_datestamp
46
47
  data_type: 'datetime'
48
  datetime_undef_if_invalid: 1
49
  is_nullable: 0
50
51
=head2 header_status
52
53
  data_type: 'varchar'
54
  is_nullable: 1
55
  size: 45
56
57
=head2 record
58
59
  data_type: 'longtext'
60
  is_nullable: 0
61
62
=head2 upload_timestamp
63
64
  data_type: 'timestamp'
65
  datetime_undef_if_invalid: 1
66
  default_value: current_timestamp
67
  is_nullable: 0
68
69
=head2 status
70
71
  data_type: 'varchar'
72
  is_nullable: 0
73
  size: 45
74
75
=head2 filter
76
77
  data_type: 'text'
78
  is_nullable: 0
79
80
=head2 framework
81
82
  data_type: 'varchar'
83
  is_nullable: 0
84
  size: 4
85
86
=head2 record_type
87
88
  data_type: 'enum'
89
  extra: {list => ["biblio","auth","holdings"]}
90
  is_nullable: 0
91
92
=head2 matcher_code
93
94
  data_type: 'varchar'
95
  is_nullable: 1
96
  size: 10
97
98
=cut
99
100
__PACKAGE__->add_columns(
101
  "import_oai_id",
102
  {
103
    data_type => "integer",
104
    extra => { unsigned => 1 },
105
    is_auto_increment => 1,
106
    is_nullable => 0,
107
  },
108
  "repository",
109
  { data_type => "varchar", is_nullable => 1, size => 255 },
110
  "header_identifier",
111
  { data_type => "varchar", is_nullable => 1, size => 255 },
112
  "header_datestamp",
113
  {
114
    data_type => "datetime",
115
    datetime_undef_if_invalid => 1,
116
    is_nullable => 0,
117
  },
118
  "header_status",
119
  { data_type => "varchar", is_nullable => 1, size => 45 },
120
  "record",
121
  { data_type => "longtext", is_nullable => 0 },
122
  "upload_timestamp",
123
  {
124
    data_type => "timestamp",
125
    datetime_undef_if_invalid => 1,
126
    default_value => \"current_timestamp",
127
    is_nullable => 0,
128
  },
129
  "status",
130
  { data_type => "varchar", is_nullable => 0, size => 45 },
131
  "filter",
132
  { data_type => "text", is_nullable => 0 },
133
  "framework",
134
  { data_type => "varchar", is_nullable => 0, size => 4 },
135
  "record_type",
136
  {
137
    data_type => "enum",
138
    extra => { list => ["biblio", "auth", "holdings"] },
139
    is_nullable => 0,
140
  },
141
  "matcher_code",
142
  { data_type => "varchar", is_nullable => 1, size => 10 },
143
);
144
145
=head1 PRIMARY KEY
146
147
=over 4
148
149
=item * L</import_oai_id>
150
151
=back
152
153
=cut
154
155
__PACKAGE__->set_primary_key("import_oai_id");
156
157
158
# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-03-29 12:23:43
159
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Rp/ZEZsKVlLo2vaM3M37ow
160
161
162
# You can replace this text with custom code or comments, and it will be preserved on regeneration
163
1;
(-)a/Koha/Schema/Result/OaiHarvesterImportQueue.pm (+106 lines)
Line 0 Link Here
1
use utf8;
2
package Koha::Schema::Result::OaiHarvesterImportQueue;
3
4
# Created by DBIx::Class::Schema::Loader
5
# DO NOT MODIFY THE FIRST PART OF THIS FILE
6
7
=head1 NAME
8
9
Koha::Schema::Result::OaiHarvesterImportQueue
10
11
=cut
12
13
use strict;
14
use warnings;
15
16
use base 'DBIx::Class::Core';
17
18
=head1 TABLE: C<oai_harvester_import_queue>
19
20
=cut
21
22
__PACKAGE__->table("oai_harvester_import_queue");
23
24
=head1 ACCESSORS
25
26
=head2 id
27
28
  data_type: 'integer'
29
  extra: {unsigned => 1}
30
  is_auto_increment: 1
31
  is_nullable: 0
32
33
=head2 uuid
34
35
  data_type: 'varchar'
36
  is_nullable: 0
37
  size: 45
38
39
=head2 status
40
41
  data_type: 'varchar'
42
  default_value: 'new'
43
  is_nullable: 0
44
  size: 45
45
46
=head2 result
47
48
  data_type: 'text'
49
  is_nullable: 0
50
51
=head2 result_timestamp
52
53
  data_type: 'timestamp'
54
  datetime_undef_if_invalid: 1
55
  default_value: current_timestamp
56
  is_nullable: 0
57
58
=cut
59
60
__PACKAGE__->add_columns(
61
  "id",
62
  {
63
    data_type => "integer",
64
    extra => { unsigned => 1 },
65
    is_auto_increment => 1,
66
    is_nullable => 0,
67
  },
68
  "uuid",
69
  { data_type => "varchar", is_nullable => 0, size => 45 },
70
  "status",
71
  {
72
    data_type => "varchar",
73
    default_value => "new",
74
    is_nullable => 0,
75
    size => 45,
76
  },
77
  "result",
78
  { data_type => "text", is_nullable => 0 },
79
  "result_timestamp",
80
  {
81
    data_type => "timestamp",
82
    datetime_undef_if_invalid => 1,
83
    default_value => \"current_timestamp",
84
    is_nullable => 0,
85
  },
86
);
87
88
=head1 PRIMARY KEY
89
90
=over 4
91
92
=item * L</id>
93
94
=back
95
96
=cut
97
98
__PACKAGE__->set_primary_key("id");
99
100
101
# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-03-29 12:23:43
102
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zBD+hMawbvu7sonuLRHnCA
103
104
105
# You can replace this text with custom code or comments, and it will be preserved on regeneration
106
1;
(-)a/Koha/Schema/Result/OaiHarvesterRequest.pm (+209 lines)
Line 0 Link Here
1
use utf8;
2
package Koha::Schema::Result::OaiHarvesterRequest;
3
4
# Created by DBIx::Class::Schema::Loader
5
# DO NOT MODIFY THE FIRST PART OF THIS FILE
6
7
=head1 NAME
8
9
Koha::Schema::Result::OaiHarvesterRequest
10
11
=cut
12
13
use strict;
14
use warnings;
15
16
use base 'DBIx::Class::Core';
17
18
=head1 TABLE: C<oai_harvester_requests>
19
20
=cut
21
22
__PACKAGE__->table("oai_harvester_requests");
23
24
=head1 ACCESSORS
25
26
=head2 id
27
28
  data_type: 'integer'
29
  extra: {unsigned => 1}
30
  is_auto_increment: 1
31
  is_nullable: 0
32
33
=head2 uuid
34
35
  data_type: 'varchar'
36
  is_nullable: 0
37
  size: 45
38
39
=head2 oai_verb
40
41
  data_type: 'varchar'
42
  is_nullable: 0
43
  size: 45
44
45
=head2 oai_metadataPrefix
46
47
  accessor: 'oai_metadata_prefix'
48
  data_type: 'varchar'
49
  is_nullable: 0
50
  size: 255
51
52
=head2 oai_identifier
53
54
  data_type: 'varchar'
55
  is_nullable: 1
56
  size: 255
57
58
=head2 oai_from
59
60
  data_type: 'varchar'
61
  is_nullable: 1
62
  size: 45
63
64
=head2 oai_until
65
66
  data_type: 'varchar'
67
  is_nullable: 1
68
  size: 45
69
70
=head2 oai_set
71
72
  data_type: 'varchar'
73
  is_nullable: 1
74
  size: 255
75
76
=head2 http_url
77
78
  data_type: 'varchar'
79
  is_nullable: 1
80
  size: 255
81
82
=head2 http_username
83
84
  data_type: 'varchar'
85
  is_nullable: 1
86
  size: 255
87
88
=head2 http_password
89
90
  data_type: 'varchar'
91
  is_nullable: 1
92
  size: 255
93
94
=head2 http_realm
95
96
  data_type: 'varchar'
97
  is_nullable: 1
98
  size: 255
99
100
=head2 import_filter
101
102
  data_type: 'varchar'
103
  is_nullable: 0
104
  size: 255
105
106
=head2 import_framework_code
107
108
  data_type: 'varchar'
109
  is_nullable: 0
110
  size: 4
111
112
=head2 import_record_type
113
114
  data_type: 'enum'
115
  extra: {list => ["biblio","auth","holdings"]}
116
  is_nullable: 0
117
118
=head2 import_matcher_code
119
120
  data_type: 'varchar'
121
  is_nullable: 1
122
  size: 10
123
124
=head2 interval
125
126
  data_type: 'integer'
127
  extra: {unsigned => 1}
128
  is_nullable: 0
129
130
=head2 name
131
132
  data_type: 'varchar'
133
  is_nullable: 0
134
  size: 45
135
136
=cut
137
138
__PACKAGE__->add_columns(
139
  "id",
140
  {
141
    data_type => "integer",
142
    extra => { unsigned => 1 },
143
    is_auto_increment => 1,
144
    is_nullable => 0,
145
  },
146
  "uuid",
147
  { data_type => "varchar", is_nullable => 0, size => 45 },
148
  "oai_verb",
149
  { data_type => "varchar", is_nullable => 0, size => 45 },
150
  "oai_metadataPrefix",
151
  {
152
    accessor => "oai_metadata_prefix",
153
    data_type => "varchar",
154
    is_nullable => 0,
155
    size => 255,
156
  },
157
  "oai_identifier",
158
  { data_type => "varchar", is_nullable => 1, size => 255 },
159
  "oai_from",
160
  { data_type => "varchar", is_nullable => 1, size => 45 },
161
  "oai_until",
162
  { data_type => "varchar", is_nullable => 1, size => 45 },
163
  "oai_set",
164
  { data_type => "varchar", is_nullable => 1, size => 255 },
165
  "http_url",
166
  { data_type => "varchar", is_nullable => 1, size => 255 },
167
  "http_username",
168
  { data_type => "varchar", is_nullable => 1, size => 255 },
169
  "http_password",
170
  { data_type => "varchar", is_nullable => 1, size => 255 },
171
  "http_realm",
172
  { data_type => "varchar", is_nullable => 1, size => 255 },
173
  "import_filter",
174
  { data_type => "varchar", is_nullable => 0, size => 255 },
175
  "import_framework_code",
176
  { data_type => "varchar", is_nullable => 0, size => 4 },
177
  "import_record_type",
178
  {
179
    data_type => "enum",
180
    extra => { list => ["biblio", "auth", "holdings"] },
181
    is_nullable => 0,
182
  },
183
  "import_matcher_code",
184
  { data_type => "varchar", is_nullable => 1, size => 10 },
185
  "interval",
186
  { data_type => "integer", extra => { unsigned => 1 }, is_nullable => 0 },
187
  "name",
188
  { data_type => "varchar", is_nullable => 0, size => 45 },
189
);
190
191
=head1 PRIMARY KEY
192
193
=over 4
194
195
=item * L</id>
196
197
=back
198
199
=cut
200
201
__PACKAGE__->set_primary_key("id");
202
203
204
# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-04-07 11:26:24
205
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Vm/b4yurmr8WF7z+Fo6KEw
206
207
208
# You can replace this text with custom code or comments, and it will be preserved on regeneration
209
1;
(-)a/Makefile.PL (+14 lines)
Lines 360-365 my $target_map = { Link Here
360
  './skel/var/lib/koha/zebradb/biblios/tmp'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
360
  './skel/var/lib/koha/zebradb/biblios/tmp'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
361
  './skel/var/lock/koha/zebradb/rebuild' => { target => 'ZEBRA_LOCK_DIR', trimdir => 6 },
361
  './skel/var/lock/koha/zebradb/rebuild' => { target => 'ZEBRA_LOCK_DIR', trimdir => 6 },
362
  './skel/var/lib/koha/plugins' => { target => 'PLUGINS_DIR', trimdir => 6 },
362
  './skel/var/lib/koha/plugins' => { target => 'PLUGINS_DIR', trimdir => 6 },
363
  './skel/var/lib/koha/oai-pmh-harvester' => { target => 'OAI_LIB_DIR', trimdir => 6 },
364
  './skel/var/run/koha/oai-pmh-harvester' => { target => 'OAI_RUN_DIR', trimdir => 6 },
365
  './skel/var/spool/koha/oai-pmh-harvester' => { target => 'OAI_SPOOL_DIR', trimdir => 6 },
363
  './sms'                       => 'INTRANET_CGI_DIR',
366
  './sms'                       => 'INTRANET_CGI_DIR',
364
  './suggestion'                => 'INTRANET_CGI_DIR',
367
  './suggestion'                => 'INTRANET_CGI_DIR',
365
  './svc'                       => 'INTRANET_CGI_DIR',
368
  './svc'                       => 'INTRANET_CGI_DIR',
Lines 594-599 my $pl_files = { Link Here
594
         'blib/KOHA_CONF_DIR/koha-conf.xml',
597
         'blib/KOHA_CONF_DIR/koha-conf.xml',
595
         'blib/KOHA_CONF_DIR/koha-httpd.conf',
598
         'blib/KOHA_CONF_DIR/koha-httpd.conf',
596
         'blib/KOHA_CONF_DIR/log4perl.conf',
599
         'blib/KOHA_CONF_DIR/log4perl.conf',
600
         'blib/KOHA_CONF_DIR/oai-pmh-harvester.yaml',
597
         'blib/ZEBRA_CONF_DIR/etc/default.idx',
601
         'blib/ZEBRA_CONF_DIR/etc/default.idx',
598
         'blib/MISC_DIR/koha-install-log'
602
         'blib/MISC_DIR/koha-install-log'
599
         ],
603
         ],
Lines 1355-1360 sub get_target_directories { Link Here
1355
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'koha', 'plugins');
1359
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'koha', 'plugins');
1356
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1360
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1357
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
1361
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
1362
        $dirmap{'OAI_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'oai-pmh-harvester');
1363
        $dirmap{'OAI_LIB_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'oai-pmh-harvester');
1364
        $dirmap{'OAI_SPOOL_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'spool', 'oai-pmh-harvester');
1358
    } elsif ($mode eq 'dev') {
1365
    } elsif ($mode eq 'dev') {
1359
        my $curdir = File::Spec->rel2abs(File::Spec->curdir());
1366
        my $curdir = File::Spec->rel2abs(File::Spec->curdir());
1360
        $dirmap{'API_CGI_DIR'} = File::Spec->catdir($curdir, 'api');
1367
        $dirmap{'API_CGI_DIR'} = File::Spec->catdir($curdir, 'api');
Lines 1390-1395 sub get_target_directories { Link Here
1390
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'plugins');
1397
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'plugins');
1391
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1398
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1392
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
1399
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
1400
        $dirmap{'OAI_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'oai-pmh-harvester');
1401
        $dirmap{'OAI_LIB_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'oai-pmh-harvester');
1402
        $dirmap{'OAI_SPOOL_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'spool', 'oai-pmh-harvester');
1403
1393
    } else {
1404
    } else {
1394
        # mode is standard, i.e., 'fhs'
1405
        # mode is standard, i.e., 'fhs'
1395
        $dirmap{'API_CGI_DIR'} = File::Spec->catdir(@basedir, $package, 'api');
1406
        $dirmap{'API_CGI_DIR'} = File::Spec->catdir(@basedir, $package, 'api');
Lines 1414-1419 sub get_target_directories { Link Here
1414
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'plugins');
1425
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'plugins');
1415
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'zebradb');
1426
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'zebradb');
1416
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'run', $package, 'zebradb');
1427
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'run', $package, 'zebradb');
1428
        $dirmap{'OAI_RUN_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'run', $package, 'oai-pmh-harvester');
1429
        $dirmap{'OAI_LIB_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'oai-pmh-harvester');
1430
        $dirmap{'OAI_SPOOL_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'spool', $package, 'oai-pmh-harvester');
1417
    }
1431
    }
1418
1432
1419
    _get_env_overrides(\%dirmap);
1433
    _get_env_overrides(\%dirmap);
(-)a/debian/scripts/koha-create (+7 lines)
Lines 113-118 generate_config_file() { Link Here
113
        -e "s/__PLUGINS_DIR__/\/var\/lib\/koha\/$name\/plugins/g" \
113
        -e "s/__PLUGINS_DIR__/\/var\/lib\/koha\/$name\/plugins/g" \
114
        -e "s/__MEMCACHED_NAMESPACE__/$MEMCACHED_NAMESPACE/g" \
114
        -e "s/__MEMCACHED_NAMESPACE__/$MEMCACHED_NAMESPACE/g" \
115
        -e "s/__MEMCACHED_SERVERS__/$MEMCACHED_SERVERS/g" \
115
        -e "s/__MEMCACHED_SERVERS__/$MEMCACHED_SERVERS/g" \
116
        -e "s/__OAI_RUN_DIR__/\/var\/run\/koha\/$name\/oai-pmh-harvester/g" \
117
        -e "s/__OAI_LIB_DIR__/\/var\/lib\/koha\/$name\/oai-pmh-harvester/g" \
118
        -e "s/__OAI_SPOOL_DIR__/\/var\/spool\/koha\/$name\/oai-pmh-harvester/g" \
116
        "/etc/koha/$1" > "$2"
119
        "/etc/koha/$1" > "$2"
117
120
118
}
121
}
Lines 652-657 eof Link Here
652
    generate_config_file zebra.passwd.in \
655
    generate_config_file zebra.passwd.in \
653
        "/etc/koha/sites/$name/zebra.passwd"
656
        "/etc/koha/sites/$name/zebra.passwd"
654
657
658
    # Generate and install OAI-PMH harvester config file
659
    generate_config_file oai-pmh-harvester.yaml.in \
660
        "/etc/koha/sites/$name/oai-pmh-harvester.yaml"
661
655
    # Create a GPG-encrypted file for requesting a DB to be set up.
662
    # Create a GPG-encrypted file for requesting a DB to be set up.
656
    if [ "$op" = request ]
663
    if [ "$op" = request ]
657
    then
664
    then
(-)a/debian/scripts/koha-create-dirs (+3 lines)
Lines 56-66 do Link Here
56
    userdir "$name" "/var/lib/koha/$name/plugins"
56
    userdir "$name" "/var/lib/koha/$name/plugins"
57
    userdir "$name" "/var/lib/koha/$name/uploads"
57
    userdir "$name" "/var/lib/koha/$name/uploads"
58
    userdir "$name" "/var/lib/koha/$name/tmp"
58
    userdir "$name" "/var/lib/koha/$name/tmp"
59
    userdir "$name" "/var/lib/koha/$name/oai-pmh-harvester"
59
    userdir "$name" "/var/lock/koha/$name"
60
    userdir "$name" "/var/lock/koha/$name"
60
    userdir "$name" "/var/lock/koha/$name/authorities"
61
    userdir "$name" "/var/lock/koha/$name/authorities"
61
    userdir "$name" "/var/lock/koha/$name/biblios"
62
    userdir "$name" "/var/lock/koha/$name/biblios"
62
    userdir "$name" "/var/run/koha/$name"
63
    userdir "$name" "/var/run/koha/$name"
63
    userdir "$name" "/var/run/koha/$name/authorities"
64
    userdir "$name" "/var/run/koha/$name/authorities"
64
    userdir "$name" "/var/run/koha/$name/biblios"
65
    userdir "$name" "/var/run/koha/$name/biblios"
66
    userdir "$name" "/var/run/koha/$name/oai-pmh-harvester"
67
    userdir "$name" "/var/spool/koha/$name/oai-pmh-harvester"
65
done
68
done
66
69
(-)a/debian/templates/koha-conf-site.xml.in (+1 lines)
Lines 298-303 __END_SRU_PUBLICSERVER__ Link Here
298
 <use_zebra_facets>1</use_zebra_facets>
298
 <use_zebra_facets>1</use_zebra_facets>
299
 <queryparser_config>/etc/koha/searchengine/queryparser.yaml</queryparser_config>
299
 <queryparser_config>/etc/koha/searchengine/queryparser.yaml</queryparser_config>
300
 <log4perl_conf>__KOHA_CONF_DIR__/log4perl.conf</log4perl_conf>
300
 <log4perl_conf>__KOHA_CONF_DIR__/log4perl.conf</log4perl_conf>
301
 <oai_pmh_harvester_config>/etc/koha/sites/__KOHASITE__/oai-pmh-harvester.yaml</oai_pmh_harvester_config>
301
 <!-- Uncomment/edit next setting if you want to adjust zebra log levels.
302
 <!-- Uncomment/edit next setting if you want to adjust zebra log levels.
302
      Default is: none,fatal,warn.
303
      Default is: none,fatal,warn.
303
      You can also include: debug,log,malloc,all,request.
304
      You can also include: debug,log,malloc,all,request.
(-)a/debian/templates/oai-pmh-harvester-site.yaml.in (+22 lines)
Line 0 Link Here
1
---
2
#Harvester
3
socket: 'unix:__OAI_RUN_DIR__/harvesterd.sock'
4
5
logfile: '__LOG_DIR__/oai-pmh-harvester.log'
6
loglevel: 'WARN'
7
8
pidfile: '__OAI_RUN_DIR__/harvesterd.pid'
9
10
statefile: '__OAI_LIB_DIR__/harvesterd.state'
11
12
spooldir: '__OAI_SPOOL_DIR__'
13
14
#Downloader
15
download_workers: 1
16
download_module: 'Koha::OAI::Harvester::Worker::Download::Stream'
17
download_batch: 100
18
19
#Importer
20
import_workers: 1
21
import_poll: 5
22
import_module: 'Koha::OAI::Harvester::Worker::Import'
(-)a/etc/koha-conf.xml (+1 lines)
Lines 119-124 __PAZPAR2_TOGGLE_XML_POST__ Link Here
119
 <use_zebra_facets>1</use_zebra_facets>
119
 <use_zebra_facets>1</use_zebra_facets>
120
 <queryparser_config>__KOHA_CONF_DIR__/searchengine/queryparser.yaml</queryparser_config>
120
 <queryparser_config>__KOHA_CONF_DIR__/searchengine/queryparser.yaml</queryparser_config>
121
 <log4perl_conf>__KOHA_CONF_DIR__/log4perl.conf</log4perl_conf>
121
 <log4perl_conf>__KOHA_CONF_DIR__/log4perl.conf</log4perl_conf>
122
 <oai_pmh_harvester_config>__KOHA_CONF_DIR__/oai-pmh-harvester.yaml</oai_pmh_harvester_config>
122
 <memcached_servers>__MEMCACHED_SERVERS__</memcached_servers>
123
 <memcached_servers>__MEMCACHED_SERVERS__</memcached_servers>
123
 <memcached_namespace>__MEMCACHED_NAMESPACE__</memcached_namespace>
124
 <memcached_namespace>__MEMCACHED_NAMESPACE__</memcached_namespace>
124
 <template_cache_dir>__TEMPLATE_CACHE_DIR__</template_cache_dir>
125
 <template_cache_dir>__TEMPLATE_CACHE_DIR__</template_cache_dir>
(-)a/etc/oai-pmh-harvester.yaml (+22 lines)
Line 0 Link Here
1
---
2
#Harvester
3
socket: 'unix:__OAI_RUN_DIR__/harvesterd.sock'
4
5
logfile: '__LOG_DIR__/oai-pmh-harvester.log'
6
loglevel: 'WARN'
7
8
pidfile: '__OAI_RUN_DIR__/harvesterd.pid'
9
10
statefile: '__OAI_LIB_DIR__/harvesterd.state'
11
12
spooldir: '__OAI_SPOOL_DIR__'
13
14
#Downloader
15
download_workers: 1
16
download_module: 'Koha::OAI::Harvester::Worker::Download::Stream'
17
download_batch: 100
18
19
#Importer
20
import_workers: 1
21
import_poll: 5
22
import_module: 'Koha::OAI::Harvester::Worker::Import'
(-)a/installer/data/mysql/atomicupdate/bug_10662.sql (+73 lines)
Line 0 Link Here
1
--
2
-- Table structure for table 'oai_harvester_biblios'
3
--
4
5
CREATE TABLE IF NOT EXISTS `oai_harvester_biblios` (
6
  `import_oai_biblio_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
7
  `oai_repository` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
8
  `oai_identifier` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
9
  `biblionumber` int(11) NOT NULL,
10
  PRIMARY KEY (`import_oai_biblio_id`),
11
  UNIQUE KEY `oai_record` (`oai_identifier`,`oai_repository`) USING BTREE,
12
  KEY `FK_import_oai_biblio_1` (`biblionumber`),
13
  CONSTRAINT `FK_import_oai_biblio_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE NO ACTION
14
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
15
16
--
17
-- Table structure for table 'oai_harvester_history'
18
--
19
20
CREATE TABLE IF NOT EXISTS  `oai_harvester_history` (
21
  `import_oai_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
22
  `repository` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
23
  `header_identifier` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
24
  `header_datestamp` datetime NOT NULL,
25
  `header_status` varchar(45) COLLATE utf8_unicode_ci DEFAULT NULL,
26
  `record` longtext COLLATE utf8_unicode_ci NOT NULL,
27
  `upload_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
28
  `status` varchar(45) COLLATE utf8_unicode_ci NOT NULL,
29
  `filter` text COLLATE utf8_unicode_ci NOT NULL,
30
  `framework` varchar(4) COLLATE utf8_unicode_ci NOT NULL,
31
  `record_type` enum('biblio','auth','holdings') COLLATE utf8_unicode_ci NOT NULL,
32
  `matcher_code` varchar(10) COLLATE utf8_unicode_ci DEFAULT NULL,
33
  PRIMARY KEY (`import_oai_id`)
34
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
35
36
--
37
-- Table structure for table 'oai_harvester_import_queue'
38
--
39
40
CREATE TABLE IF NOT EXISTS `oai_harvester_import_queue` (
41
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
42
  `uuid` varchar(45) CHARACTER SET utf8 NOT NULL,
43
  `status` varchar(45) CHARACTER SET utf8 NOT NULL DEFAULT 'new',
44
  `result` text CHARACTER SET utf8 NOT NULL,
45
  `result_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
46
  PRIMARY KEY (`id`)
47
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
48
49
--
50
-- Table structure for table 'oai_harvester_requests'
51
--
52
53
CREATE TABLE IF NOT EXISTS `oai_harvester_requests` (
54
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
55
  `uuid` varchar(45) NOT NULL,
56
  `oai_verb` varchar(45) NOT NULL,
57
  `oai_metadataPrefix` varchar(255) NOT NULL,
58
  `oai_identifier` varchar(255) DEFAULT NULL,
59
  `oai_from` varchar(45) DEFAULT NULL,
60
  `oai_until` varchar(45) DEFAULT NULL,
61
  `oai_set` varchar(255) DEFAULT NULL,
62
  `http_url` varchar(255) DEFAULT NULL,
63
  `http_username` varchar(255) DEFAULT NULL,
64
  `http_password` varchar(255) DEFAULT NULL,
65
  `http_realm` varchar(255) DEFAULT NULL,
66
  `import_filter` varchar(255) NOT NULL,
67
  `import_framework_code` varchar(4) NOT NULL,
68
  `import_record_type` enum('biblio','auth','holdings') NOT NULL,
69
  `import_matcher_code` varchar(10) DEFAULT NULL,
70
  `interval` int(10) unsigned NOT NULL,
71
  `name` varchar(45) NOT NULL,
72
  PRIMARY KEY (`id`)
73
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
(-)a/installer/data/mysql/kohastructure.sql (+74 lines)
Lines 4192-4197 CREATE TABLE `oauth_access_tokens` ( Link Here
4192
    PRIMARY KEY (`access_token`)
4192
    PRIMARY KEY (`access_token`)
4193
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
4193
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
4194
4194
4195
--
4196
-- Table structure for table 'oai_harvester_biblios'
4197
--
4198
4199
CREATE TABLE IF NOT EXISTS `oai_harvester_biblios` (
4200
  `import_oai_biblio_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
4201
  `oai_repository` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
4202
  `oai_identifier` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
4203
  `biblionumber` int(11) NOT NULL,
4204
  PRIMARY KEY (`import_oai_biblio_id`),
4205
  UNIQUE KEY `oai_record` (`oai_identifier`,`oai_repository`) USING BTREE,
4206
  KEY `FK_import_oai_biblio_1` (`biblionumber`),
4207
  CONSTRAINT `FK_import_oai_biblio_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE NO ACTION
4208
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
4209
4210
--
4211
-- Table structure for table 'oai_harvester_history'
4212
--
4213
4214
CREATE TABLE IF NOT EXISTS  `oai_harvester_history` (
4215
  `import_oai_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
4216
  `repository` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
4217
  `header_identifier` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
4218
  `header_datestamp` datetime NOT NULL,
4219
  `header_status` varchar(45) COLLATE utf8_unicode_ci DEFAULT NULL,
4220
  `record` longtext COLLATE utf8_unicode_ci NOT NULL,
4221
  `upload_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
4222
  `status` varchar(45) COLLATE utf8_unicode_ci NOT NULL,
4223
  `filter` text COLLATE utf8_unicode_ci NOT NULL,
4224
  `framework` varchar(4) COLLATE utf8_unicode_ci NOT NULL,
4225
  `record_type` enum('biblio','auth','holdings') COLLATE utf8_unicode_ci NOT NULL,
4226
  `matcher_code` varchar(10) COLLATE utf8_unicode_ci DEFAULT NULL,
4227
  PRIMARY KEY (`import_oai_id`)
4228
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
4229
4230
--
4231
-- Table structure for table 'oai_harvester_import_queue'
4232
--
4233
4234
CREATE TABLE IF NOT EXISTS `oai_harvester_import_queue` (
4235
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
4236
  `uuid` varchar(45) CHARACTER SET utf8 NOT NULL,
4237
  `status` varchar(45) CHARACTER SET utf8 NOT NULL DEFAULT 'new',
4238
  `result` text CHARACTER SET utf8 NOT NULL,
4239
  `result_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
4240
  PRIMARY KEY (`id`)
4241
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
4242
4243
--
4244
-- Table structure for table 'oai_harvester_requests'
4245
--
4246
4247
CREATE TABLE IF NOT EXISTS `oai_harvester_requests` (
4248
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
4249
  `uuid` varchar(45) NOT NULL,
4250
  `oai_verb` varchar(45) NOT NULL,
4251
  `oai_metadataPrefix` varchar(255) NOT NULL,
4252
  `oai_identifier` varchar(255) DEFAULT NULL,
4253
  `oai_from` varchar(45) DEFAULT NULL,
4254
  `oai_until` varchar(45) DEFAULT NULL,
4255
  `oai_set` varchar(255) DEFAULT NULL,
4256
  `http_url` varchar(255) DEFAULT NULL,
4257
  `http_username` varchar(255) DEFAULT NULL,
4258
  `http_password` varchar(255) DEFAULT NULL,
4259
  `http_realm` varchar(255) DEFAULT NULL,
4260
  `import_filter` varchar(255) NOT NULL,
4261
  `import_framework_code` varchar(4) NOT NULL,
4262
  `import_record_type` enum('biblio','auth','holdings') NOT NULL,
4263
  `import_matcher_code` varchar(10) DEFAULT NULL,
4264
  `interval` int(10) unsigned NOT NULL,
4265
  `name` varchar(45) NOT NULL,
4266
  PRIMARY KEY (`id`)
4267
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
4268
4195
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
4269
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
4196
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
4270
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
4197
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
4271
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-menu.inc (-16 / +19 lines)
Lines 6-27 Link Here
6
<h5>Patrons and circulation</h5>
6
<h5>Patrons and circulation</h5>
7
<ul>
7
<ul>
8
    [% IF ( CAN_user_tools_manage_patron_lists ) %]
8
    [% IF ( CAN_user_tools_manage_patron_lists ) %]
9
	<li><a href="/cgi-bin/koha/patron_lists/lists.pl">Patron lists</a></li>
9
    <li><a href="/cgi-bin/koha/patron_lists/lists.pl">Patron lists</a></li>
10
    [% END %]
10
    [% END %]
11
    [% IF (CAN_user_clubs) %]
11
    [% IF (CAN_user_clubs) %]
12
        <li><a href="/cgi-bin/koha/clubs/clubs.pl">Patron clubs</a></li>
12
        <li><a href="/cgi-bin/koha/clubs/clubs.pl">Patron clubs</a></li>
13
    [% END %]
13
    [% END %]
14
    [% IF ( CAN_user_tools_moderate_comments ) %]
14
    [% IF ( CAN_user_tools_moderate_comments ) %]
15
	<li><a href="/cgi-bin/koha/reviews/reviewswaiting.pl">Comments</a></li>
15
    <li><a href="/cgi-bin/koha/reviews/reviewswaiting.pl">Comments</a></li>
16
    [% END %]
16
    [% END %]
17
    [% IF ( CAN_user_tools_import_patrons ) %]
17
    [% IF ( CAN_user_tools_import_patrons ) %]
18
	<li><a href="/cgi-bin/koha/tools/import_borrowers.pl">Import patrons</a></li>
18
    <li><a href="/cgi-bin/koha/tools/import_borrowers.pl">Import patrons</a></li>
19
    [% END %]
19
    [% END %]
20
    [% IF ( CAN_user_tools_edit_notices ) %]
20
    [% IF ( CAN_user_tools_edit_notices ) %]
21
    <li><a href="/cgi-bin/koha/tools/letter.pl">Notices &amp; slips</a></li>
21
    <li><a href="/cgi-bin/koha/tools/letter.pl">Notices &amp; slips</a></li>
22
    [% END %]
22
    [% END %]
23
    [% IF ( CAN_user_tools_edit_notice_status_triggers ) %]
23
    [% IF ( CAN_user_tools_edit_notice_status_triggers ) %]
24
	<li><a href="/cgi-bin/koha/tools/overduerules.pl">Overdue notice/status triggers</a></li>
24
    <li><a href="/cgi-bin/koha/tools/overduerules.pl">Overdue notice/status triggers</a></li>
25
    [% END %]
25
    [% END %]
26
    [% IF ( CAN_user_tools_label_creator ) %]
26
    [% IF ( CAN_user_tools_label_creator ) %]
27
    <li><a href="/cgi-bin/koha/patroncards/home.pl">Patron card creator</a></li>
27
    <li><a href="/cgi-bin/koha/patroncards/home.pl">Patron card creator</a></li>
Lines 36-51 Link Here
36
    <li><a href="/cgi-bin/koha/tags/review.pl">Tag moderation</a></li>
36
    <li><a href="/cgi-bin/koha/tags/review.pl">Tag moderation</a></li>
37
    [% END %]
37
    [% END %]
38
    [% IF ( CAN_user_tools_batch_upload_patron_images ) %]
38
    [% IF ( CAN_user_tools_batch_upload_patron_images ) %]
39
	<li><a href="/cgi-bin/koha/tools/picture-upload.pl">Upload patron images</a></li>
39
    <li><a href="/cgi-bin/koha/tools/picture-upload.pl">Upload patron images</a></li>
40
    [% END %]
40
    [% END %]
41
</ul>
41
</ul>
42
<h5>Catalog</h5>
42
<h5>Catalog</h5>
43
<ul>
43
<ul>
44
    [% IF ( CAN_user_tools_items_batchdel ) %]
44
    [% IF ( CAN_user_tools_items_batchdel ) %]
45
	<li><a href="/cgi-bin/koha/tools/batchMod.pl?del=1">Batch item deletion</a></li>
45
    <li><a href="/cgi-bin/koha/tools/batchMod.pl?del=1">Batch item deletion</a></li>
46
    [% END %]
46
    [% END %]
47
    [% IF ( CAN_user_tools_items_batchmod ) %]
47
    [% IF ( CAN_user_tools_items_batchmod ) %]
48
	<li><a href="/cgi-bin/koha/tools/batchMod.pl">Batch item modification</a></li>
48
    <li><a href="/cgi-bin/koha/tools/batchMod.pl">Batch item modification</a></li>
49
    [% END %]
49
    [% END %]
50
    [% IF CAN_user_tools_records_batchdel %]
50
    [% IF CAN_user_tools_records_batchdel %]
51
      <li><a href="/cgi-bin/koha/tools/batch_delete_records.pl">Batch record deletion</a></li>
51
      <li><a href="/cgi-bin/koha/tools/batch_delete_records.pl">Batch record deletion</a></li>
Lines 63-70 Link Here
63
        <li><a href="/cgi-bin/koha/tools/inventory.pl">Inventory</a></li>
63
        <li><a href="/cgi-bin/koha/tools/inventory.pl">Inventory</a></li>
64
    [% END %]
64
    [% END %]
65
    [% IF ( CAN_user_tools_label_creator ) %]
65
    [% IF ( CAN_user_tools_label_creator ) %]
66
	<li><a href="/cgi-bin/koha/labels/label-home.pl">Label creator</a></li>
66
    <li><a href="/cgi-bin/koha/labels/label-home.pl">Label creator</a></li>
67
	<li><a href="/cgi-bin/koha/labels/spinelabel-home.pl">Quick spine label creator</a></li>
67
    <li><a href="/cgi-bin/koha/labels/spinelabel-home.pl">Quick spine label creator</a></li>
68
    [% END %]
68
    [% END %]
69
    [% IF ( CAN_user_tools_rotating_collections ) %]
69
    [% IF ( CAN_user_tools_rotating_collections ) %]
70
    <li><a href="/cgi-bin/koha/rotating_collections/rotatingCollections.pl">Rotating collections</a></li>
70
    <li><a href="/cgi-bin/koha/rotating_collections/rotatingCollections.pl">Rotating collections</a></li>
Lines 73-103 Link Here
73
        <li><a href="/cgi-bin/koha/tools/marc_modification_templates.pl">Manage MARC modification templates</a></li>
73
        <li><a href="/cgi-bin/koha/tools/marc_modification_templates.pl">Manage MARC modification templates</a></li>
74
    [% END %]
74
    [% END %]
75
    [% IF ( CAN_user_tools_stage_marc_import ) %]
75
    [% IF ( CAN_user_tools_stage_marc_import ) %]
76
	<li><a href="/cgi-bin/koha/tools/stage-marc-import.pl">Stage MARC for import</a></li>
76
    <li><a href="/cgi-bin/koha/tools/stage-marc-import.pl">Stage MARC for import</a></li>
77
    [% END %]
77
    [% END %]
78
    [% IF ( CAN_user_tools_manage_staged_marc ) %]
78
    [% IF ( CAN_user_tools_manage_staged_marc ) %]
79
	<li><a href="/cgi-bin/koha/tools/manage-marc-import.pl">Staged MARC management</a></li>
79
    <li><a href="/cgi-bin/koha/tools/manage-marc-import.pl">Staged MARC management</a></li>
80
    [% END %]
80
    [% END %]
81
    [% IF ( CAN_user_tools_upload_local_cover_images ) %]
81
    [% IF ( CAN_user_tools_upload_local_cover_images ) %]
82
    <li><a href="/cgi-bin/koha/tools/upload-cover-image.pl">Upload local cover image</a></li>
82
    <li><a href="/cgi-bin/koha/tools/upload-cover-image.pl">Upload local cover image</a></li>
83
    [% END %]
83
    [% END %]
84
    [% IF ( CAN_user_tools_manage_staged_marc ) %]
85
    <li><a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a></li>
86
    [% END %]
84
</ul>
87
</ul>
85
<h5>Additional tools</h5>
88
<h5>Additional tools</h5>
86
<ul>
89
<ul>
87
    [% IF ( CAN_user_tools_edit_calendar ) %]
90
    [% IF ( CAN_user_tools_edit_calendar ) %]
88
	<li><a href="/cgi-bin/koha/tools/holidays.pl">Calendar</a></li>
91
    <li><a href="/cgi-bin/koha/tools/holidays.pl">Calendar</a></li>
89
    [% END %]
92
    [% END %]
90
    [% IF ( CAN_user_tools_manage_csv_profiles ) %]
93
    [% IF ( CAN_user_tools_manage_csv_profiles ) %]
91
	<li><a href="/cgi-bin/koha/tools/csv-profiles.pl">CSV profiles</a></li>
94
    <li><a href="/cgi-bin/koha/tools/csv-profiles.pl">CSV profiles</a></li>
92
    [% END %]
95
    [% END %]
93
    [% IF ( CAN_user_tools_view_system_logs ) %]
96
    [% IF ( CAN_user_tools_view_system_logs ) %]
94
	<li><a href="/cgi-bin/koha/tools/viewlog.pl">Log viewer</a></li>
97
    <li><a href="/cgi-bin/koha/tools/viewlog.pl">Log viewer</a></li>
95
    [% END %]
98
    [% END %]
96
    [% IF ( CAN_user_tools_edit_news ) %]
99
    [% IF ( CAN_user_tools_edit_news ) %]
97
	<li><a href="/cgi-bin/koha/tools/koha-news.pl">News</a></li>
100
    <li><a href="/cgi-bin/koha/tools/koha-news.pl">News</a></li>
98
    [% END %]
101
    [% END %]
99
    [% IF ( CAN_user_tools_schedule_tasks ) %]
102
    [% IF ( CAN_user_tools_schedule_tasks ) %]
100
	<li><a href="/cgi-bin/koha/tools/scheduler.pl">Task scheduler</a></li>
103
    <li><a href="/cgi-bin/koha/tools/scheduler.pl">Task scheduler</a></li>
101
    [% END %]
104
    [% END %]
102
    [% IF ( CAN_user_tools_edit_quotes ) %]
105
    [% IF ( CAN_user_tools_edit_quotes ) %]
103
       <li><a href="/cgi-bin/koha/tools/quotes.pl">Quote editor</a></li>
106
       <li><a href="/cgi-bin/koha/tools/quotes.pl">Quote editor</a></li>
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/dashboard.tt (+369 lines)
Line 0 Link Here
1
[% INCLUDE 'doc-head-open.inc' %]
2
<title>Koha &rsaquo; Tools &rsaquo; OAI-PMH harvester</title>
3
[% INCLUDE 'doc-head-close.inc' %]
4
[% INCLUDE 'datatables.inc' %]
5
[% dashboard_page = '/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl' %]
6
[% request_page = '/cgi-bin/koha/tools/oai-pmh-harvester/request.pl' %]
7
<script type="text/javascript">
8
//<![CDATA[
9
    $(document).ready(function() {
10
        $('#dashboard-items').tabs();
11
        [% IF ( result.start.defined || result.stop.defined || result.delete.defined ) %]
12
            $('#dashboard-items').tabs("option","active",1);
13
        [% END %]
14
15
        var saved_table = $('#saved-table').DataTable({});
16
        var submitted_table = $('#submitted-table').DataTable({});
17
        var history_table = $('#history-table').DataTable({
18
            serverSide: true,
19
            searching: true,
20
            processing: true,
21
            ajax: {
22
                "url": '/cgi-bin/koha/svc/oai-pmh-harvester/history',
23
                contentType: 'application/json',
24
                type: 'POST',
25
                data: function ( d ) {
26
                    return JSON.stringify( d );
27
                },
28
                dataSrc: function (json){
29
                    var recordsTotal = json.recordsTotal;
30
                    if(recordsTotal){
31
                        $('#import_count').text( "("+recordsTotal+")" );
32
                    }
33
                    return json.data;
34
                }
35
            },
36
            columns: [
37
                { data: 'import_oai_id', },
38
                { data: 'repository', },
39
                { data: 'header_identifier', },
40
                { data: 'header_datestamp', },
41
                {
42
                    data: 'header_status', render: function (data, type, full, meta){
43
                        var display_string = _("Active");
44
                        if (data == "deleted"){
45
                            display_string = _("Deleted");
46
                        }
47
                        return display_string;
48
                    }
49
                },
50
                {
51
                    data: 'status', render: function (data, type, full, meta){
52
                        var display_string = data;
53
                        if (data == "added"){
54
                            display_string = _("Added");
55
                        }
56
                        else if (data == "error"){
57
                            display_string = _("Error");
58
                        }
59
                        else if (data == "not_found"){
60
                            display_string = _("Not found");
61
                        }
62
                        else if (data == "updated"){
63
                            display_string = _("Updated");
64
                        }
65
                        return display_string;
66
                    }
67
                },
68
                { data: 'upload_timestamp', },
69
                {
70
                    data: 'imported_record', render: function (data, type, full, meta){
71
                        var display_string = data;
72
                        var record_type = (full.record_type) ? full.record_type : 'biblio';
73
                        if (data && record_type == 'biblio'){
74
                            var link_text = _("View biblio record");
75
                            var link = '<a href="/cgi-bin/koha/catalogue/detail.pl?biblionumber='+data+'">'+link_text+'</a>';
76
                            display_string = link;
77
                        }
78
                        return display_string;
79
                    }, searchable: false
80
81
                },
82
                {
83
                    data: 'import_oai_id', render: function (data, type, full, meta){
84
                        var display_string = data;
85
                        var link_text = _("View record");
86
                        var link = '<a href="/cgi-bin/koha/tools/oai-pmh-harvester/record.pl?import_oai_id='+data+'">'+link_text+'</a>';
87
                        display_string = link;
88
                        return display_string;
89
                    }, searchable: false
90
                },
91
            ],
92
            //In theory, it would be nice to sort in descending order, but
93
            //it causes severe paging issues the data is frequently updated.
94
            //Caching would make the paging more stable, but the results would be stale.
95
            order: [
96
                [ 0, 'asc' ],
97
            ]
98
        });
99
        $('#refresh-button').click(function(){
100
            history_table.ajax.reload( null, false );
101
        });
102
    });
103
//]]>
104
</script>
105
<style type="text/css">
106
    a.paginate_button {
107
        padding: 2px;
108
    }
109
</style>
110
</head>
111
<body id="tools_oai-pmh-harvester" class="tools">
112
[% INCLUDE 'header.inc' %]
113
[% INCLUDE 'cat-search.inc' %]
114
    <div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> &rsaquo; OAI-PMH harvester</div>
115
    <div id="doc3" class="yui-t2">
116
        <div id="bd">
117
            <div id="yui-main">
118
                <div class="yui-b">
119
                    <h1>OAI-PMH harvester</h1>
120
                    <div id="toolbar" class="btn-toolbar">
121
                        <a id="new-request" class="btn btn-default btn-sm" href="[% request_page %]?op=new"><i class="fa fa-plus"></i> New request</a>
122
                    </div>
123
                    [% IF ( harvester.offline ) %]
124
                        <div class="alert">
125
                            <span>OAI-PMH harvester offline</span>
126
                        </div>
127
                    [% END %]
128
                    <div id="dashboard-items" class="toptabs">
129
                        <ul>
130
                            <li>
131
                                <a href="#saved_requests">Saved requests <span id="saved_count">([% saved_requests.size %])</span></a>
132
                            </li>
133
                            <li>
134
                                <a href="#submitted_requests">Submitted requests <span id="submitted_count">([% submitted_requests.size %])</span></a>
135
                            </li>
136
                            <li>
137
                                <a href="#imports">Import history <span id="import_count">(0)</span></a>
138
                            </li>
139
                        </ul>
140
                        <div id="submitted_requests">
141
                            [% IF ( result.defined("start") ) %]
142
                                <div class="alert">
143
                                    [% IF ( result.start ) %]
144
                                        <span>Start succeeded</span>
145
                                    [% ELSE %]
146
                                        <span>Start failed</span>
147
                                    [% END %]
148
                                </div>
149
                            [% ELSIF ( result.defined("stop") ) %]
150
                                 <div class="alert">
151
                                    [% IF ( result.stop ) %]
152
                                        <span>Stop succeeded</span>
153
                                    [% ELSE %]
154
                                        <span>Stop failed</span>
155
                                    [% END %]
156
                                </div>
157
                            [% ELSIF ( result.defined("delete") ) %]
158
                                 <div class="alert">
159
                                    [% IF ( result.delete ) %]
160
                                        <span>Delete succeeded</span>
161
                                    [% ELSE %]
162
                                        <span>Delete failed</span>
163
                                    [% END %]
164
                                </div>
165
                            [% END %]
166
                            <table id="submitted-table">
167
                                <thead>
168
                                    <tr>
169
                                        <th>Name</th>
170
                                        <th>URL</th>
171
                                        <th>Set</th>
172
                                        <th>From</th>
173
                                        <th>Until</th>
174
                                        <th>Interval</th>
175
                                        <th>Effective from</th>
176
                                        <th>Effective until</th>
177
                                        <th>Pending imports</th>
178
                                        <th>Status</th>
179
                                        <th>Error</th>
180
                                        <th></th>
181
                                    </tr>
182
                                </thead>
183
                                <tbody>
184
                                [% IF ( submitted_requests ) %]
185
                                    [% FOREACH submitted_request IN submitted_requests %]
186
                                        <tr>
187
                                            <td>[% submitted_request.name %]</td>
188
                                            <td>[% submitted_request.parameters.oai_pmh.baseURL %]</td>
189
                                            <td>[% submitted_request.parameters.oai_pmh.set %]</td>
190
                                            <td>[% submitted_request.parameters.oai_pmh.from %]</td>
191
                                            <td>[% submitted_request.parameters.oai_pmh.until %]</td>
192
                                            <td>[% submitted_request.interval %]</td>
193
                                            <td>[% submitted_request.effective_from %]</td>
194
                                            <td>[% submitted_request.effective_until %]</td>
195
                                            <td>[% submitted_request.pending_imports %]</td>
196
                                            <td>
197
                                                [% IF ( submitted_status = submitted_request.status ) %]
198
                                                    [% IF ( submitted_status == "new" ) %]
199
                                                        <span>New</span>
200
                                                    [% ELSIF ( submitted_status == "active" ) %]
201
                                                        <span>Active</span>
202
                                                    [% ELSIF ( submitted_status == "stopped" ) %]
203
                                                        <span>Stopped</span>
204
                                                    [% END %]
205
                                                [% END %]
206
                                            </td>
207
                                            <td>
208
                                                [% IF ( submitted_request.error ) %]
209
                                                    <span>Harvest failure</span>
210
                                                [% END %]
211
                                            </td>
212
                                            <td>
213
                                                <div class="dropdown">
214
                                                    <a class="btn btn-default btn-xs dropdown-toggle" role="button" data-toggle="dropdown" href="#">Actions <span class="caret"></span></a>
215
                                                    <ul class="dropdown-menu pull-right" role="menu">
216
                                                          <li>
217
                                                              <a href="[% dashboard_page %]?op=start&uuid=[% submitted_request.uuid %]"><i class="fa fa-play"></i> Start</a>
218
                                                          </li>
219
                                                          <li>
220
                                                              <a href="[% dashboard_page %]?op=stop&uuid=[% submitted_request.uuid %]"><i class="fa fa-stop"></i> Stop</a>
221
                                                          </li>
222
                                                          <li>
223
                                                              <a href="[% dashboard_page %]?op=delete&uuid=[% submitted_request.uuid %]"><i class="fa fa-trash"></i> Delete</a>
224
                                                          </li>
225
                                                    </ul>
226
                                                </div>
227
                                            </td>
228
                                        </tr>
229
                                    [% END %]
230
                                [% END %]
231
                                </tbody>
232
                            </table>
233
                        </div>
234
                        <div id="saved_requests">
235
                            [% IF ( result.send.defined ) %]
236
                                <div class="alert">
237
                                    [% IF ( result.send ) %]
238
                                        <span>Submit succeeded</span>
239
                                    [% ELSE %]
240
                                        <span>Submit failed</span>
241
                                    [% END %]
242
                                </div>
243
                            [% END %]
244
                            <table id="saved-table">
245
                                <thead>
246
                                    <tr>
247
                                        <th>Name</th>
248
                                        <th>URL</th>
249
                                   <!-- <th>Verb</th>
250
                                        <th>Metadata prefix</th>
251
                                        <th>Identifier</th> -->
252
                                        <th>Set</th>
253
                                        <th>From</th>
254
                                        <th>Until</th>
255
                                        <th>Interval</th>
256
                                   <!-- <th>Filter</th>
257
                                        <th>Framework code</th>
258
                                        <th>Record type</th>
259
                                        <th>Matcher code</th> -->
260
                                        <th></th>
261
                                    </tr>
262
                                </thead>
263
                                <tbody>
264
                                [% IF ( saved_requests ) %]
265
                                    [% FOREACH saved_request IN saved_requests %]
266
                                        <tr>
267
                                            <td>[% saved_request.name %]</td>
268
                                            <td>[% saved_request.http_url %]</td>
269
                                       <!-- <td>[% saved_request.oai_verb %]</td>
270
                                            <td>[% saved_request.oai_metadataPrefix %]</td>
271
                                            <td>[% saved_request.oai_identifier %]</td> -->
272
                                            <td>[% saved_request.oai_set %]</td>
273
                                            <td>[% saved_request.oai_from %]</td>
274
                                            <td>[% saved_request.oai_until %]</td>
275
                                            <td>[% saved_request.interval %]</td>
276
                                       <!-- <td>
277
                                                [% IF ( saved_request.import_filter == "default" ) %]
278
                                                    <span>Default</span>
279
                                                [% ELSE %]
280
                                                    [% saved_request.import_filter %]
281
                                                [% END %]
282
                                            </td>
283
                                            <td>
284
                                                [% display_framework = "" %]
285
                                                [% FOREACH framework IN frameworks %]
286
                                                    [% IF ( framework.frameworkcode == saved_request.import_framework_code ) %]
287
                                                        [% display_framework = framework %]
288
                                                    [% END %]
289
                                                [% END %]
290
                                                [% IF ( display_framework ) %]
291
                                                    [% display_framework.frameworktext %]
292
                                                [% ELSE %]
293
                                                    [% saved_request.import_framework_code %]
294
                                                [% END %]
295
                                            </td>
296
                                            <td>
297
                                                [% IF ( saved_request.import_record_type == "biblio" ) %]
298
                                                    <span>Bibliographic</span>
299
                                                [% ELSE %]
300
                                                    [% saved_request.import_record_type %]
301
                                                [% END %]
302
                                            </td>
303
                                            <td>
304
                                                [% display_matcher = "" %]
305
                                                [% FOREACH matcher IN matchers %]
306
                                                    [% IF ( matcher.code == saved_request.import_matcher_code ) %]
307
                                                        [% display_matcher = matcher %]
308
                                                    [% END %]
309
                                                [% END %]
310
                                                [% IF ( display_matcher ) %]
311
                                                    [% display_matcher.description %]
312
                                                [% ELSE %]
313
                                                    [% saved_request.import_matcher_code %]
314
                                                [% END %]
315
                                            </td> -->
316
                                            <td>
317
                                                <div class="dropdown">
318
                                                    <a class="btn btn-default btn-xs dropdown-toggle" role="button" data-toggle="dropdown" href="#">Actions <span class="caret"></span></a>
319
                                                    <ul class="dropdown-menu pull-right" role="menu">
320
                                                          <li>
321
                                                              <a href="[% request_page %]?op=edit&id=[% saved_request.id %]"><i class="fa fa-pencil"></i> Edit</a>
322
                                                          </li>
323
                                                          <li>
324
                                                              <a href="[% request_page %]?op=new&id=[% saved_request.id %]"><i class="fa fa-copy"></i> Copy</a>
325
                                                          </li>
326
                                                          <li>
327
                                                              <a href="[% dashboard_page %]?op=send&id=[% saved_request.id %]"><i class="fa fa-send"></i> Submit</a>
328
                                                          </li>
329
                                                          <li>
330
                                                            <a href="[% request_page %]?op=delete&id=[% saved_request.id %]"><i class="fa fa-trash"></i> Delete</a>
331
                                                          </li>
332
                                                    </ul>
333
                                                </div>
334
                                            </td>
335
                                        </tr>
336
                                    [% END %]
337
                                [% END %]
338
                                </tbody>
339
                            </table>
340
                        </div>
341
                        <div id="imports">
342
                            <div class="btn-toolbar">
343
                                <button id="refresh-button" type="button" class="btn btn-default btn-sm">Refresh import history</button>
344
                            </div>
345
                            <table id="history-table">
346
                                <thead>
347
                                    <tr>
348
                                        <th>Id</th>
349
                                        <th>Repository</th>
350
                                        <th>Identifier</th>
351
                                        <th>Datestamp</th>
352
                                        <th>Upstream status</th>
353
                                        <th>Import status</th>
354
                                        <th>Import timestamp</th>
355
                                        <th>Imported record</th>
356
                                        <th>Downloaded record</th>
357
                                    </tr>
358
                                </thead>
359
                            </table>
360
361
                        </div>
362
                    </div>
363
                </div>
364
            </div>
365
            <div class="yui-b">
366
                [% INCLUDE 'tools-menu.inc' %]
367
            </div>
368
        </div>
369
[% INCLUDE 'intranet-bottom.inc' %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/record.tt (+23 lines)
Line 0 Link Here
1
[% INCLUDE 'doc-head-open.inc' %]
2
<title>Koha &rsaquo; Tools &rsaquo; OAI-PMH harvester &rsaquo; Downloaded record</title>
3
[% INCLUDE 'doc-head-close.inc' %]
4
</head>
5
<body id="tools_oai-pmh-harvester_request" class="tools">
6
[% INCLUDE 'header.inc' %]
7
[% INCLUDE 'cat-search.inc' %]
8
    <div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> &rsaquo; <a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a> &rsaquo; Downloaded record</div>
9
    <div id="doc3" class="yui-t2">
10
        <div id="bd">
11
            <div id="yui-main">
12
                <div class="yui-b">
13
                    <h1>Downloaded record</h1>
14
                    [% IF ( record ) %]
15
                        <div style="white-space:pre">[% record | xml %]</div>
16
                    [% END %]
17
                </div>
18
            </div>
19
            <div class="yui-b">
20
                [% INCLUDE 'tools-menu.inc' %]
21
            </div>
22
        </div>
23
[% INCLUDE 'intranet-bottom.inc' %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/request.tt (+241 lines)
Line 0 Link Here
1
[% INCLUDE 'doc-head-open.inc' %]
2
<title>Koha &rsaquo; Tools &rsaquo; OAI-PMH harvester &rsaquo; Request</title>
3
[% INCLUDE 'doc-head-close.inc' %]
4
[% INCLUDE 'calendar.inc' %]
5
<script type="text/javascript" src="[% interface %]/lib/jquery/plugins/jquery-ui-timepicker-addon.min.js"></script>
6
[% INCLUDE 'timepicker.inc' %]
7
<script type="text/javascript">
8
//<![CDATA[
9
    $(document).ready(function() {
10
        toggle_identifier();
11
        $("#oai_verb").on("click",toggle_identifier);
12
        $(".datetime_utc").datetimepicker({
13
            separator: "T",
14
            timeSuffix: 'Z',
15
            dateFormat: "yy-mm-dd",
16
            timeFormat: "HH:mm:ss",
17
            hour: 0,
18
            minute: 0,
19
            second: 0,
20
            showSecond: 1,
21
            // timezone doesn't work with the "Now" button in v1.4.3 although it appears to in v1.6.1
22
            // timezone: '+000'
23
        });
24
    });
25
    function toggle_identifier (){
26
        var verb = $("#oai_verb").find(":selected").val();
27
        var oai_identifier = $("#oai_identifier");
28
        var oai_set = $("#oai_set");
29
        var oai_from = $("#oai_from");
30
        var oai_until = $("#oai_until");
31
        if (verb == 'ListRecords'){
32
            oai_identifier.prop('disabled', true);
33
            oai_set.prop('disabled', false);
34
            oai_from.prop('disabled', false);
35
            oai_until.prop('disabled', false);
36
        }
37
        else if (verb == 'GetRecord'){
38
            oai_identifier.prop('disabled', false);
39
            oai_set.prop('disabled', true);
40
            oai_from.prop('disabled', true);
41
            oai_until.prop('disabled', true);
42
        }
43
    }
44
//]]>
45
</script>
46
<style type="text/css">
47
    /* Override staff-global.css which hides second, millisecond, and microsecond sliders */
48
    .ui_tpicker_second {
49
        display: block;
50
    }
51
</style>
52
</head>
53
<body id="tools_oai-pmh-harvester_request" class="tools">
54
[% INCLUDE 'header.inc' %]
55
[% INCLUDE 'cat-search.inc' %]
56
    <div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> &rsaquo; <a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a> &rsaquo; Request</div>
57
    <div id="doc3" class="yui-t2">
58
        <div id="bd">
59
            <div id="yui-main">
60
                <div class="yui-b">
61
                    [% IF ( op == "edit" ) %]
62
                        <h1>Edit OAI-PMH request</h1>
63
                    [% ELSE %]
64
                        <h1>New OAI-PMH request</h1>
65
                    [% END %]
66
                    [% IF ( test_parameters ) %]
67
                        [% IF ( errors.size ) %]
68
                            <div class="dialog message"><span class="text-danger">Tests failed!</span></div>
69
                        [% ELSE %]
70
                            <div class="dialog message"><span class="text-success">Tests succeeded!</span></div>
71
                        [% END %]
72
                    [% END %]
73
                    <form action="/cgi-bin/koha/tools/oai-pmh-harvester/request.pl" method="post" name="entry-form">
74
                        [% IF ( op == "new" ) %]
75
                            <input type="hidden" name="op" value="create" />
76
                        [% ELSIF ( op == "edit" ) %]
77
                             <input type="hidden" name="op" value="update" />
78
                        [% ELSE %]
79
                            <input type="hidden" name="op" value="[% op %]" />
80
                        [% END %]
81
                        [% IF ( id ) %]
82
                            <input type="hidden" name="id" value="[% id %]" />
83
                        [% END %]
84
                        <fieldset class="rows">
85
                            <ol>
86
                                <li>
87
                                    <label for="name">Name:</label>
88
                                    <input type="text" size="30" id="name" name="name" value="[% oai_pmh_request.name %]"/>
89
                                    <span class="help">This is just a short name to help in managing requests.</span>
90
                                </li>
91
                            </ol>
92
                        </fieldset>
93
                        <fieldset class="rows">
94
                            <legend>HTTP parameters:</legend>
95
                            <ol>
96
                                <li>
97
                                    <label for="http_url">URL:</label>
98
                                    <input type="text" size="30" id="http_url" name="http_url" value="[% oai_pmh_request.http_url %]">
99
                                    [% IF (errors.http_url.malformed) %]<span class="error">[This must be a properly formatted HTTP or HTTPS URL.]</span>[% END %]
100
                                    [% IF (errors.http.404) %]<span class="error">[Cannot find address specified by this URL.]</span>[% END %]
101
                                    [% IF (errors.http.401) %]<span class="error">[Permission denied to access this URL.]</span>[% END %]
102
                                    [% IF (errors.http.generic) %]<span class="error">[Unable to access this URL.]</span>[% END %]
103
                                </li>
104
                            </ol>
105
                            <span class="help">The following parameters are not required by all OAI-PMH repositories, so they may be optional for this task.</span>
106
                            <ol>
107
                                <li>
108
                                    <label for="http_username">Username:</label>
109
                                    <input type="text" size="30" id="http_username" name="http_username" value="[% oai_pmh_request.http_username %]">
110
                                </li>
111
                                <li>
112
                                    <label for="http_password">Password:</label>
113
                                    <input type="text" size="30" id="http_password" name="http_password" value="[% oai_pmh_request.http_password %]">
114
                                </li>
115
                                <li>
116
                                    <label for="http_realm">Realm:</label>
117
                                    <input type="text" size="30" id="http_realm" name="http_realm" value="[% oai_pmh_request.http_realm %]">
118
                                </li>
119
                            </ol>
120
                        </fieldset>
121
                        <fieldset class="rows">
122
                            <legend>OAI-PMH parameters:</legend>
123
                            <ol>
124
                                <li>
125
                                    <label for="oai_verb">Verb:</label>
126
                                    <select id="oai_verb" name="oai_verb">
127
                                        [% IF ( oai_pmh_request.oai_verb == "ListRecords" ) %]
128
                                        <option value="ListRecords" selected="selected">ListRecords</option>
129
                                        [% ELSE %]
130
                                        <option value="ListRecords">ListRecords</option>
131
                                        [% END %]
132
                                        [% IF ( oai_pmh_request.oai_verb == "GetRecord" ) %]
133
                                        <option value="GetRecord" selected="selected">GetRecord</option>
134
                                        [% ELSE %]
135
                                        <option value="GetRecord">GetRecord</option>
136
                                        [% END %]
137
                                    </select>
138
                                </li>
139
                                <li>
140
                                    <label for="oai_metadataPrefix">Metadata prefix:</label>
141
                                    <input type="text" size="30" id="oai_metadataPrefix" name="oai_metadataPrefix" value="[% oai_pmh_request.oai_metadataPrefix %]">
142
                                    [% IF (errors.oai_metadataPrefix.unavailable) %]<span class="error">[This metadataPrefix is unavailable from this OAI-PMH provider.]</span>[% END %]
143
                                    [% IF (errors.oai_metadataPrefix.missing) %]<span class="error">[metadataPrefix is a required field for an OAI-PMH request.]</span>[% END %]
144
                                </li>
145
                                <li>
146
                                    <label for="oai_identifier">Identifier:</label>
147
                                    <input type="text" size="30" id="oai_identifier" name="oai_identifier" value="[% oai_pmh_request.oai_identifier %]">
148
                                    [% IF (errors.oai_identifier.missing) %]<span class="error">[Identifier is a required field when using GetRecord for an OAI-PMH request.]</span>[% END %]
149
                                </li>
150
                                <li>
151
                                    <label for="oai_set">Set:</label>
152
                                    <input type="text" size="30" id="oai_set" name="oai_set" value="[% oai_pmh_request.oai_set %]">
153
                                    [% IF (errors.oai_set.unavailable) %]<span class="error">[This set is unavailable from this OAI-PMH provider.]</span>[% END %]
154
                                </li>
155
                                [% IF (errors.oai.granularity_mismatch) %]<span class="error">[You must specify the same granularity for both From and Until.]</span>[% END %]
156
                                <li>
157
                                    <label for="oai_from">From:</label>
158
                                    <input type="text" size="30" class="datetime_utc" id="oai_from" name="oai_from" value="[% oai_pmh_request.oai_from %]">
159
                                    <span class="help">This value will be treated as UTC time. Note that some repositories only support YYYY-MM-DD datestamps.</span>
160
                                    [% IF (errors.oai_from.malformed) %]<span class="error">[This must be in YYYY-MM-DD or YYYY-MM-DDThh:mm:ssZ format.]</span>[% END %]
161
                                    [% IF (errors.oai_from.unavailable) %]<span class="error">[This granularity is unsupported by this OAI-PMH provider.]</span>[% END %]
162
                                </li>
163
                                <li>
164
                                    <label for="oai_until">Until:</label>
165
                                    <input type="text" size="30" class="datetime_utc" id="oai_until" name="oai_until" value="[% oai_pmh_request.oai_until %]">
166
                                    <span class="help">This value will be treated as UTC time. Note that some repositories only support YYYY-MM-DD datestamps.</span>
167
                                    [% IF (errors.oai_until.malformed) %]<span class="error">[This must be in YYYY-MM-DD or YYYY-MM-DDThh:mm:ssZ format.]</span>[% END %]
168
                                    [% IF (errors.oai_until.unavailable) %]<span class="error">[This granularity is unsupported by this OAI-PMH provider.]</span>[% END %]
169
                                </li>
170
                            </ol>
171
                        </fieldset>
172
                        <fieldset class="rows">
173
                            <legend>Import parameters:</legend>
174
                            <ol>
175
                                <li>
176
                                    <label for="import_filter">Filter:</label>
177
                                    [% IF ( oai_pmh_request.import_filter == "default" ) %]
178
                                        <input type="text" size="30" id="import_filter" name="import_filter" value="default">
179
                                    [% ELSE %]
180
                                        <input type="text" size="30" id="import_filter" name="import_filter" value="[% oai_pmh_request.import_filter %]">
181
                                    [% END %]
182
                                    <span class="help">If no filter is entered, the default filter will be used.</span>
183
                                </li>
184
                                <li>
185
                                    <label for="import_framework_code">Framework code:</label>
186
                                    <select id="import_framework_code" name="import_framework_code">
187
                                        <option value="">Default framework</option>
188
                                        [% FOREACH framework IN frameworks %]
189
                                            [% IF ( oai_pmh_request.import_framework_code == framework.frameworkcode ) %]
190
                                                <option selected="selected" value="[% framework.frameworkcode %]">[% framework.frameworktext %]</option>
191
                                            [% ELSE %]
192
                                                <option value="[% framework.frameworkcode %]">[% framework.frameworktext %]</option>
193
                                            [% END %]
194
                                        [% END %]
195
                                    </select>
196
                                </li>
197
                                <li>
198
                                    <label for="import_record_type">Record type:</label>
199
                                    <select id="import_record_type" name="import_record_type">
200
                                        <option value="biblio">Bibliographic</option>
201
                                    </select>
202
                                </li>
203
                                <li>
204
                                    <label for="import_matcher_code">Record matcher:</label>
205
                                    <select id="import_matcher_code" name="import_matcher_code">
206
                                        <option value="">None</option>
207
                                        [% FOREACH matcher IN matchers %]
208
                                            [% IF ( oai_pmh_request.import_matcher_code == matcher.code ) %]
209
                                                <option value="[% matcher.code %]" selected="selected">[% matcher.description %]</option>
210
                                            [% ELSE %]
211
                                                <option value="[% matcher.code %]">[% matcher.description %]</option>
212
                                            [% END %]
213
                                        [% END %]
214
                                    </select>
215
                                    <span class="help">See <a href="/cgi-bin/koha/admin/matching-rules.pl">record matching rules</a> to add or edit rules.</span>
216
                                </li>
217
                            </ol>
218
                        </fieldset>
219
                        <fieldset class="rows">
220
                            <legend>Download parameters:</legend>
221
                            <ol>
222
                                <li>
223
                                    <label for="interval">Interval (seconds): </label>
224
                                    <input type="text" id="interval" name="interval" value="[% oai_pmh_request.interval %]" size="4">
225
                                    <span class="help">The download request will be repeated in intervals of this many seconds. Enter "0" if you want the task to only happen once.</span>
226
                                </li>
227
                            </ol>
228
                        </fieldset>
229
                        <fieldset class="action">
230
                            <input type="submit" name="test_parameters" value="Test parameters">
231
                            <input type="submit" name="save" value="Save">
232
                            <a class="cancel" href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">Cancel</a>
233
                        </fieldset>
234
                    </form>
235
                </div>
236
            </div>
237
            <div class="yui-b">
238
                [% INCLUDE 'tools-menu.inc' %]
239
            </div>
240
        </div>
241
[% INCLUDE 'intranet-bottom.inc' %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/tools-home.tt (+5 lines)
Lines 195-200 Link Here
195
    <dd>Utility to upload scanned cover images for display in OPAC</dd>
195
    <dd>Utility to upload scanned cover images for display in OPAC</dd>
196
    [% END %]
196
    [% END %]
197
197
198
    [% IF ( CAN_user_tools_manage_staged_marc ) %]
199
    <dt><a href="/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl">OAI-PMH harvester</a></dt>
200
    <dd>Harvest (ie download and import) records from remote sources using the OAI-PMH protocol</dd>
201
    [% END %]
202
198
</dl>
203
</dl>
199
</div>
204
</div>
200
205
(-)a/koha-tmpl/intranet-tmpl/prog/en/xslt/StripOAIPMH.xsl (+28 lines)
Line 0 Link Here
1
<?xml version="1.0" encoding="UTF-8"?>
2
<xsl:stylesheet version="1.0"
3
    xmlns:oai="http://www.openarchives.org/OAI/2.0/"
4
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
5
    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>
6
    <!-- NOTE: This XSLT strips the OAI-PMH wrapper from the metadata. -->
7
8
    <!-- Match the root oai:record element -->
9
    <xsl:template match="oai:record">
10
        <!-- Apply templates only to the child metadata element(s) -->
11
        <xsl:apply-templates select="oai:metadata" />
12
    </xsl:template>
13
14
    <!-- Matches an oai:metadata element -->
15
    <xsl:template match="oai:metadata">
16
        <!-- Create a copy of child attributes and nodes -->
17
        <xsl:apply-templates select="@* | node()" />
18
    </xsl:template>
19
20
    <!-- Identity transformation: this template copies attributes and nodes -->
21
    <xsl:template match="@* | node()">
22
        <!-- Create a copy of this attribute or node -->
23
        <xsl:copy>
24
            <!-- Recursively apply this template to the attributes and child nodes of this element -->
25
            <xsl:apply-templates select="@* | node()" />
26
        </xsl:copy>
27
    </xsl:template>
28
</xsl:stylesheet>
(-)a/misc/cronjobs/cleanup_database.pl (-1 / +14 lines)
Lines 26-31 use constant DEFAULT_LOGS_PURGEDAYS => 180; Link Here
26
use constant DEFAULT_SEARCHHISTORY_PURGEDAYS      => 30;
26
use constant DEFAULT_SEARCHHISTORY_PURGEDAYS      => 30;
27
use constant DEFAULT_SHARE_INVITATION_EXPIRY_DAYS => 14;
27
use constant DEFAULT_SHARE_INVITATION_EXPIRY_DAYS => 14;
28
use constant DEFAULT_DEBARMENTS_PURGEDAYS         => 30;
28
use constant DEFAULT_DEBARMENTS_PURGEDAYS         => 30;
29
use constant DEFAULT_IMPORTOAI_PURGEDAYS          => 30;
29
30
30
BEGIN {
31
BEGIN {
31
    # find Koha's Perl modules
32
    # find Koha's Perl modules
Lines 44-50 use Koha::UploadedFiles; Link Here
44
45
45
sub usage {
46
sub usage {
46
    print STDERR <<USAGE;
47
    print STDERR <<USAGE;
47
Usage: $0 [-h|--help] [--sessions] [--sessdays DAYS] [-v|--verbose] [--zebraqueue DAYS] [-m|--mail] [--merged] [--import DAYS] [--logs DAYS] [--searchhistory DAYS] [--restrictions DAYS] [--all-restrictions] [--fees DAYS] [--temp-uploads] [--temp-uploads-days DAYS] [--uploads-missing 0|1 ]
48
Usage: $0 [-h|--help] [--sessions] [--sessdays DAYS] [-v|--verbose] [--zebraqueue DAYS] [-m|--mail] [--merged] [--import DAYS] [--logs DAYS] [--searchhistory DAYS] [--restrictions DAYS] [--all-restrictions] [--fees DAYS] [--temp-uploads] [--temp-uploads-days DAYS] [--uploads-missing 0|1 ] [--importoai DAYS ]
48
49
49
   -h --help          prints this help message, and exits, ignoring all
50
   -h --help          prints this help message, and exits, ignoring all
50
                      other options
51
                      other options
Lines 82-87 Usage: $0 [-h|--help] [--sessions] [--sessdays DAYS] [-v|--verbose] [--zebraqueu Link Here
82
   --temp-uploads-days DAYS Override the corresponding preference value.
83
   --temp-uploads-days DAYS Override the corresponding preference value.
83
   --uploads-missing FLAG Delete upload records for missing files when FLAG is true, count them otherwise
84
   --uploads-missing FLAG Delete upload records for missing files when FLAG is true, count them otherwise
84
   --oauth-tokens     Delete expired OAuth2 tokens
85
   --oauth-tokens     Delete expired OAuth2 tokens
86
   --importoai DAYS    purge OAI-PMH records from import_oai table older than DAYS
87
                       days. Defaults to 30 days if no days specified.
85
USAGE
88
USAGE
86
    exit $_[0];
89
    exit $_[0];
87
}
90
}
Lines 108-113 my $temp_uploads; Link Here
108
my $temp_uploads_days;
111
my $temp_uploads_days;
109
my $uploads_missing;
112
my $uploads_missing;
110
my $oauth_tokens;
113
my $oauth_tokens;
114
my $importoai_days;
111
115
112
GetOptions(
116
GetOptions(
113
    'h|help'            => \$help,
117
    'h|help'            => \$help,
Lines 132-137 GetOptions( Link Here
132
    'temp-uploads-days:i' => \$temp_uploads_days,
136
    'temp-uploads-days:i' => \$temp_uploads_days,
133
    'uploads-missing:i' => \$uploads_missing,
137
    'uploads-missing:i' => \$uploads_missing,
134
    'oauth-tokens'      => \$oauth_tokens,
138
    'oauth-tokens'      => \$oauth_tokens,
139
    'importoai:i'       => \$importoai_days,
135
) || usage(1);
140
) || usage(1);
136
141
137
# Use default values
142
# Use default values
Lines 143-148 $mail = DEFAULT_MAIL_PURGEDAYS if defined($mail) Link Here
143
$pSearchhistory    = DEFAULT_SEARCHHISTORY_PURGEDAYS      if defined($pSearchhistory)    && $pSearchhistory == 0;
148
$pSearchhistory    = DEFAULT_SEARCHHISTORY_PURGEDAYS      if defined($pSearchhistory)    && $pSearchhistory == 0;
144
$pListShareInvites = DEFAULT_SHARE_INVITATION_EXPIRY_DAYS if defined($pListShareInvites) && $pListShareInvites == 0;
149
$pListShareInvites = DEFAULT_SHARE_INVITATION_EXPIRY_DAYS if defined($pListShareInvites) && $pListShareInvites == 0;
145
$pDebarments       = DEFAULT_DEBARMENTS_PURGEDAYS         if defined($pDebarments)       && $pDebarments == 0;
150
$pDebarments       = DEFAULT_DEBARMENTS_PURGEDAYS         if defined($pDebarments)       && $pDebarments == 0;
151
$importoai_days    = DEFAULT_IMPORTOAI_PURGEDAYS          if defined($importoai_days)    && $importoai_days == 0;
146
152
147
if ($help) {
153
if ($help) {
148
    usage(0);
154
    usage(0);
Lines 166-171 unless ( $sessions Link Here
166
    || $temp_uploads
172
    || $temp_uploads
167
    || defined $uploads_missing
173
    || defined $uploads_missing
168
    || $oauth_tokens
174
    || $oauth_tokens
175
    || $importoai_days
169
) {
176
) {
170
    print "You did not specify any cleanup work for the script to do.\n\n";
177
    print "You did not specify any cleanup work for the script to do.\n\n";
171
    usage(1);
178
    usage(1);
Lines 344-349 if ($oauth_tokens) { Link Here
344
    say "Removed $count expired OAuth2 tokens" if $verbose;
351
    say "Removed $count expired OAuth2 tokens" if $verbose;
345
}
352
}
346
353
354
if ($importoai_days){
355
    my $sql = "DELETE FROM import_oai WHERE date(upload_timestamp) < (date_sub(curdate(), INTERVAL ? DAY))";
356
    my $sth = $dbh->prepare($sql);
357
    $sth->execute($importoai_days) or die $dbh->errstr;
358
}
359
347
exit(0);
360
exit(0);
348
361
349
sub RemoveOldSessions {
362
sub RemoveOldSessions {
(-)a/misc/harvesterd.pl (+207 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
use Modern::Perl;
21
use Getopt::Long;
22
use Pod::Usage;
23
use Module::Load;
24
use Log::Log4perl qw(:easy);
25
use POE;
26
use YAML;
27
28
use C4::Context;
29
use Koha::Daemon;
30
use Koha::OAI::Harvester;
31
use Koha::OAI::Harvester::Listener;
32
33
binmode(STDOUT,':encoding(UTF-8)');
34
$|++;
35
36
my $help = 0;
37
my $daemonize = 0;
38
39
my ($socket_addr,$pidfile,$statefile,$spooldir);
40
my ($download_module,$import_module);
41
my ($batch,$download_workers,$import_workers,$import_poll);
42
my ($logfile,$log_level);
43
my $log_levels = {
44
    FATAL => $FATAL,
45
    ERROR => $ERROR,
46
    WARN => $WARN,
47
    INFO => $INFO,
48
    DEBUG => $DEBUG,
49
    TRACE => $TRACE,
50
};
51
52
my $context = C4::Context->new();
53
my $config_filename = $context->{config}->{oai_pmh_harvester_config};
54
if ($config_filename){
55
    my $config = YAML::LoadFile($config_filename);
56
    if ($config){
57
        $socket_addr = $config->{socket};
58
        $pidfile = $config->{pidfile};
59
        $statefile = $config->{statefile};
60
        $spooldir = $config->{spooldir};
61
        $logfile = $config->{logfile};
62
        $log_level = $config->{loglevel};
63
        $download_module = $config->{download_module};
64
        $import_module = $config->{import_module};
65
        $batch = $config->{download_batch};
66
        $download_workers = $config->{download_workers};
67
        $import_workers = $config->{import_workers};
68
        $import_poll = $config->{import_poll};
69
    }
70
}
71
72
GetOptions(
73
    "help|?"            => \$help,
74
    "daemon"            => \$daemonize,
75
    "socket-uri=s"          => \$socket_addr,
76
    "pid-file=s"             => \$pidfile,
77
    "state-file=s"      => \$statefile,
78
    "spool-dir=s"       => \$spooldir,
79
    "log-file=s"             => \$logfile,
80
    "log-level=s"       => \$log_level,
81
    "download-module=s" => \$download_module,
82
    "import-module=s"   => \$import_module,
83
) or pod2usage(2);
84
pod2usage(1) if $help;
85
86
my $level = ( $log_level && $log_levels->{$log_level} ) ? $log_levels->{$log_level} : $log_levels->{WARN};
87
Log::Log4perl->easy_init(
88
    {
89
        level => $level,
90
        file => "STDOUT",
91
        layout   => '[%d{yyyy-MM-dd HH:mm:ss}][%p] %m%n',
92
    }
93
);
94
my $logger = Log::Log4perl->get_logger();
95
96
unless($download_module){
97
    $download_module = "Koha::OAI::Harvester::Worker::Download::Stream";
98
}
99
unless($import_module){
100
    $import_module = "Koha::OAI::Harvester::Worker::Import";
101
}
102
103
foreach my $module ( $download_module, $import_module ){
104
    load $module;
105
}
106
my $downloader = $download_module->new({
107
    logger => $logger,
108
    batch => $batch,
109
});
110
my $importer = $import_module->new({
111
    logger => $logger,
112
});
113
114
my $daemon = Koha::Daemon->new({
115
    pidfile => $pidfile,
116
    logfile => $logfile,
117
    daemonize => $daemonize,
118
});
119
$daemon->run();
120
121
my $harvester = Koha::OAI::Harvester->spawn({
122
    Downloader => $downloader,
123
    DownloaderWorkers => $download_workers,
124
    Importer => $importer,
125
    ImporterWorkers => $import_workers,
126
    ImportQueuePoll => $import_poll,
127
    logger => $logger,
128
    state_file => $statefile,
129
    spooldir => $spooldir,
130
});
131
132
my $listener = Koha::OAI::Harvester::Listener->spawn({
133
    logger => $logger,
134
    socket => $socket_addr,
135
});
136
137
$logger->info("OAI-PMH harvester started.");
138
139
POE::Kernel->run();
140
141
exit;
142
143
=head1 NAME
144
145
harvesterd.pl - a daemon that asynchronously sends OAI-PMH requests and imports OAI-PMH records
146
147
=head1 SYNOPSIS
148
149
KOHA_CONF=/path/to/koha-conf.xml ./harvesterd.pl
150
151
=head1 OPTIONS
152
153
=over 8
154
155
=item B<--help>
156
157
Print a brief help message and exits.
158
159
=item B<--daemon>
160
161
Run program as a daemon (ie fork process, setsid, chdir to root, reset umask,
162
and close STDIN, STDOUT, and STDERR).
163
164
=item B<--log-file>
165
166
Specify a file to which to log STDOUT and STDERR.
167
168
=item B<--pid-file>
169
170
Specify a file to store the process id (this prevents multiple copies of the program
171
from running at the same time).
172
173
=item B<--socket-uri>
174
175
Specify a URI to use for the UNIX socket used to communicate with the daemon.
176
(e.g. unix:/path/to/socket.sock)
177
178
=item B<--state-file>
179
180
Specify a filename to use for storing the harvester's in-memory state.
181
182
In the event that the harvester crashes, it can resume from where it stopped.
183
184
=item B<--spool-dir>
185
186
Specify a directory to store downloaded OAI-PMH records prior to import.
187
188
=item B<--log-level>
189
190
Specify a log level for logging. The logger uses Log4Perl, which provides
191
FATAL, ERROR, WARN, INFO, DEBUG, and TRACE in order of descending priority.
192
193
Defaults to WARN level.
194
195
=item B<--download-module>
196
197
Specify a Perl module to use for downloading records. This is a specialty module,
198
which has particular requirements, so only advanced users should use this option.
199
200
=item B<--import-module>
201
202
Specify a Perl module to use for importing records. This is a specialty module,
203
which has particular requirements, so only advanced users should use this option.
204
205
=back
206
207
=cut
(-)a/rewrite-config.PL (+3 lines)
Lines 152-157 $prefix = $ENV{'INSTALL_BASE'} || "/usr"; Link Here
152
  "__MEMCACHED_NAMESPACE__" => "",
152
  "__MEMCACHED_NAMESPACE__" => "",
153
  "__FONT_DIR__" => "/usr/share/fonts/truetype/ttf-dejavu",
153
  "__FONT_DIR__" => "/usr/share/fonts/truetype/ttf-dejavu",
154
  "__TEMPLATE_CACHE_DIR__" => "/tmp/koha"
154
  "__TEMPLATE_CACHE_DIR__" => "/tmp/koha"
155
  "__OAI_RUN_DIR__" => "",
156
  "__OAI_LIB_DIR__" => "",
157
  "__OAI_SPOOL_DIR__" => "",
155
);
158
);
156
159
157
# Override configuration from the environment
160
# Override configuration from the environment
(-)a/svc/oai-pmh-harvester/history (+132 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use C4::Auth qw(check_cookie_auth haspermission get_session);
22
use JSON;
23
use Koha::Database;
24
25
my $input = new CGI;
26
27
my ( $auth_status, $sessionID ) =
28
  check_cookie_auth( $input->cookie('CGISESSID'));
29
30
if ( $auth_status ne "ok" ) {
31
    exit 0;
32
}
33
34
my $data = {
35
    data => [],
36
    recordsTotal => 0,
37
    recordsFiltered => 0,
38
    draw => undef,
39
};
40
41
my $length = 10;
42
my $start = 0;
43
my @order_by = ();
44
my @search = ();
45
46
if ($input->request_method eq "POST"){
47
    my $postdata = $input->param('POSTDATA');
48
    my $request = from_json($postdata);
49
    $data->{draw} = int( $request->{draw} ) if $request->{draw};
50
    $length = $request->{length} if $request->{length};
51
    $start = $request->{start} if $request->{start};
52
    if (my $search = $request->{search}){
53
        my $value = $search->{value};
54
        if ($value){
55
            foreach my $column (@{$request->{columns}}){
56
                if ($column->{data} && $column->{searchable}){
57
                    my $search_element = {
58
                        $column->{data} => { 'like', "%".$value."%" },
59
                    };
60
                    push(@search,$search_element);
61
                }
62
            }
63
        }
64
    }
65
    if (my $order = $request->{order}){
66
        foreach my $element (@$order){
67
            my $dir = $element->{dir};
68
            my $column_index = $element->{column};
69
            my $column = $request->{columns}->[$column_index];
70
            my $orderable = $column->{orderable};
71
            if ($orderable){
72
                my $column_name = $column->{data};
73
                my $direction;
74
                if ($dir){
75
                    if ($dir eq "asc" || $dir eq "desc"){
76
                        $direction = "-$dir";
77
                    }
78
                }
79
                if ($column_name && $direction){
80
                    my $single_order = {
81
                        $direction => $column_name,
82
                    };
83
                    push(@order_by,$single_order);
84
                }
85
            }
86
        }
87
    }
88
}
89
90
my $page = ( $start / $length ) + 1;
91
my $schema = Koha::Database->new()->schema();
92
if ($schema){
93
    my $rs = $schema->resultset("OaiHarvesterHistory");
94
    my $results = $rs->search(
95
        \@search,
96
        {
97
            result_class => 'DBIx::Class::ResultClass::HashRefInflator',
98
            page => $page,
99
            rows => $length,
100
            order_by => \@order_by,
101
        },
102
    );
103
    my $count = $rs->count;
104
    my $filtered_count = $results->pager->total_entries;
105
    my @rows = ();
106
    while (my $row = $results->next){
107
        $row->{imported_record} = '';
108
        if ($row->{record_type} eq "biblio"){
109
            my $harvested_biblio = $schema->resultset("OaiHarvesterBiblio")->find(
110
                    {
111
                        oai_repository => $row->{repository},
112
                        oai_identifier => $row->{header_identifier},
113
                    },
114
                    { key => "oai_record" },
115
            );
116
            $row->{imported_record} = $harvested_biblio->biblionumber->id if $harvested_biblio;
117
        }
118
        push(@rows,$row);
119
    }
120
    if ($count){
121
        $data->{recordsTotal} = $count;
122
        $data->{recordsFiltered} = $filtered_count;
123
        $data->{data} = \@rows if @rows;
124
    }
125
}
126
127
binmode STDOUT, ":encoding(UTF-8)";
128
print $input->header(
129
    -type => 'application/json',
130
    -charset => 'UTF-8'
131
);
132
print to_json($data, { pretty => 1, });
(-)a/tools/oai-pmh-harvester/dashboard.pl (+134 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
use Modern::Perl;
21
use YAML;
22
23
use C4::Auth;
24
use C4::Context;
25
use C4::Output;
26
use Koha::OAI::Harvester::Client;
27
use Koha::OAI::Harvester::Requests;
28
use Koha::BiblioFrameworks;
29
use Koha::Database;
30
31
my $context = C4::Context->new();
32
my $config_filename = $context->{config}->{oai_pmh_harvester_config};
33
my $client_config = {};
34
if ($config_filename){
35
    my $config = YAML::LoadFile($config_filename);
36
    if ($config && $config->{socket}){
37
        $client_config->{socket_uri} = $config->{socket};
38
    }
39
}
40
41
my $input = new CGI;
42
43
my ($template, $loggedinuser, $cookie) =
44
    get_template_and_user({template_name => "tools/oai-pmh-harvester/dashboard.tt",
45
        query => $input,
46
        type => "intranet",
47
        authnotrequired => 0,
48
        flagsrequired => {tools => 'manage_staged_marc'},
49
    });
50
51
my $op = $input->param('op') // 'list';
52
my $id = $input->param('id');
53
my $uuid = $input->param('uuid');
54
55
my $client = Koha::OAI::Harvester::Client->new($client_config);
56
my $is_connected = $client->connect;
57
58
if ( ($op eq "send") && $id ){
59
    if ($is_connected){
60
        my $request = Koha::OAI::Harvester::Requests->find($id);
61
        if ($request){
62
            my $task = {
63
                name => $request->name,
64
                uuid => $request->uuid,
65
                interval => $request->interval,
66
                parameters => {
67
                    oai_pmh => {
68
                        baseURL => $request->http_url,
69
                        verb => $request->oai_verb,
70
                        metadataPrefix => $request->oai_metadataPrefix,
71
                        identifier => $request->oai_identifier,
72
                        set => $request->oai_set,
73
                        from => $request->oai_from,
74
                        until => $request->oai_until,
75
                    },
76
                    import => {
77
                        filter => $request->import_filter,
78
                        frameworkcode => $request->import_framework_code,
79
                        matcher_code => $request->import_matcher_code,
80
                        record_type => $request->import_record_type,
81
                    },
82
                },
83
            };
84
            if ($request->http_username && $request->http_password && $request->http_realm){
85
                $task->{parameters}->{http_basic_auth} = {
86
                    username => $request->http_username,
87
                    password => $request->http_password,
88
                    realm => $request->http_realm,
89
                };
90
            }
91
            my $is_created = $client->create($task);
92
            $template->{VARS}->{ result }->{ send } = $is_created;
93
        }
94
    }
95
}
96
elsif ( ($op eq "start") && ($uuid) ){
97
    if ($is_connected){
98
        my $is_started = $client->start($uuid);
99
        $template->{VARS}->{ result }->{ start } = $is_started;
100
    }
101
}
102
elsif ( ($op eq "stop") && ($uuid) ){
103
    if ($is_connected){
104
        my $is_stopped = $client->stop($uuid);
105
        $template->{VARS}->{ result }->{ stop } = $is_stopped;
106
    }
107
}
108
elsif ( ($op eq "delete") && ($uuid) ){
109
    if ($is_connected){
110
        my $is_deleted = $client->delete($uuid);
111
        $template->{VARS}->{ result }->{ delete } = $is_deleted;
112
    }
113
}
114
115
my $requests = Koha::OAI::Harvester::Requests->as_list;
116
$template->{VARS}->{ saved_requests } = $requests;
117
118
my $frameworks = Koha::BiblioFrameworks->as_list();
119
$template->{VARS}->{ frameworks } = $frameworks;
120
121
my $schema = Koha::Database->new()->schema();
122
my $matcher_rs = $schema->resultset("MarcMatcher");
123
my @matchers = $matcher_rs->all;
124
$template->{VARS}->{ matchers } = \@matchers;
125
126
if ($is_connected){
127
    my $submitted_requests = $client->list;
128
    $template->{VARS}->{ submitted_requests } = $submitted_requests;
129
}
130
else {
131
    $template->{VARS}->{ harvester }->{ offline } = 1;
132
}
133
134
output_html_with_http_headers($input, $cookie, $template->output);
(-)a/tools/oai-pmh-harvester/record.pl (+53 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
use Modern::Perl;
21
22
use C4::Auth;
23
use C4::Output;
24
use Koha::Database;
25
26
my $input = new CGI;
27
28
my ($template, $loggedinuser, $cookie) =
29
    get_template_and_user({template_name => "tools/oai-pmh-harvester/record.tt",
30
        query => $input,
31
        type => "intranet",
32
        authnotrequired => 0,
33
        flagsrequired => {tools => 'manage_staged_marc'},
34
    });
35
36
my $import_oai_id = $input->param('import_oai_id');
37
if ($import_oai_id){
38
    my $schema = Koha::Database->new()->schema();
39
    if ($schema){
40
        my $rs = $schema->resultset("OaiHarvesterHistory");
41
        if ($rs){
42
            my $row = $rs->find($import_oai_id);
43
            if ($row){
44
                my $record = $row->record;
45
                if ($record){
46
                    $template->{VARS}->{ record } = $record;
47
                }
48
            }
49
        }
50
    }
51
}
52
53
output_html_with_http_headers($input, $cookie, $template->output);
(-)a/tools/oai-pmh-harvester/request.pl (-1 / +142 lines)
Line 0 Link Here
0
- 
1
#!/usr/bin/perl
2
3
# Copyright 2017 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
use Modern::Perl;
21
use UUID;
22
23
use C4::Auth;
24
use C4::Output;
25
use Koha::OAI::Harvester::Requests;
26
use Koha::BiblioFrameworks;
27
use Koha::Database;
28
29
my $input = new CGI;
30
31
my ($template, $loggedinuser, $cookie) =
32
    get_template_and_user({template_name => "tools/oai-pmh-harvester/request.tt",
33
        query => $input,
34
        type => "intranet",
35
        authnotrequired => 0,
36
        flagsrequired => {tools => 'manage_staged_marc'},
37
    });
38
39
my $op = $input->param('op');
40
my $id = $input->param('id');
41
42
my @frameworks = Koha::BiblioFrameworks->as_list();
43
$template->{VARS}->{ frameworks } = \@frameworks;
44
45
my $schema = Koha::Database->new()->schema();
46
my $rs = $schema->resultset("MarcMatcher");
47
my @matchers = $rs->all;
48
$template->{VARS}->{ matchers } = \@matchers;
49
50
my $http_url = $input->param('http_url');
51
my $http_username = $input->param('http_username');
52
my $http_password = $input->param('http_password');
53
my $http_realm = $input->param('http_realm');
54
55
my $oai_verb = $input->param('oai_verb');
56
my $oai_metadataPrefix = $input->param('oai_metadataPrefix');
57
my $oai_identifier = $input->param('oai_identifier');
58
my $oai_from = $input->param('oai_from');
59
my $oai_until = $input->param('oai_until');
60
my $oai_set = $input->param('oai_set');
61
62
my $import_filter = $input->param('import_filter') // 'default';
63
my $import_framework_code = $input->param('import_framework_code');
64
my $import_record_type = $input->param('import_record_type');
65
my $import_matcher_code = $input->param('import_matcher_code');
66
67
my $interval = $input->param("interval") ? int ( $input->param("interval") ) : 0;
68
my $name = $input->param("name");
69
70
my $save = $input->param('save');
71
my $test_parameters = $input->param('test_parameters');
72
73
my $request = $id ? Koha::OAI::Harvester::Requests->find($id) : Koha::OAI::Harvester::Request->new();
74
if ($request){
75
    if ($op eq "create" || $op eq "update"){
76
        $request->set({
77
            name => $name,
78
            http_url => $http_url,
79
            http_username => $http_username,
80
            http_password => $http_password,
81
            http_realm => $http_realm,
82
            oai_verb => $oai_verb,
83
            oai_metadataPrefix => $oai_metadataPrefix,
84
            oai_identifier => $oai_identifier,
85
            oai_from => $oai_from,
86
            oai_until => $oai_until,
87
            oai_set => $oai_set,
88
            import_filter => $import_filter,
89
            import_framework_code => $import_framework_code,
90
            import_record_type => $import_record_type,
91
            import_matcher_code => $import_matcher_code,
92
            interval => $interval,
93
        });
94
    }
95
}
96
97
if ($test_parameters){
98
    my $errors = $request->validate();
99
    $template->{VARS}->{ errors } = $errors;
100
    $template->{VARS}->{ test_parameters } = 1;
101
}
102
103
if ($op eq "new"){
104
    #Empty form with some defaults
105
    $request->import_filter("default") unless $request->import_filter;
106
    $request->interval(0) unless $request->interval;
107
}
108
elsif ($op eq "create"){
109
    if ($save){
110
        my ($uuid,$uuid_string);
111
        UUID::generate($uuid);
112
        UUID::unparse($uuid, $uuid_string);
113
        $request->uuid($uuid_string);
114
        $request->store;
115
        print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
116
        exit;
117
    }
118
}
119
elsif ( $op eq "edit"){
120
    $template->{VARS}->{ id } = $id;
121
}
122
elsif ($op eq "update"){
123
    $template->{VARS}->{ id } = $id;
124
    if ($save){
125
        $request->store;
126
        print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
127
        exit;
128
    }
129
}
130
elsif ($op eq "delete"){
131
    if ($request){
132
        $request->delete;
133
        print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
134
    }
135
}
136
else {
137
    print $input->redirect('/cgi-bin/koha/tools/oai-pmh-harvester/dashboard.pl#saved_results');
138
}
139
$template->{VARS}->{ op } = $op;
140
$template->{VARS}->{ oai_pmh_request } = $request;
141
142
output_html_with_http_headers($input, $cookie, $template->output);

Return to bug 10662