View | Details | Raw Unified | Return to bug 10662
Collapse All | Expand All

(-)a/Koha/Icarus.pm (+177 lines)
Line 0 Link Here
1
package Koha::Icarus;
2
3
# Copyright 2016 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use IO::Socket::UNIX;
22
use IO::Select;
23
use URI;
24
use JSON;
25
26
sub new {
27
    my ($class, $args) = @_;
28
    $args = {} unless defined $args;
29
    return bless ($args, $class);
30
}
31
32
sub connected {
33
    my ($self) = @_;
34
    if ($self->{_connected}){
35
        return 1;
36
    }
37
}
38
39
sub connect {
40
    my ($self) =  @_;
41
    my $socket_uri = $self->{socket_uri};
42
    if ($socket_uri){
43
        my $uri = URI->new($socket_uri);
44
        if ($uri && $uri->scheme eq 'unix'){
45
            my $socket_path = $uri->path;
46
            my $socket = IO::Socket::UNIX->new(
47
                Type => IO::Socket::UNIX::SOCK_STREAM(),
48
                Peer => $socket_path,
49
            );
50
            if ($socket){
51
                my $socketio = new IO::Select();
52
                $socketio->add($socket);
53
                #FIXME: Should probably fix these return values...
54
                $self->{_socketio} = $socketio;
55
                $self->{_socket} = $socket;
56
                my $message = $self->_read();
57
                if ($message eq 'HELLO'){
58
                    $self->{_connected} = 1;
59
                    return 1;
60
                }
61
            }
62
        }
63
    }
64
    return 0;
65
}
66
67
sub add_task {
68
    my ($self, $args) = @_;
69
    my $task = $args->{task};
70
    if ($task && %$task){
71
        my $response = $self->command("add task", undef, $task);
72
        if ($response){
73
            return $response;
74
        }
75
    }
76
}
77
78
sub start_task {
79
    my ($self, $args) = @_;
80
    my $task_id = $args->{task_id};
81
    if ($task_id){
82
        my $response = $self->command("start task", $task_id);
83
        if ($response){
84
            return $response;
85
        }
86
    }
87
}
88
89
sub remove_task {
90
    my ($self, $args) = @_;
91
    my $task_id = $args->{task_id};
92
    if ($task_id){
93
        my $response = $self->command("remove task", $task_id);
94
        if ($response){
95
            return $response;
96
        }
97
    }
98
}
99
100
sub list_tasks {
101
   my ($self) = @_;
102
   my $response = $self->command("list tasks");
103
    if ($response){
104
        if (my $tasks = $response->{tasks}){
105
            return $tasks;
106
        }
107
    }
108
}
109
110
sub shutdown {
111
    my ($self) = @_;
112
    my $response = $self->command("shutdown");
113
    if ($response){
114
        return $response;
115
    }
116
}
117
118
119
120
121
122
sub command {
123
    my ($self, $command, $task_id, $task) = @_;
124
    my $serialized = $self->_serialize({ "command" => $command, "task_id" => $task_id, "task" => $task });
125
    if ($serialized){
126
        $self->_write({ serialized => $serialized });
127
        my $json = $self->_read();
128
        if ($json){
129
            my $response = from_json($json);
130
            if ($response){
131
                return $response;
132
            }
133
        }
134
    }
135
}
136
137
sub _serialize {
138
    my ($self, $output) = @_;
139
    my $serialized = to_json($output);
140
    return $serialized;
141
}
142
143
sub _write {
144
    my ($self, $args) = @_;
145
    my $socket = $self->{_socket};
146
    my $output = $args->{serialized};
147
    if ($output){
148
        if (my $socketio = $self->{_socketio}){
149
            if (my @filehandles = $socketio->can_write(5)){
150
                foreach my $filehandle (@filehandles){
151
                    #Localize output record separator as null
152
                    local $\ = "\x00";
153
                    print $socket $output;
154
                }
155
            }
156
        }
157
    }
158
}
159
160
sub _read {
161
    my ($self) = @_;
162
    if (my $socketio = $self->{_socketio}){
163
        if (my @filehandles = $socketio->can_read(5)){
164
            foreach my $filehandle (@filehandles){
165
                #Localize input record separator as null
166
                local $/ = "\x00";
167
                my $message = <$filehandle>;
168
                chomp($message) if $message;
169
                return $message;
170
            }
171
        }
172
    }
173
}
174
175
176
177
1;
(-)a/Koha/Icarus/Base.pm (+22 lines)
Line 0 Link Here
1
package Koha::Icarus::Base;
2
3
use Modern::Perl;
4
use DateTime;
5
6
sub new {
7
    my ($class, $args) = @_;
8
    $args = {} unless defined $args;
9
    return bless ($args, $class);
10
}
11
12
sub log {
13
    my ($self,$message) = @_;
14
    my $id = $self->{_id};
15
    my $component = $self->{_component} // "component";
16
    if ( ($self->{Verbosity}) && ($self->{Verbosity} > 0) ){
17
        my $now = DateTime->now(time_zone => "local");
18
        say "[$now] [$component $id] $message";
19
    }
20
}
21
22
1;
(-)a/Koha/Icarus/Listener.pm (+327 lines)
Line 0 Link Here
1
package Koha::Icarus::Listener;
2
3
use Modern::Perl;
4
use parent 'Koha::Icarus::Base';
5
6
use POE qw(Wheel::ReadWrite Wheel::SocketFactory Wheel::Run);
7
use IO::Socket qw(AF_UNIX);
8
use URI;
9
use Koha::Icarus::Task;
10
use JSON; #For "on_client_input"
11
12
my $null_filter = POE::Filter::Line->new(
13
     Literal => chr(0),
14
);
15
16
sub new {
17
    my ($class, $args) = @_;
18
    $args = {} unless defined $args;
19
    $args->{_component} = "server";
20
    $args->{_id} = "undefined";
21
    return bless ($args, $class);
22
}
23
24
#NOTE: "spawn" inspired by http://poe.perl.org/?POE_Cookbook/Object_Methods
25
sub spawn {
26
    my ($class, $args) = @_;
27
    my $self = $class->new($args);
28
    POE::Session->create(
29
        object_states => [
30
            $self => {
31
                _start => "on_server_start",
32
                shutdown => "shutdown",
33
                set_verbosity => "set_verbosity",
34
                _child => "on_task_event",
35
                got_list_tasks => "on_list_tasks",
36
                graceful_shutdown => "graceful_shutdown",
37
                got_client_accept => "on_client_accept",
38
                got_client_error => "on_client_error",
39
                got_server_error => "on_server_error",
40
                got_add_task => "on_add_task",
41
                got_client_input => "on_client_input",
42
            },
43
        ],
44
    );
45
}
46
47
#Methods for POE::Session
48
49
sub on_server_start {
50
    my ($self, $kernel,$heap,$session) = @_[OBJECT, KERNEL,HEAP,SESSION];
51
    my $server_id = $session->ID;
52
    $self->{_id} = $server_id; #Set internal id for logging purposes
53
54
    my $bind_address_uri = $self->{Socket};
55
    my $max_tasks = $self->{MaxTasks};
56
57
    $kernel->sig(INT => "graceful_shutdown");
58
    $kernel->sig(TERM => "graceful_shutdown");
59
60
    $heap->{max_tasks} = $max_tasks // 25; #Default maximum of 25 unless otherwise specified
61
62
    $self->log("Maximum number of tasks allowed: $heap->{max_tasks}");
63
    $self->log("Starting server...");
64
65
    my %server_params = (
66
        SuccessEvent => "got_client_accept",
67
        FailureEvent => "got_server_error",
68
    );
69
70
    #TODO: At this time, only "unix" sockets are supported. In future, perhaps TCP/IP sockets could also be supported.
71
    my $uri = URI->new($bind_address_uri);
72
    my $scheme = $uri->scheme;
73
74
    if ($scheme eq 'unix'){
75
        my $bind_address = $uri->path;
76
        $server_params{SocketDomain} = AF_UNIX;
77
        $server_params{BindAddress} = $bind_address;
78
        #When starting a unix socket server, you need to remove any existing references to that socket file.
79
        if ($bind_address && (-e $bind_address) ){
80
            unlink $bind_address;
81
        }
82
    }
83
84
    $heap->{server} = POE::Wheel::SocketFactory->new(%server_params);
85
86
    if ($scheme eq 'unix'){
87
        #FIXME/DEBUGGING: This is a way to force a permission denied error...
88
        #chmod 0755, $uri->path;
89
        #Make the socket writeable to other users like Apache
90
        chmod 0666, $uri->path;
91
    }
92
93
}
94
95
sub shutdown {
96
    my ($self,$heap,$session,$kernel) = @_[OBJECT, HEAP,SESSION,KERNEL];
97
98
    if ($heap->{server}){
99
        $self->log("Shutting down server...");
100
        #Delete the server, so that you can't get any new connections
101
        delete $heap->{server} if $heap->{server};
102
    }
103
104
    if ($heap->{client}){
105
        $self->log("Shutting down any remaining clients...");
106
        #Delete the clients, so that you bring down the existing connections
107
        delete $heap->{client}; #http://www.perlmonks.org/?node_id=176971
108
    }
109
}
110
111
sub on_task_event {
112
    my ($self, $kernel, $heap,$session) = @_[OBJECT,KERNEL, HEAP,SESSION];
113
    my ($action,$child_session,$task) = @_[ARG0,ARG1,ARG2];
114
115
    my $child_id = $child_session->ID;
116
    $self->log("$action child $child_id");
117
118
    if ($action eq 'create'){
119
        #NOTE: The $task variable is returned by the child POE session's _start event
120
        my $task_id = $child_session->ID;
121
        $heap->{tasks}->{$task_id}->{task} = $task;
122
123
    } elsif ($action eq 'lose'){
124
        my $task_id = $child_session->ID;
125
        delete $heap->{tasks}->{$task_id};
126
    }
127
}
128
129
#TODO: Put this in a parent class?
130
sub set_verbosity {
131
    my ($self,$session,$kernel,$new_verbosity) = @_[OBJECT,SESSION,KERNEL,ARG0];
132
    if (defined $new_verbosity){
133
        $self->{Verbosity} = $new_verbosity;
134
    }
135
}
136
137
sub on_list_tasks {
138
    my ($self, $kernel, $heap,$session) = @_[OBJECT, KERNEL, HEAP,SESSION];
139
140
    #DEBUG: You can access the POE::Kernel's sessions with "$POE::Kernel::poe_kernel->[POE::Kernel::KR_SESSIONS]".
141
    #While it's black magic you shouldn't touch, it can be helpful when debugging.
142
143
    my @tasks = ();
144
    foreach my $task_id (keys %{$heap->{tasks}} ){
145
        push(@tasks,{ task_id => $task_id, task => $heap->{tasks}->{$task_id}->{task} });
146
    }
147
    return \@tasks;
148
}
149
150
sub graceful_shutdown {
151
    my ($self, $heap,$session,$kernel,$signal) = @_[OBJECT, HEAP,SESSION,KERNEL,ARG0];
152
153
    #Tell the kernel that you're handling the signal sent to this session
154
    $kernel->sig_handled();
155
    $kernel->sig($signal);
156
157
    my $tasks = $kernel->call($session,"got_list_tasks");
158
159
160
    if ( $heap->{tasks} && %{$heap->{tasks}} ){
161
        $self->log("Waiting for tasks to finish...");
162
        foreach my $task_id (keys %{$heap->{tasks}}){
163
            $self->log("Task $task_id still exists...");
164
            $kernel->post($task_id,"got_task_stop");
165
        }
166
    } else {
167
        $self->log("All tasks have finished");
168
        $kernel->yield("shutdown");
169
        return;
170
    }
171
172
    $self->log("Attempting graceful shutdown in 1 second...");
173
    #NOTE: Basically, we just try another graceful shutdown on the next tick.
174
    $kernel->delay("graceful_shutdown" => 1);
175
}
176
177
#Accept client connection to listener
178
sub on_client_accept {
179
    my ($self, $client_socket, $server_wheel_id, $heap, $session) = @_[OBJECT, ARG0, ARG3, HEAP,SESSION];
180
181
    my $client_wheel = POE::Wheel::ReadWrite->new(
182
      Handle => $client_socket,
183
      InputEvent => "got_client_input",
184
      ErrorEvent => "got_client_error",
185
      InputFilter => $null_filter,
186
      OutputFilter => $null_filter,
187
    );
188
189
    $client_wheel->put("HELLO");
190
    $heap->{client}->{ $client_wheel->ID() } = $client_wheel;
191
    $self->log("Connection ".$client_wheel->ID()." started.$server_wheel_id");
192
}
193
194
#Handle server error - shutdown server
195
sub on_server_error {
196
    my ($self, $operation, $errnum, $errstr, $heap, $session) = @_[OBJECT, ARG0, ARG1, ARG2,HEAP, SESSION];
197
    $self->log("Server $operation error $errnum: $errstr\n");
198
    delete $heap->{server};
199
}
200
201
#Handle client error - including disconnect
202
sub on_client_error {
203
    my ($self, $wheel_id,$heap,$session) = @_[OBJECT, ARG3,HEAP,SESSION];
204
205
    $self->log("Connection $wheel_id failed or ended.");
206
    delete $heap->{client}->{$wheel_id};
207
208
}
209
210
sub on_add_task {
211
    my ($self, $message, $kernel, $heap, $session) = @_[OBJECT, ARG0, KERNEL, HEAP,SESSION];
212
213
    #Fetch a list of all tasks
214
    my @task_keys = keys %{$heap->{tasks}};
215
216
    #If the number in the list is less than the max, add a new task
217
    #else die.
218
    if (scalar @task_keys < $heap->{max_tasks}){
219
        my $server_id = $session->ID;
220
        my $task_session = Koha::Icarus::Task->spawn({ message => $message, server_id => $server_id, Verbosity => 1, });
221
        return $task_session->ID;
222
    } else {
223
        #This die should be caught by the event caller...
224
        die "Maximum number of tasks already reached.\n";
225
    }
226
}
227
228
sub on_client_input {
229
    my ($self, $input, $wheel_id, $session, $kernel, $heap) = @_[OBJECT, ARG0, ARG1, SESSION, KERNEL, HEAP];
230
231
    #Store server id more explicitly
232
    my $server_id = $session->ID;
233
234
    #Server listener has received input from client
235
    my $client = $heap->{client}->{$wheel_id};
236
237
    #FIXME: you probably don't want to log this as it can have auth info...
238
    #$self->log("Input = $input");
239
240
    #Parse input from client
241
    my $message = from_json($input);
242
243
    if ( ref $message eq 'HASH' ){
244
        #Read "command" from client
245
        if (my $command = $message->{command}){
246
            $self->log("Message received with command \"$command\".");
247
            if ($command eq 'add task'){
248
                my $output = {};
249
250
                #Create a task session
251
                eval {
252
                   #NOTE: The server automatically keeps track of its child tasks
253
                    my $task_id = $kernel->call($server_id,"got_add_task",$message);
254
255
                    $output->{action} = "added";
256
                    $output->{task_id} = $task_id;
257
                };
258
                if ($@){
259
                    #FIXME: You might be able to remove this log...
260
                    $self->log("$@");
261
                    chomp($@);
262
                    $output->{action} = "error";
263
                    $output->{error_message} = $@;
264
                }
265
                my $server_output = to_json($output);
266
                $client->put($server_output);
267
                return;
268
269
            } elsif ( ($command eq 'remove task') || ($command eq 'start task' ) ){
270
271
                my $task_id = $message->{task_id};
272
273
                my $output = {
274
                    task_id => $task_id,
275
                };
276
277
                if ($command eq 'remove task'){
278
                    $kernel->call($task_id,"got_task_stop");
279
                    $output->{action} = "removed";
280
                } elsif ($command eq 'start task'){
281
                    my $response = $kernel->call($task_id, "on_task_init");
282
                    $output->{action} = $response;
283
                }
284
285
                if ($!){
286
                    $output->{action} = "error";
287
                    $output->{error_message} = $!;
288
                }
289
290
                #FIXME: What do we actually want to send back to the client?
291
                my $server_output = to_json($output);
292
                $client->put($server_output);
293
                return;
294
295
            } elsif ($command eq 'list tasks'){
296
297
                #Get tasks from listener (ie self)
298
                my $tasks = $kernel->call($server_id, "got_list_tasks");
299
300
                #Prepare output for client
301
                my $server_output = to_json({tasks => $tasks}, {pretty => 1});
302
303
                #Send output to client
304
                $client->put($server_output);
305
                return;
306
307
            } elsif ($command eq 'shutdown'){
308
                $kernel->post($server_id, "graceful_shutdown");
309
                my $server_output = to_json({action => 'shutting down'});
310
                $client->put($server_output);
311
                return;
312
            } else {
313
                $self->log("The message contained an invalid command!");
314
                $client->put("Sorry! That is an invalid command!");
315
                return;
316
            }
317
        } else {
318
            $self->log("The message was missing a command!");
319
        }
320
    } else {
321
        $self->log("The message was malformed!");
322
    }
323
    $client->put("Sorry! That is an invalid message!");
324
    return;
325
}
326
327
1;
(-)a/Koha/Icarus/Task.pm (+302 lines)
Line 0 Link Here
1
package Koha::Icarus::Task;
2
3
use Modern::Perl;
4
use parent 'Koha::Icarus::Base';
5
6
use POE qw(Wheel::Run);
7
use DateTime;
8
use DateTime::Format::Strptime;
9
use JSON;
10
use Module::Load::Conditional qw/can_load/;
11
12
my $datetime_pattern = DateTime::Format::Strptime->new(
13
    pattern   => '%F %T',
14
    time_zone => 'local',
15
);
16
my $epoch_pattern = DateTime::Format::Strptime->new(
17
    pattern   => '%s',
18
);
19
20
sub new {
21
    my ($class, $args) = @_;
22
    $args = {} unless defined $args;
23
    $args->{_component} = "task";
24
    $args->{_id} = "undefined";
25
    return bless ($args, $class);
26
}
27
28
#NOTE: "spawn" inspired by http://poe.perl.org/?POE_Cookbook/Object_Methods
29
sub spawn {
30
    my ($class, $args) = @_;
31
    my $self = $class->new($args);
32
    my $task_session = POE::Session->create(
33
        object_states => [
34
            $self => {
35
                _start => "on_task_create",
36
                 "got_child_stdout" => "on_child_stdout",
37
                 "got_child_stderr" => "on_child_stderr",
38
                 "got_child_close"  => "on_child_close",
39
                 "got_child_signal" => "on_child_signal",
40
                 "got_terminal_signal" => "on_terminal_signal",
41
                 "child_process_success" => "child_process_success",
42
                 "got_task_stop" => "on_task_stop",
43
                 "on_task_init" => "on_task_init",
44
                 "on_task_start" => "on_task_start",
45
            },
46
        ],
47
    );
48
    return $task_session;
49
}
50
51
sub on_task_create {
52
    my ($self, $session, $kernel, $heap) = @_[OBJECT, SESSION, KERNEL, HEAP];
53
54
    #Trap terminal signals so that the task can stop gracefully.
55
    $kernel->sig(INT => "got_terminal_signal");
56
    $kernel->sig(TERM => "got_terminal_signal");
57
58
    my $task_id = $session->ID;
59
    if ($task_id){
60
        #Tell the kernel that this task is waiting for an external action (ie keepalive counter)
61
        $kernel->refcount_increment($task_id,"waiting task");
62
        $self->{_id} = $task_id; #Set internal id for logging purposes
63
    }
64
65
    my $server_id = $self->{server_id};
66
    if ($server_id){
67
        $heap->{server_id} = $server_id;
68
    }
69
70
    my $task = undef;
71
    my $message = $self->{message};
72
    if ($message){
73
        $task = $message->{task};
74
        if ($task){
75
            $task->{status} = 'new';
76
            $heap->{task} = $task;
77
        }
78
    }
79
    return $task; #This return value is used by the parent POE session's _child handler
80
}
81
82
#This sub is just to start it now, or set it to start in the future... if the time is now or in the past, it starts now... if it's in the future, it starts in the future...
83
sub on_task_init {
84
    my ($self, $session, $kernel, $heap) = @_[OBJECT, SESSION, KERNEL, HEAP];
85
    my $response = 'pending';
86
    my $task = $heap->{task};
87
    my $status = $task->{status};
88
    if ($status){
89
        if ($status eq 'started'){
90
            $response = 'already started';
91
        } elsif ($status eq 'pending'){
92
            $response = 'already pending';
93
        } else {
94
            $task->{status} = 'pending';
95
96
            my $start = $task->{start};
97
            if ( my $dt = $datetime_pattern->parse_datetime($start) ){
98
                $start = $dt->epoch;
99
            } elsif ( $epoch_pattern->parse_datetime($start) ){
100
                #No change required
101
            } else {
102
                #If we don't match the datetime_pattern or epoch_pattern, then we start right now.
103
                $start = time(); #time() returns a UNIX epoch time value
104
            }
105
106
            $self->log("Start task at $start");
107
            #NOTE: $start must be in UNIX epoch time (ie number of seconds that have elapsed since 00:00:00 UTC Thursday 1 January 1970)
108
            $kernel->alarm("on_task_start",$start);
109
        }
110
    }
111
    return $response;
112
}
113
114
sub on_task_start {
115
    my ($self, $session, $kernel, $heap) = @_[OBJECT, SESSION, KERNEL, HEAP];
116
    my $task = $heap->{task};
117
    $task->{status} = 'started';
118
119
    if (my $repeat_interval = $task->{repeat_interval}){
120
        #NOTE: Reset the start time with a human readable timestamp
121
        my $dt = DateTime->now( time_zone => 'local', );
122
        $dt->add( seconds => $repeat_interval );
123
        $task->{start} = $dt->strftime("%F %T");
124
    }
125
    #FIXME: You need to impose child process limits here! How many child processes are allowed to be running at any given time? Well, you can only have one child process per task...
126
    #so it's really more of a limit on the number of tasks... you probably need to have an internal task queue... that's easy enough though.
127
    my $child = POE::Wheel::Run->new(
128
        ProgramArgs => [ $task, ],
129
        Program => sub {
130
            my ($task) = @_;
131
132
            #Perform some last minute POE calls before running the task module plugin
133
            my $session = $poe_kernel->get_active_session();
134
            if ($session){
135
                my $heap = $session->get_heap();
136
                $poe_kernel->call($heap->{server_id},"set_verbosity",0); #This turns off the server logging in this forked process, so the following call() doesn't mess up our logs
137
                $poe_kernel->call($heap->{server_id},"shutdown"); #Shutdown the socket listener on the child process, so there's zero chance of writing to or reading from the socket in the child process
138
            }
139
140
            #NOTE: I don't know if this really needs to be run, but it shouldn't hurt.
141
            $poe_kernel->stop();
142
143
            #Try to load the task type module.
144
            my $task_type = $task->{type};
145
            if ( can_load ( modules => { $task_type => undef, }, ) ){
146
                #Create the object
147
                my $task_object = $task_type->new({task => $task});
148
                if ($task_object){
149
                    #Synchronous action: run the task module
150
                    $task_object->run;
151
                }
152
            } else {
153
                die "Couldn't load module $task_type: $Module::Load::Conditional::ERROR"
154
            }
155
        },
156
        StdoutEvent  => "got_child_stdout",
157
        StderrEvent  => "got_child_stderr",
158
        CloseEvent   => "got_child_close",
159
        NoSetPgrp => 1, #Keep child processes in same group as parent. This is especially useful when using Ctrl+C to kill the whole group.
160
    );
161
162
    $kernel->sig_child($child->PID, "got_child_signal");
163
    # Wheel events include the wheel's ID.
164
    $_[HEAP]{children_by_wid}{$child->ID} = $child;
165
    # Signal events include the process ID.
166
    $_[HEAP]{children_by_pid}{$child->PID} = $child;
167
168
    $self->log("child pid ".$child->PID." started as wheel ".$child->ID);
169
}
170
171
sub on_task_stop {
172
    my ($self, $session, $kernel, $heap) = @_[OBJECT, SESSION, KERNEL, HEAP];
173
    my $task = $heap->{task};
174
    $task->{status} = 'stopping';
175
    my $task_id = $session->ID;
176
    my $server_id = $heap->{server_id};
177
178
    if ($heap->{stopping}){
179
        $self->log("Task is already in the process of stopping...");
180
181
    } else {
182
        $self->log("Trying to stop task.");
183
184
        #Mark this task as stopping
185
        $heap->{stopping} = 1;
186
187
        #Stop the task from spawning new jobs
188
        $kernel->alarm("on_task_start");
189
190
        my $children_by_pid = $heap->{children_by_pid};
191
        if ($children_by_pid && %$children_by_pid){
192
193
            $self->log("Child processes in progres...");
194
            my $child_processes = $heap->{children_by_pid};
195
            foreach my $child_pid (keys %$child_processes){
196
                my $child = $child_processes->{$child_pid};
197
                $self->log("Telling child pid $child_pid to stop");
198
                $child->put("quit");
199
                #TODO: Perhaps it would be worthwhile having a kill switch too?
200
                # my $rv = $child->kill("TERM");
201
            }
202
        }
203
        $self->log("Removing task keepalive.");
204
        $kernel->refcount_decrement($task_id,"waiting task");
205
    }
206
}
207
208
sub on_terminal_signal {
209
    my ($self, $signal,$session,$kernel) = @_[OBJECT, ARG0,SESSION,KERNEL];
210
    $self->log("Trapped SIGNAL: $signal.");
211
    #Gracefully stop the task
212
    $kernel->call($session, "got_task_stop");
213
}
214
215
sub child_process_success {
216
    my ($self, $heap,$session,$kernel) = @_[OBJECT, HEAP,SESSION,KERNEL];
217
    my $task = $heap->{task};
218
    if (my $repeat_interval = $task->{repeat_interval}){
219
        if ($heap->{stopping}){
220
            $self->log("Will skip repeating the task, as task is stopping.");
221
        } else {
222
            $self->log("Will repeat the task");
223
            $task->{status} = "restarting";
224
            $kernel->yield("on_task_init");
225
        }
226
    } else {
227
        $self->log("I'm going to stop this task");
228
        $kernel->yield("got_task_stop");
229
    }
230
}
231
232
#############################################################
233
#                                                           #
234
#      Methods for communicating with child processes       #
235
#                                                           #
236
#############################################################
237
# Originally inspired by the POE::Wheel::Run perldoc example
238
239
# Wheel event, including the wheel's ID
240
sub on_child_stdout {
241
    my ($self, $stdout_line, $wheel_id, $session) = @_[OBJECT, ARG0, ARG1, SESSION];
242
    my $child = $_[HEAP]{children_by_wid}{$wheel_id};
243
    #NOTE: Log everything child process sends to STDOUT
244
    $self->log("[pid ".$child->PID."] STDOUT: $stdout_line");
245
246
    #If the child outputs a line to STDOUT which starts with UPDATE_PARAMS=, we capture the data,
247
    #and update the task params.
248
    if ($stdout_line =~ /^UPDATE_PARAMS=(.*)$/){
249
        my $json_string = $1;
250
        my $json = from_json($json_string);
251
        my $task = $_[HEAP]->{task};
252
        my $params = $task->{params};
253
        foreach my $key (%$json){
254
            if (defined $params->{$key}){
255
                #FIXME: Don't just overwrite? Only update differences?
256
                $params->{$key} = $json->{$key};
257
            }
258
        }
259
        $_[HEAP]->{task} = $task;
260
    }
261
}
262
263
# Wheel event, including the wheel's ID.
264
sub on_child_stderr {
265
    my ($self, $stderr_line, $wheel_id, $session) = @_[OBJECT, ARG0, ARG1,SESSION];
266
    my $child = $_[HEAP]{children_by_wid}{$wheel_id};
267
    #NOTE: Log everything child process sends to STDERR
268
    $self->log("[pid ".$child->PID."] STDERR: $stderr_line");
269
}
270
271
# Wheel event, including the wheel's ID.
272
sub on_child_close {
273
    my ($self, $wheel_id,$session,$kernel) = @_[OBJECT, ARG0,SESSION,KERNEL];
274
275
    my $child = delete $_[HEAP]{children_by_wid}{$wheel_id};
276
277
    # May have been reaped by on_child_signal().
278
    unless (defined $child) {
279
        $self->log("[wid $wheel_id] closed all pipes.");
280
        return;
281
    }
282
    $self->log("[pid ".$child->PID."] closed all pipes.");
283
    delete $_[HEAP]{children_by_pid}{$child->PID};
284
}
285
286
sub on_child_signal {
287
    my ($self, $heap,$kernel,$pid,$exit_code,$session) = @_[OBJECT, HEAP,KERNEL,ARG1,ARG2,SESSION];
288
289
    #If the child's exit code is 0, handle this successful exit status
290
    if ($exit_code == 0){
291
        $kernel->yield("child_process_success");
292
    }
293
    $self->log("pid $pid exited with status $exit_code.");
294
    my $child = delete $_[HEAP]{children_by_pid}{$pid};
295
296
    # May have been reaped by on_child_close().
297
    return unless defined $child;
298
299
    delete $_[HEAP]{children_by_wid}{$child->ID};
300
}
301
302
1;
(-)a/Koha/Icarus/Task/Base.pm (+24 lines)
Line 0 Link Here
1
package Koha::Icarus::Task::Base;
2
3
use Modern::Perl;
4
use IO::Select;
5
6
sub new {
7
    my ($class, $args) = @_;
8
    $args = {} unless defined $args;
9
    return bless ($args, $class);
10
}
11
12
sub listen_for_instruction {
13
    my ($self) = @_;
14
    my $select = $self->{_select} ||= IO::Select->new(\*STDIN);
15
    if (my @ready_FHs  = $select->can_read(0) ){
16
        foreach my $FH (@ready_FHs){
17
            my $line = $FH->getline();
18
            chomp($line);
19
            return $line;
20
        }
21
    }
22
}
23
24
1;
(-)a/Koha/Icarus/Task/Dequeue/OAIPMH/Biblio.pm (+111 lines)
Line 0 Link Here
1
package Koha::Icarus::Task::Dequeue::OAIPMH::Biblio;
2
3
use Modern::Perl;
4
use parent 'Koha::Icarus::Task::Base';
5
use URI;
6
use LWP::UserAgent;
7
use HTTP::Status qw(:constants);
8
9
my $ua = LWP::UserAgent->new;
10
11
#FIXME: If we store the cookie jar on disk, we can prevent unnecessary HTTP requests...
12
$ua->cookie_jar({});
13
14
sub new {
15
    my ($class, $args) = @_;
16
    $args = {} unless defined $args;
17
    return bless ($args, $class);
18
}
19
20
sub run {
21
    my ( $self ) = @_;
22
23
    my $task = $self->{task};
24
25
    #DEBUGGING/FIXME: Remove these lines
26
    use Data::Dumper;
27
    warn Dumper($task);
28
29
    my $params = $task->{params};
30
31
    my $auth_uri = $params->{auth_uri};
32
    my $target_uri = $params->{target_uri};
33
34
    my $queue = $params->{queue};
35
    my $queue_uri = URI->new($queue);
36
37
    if ($queue_uri->scheme eq 'file'){
38
39
        my $path = $queue_uri->path;
40
        opendir(my $dh, $path);
41
        my @files = sort readdir($dh);
42
        foreach my $file (@files){
43
            #NOTE: This is plugin specific as the plugins define when they stop to listen for instructions...
44
            my $instruction = $self->listen_for_instruction();
45
            if ($instruction eq 'quit'){
46
                warn "I was asked to quit!";
47
                return;
48
            }
49
50
            next if $file =~ /^\.+$/;
51
            my $filepath = "$path/$file";
52
            if ( -d $filepath ){
53
                warn "Directory: $file";
54
            } elsif ( -e $filepath ){
55
                warn "File: $file";
56
57
                #Slurp mode
58
                local $/;
59
                #TODO: Check flock on $filepath first
60
                open( my $fh, '<', $filepath );
61
                my $data   = <$fh>;
62
63
                #TODO: Improve this section...
64
                #Send to Koha API... (we could speed this up using Asynchronous HTTP requests with AnyEvent::HTTP...)
65
                my $resp = $ua->post( $target_uri,
66
                              {'nomatch_action' => $params->{nomatch_action},
67
                               'overlay_action' => $params->{overlay_action},
68
                               'match'          => $params->{match},
69
                               'import_mode'    => $params->{import_mode},
70
                               'framework'      => $params->{framework},
71
                               'item_action'    => $params->{item_action},
72
                               'filter'         => $params->{filter},
73
                               'xml'            => $data});
74
75
                my $status = $resp->code;
76
                #FIXME: DEBUGGING
77
                warn $status;
78
                warn $resp->code;
79
                warn $resp->decoded_content;
80
81
                if ($status == HTTP_UNAUTHORIZED || $status == HTTP_FORBIDDEN) {
82
                    my $user = $params->{auth_username};
83
                    my $password = $params->{auth_password};
84
                    $resp = $ua->post( $auth_uri, { userid => $user, password => $password } );
85
                    #FIXME: DEBUGGING
86
                    warn $resp->code;
87
                    warn $resp->decoded_content;
88
89
                    $resp = $ua->post( $target_uri,
90
                                          {'nomatch_action' => $params->{nomatch_action},
91
                                           'overlay_action' => $params->{overlay_action},
92
                                           'match'          => $params->{match},
93
                                           'import_mode'    => $params->{import_mode},
94
                                           'framework'      => $params->{framework},
95
                                           'item_action'    => $params->{item_action},
96
                                           'filter'         => $params->{filter},
97
                                           'xml'            => $data})
98
                      if $resp->is_success;
99
                    #FIXME: DEBUGGING
100
                    warn $resp->code;
101
                    warn $resp->decoded_content;
102
                }
103
                if ($resp->code == 200){
104
                    unlink $filepath;
105
                }
106
            }
107
        }
108
    }
109
}
110
111
1;
(-)a/Koha/Icarus/Task/Enqueue/OAIPMH/Biblio.pm (+311 lines)
Line 0 Link Here
1
package Koha::Icarus::Task::Enqueue::OAIPMH::Biblio;
2
3
use Modern::Perl;
4
use parent 'Koha::Icarus::Task::Base';
5
6
use DateTime;
7
use DateTime::Format::Strptime;
8
use HTTP::OAI;
9
use File::Path qw(make_path);
10
use Digest::MD5;
11
use JSON;
12
use URI;
13
14
my $strp = DateTime::Format::Strptime->new(
15
        pattern   => '%Y%m%dT%H%M%S.%NZ',
16
);
17
18
my $oai_second_granularity = DateTime::Format::Strptime->new(
19
        pattern   => '%Y-%m-%dT%H:%M:%SZ',
20
);
21
22
my $oai_day_granularity = DateTime::Format::Strptime->new(
23
        pattern   => '%Y-%m-%d',
24
);
25
26
sub validate_parameter_names {
27
28
}
29
sub validate_repeat_interval {
30
    my ($self,$repeat_interval) = @_;
31
    if ($repeat_interval && $repeat_interval =~ /^\d*$/){
32
        return undef;
33
    }
34
    $self->{invalid_data}++;
35
    return { not_numeric => 1, };
36
}
37
38
sub validate_url {
39
    my ($self,$url) = @_;
40
    my $response = {};
41
    if (my $url_obj = URI->new($url)){
42
        if ($url_obj->scheme ne "http"){
43
            $response->{not_http} = 1;
44
            $self->{invalid_data}++;
45
        }
46
        if ( ! $url_obj->path){
47
            $response->{no_path} = 1;
48
            $self->{invalid_data}++;
49
        }
50
    } else {
51
        $response->{not_a_url} = 1;
52
        $self->{invalid_data}++;
53
    }
54
55
    return $response;
56
}
57
58
sub validate {
59
    my ($self, $args) = @_;
60
    #Reset the invalid data counter...
61
    $self->{invalid_data} = 0;
62
    my $errors = { };
63
    my $task = $self->{task};
64
    my $tests = $args->{tests};
65
    if ($task){
66
        if ($tests && $tests eq 'all'){
67
            #warn "PARAMS = ".$task->{params};
68
        }
69
    }
70
    my $params = $task->{params};
71
72
    #validate_start_time
73
    $errors->{"repeat_interval"} = $self->validate_repeat_interval($task->{repeat_interval});
74
75
    $errors->{"url"} = $self->validate_url($params->{url});
76
77
    #NOTE: You don't need to validate these 3 HTTP Basic Auth parameters
78
    #validate_username
79
    #validate_password
80
    #validate_realm
81
82
    #OAI-PMH parameters
83
    #validate_verb
84
    #validate_sets
85
    #validate_marcxml
86
    #validate_from
87
    #validate_until
88
89
    #Download parameters
90
    #validate_queue
91
92
    return $errors;
93
}
94
95
sub new {
96
    my ($class, $args) = @_;
97
    $args = {} unless defined $args;
98
    $args->{invalid_data} = 0;
99
    return bless ($args, $class);
100
}
101
102
sub validate_queue {
103
    my ( $self ) = @_;
104
    my $task = $self->{task};
105
    if (my $queue = $task->{params}->{queue}){
106
107
        my $queue_uri = URI->new($queue);
108
        #TODO: In theory, you could even use a DBI DSN like DBI:mysql:database=koha;host=koha.db;port=3306.
109
        #Then you could provide the table, username, and password in the params as well...
110
111
        #NOTE: If the queue directory doesn't exist on the filesystem, we try to make it and change to it.
112
        if ($queue_uri->scheme eq 'file'){
113
            my $filepath = $queue_uri->file;
114
            if ( ! -d $filepath ){
115
                make_path($filepath,{ mode => 0755 });
116
            }
117
            if ( -d $filepath ){
118
                chdir $filepath or die "$!";
119
            }
120
        }
121
122
    }
123
}
124
125
sub run {
126
    my ( $self ) = @_;
127
    $self->validate_queue;
128
129
    my $task = $self->{task};
130
131
    #DEBUGGING/FIXME: Remove these lines
132
    use Data::Dumper;
133
    warn Dumper($task);
134
135
    my $params = $task->{params};
136
137
    my $now = DateTime->now(); #This is in UTC time, which is required by the OAI-PMH protocol.
138
    if ( $oai_second_granularity->parse_datetime($params->{from}) ){
139
        $now->set_formatter($oai_second_granularity);
140
    } else {
141
        $now->set_formatter($oai_day_granularity);
142
    }
143
144
    $params->{until}  = "$now" if $task->{repeat_interval};
145
146
    $self->{digester} = Digest::MD5->new();
147
    $self->create_harvester;
148
    my $sets = $self->prepare_sets;
149
150
    #Send a OAI-PMH request for each set
151
    foreach my $set (@{$sets}){
152
        my $response = $self->send_request({set => $set});
153
        $self->handle_response({ response => $response, set => $set,});
154
    }
155
156
    #FIXME: Do you want to update the task only when the task is finished, or
157
    #also after each resumption?
158
    #Update the task params in Icarus after the task is finished...
159
    #TODO: This really does make it seem like you should be handling the repeat_interval within the child process rather than the parent...
160
    if ($task->{repeat_interval}){
161
        $params->{from} = "$now";
162
        $params->{until} = "";
163
        my $json_update = to_json($params);
164
        say STDOUT "UPDATE_PARAMS=$json_update";
165
    }
166
167
}
168
169
#FIXME: I wonder if it would be faster to send your own HTTP requests and not use HTTP::OAI...
170
sub send_request {
171
    my ( $self, $args ) = @_;
172
173
    #NOTE: This is plugin specific as the plugins define when they stop to listen for instructions...
174
    #NOTE: Before sending a new request, check if Icarus has already asked us to quit.
175
    my $instruction = $self->listen_for_instruction();
176
    if ($instruction eq 'quit'){
177
        warn "I was asked to quit!";
178
        return;
179
    }
180
181
    my $set = $args->{set};
182
    my $resumptionToken = $args->{resumptionToken};
183
184
    my $response;
185
    my $task_params = $self->{task}->{params};
186
187
    my $harvester = $self->{harvester};
188
    my $verb = $task_params->{verb};
189
    if ($verb eq 'GetRecord'){
190
        $response = $harvester->GetRecord(
191
            metadataPrefix => $task_params->{metadataPrefix},
192
            identifier => $task_params->{identifier},
193
         );
194
    } elsif ($verb eq 'ListRecords'){
195
        $response = $harvester->ListRecords(
196
            metadataPrefix => $task_params->{metadataPrefix},
197
            from => $task_params->{from},
198
            until => $task_params->{until},
199
            set => $set,
200
            resumptionToken => $resumptionToken,
201
        );
202
    }
203
    return $response;
204
}
205
206
sub create_harvester {
207
    my ( $self ) = @_;
208
    my $task_params = $self->{task}->{params};
209
210
    #FIXME: DEBUGGING
211
    #use HTTP::OAI::Debug qw(+);
212
213
    #Create HTTP::OAI::Harvester object
214
    my $harvester = new HTTP::OAI::Harvester( baseURL => $task_params->{url} );
215
    if ($harvester){
216
        $harvester->timeout(5); #NOTE: the default timeout is 180
217
        #Set HTTP Basic Authentication Credentials
218
        my $uri = URI->new($task_params->{url});
219
        my $host = $uri->host;
220
        my $port = $uri->port;
221
        $harvester->credentials($host.":".$port, $task_params->{realm}, $task_params->{username}, $task_params->{password});
222
    }
223
    $self->{harvester} = $harvester;
224
}
225
226
sub prepare_sets {
227
    my ( $self ) = @_;
228
    my $task_params = $self->{task}->{params};
229
    my @sets = split(/\|/, $task_params->{sets});
230
    #If no sets are defined, create a null element to force the foreach loop to run once
231
    if (!@sets){
232
        push(@sets,undef)
233
    }
234
    return \@sets;
235
}
236
237
sub handle_response {
238
    my ( $self, $args ) = @_;
239
    my $params = $self->{task}->{params};
240
    my $response = $args->{response};
241
    my $set = $args->{set};
242
    if ($response){
243
        #NOTE: We have options at this point
244
        #Option 1: Use $response->toDOM() to handle the XML response as a single document
245
        #Option 2: Use $response->next() to handle each record individually. You would need to create a new document using $rec->header->dom() and $rec->metadata->dom() anyway.
246
247
        #NOTE: I wonder which option would be the fastest. For now, we're going with Option 1:
248
        my $dom = $response->toDOM;
249
        my $root = $dom->documentElement;
250
251
        #FIXME: Provide these as arguments so you're not re-creating them for each response
252
        my $xpc = XML::LibXML::XPathContext->new();
253
        $xpc->registerNs('oai','http://www.openarchives.org/OAI/2.0/');
254
        my $xpath = XML::LibXML::XPathExpression->new("(oai:GetRecord|oai:ListRecords)/oai:record");
255
256
257
        my @records = $xpc->findnodes($xpath,$root);
258
        my $now_pretty = DateTime->now();
259
260
        $now_pretty->set_formatter($strp);
261
        warn "Downloaded ".scalar @records." records at $now_pretty";
262
        foreach my $record (@records) {
263
264
            #FIXME: This is where you could put a filter to prevent certain records from being saved...
265
266
            #Create a new XML document from the XML fragment
267
            my $document = XML::LibXML::Document->new( "1.0", "UTF-8" );
268
            $document->setDocumentElement($record);
269
            my $record_string = $document->toString;
270
271
            #NOTE: We have options at this point.
272
            #Option 1: Write documents to disk, and have a separate importer upload the documents
273
            #Option 2: Use AnyEvent::HTTP or POE::Component::Client::HTTP to send to a HTTP API asynchronously
274
            #Option 3: Write records to a database, and have a separate importer upload the documents
275
            #Option 4: Shared memory, although that seems fragile if nothing else
276
            #Option 5: Write the records to a socket/pipe
277
278
            #NOTE: I wonder which option would be the fastest. For now, we're going to go with Option 1:
279
            $self->{digester}->add($record_string);
280
            my $digest = $self->{digester}->hexdigest;
281
            #FIXME: If a record appears more than once during the download signified by $now, you'll
282
            #overwrite the former with the latter. While this acts as a sort of heavy-handed de-duplication,
283
            #you need to take into account the importer daemon...
284
285
            require Time::HiRes;
286
            my $epoch = Time::HiRes::time();
287
            my $now = DateTime->from_epoch(epoch => $epoch);
288
            $now->set_formatter($strp);
289
290
            my $filename = "$now-$digest";
291
            #NOTE: Here is where we write the XML out to disk
292
            my $state = $document->toFile($filename);
293
        }
294
295
296
        #NOTE: Check if object has method due to bug in HTTP::OAI which causes fatal error on $response->resumptionToken if no real response is fetched from the OAI-PMH server
297
        if ($response->can("resumptionToken")){
298
            my $resumption_token = $response->resumptionToken->resumptionToken if $response->resumptionToken && $response->resumptionToken->resumptionToken;
299
            if ($resumption_token){
300
                warn "Resumption Token = $resumption_token";
301
                my $resumed_response = $self->send_request({set => $set, resumptionToken => $resumption_token});
302
                $self->handle_response({ response => $resumed_response, set => $set,});
303
            }
304
        }
305
306
        #In theory $response->resume(resumptionToken => resumptionToken) should kick off another response...
307
        warn $response->message if $response->is_error;
308
    }
309
}
310
311
1;
(-)a/Koha/SavedTask.pm (+86 lines)
Line 0 Link Here
1
package Koha::SavedTask;
2
3
# Copyright Prosentient Systems 2016
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use Carp;
23
24
use JSON;
25
26
use base qw(Koha::Object);
27
28
29
30
=head1 NAME
31
32
Koha::SavedTask -
33
34
=head1 API
35
36
=head2 Class Methods
37
38
=cut
39
40
41
42
=head3 _type
43
44
=cut
45
46
sub _type {
47
    return 'SavedTask';
48
}
49
50
sub params_as_perl {
51
    my ($self) = @_;
52
    my $perl = from_json($self->params);
53
    return $perl;
54
}
55
56
sub serialize {
57
    my ($self,$args) = @_;
58
    my $for = $args->{for};
59
    my $type = $args->{type};
60
    if ($for eq 'icarus'){
61
        my $json_params = $self->params;
62
        my $perl_params = from_json($json_params);
63
64
        my $icarus_task = {
65
            type => $self->task_type,
66
            start => $self->start_time,
67
            repeat_interval => $self->repeat_interval,
68
            params => $perl_params,
69
        };
70
        if ($type eq 'perl'){
71
            return  $icarus_task;
72
        } elsif ($type eq 'json'){
73
            my $json = to_json($icarus_task);
74
            return $json;
75
        }
76
    }
77
    return undef;
78
}
79
80
=head1 AUTHOR
81
82
David Cook <dcook@prosentient.com.au>
83
84
=cut
85
86
1;
(-)a/Koha/SavedTasks.pm (+62 lines)
Line 0 Link Here
1
package Koha::SavedTasks;
2
3
# Copyright Prosentient Systems 2016
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use Carp;
23
24
use Koha::Database;
25
26
use Koha::SavedTask;
27
28
use base qw(Koha::Objects);
29
30
=head1 NAME
31
32
Koha::SavedTasks -
33
34
=head1 API
35
36
=head2 Class Methods
37
38
=cut
39
40
=head3 _type
41
42
=cut
43
44
sub _type {
45
    return 'SavedTask';
46
}
47
48
=head3 object_class
49
50
=cut
51
52
sub object_class {
53
    return 'Koha::SavedTask';
54
}
55
56
=head1 AUTHOR
57
58
David Cook <dcook@prosentient.com.au>
59
60
=cut
61
62
1;
(-)a/Koha/Schema/Result/SavedTask.pm (+98 lines)
Line 0 Link Here
1
use utf8;
2
package Koha::Schema::Result::SavedTask;
3
4
# Created by DBIx::Class::Schema::Loader
5
# DO NOT MODIFY THE FIRST PART OF THIS FILE
6
7
=head1 NAME
8
9
Koha::Schema::Result::SavedTask
10
11
=cut
12
13
use strict;
14
use warnings;
15
16
use base 'DBIx::Class::Core';
17
18
=head1 TABLE: C<saved_tasks>
19
20
=cut
21
22
__PACKAGE__->table("saved_tasks");
23
24
=head1 ACCESSORS
25
26
=head2 task_id
27
28
  data_type: 'integer'
29
  extra: {unsigned => 1}
30
  is_auto_increment: 1
31
  is_nullable: 0
32
33
=head2 start_time
34
35
  data_type: 'datetime'
36
  datetime_undef_if_invalid: 1
37
  is_nullable: 0
38
39
=head2 repeat_interval
40
41
  data_type: 'integer'
42
  extra: {unsigned => 1}
43
  is_nullable: 0
44
45
=head2 task_type
46
47
  data_type: 'varchar'
48
  is_nullable: 0
49
  size: 255
50
51
=head2 params
52
53
  data_type: 'text'
54
  is_nullable: 0
55
56
=cut
57
58
__PACKAGE__->add_columns(
59
  "task_id",
60
  {
61
    data_type => "integer",
62
    extra => { unsigned => 1 },
63
    is_auto_increment => 1,
64
    is_nullable => 0,
65
  },
66
  "start_time",
67
  {
68
    data_type => "datetime",
69
    datetime_undef_if_invalid => 1,
70
    is_nullable => 0,
71
  },
72
  "repeat_interval",
73
  { data_type => "integer", extra => { unsigned => 1 }, is_nullable => 0 },
74
  "task_type",
75
  { data_type => "varchar", is_nullable => 0, size => 255 },
76
  "params",
77
  { data_type => "text", is_nullable => 0 },
78
);
79
80
=head1 PRIMARY KEY
81
82
=over 4
83
84
=item * L</task_id>
85
86
=back
87
88
=cut
89
90
__PACKAGE__->set_primary_key("task_id");
91
92
93
# Created by DBIx::Class::Schema::Loader v0.07042 @ 2016-01-27 13:35:22
94
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gnoi7I9fiXM3IfDysMTm+A
95
96
97
# You can replace this text with custom code or comments, and it will be preserved on regeneration
98
1;
(-)a/admin/saved_tasks.pl (+338 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright Prosentient Systems 2016
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
=head1 NAME
21
22
saved_tasks.pl
23
24
=head1 DESCRIPTION
25
26
Admin page to manage saved tasks
27
28
=cut
29
30
use Modern::Perl;
31
use CGI qw ( -utf8 );
32
use C4::Auth;
33
use C4::Output;
34
use C4::Context;
35
36
use Koha::SavedTasks;
37
use Koha::Icarus;
38
use Module::Load::Conditional qw/can_load check_install/;
39
use JSON;
40
41
my $input = new CGI;
42
my ($template, $loggedinuser, $cookie, $flags) = get_template_and_user( {
43
    template_name   => 'admin/saved_tasks.tt',
44
    query           => $input,
45
    type            => 'intranet',
46
    authnotrequired => 0,
47
    flagsrequired   => { 'parameters' => 'parameters_remaining_permissions' },
48
} );
49
50
my $filename = "saved_tasks.pl";
51
$template->param(
52
    filename => $filename,
53
);
54
55
my $context = C4::Context->new();
56
57
58
my $task_server = $input->param("task_server") // "icarus";
59
60
61
my $socket_uri = $context->{"icarus"}->{"socket"};
62
63
my @available_plugins = ();
64
my $task_plugins = $context->{"icarus"}->{"task_plugin"};
65
if ($task_plugins && ref $task_plugins eq 'ARRAY'){
66
    #FIXME: This should probably be a module method... validation that a plugin is installed...
67
    foreach my $task_plugin (@$task_plugins){
68
        #Check that plugin module is installed
69
        if ( check_install( module => $task_plugin ) ){
70
                push(@available_plugins,$task_plugin);
71
        }
72
    }
73
}
74
75
$template->param(
76
    available_plugins => \@available_plugins,
77
);
78
79
#Server action and task id
80
my $server_action = $input->param("server_action");
81
my $server_task_id = $input->param('server_task_id');
82
83
#Saved task op
84
my $op = $input->param('op');
85
my $step = $input->param('step');
86
87
#Saved task id
88
my $saved_task_id = $input->param('saved_task_id');
89
90
91
#Create Koha-Icarus interface object
92
my $icarus = Koha::Icarus->new({ socket_uri => $socket_uri });
93
my $daemon_status = "";
94
95
#Connect to Icarus
96
if ( $icarus->connect() ){
97
    $daemon_status = "online";
98
    if ($server_action){
99
        if ($server_action eq 'shutdown'){
100
            my $response = $icarus->shutdown;
101
            if ( $response && (my $action = $response->{action}) ){
102
                $daemon_status = $action;
103
            }
104
        } elsif ($server_action eq 'start' && $server_task_id){
105
            my $response = $icarus->start_task({ task_id => $server_task_id });
106
            $template->param(
107
                task_response => $response,
108
            );
109
        } elsif ($server_action eq 'remove' && $server_task_id){
110
            my $response = $icarus->remove_task({ task_id => $server_task_id });
111
            $template->param(
112
                task_response => $response,
113
            );
114
        }
115
    }
116
} else {
117
    $daemon_status = $!;
118
}
119
$template->param(
120
    daemon_status => $daemon_status,
121
);
122
123
124
125
my $params = $input->param("params");
126
127
#NOTE: Parse the parameters manually, so that you can "name[]" style of parameter, which we use in the special plugin templates...
128
my $saved_params = {};
129
#Fetch the names of all the parameters passed to your script
130
my @parameter_names = $input->param;
131
#Iterate through these parameter names and look for "params[]"
132
foreach my $parameter_name (@parameter_names){
133
    if ($parameter_name =~ /^params\[(.*)\]$/){
134
        #Capture the hash key
135
        my $key = $1;
136
        #Fetch the actual individual value
137
        my $parameter_value = $input->param($parameter_name);
138
        if ($parameter_value){
139
            $saved_params->{$key} = $parameter_value;
140
        }
141
    }
142
}
143
if (%$saved_params){
144
    my $json = to_json($saved_params, { pretty => 1, });
145
    if ($json){
146
        $params = $json;
147
    }
148
}
149
150
my $start_time = $input->param("start_time");
151
my $repeat_interval = $input->param("repeat_interval");
152
my $task_type = $input->param("task_type");
153
if ($task_type){
154
    my $task_template = $task_type;
155
    #Create the template name by stripping the colons out of the task type text
156
    $task_template =~ s/://g;
157
    $template->param(
158
        task_template => "tasks/$task_template.inc",
159
    );
160
}
161
162
163
if ($op){
164
    if ($op eq 'new'){
165
166
    } elsif ($op eq 'create'){
167
168
        #Validate the $task here
169
        if ($step){
170
            if ($step eq "one"){
171
172
                $op = "new";
173
                $template->param(
174
                    step => "two",
175
                    task_type => $task_type,
176
                );
177
            } elsif ($step eq "two"){
178
                my $new_task = Koha::SavedTask->new({
179
                    start_time => $start_time,
180
                    repeat_interval => $repeat_interval,
181
                    task_type => $task_type,
182
                    params => $params,
183
                });
184
185
                #Serialize the data as an Icarus task
186
                my $icarus_task = $new_task->serialize({ for => "icarus", type => "perl", });
187
188
                my $valid = 1;
189
                #Load the plugin module, and create an object instance in order to validate user-entered data
190
                if ( can_load( modules => { $task_type => undef, }, ) ){
191
                    my $plugin = $task_type->new({ task => $icarus_task, });
192
                    if ($plugin->can("validate")){
193
                        my $errors = $plugin->validate({
194
                            "tests" => "all",
195
                        });
196
                        if (%$errors){
197
                            $template->param(
198
                                errors => $errors,
199
                            );
200
                        }
201
                        if ($plugin->{invalid_data} > 0){
202
                            $valid = 0;
203
                        }
204
                    }
205
                }
206
207
                if ($valid){
208
                    $new_task->store();
209
                    $op = "list";
210
                } else {
211
                    $op = "new";
212
                    #Create a Perl data structure from the JSON
213
                    my $editable_params = from_json($params);
214
                    $template->param(
215
                        step => "two",
216
                        task_type => $task_type,
217
                        saved_task => $new_task,
218
                        params => $editable_params,
219
                    );
220
                }
221
            }
222
        }
223
224
    } elsif ($op eq 'edit'){
225
        my $task = Koha::SavedTasks->find($saved_task_id);
226
        if ($task){
227
            #Check if the task's saved task type is actually available...
228
            #FIXME: This should be a Koha::Icarus method...
229
            my $task_type_is_valid = grep { $task->task_type eq $_ } @available_plugins;
230
            $template->param(
231
                task_type_is_valid => $task_type_is_valid,
232
                saved_task => $task,
233
            );
234
        }
235
    } elsif ($op eq 'update'){
236
        if ($step){
237
            my $task = Koha::SavedTasks->find($saved_task_id);
238
            if ($task){
239
                if ($step eq "one"){
240
                    #We've completed step one, which is choosing the task type,
241
                    #so now we're going to populate the form for editing the rest of the values
242
                    $op = "edit";
243
                    #This is the JSON string that we've saved in the database
244
                    my $current_params_string = $task->params;
245
                    my $editable_params = from_json($current_params_string);
246
247
                    $template->param(
248
                        step => "two",
249
                        task_type => $task_type,
250
                        saved_task => $task,
251
                        params => $editable_params,
252
253
                    );
254
                } elsif ($step eq "two"){
255
                    #We've completed step two, so we're storing the data now...
256
                    $task->set({
257
                        start_time => $start_time,
258
                        repeat_interval => $repeat_interval,
259
                        task_type => $task_type,
260
                        params => $params,
261
                    });
262
                    $task->store;
263
                    #FIXME: Validate the $task here...
264
                    if (my $valid = 1){
265
                        $op = "list";
266
                    } else {
267
                        $op = "edit";
268
                        $template->param(
269
                            step => "two",
270
                            task_type => $task_type,
271
                            saved_task => $task,
272
                        );
273
                    }
274
                }
275
            }
276
        }
277
    } elsif ($op eq 'send'){
278
        if ($icarus->connected){
279
            if ($saved_task_id){
280
                #Look up task
281
                my $task = Koha::SavedTasks->find($saved_task_id);
282
                if ($task){
283
                    #Create a task for Icarus, and send it to Icarus
284
                    my $icarus_task = $task->serialize({ for => "icarus", type => "perl", });
285
                    if ($icarus_task){
286
                        $icarus->add_task({ task => $icarus_task, });
287
                        $op = "list";
288
                    }
289
                }
290
            }
291
        }
292
    } elsif ($op eq 'delete'){
293
        my $saved_response = "delete_failure";
294
        if ($saved_task_id){
295
            #Look up task
296
            my $task = Koha::SavedTasks->find($saved_task_id);
297
            if ($task){
298
                if (my $something = $task->delete){
299
                    $saved_response = "delete_success";
300
                }
301
            }
302
        }
303
        $template->param(
304
            saved_response => $saved_response,
305
        );
306
        $op = "list";
307
    } else {
308
        #Don't recognize $op, so fallback to list
309
        $op = "list";
310
    }
311
} else {
312
    #No $op, so fallback to list
313
    $op = "list";
314
}
315
316
if ($op eq 'list'){
317
    #Get active tasks from Icarus
318
    if ($icarus->connected){
319
        my $tasks = $icarus->list_tasks();
320
        if ($tasks && @$tasks){
321
            $template->param(
322
                tasks => $tasks,
323
            );
324
        }
325
    }
326
327
    #Get saved tasks from Koha
328
    my @saved_tasks = Koha::SavedTasks->as_list();
329
    $template->param(
330
        saved_tasks => \@saved_tasks,
331
    );
332
}
333
334
$template->param(
335
    op => $op,
336
);
337
338
output_html_with_http_headers $input, $cookie, $template->output;
(-)a/docs/Icarus/README (+72 lines)
Line 0 Link Here
1
TODO:
2
    - Feature "svc/import_oai"
3
        - ***Process deletions
4
            - Check if there is a status and if the status is "deleted"...
5
            - Create an empty MARCXML record, add the OAI-PMH identifier in as a 024$a with a 024$2 of "uri" ( set 942$n=1 for OpacSuppression, set LDR05=d for deleted status...)
6
            - You'll want to use the C4::Matcher to get the matches... and then try deleting them. If there's a problem, you'll need to note it in the import_oai table somehow...
7
            - NOTE: Deletion support REQUIRES Bug 15541 && Bug 15555 && Bug 15745
8
        - Add a "file_name" which makes it useful... maybe something like saved_task_id:1
9
            - That way, you could have an interface for viewing all records harvested from a certain saved_task...
10
            - You can't really provide a good way of undoing a whole harvest, since harvests are done incrementally every few seconds...
11
                - But for testing purposes, you could make it a bit easier...
12
        - Do something with the status field?
13
14
    - Validation:
15
        "Koha::Icarus::Task::Dequeue::OAIPMH::Biblio":
16
            - Validate HTTP URLs and filepaths...
17
        - Add PLUGIN->validate("parameter_names")
18
        - Add PLUGIN->validate("parameter_values")
19
            - For the downloader, this would validate HTTP && OAI-PMH parameters...
20
21
    - Install/Configuration:
22
        - You should make Makefile.PL prompt them for koha-conf.xml configuration options (max_tasks, log?, pidfile, socket)...
23
        - The task_plugin options could be provided by default...
24
25
26
    - Cleanup:
27
        - Remove any unnecessary logging
28
        - Clean up all the code...
29
30
31
32
33
34
35
36
37
38
39
POSSIBLE IMPROVEMENTS:
40
    - Add default OAI record matching rule
41
        - I thought about adding an atomic update 'bug_10662-Add_oai_record_matching_rule.sql', but adding matching rules seems complex
42
        - Should the field include other fields like 022, 020, 245 rather than just 001 and 024a?
43
    - Add entry to Cleanupdatabase.pl cronjob
44
        - You could remove all import_oai rows older than a certain age?
45
    - Make the "Task type" prettier (and translateable) on saved_tasks.pl.
46
    - Provide more options for the Icarus dashboard
47
    - Add the ability to "edit" and "pause" active Icarus tasks
48
    - Make "Koha::Icarus::Task::Dequeue::OAIPMH::Biblio" use asynchronous HTTP requests to speed up the import
49
    - Add help pages for WEB GUI
50
    - Add documentation to all code...
51
    - Add unit tests
52
53
54
55
DESIGN CHANGES?:
56
    - WEB UI:
57
        - Add `name` to saved_tasks?
58
    - Move "Saved tasks" from Administration to Tools?
59
        - Look at existing bugs for schedulers:
60
            - https://bugs.koha-community.org/bugzilla3/show_bug.cgi?id=14712
61
            - https://bugs.koha-community.org/bugzilla3/show_bug.cgi?id=1993
62
    - Handle datestamp granularity better for OAI-PMH download tasks?
63
    - Change `import_oai` database table?
64
        - Add record_type column?
65
            - The only way you could know the record_type is if you passed it via the task data...
66
            - In the past, I used to store record_type and original_system_field...
67
        - What sort of statuses does import_oai use? Add/update/error/ignore?
68
        - Do I need to store metadata_prefix?
69
    - Misc:
70
        - Instead of using file:///kohawebs/dev/dcook/koha-dev/var/spool/oaipmh, why not use something like file:///tmp/koha-dev/oaipmh? I suppose because you might be able to access someone else's files?
71
72
PROBLEMS:
(-)a/installer/data/mysql/atomicupdate/bug_10662-Build_import_oai_table.sql (+21 lines)
Line 0 Link Here
1
DROP TABLE IF EXISTS import_oai;
2
CREATE TABLE  import_oai (
3
  import_oai_id int(10) unsigned NOT NULL AUTO_INCREMENT,
4
  header_identifier varchar(45) CHARACTER SET utf8 NOT NULL,
5
  header_datestamp datetime NOT NULL,
6
  header_status varchar(45) CHARACTER SET utf8 DEFAULT NULL,
7
  metadata longtext CHARACTER SET utf8 NOT NULL,
8
  last_modified timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
9
  status varchar(45) CHARACTER SET utf8 NOT NULL,
10
  PRIMARY KEY (import_oai_id)
11
) ENGINE=InnoDB AUTO_INCREMENT=297 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
12
13
DROP TABLE IF EXISTS saved_tasks;
14
CREATE TABLE  saved_tasks (
15
  task_id int(10) unsigned NOT NULL AUTO_INCREMENT,
16
  start_time datetime NOT NULL,
17
  repeat_interval int(10) unsigned NOT NULL,
18
  task_type varchar(255) CHARACTER SET utf8 NOT NULL,
19
  params text CHARACTER SET utf8 NOT NULL,
20
  PRIMARY KEY (task_id) USING BTREE
21
) ENGINE=InnoDB AUTO_INCREMENT=13 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
(-)a/installer/data/mysql/kohastructure.sql (+31 lines)
Lines 3723-3728 CREATE TABLE IF NOT EXISTS edifact_ean ( Link Here
3723
  CONSTRAINT efk_branchcode FOREIGN KEY ( branchcode ) REFERENCES branches ( branchcode )
3723
  CONSTRAINT efk_branchcode FOREIGN KEY ( branchcode ) REFERENCES branches ( branchcode )
3724
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
3724
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
3725
3725
3726
--
3727
-- Table structure for table 'import_oai'
3728
--
3729
3730
DROP TABLE IF EXISTS import_oai;
3731
CREATE TABLE  import_oai (
3732
  import_oai_id int(10) unsigned NOT NULL AUTO_INCREMENT,
3733
  header_identifier varchar(45) CHARACTER SET utf8 NOT NULL,
3734
  header_datestamp datetime NOT NULL,
3735
  header_status varchar(45) CHARACTER SET utf8 DEFAULT NULL,
3736
  metadata longtext CHARACTER SET utf8 NOT NULL,
3737
  last_modified timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
3738
  status varchar(45) CHARACTER SET utf8 NOT NULL,
3739
  PRIMARY KEY (import_oai_id)
3740
) ENGINE=InnoDB AUTO_INCREMENT=297 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
3741
3742
--
3743
-- Table structure for table 'saved_tasks'
3744
--
3745
3746
DROP TABLE IF EXISTS saved_tasks;
3747
CREATE TABLE  saved_tasks (
3748
  task_id int(10) unsigned NOT NULL AUTO_INCREMENT,
3749
  start_time datetime NOT NULL,
3750
  repeat_interval int(10) unsigned NOT NULL,
3751
  task_type varchar(255) CHARACTER SET utf8 NOT NULL,
3752
  params text CHARACTER SET utf8 NOT NULL,
3753
  PRIMARY KEY (task_id) USING BTREE
3754
) ENGINE=InnoDB AUTO_INCREMENT=13 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
3755
3756
3726
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
3757
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
3727
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
3758
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
3728
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
3759
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc (+1 lines)
Lines 78-83 Link Here
78
    [% IF Koha.Preference('SMSSendDriver') == 'Email' %]
78
    [% IF Koha.Preference('SMSSendDriver') == 'Email' %]
79
        <li><a href="/cgi-bin/koha/admin/sms_providers.pl">SMS cellular providers</a></li>
79
        <li><a href="/cgi-bin/koha/admin/sms_providers.pl">SMS cellular providers</a></li>
80
    [% END %]
80
    [% END %]
81
    <li><a href="/cgi-bin/koha/admin/saved_tasks.pl">Saved tasks</a></li>
81
</ul>
82
</ul>
82
</div>
83
</div>
83
</div>
84
</div>
(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/tasks/KohaIcarusTaskDequeueOAIPMHBiblio.inc (+143 lines)
Line 0 Link Here
1
[%# Use CGI plugin to create a default target URI %]
2
[%# TODO: Test if this works with Plack... %]
3
[% USE CGI %]
4
[% server = CGI.virtual_host %]
5
[% IF ( server_port = CGI.virtual_port ) %]
6
        [% IF ( server_port != '80' ) && ( server_port != '443' ) %]
7
                [% server = server _ ':' _ server_port %]
8
        [% END %]
9
[% END %]
10
[% default_auth_uri = 'http://' _ server _ '/cgi-bin/koha/svc/authentication' %]
11
[% default_target_uri = 'http://' _ server _ '/cgi-bin/koha/svc/import_oai' %]
12
<fieldset class="rows">
13
    <legend>Import source parameters:</legend>
14
    <ol>
15
        <li>
16
            <label for="queue">Queue: </label>
17
            [% IF ( params.queue ) %]
18
                <input type="text" id="queue" name="params[queue]" value="[% params.queue %]" size="30" />
19
            [% ELSE %]
20
                <input type="text" id="queue" name="params[queue]" value="file://" size="30" />
21
            [% END %]
22
            <span class="help">This is a filepath on your system like file:///var/spool/koha/libraryname/oaipmh</span>
23
        </li>
24
    </ol>
25
</fieldset>
26
<fieldset class="rows">
27
    <legend>API authentication parameters:</legend>
28
    <ol>
29
        <li>
30
            <label for="auth_uri">URL: </label>
31
            [% IF ( params.auth_uri ) %]
32
                <input type="text" id="auth_uri" name="params[auth_uri]" value="[% params.auth_uri %]" size="30" />
33
            [% ELSE %]
34
                <input type="text" id="auth_uri" name="params[auth_uri]" value="[% default_auth_uri %]" size="30" />
35
            [% END %]
36
            [% IF (errors.auth_uri.no_path) %]<span class="error">[The URL must have a path after "http://" like "koha-community.org/cgi-bin/koha/svc/authentication".]</span>[% END %]
37
            [% IF (errors.auth_uri.not_http) %]<span class="error">[The URL begin with a scheme of "http://" like "http://koha-community.org/cgi-bin/koha/svc/authentication".]</span>[% END %]
38
            [% IF (errors.auth_uri.not_a_url) %]<span class="error">[The value of this field must be a URL like "http://koha-community.org/cgi-bin/koha/svc/authentication".]</span>[% END %]
39
            <span class="help">This is a Koha authentication URL. The default value </span>
40
        </li>
41
        <li>
42
            <label for="auth_username">Username: </label>
43
            <input type="text" id="auth_username" name="params[auth_username]" value="[% params.auth_username %]" size="30" />
44
            <span class="help">This user must have permission to edit the catalogue.</span>
45
        </li>
46
        <li>
47
            <label for="auth_password">Password: </label>
48
            <input type="text" id="auth_password" name="params[auth_password]" value="[% params.auth_password %]" size="30" />
49
        </li>
50
    </ol>
51
</fieldset>
52
<fieldset class="rows">
53
    <legend>Import target parameters:</legend>
54
    <ol>
55
        <li>
56
            <label for="target_uri">URL: </label>
57
            [% IF ( params.target_uri ) %]
58
                <input type="text" id="target_uri" name="params[target_uri]" value="[% params.target_uri %]" size="30" />
59
            [% ELSE %]
60
                <input type="text" id="target_uri" name="params[target_uri]" value="[% default_target_uri %]" size="30" />
61
            [% END %]
62
            [% IF (errors.target_uri.no_path) %]<span class="error">[The URL must have a path after "http://" like "koha-community.org/cgi-bin/koha/svc/import_oai".]</span>[% END %]
63
            [% IF (errors.target_uri.not_http) %]<span class="error">[The URL begin with a scheme of "http://" like "http://koha-community.org/cgi-bin/koha/svc/import_oai".]</span>[% END %]
64
            [% IF (errors.target_uri.not_a_url) %]<span class="error">[The value of this field must be a URL like "http://koha-community.org/cgi-bin/koha/svc/import_oai".]</span>[% END %]
65
        </li>
66
67
        <li>
68
            <label for="match">Record matching rule code</label>
69
            <input type="text" id="match" name="params[match]" value="[% params.match %]" size="30" />
70
            <span class="help">This code must exist in "Record matching rules" in Administration for record matching to work. (Example code: OAI)</span>
71
        </li>
72
        <li>
73
            [%# TODO: Ideally, I'd like to use 'tools-overlay-action.inc' but the logic doesn't work here. Perhaps it would be better as a TT plugin. %]
74
            <label for="overlay_action">Action if matching record found</label>
75
            <select name="params[overlay_action]" id="overlay_action">
76
            [% IF ( params.overlay_action == "replace" ) %]
77
                <option value="replace"  selected="selected">
78
            [% ELSE %]
79
                <option value="replace">
80
            [% END %]
81
                Replace existing record with incoming record</option>
82
            [% IF ( params.overlay_action == "create_new" ) %]
83
                <option value="create_new" selected="selected">
84
            [% ELSE %]
85
                <option value="create_new">
86
            [% END %]
87
                Add incoming record</option>
88
            [% IF ( params.overlay_action == "ignore" ) %]
89
                <option value="ignore" selected="selected">
90
            [% ELSE %]
91
                <option value="ignore">
92
            [% END %]
93
                Ignore incoming record</option>
94
            </select>
95
        </li>
96
        <li>
97
            [%# TODO: Ideally, I'd like to use 'tools-nomatch-action.inc' but the logic doesn't work here. Perhaps it would be better as a TT plugin. %]
98
            <label for="nomatch_action">Action if no match is found</label>
99
            <select name="params[nomatch_action]" id="nomatch_action">
100
            [% IF ( params.nomatch_action == "create_new" ) %]
101
                <option value="create_new" selected="selected">
102
            [% ELSE %]
103
                <option value="create_new">
104
            [% END %]
105
                Add incoming record</option>
106
            [% IF ( params.nomatch_action == "ignore" ) %]
107
                <option value="ignore" selected="selected">
108
            [% ELSE %]
109
                <option value="ignore">
110
            [% END %]
111
                Ignore incoming record</option>
112
            </select>
113
        </li>
114
        <li>
115
            <label for="item_action">Item action</label>
116
            [%# TODO: Will you ever have a different mode than ignore? %]
117
            <input type="text" id="item_action"  value="ignore" size="30" disabled="disabled"/>
118
            <input type="hidden" name="params[item_action]" value="ignore" />
119
        </li>
120
        <li>
121
            <label for="import_mode">Import mode: </label>
122
            [%# TODO: Will you ever have a different mode than direct? %]
123
            <input type="text" id="import_mode" value="direct" size="30" disabled="disabled"/>
124
            <input type="hidden" name="params[import_mode]" value="direct" />
125
        </li>
126
        <li>
127
            <label>Framework</label>
128
        </li>
129
        <li>
130
            <label for="filter">Filter</label>
131
            [% IF ( params.filter ) %]
132
                <input type="text" id="filter" name="params[filter]" value="[% params.filter %]" size="30" />
133
            [% ELSE %]
134
                <input type="text" id="filter" name="params[filter]" value="file://" size="30" />
135
            [% END %]
136
            <span class="help">This is a filepath on your system like file:///etc/koha/sites/libraryname/OAI2MARC21slim.xsl or file:///usr/share/koha/intranet/htdocs/intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl</span>
137
        </li>
138
        </li>
139
        <li>
140
            <label>Record type</label>
141
        </li>
142
    </ol>
143
</fieldset>
(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/tasks/KohaIcarusTaskEnqueueOAIPMHBiblio.inc (+87 lines)
Line 0 Link Here
1
[%# USE CGI %]
2
[%# server_name = CGI.server_name; server_port = CGI.server_port; server = server_name _ ":" _ server_port; %]
3
4
<fieldset class="rows">
5
    <legend>HTTP parameters:</legend>
6
    <ol>
7
        <li>
8
            <label for="url">URL: </label>
9
            [% IF ( params.url ) %]
10
                <input type="text" id="url" name="params[url]" value="[% params.url %]" size="30" />
11
            [% ELSE %]
12
                <input type="text" id="url" name="params[url]" value="http://" size="30" />
13
            [% END %]
14
            [% IF (errors.url.no_path) %]<span class="error">[The URL must have a path after "http://" like "koha-community.org/cgi-bin/koha/oai.pl".]</span>[% END %]
15
            [% IF (errors.url.not_http) %]<span class="error">[The URL begin with a scheme of "http://" like "http://koha-community.org/cgi-bin/koha/oai.pl".]</span>[% END %]
16
            [% IF (errors.url.not_a_url) %]<span class="error">[The value of this field must be a URL like "http://koha-community.org/cgi-bin/koha/oai.pl".]</span>[% END %]
17
18
        </li>
19
    </ol>
20
    <span class="help">The following parameters are not required by all OAI-PMH repositories, so they may be optional for this task.</span>
21
    <ol>
22
        <li>
23
            <label for="username">Username: </label>
24
            <input type="text" id="username" name="params[username]" value="[% params.username %]" size="30" />
25
        </li>
26
        <li>
27
            <label for="password">Password: </label>
28
            <input type="text" id="password" name="params[password]" value="[% params.password %]" size="30" />
29
        </li>
30
        <li>
31
            <label for="realm">Realm: </label>
32
            <input type="text" id="realm" name="params[realm]" value="[% params.realm %]" size="30" />
33
        </li>
34
    </ol>
35
</fieldset>
36
<fieldset class="rows">
37
    <legend>OAI-PMH parameters:</legend>
38
    <ol>
39
        <li>
40
            <label for="verb">Verb: </label>
41
            <select id="verb" name="params[verb]">
42
            [% FOREACH verb IN [ 'GetRecord', 'ListRecords' ] %]
43
                [% IF ( params.verb ) && ( verb == params.verb ) %]
44
                    <option selected="selected" value="[% verb %]">[% verb %]</option>
45
                [% ELSE %]
46
                    <option value="[% verb %]">[% verb %]</option>
47
                [% END %]
48
            [% END %]
49
            </select>
50
        </li>
51
        <li>
52
            <label for="identifier">Identifier: </label>
53
            <input type="text" id="identifier" name="params[identifier]" value="[% params.identifier %]" size="30" />
54
            <span class="help">This identifier will only be used with the GetRecord verb.</span>
55
        </li>
56
        <li>
57
            <label for="sets">Sets: </label>
58
            <input type="text" id="sets" name="params[sets]" value="[% params.sets %]" size="30" /><span class="help">You may specify several sets by separating the sets with a pipe (e.g. set1|set2 )</span>
59
        </li>
60
        <li>
61
            <label for="metadataPrefix">Metadata Prefix: </label>
62
            <input type="text" id="metadataPrefix" name="params[metadataPrefix]" value="[% params.metadataPrefix %]" size="30" />
63
        </li>
64
        <li>
65
            <label for="opt_from">From: </label>
66
            <input type="text" class="datetime_utc" id="opt_from" name="params[from]" value="[% params.from %]" size="30" /><span class="help">This value will be treated as UTC time. Note that some repositories only support YYYY-MM-DD datestamps.</span>
67
        </li>
68
        <li>
69
            <label for="opt_until">Until: </label>
70
            <input type="text" class="datetime_utc" id="opt_until" name="params[until]" value="[% params.until %]" size="30" /><span class="help">This value will be treated as UTC time. Note that some repositories only support YYYY-MM-DD datestamps.</span>
71
        </li>
72
    </ol>
73
</fieldset>
74
<fieldset class="rows">
75
    <legend>Download parameters:</legend>
76
    <ol>
77
        <li>
78
            <label for="queue">Queue: </label>
79
            [% IF ( params.queue ) %]
80
                <input type="text" id="queue" name="params[queue]" value="[% params.queue %]" size="30" />
81
            [% ELSE %]
82
                <input type="text" id="queue" name="params[queue]" value="file://" size="30" />
83
            [% END %]
84
            <span class="help">This is a filepath on your system like file:///var/spool/koha/libraryname/oaipmh</span>
85
        </li>
86
    </ol>
87
</fieldset>
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt (+2 lines)
Lines 120-125 Link Here
120
                        <dt><a href="/cgi-bin/koha/admin/sms_providers.pl">SMS cellular providers</a></dt>
120
                        <dt><a href="/cgi-bin/koha/admin/sms_providers.pl">SMS cellular providers</a></dt>
121
                        <dd>Define a list of cellular providers for sending SMS messages via email.</dd>
121
                        <dd>Define a list of cellular providers for sending SMS messages via email.</dd>
122
                    [% END %]
122
                    [% END %]
123
                    <dt><a href="/cgi-bin/koha/admin/saved_tasks.pl">Saved tasks</a></dt>
124
                    <dd>Define tasks which may be run in the background</dd>
123
                </dl>
125
                </dl>
124
            </div>
126
            </div>
125
        </div>
127
        </div>
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/saved_tasks.tt (+333 lines)
Line 0 Link Here
1
[% INCLUDE 'doc-head-open.inc' %]
2
<title>Koha &rsaquo; Administration &rsaquo; Saved tasks</title>
3
[% INCLUDE 'doc-head-close.inc' %]
4
[% INCLUDE 'calendar.inc' %]
5
<script type="text/javascript" src="[% interface %]/lib/jquery/plugins/jquery-ui-timepicker-addon.min.js"></script>
6
[% INCLUDE 'timepicker.inc' %]
7
[% IF ( op == "list" ) %]
8
    <link rel="stylesheet" type="text/css" href="[% themelang %]/css/datatables.css" />
9
    [% INCLUDE 'datatables.inc' %]
10
    <script type="text/javascript">
11
    //<![CDATA[
12
        $(document).ready(function() {
13
            $("#taskst").dataTable($.extend(true, {}, dataTablesDefaults, {
14
                "aoColumnDefs": [
15
                    { "aTargets": [3,4,5,6], "bSortable": false },
16
                ],
17
                "sPaginationType": "four_button"
18
            }));
19
        });
20
    //]]>
21
    </script>
22
[% ELSIF ( op == "edit" ) || ( op == "new" ) %]
23
    <script type="text/javascript">
24
    //<![CDATA[
25
        $(document).ready(function() {
26
            [%# Ideally, it would be nice to record the timezone here too, but currently we use MySQL's DATETIME field which doesn't store ISO 8601 timezone designators... %]
27
            $(".datetime_local").datetimepicker({
28
                dateFormat: "yy-mm-dd",
29
                timeFormat: "HH:mm:ss",
30
                hour: 0,
31
                minute: 0,
32
                second: 0,
33
                showSecond: 1,
34
            });
35
            $(".datetime_utc").datetimepicker({
36
                separator: "T",
37
                timeSuffix: 'Z',
38
                dateFormat: "yy-mm-dd",
39
                timeFormat: "HH:mm:ss",
40
                hour: 0,
41
                minute: 0,
42
                second: 0,
43
                showSecond: 1,
44
                // timezone doesn't work with the "Now" button in v1.4.3 although it appears to in v1.6.1
45
                // timezone: +000,
46
            });
47
48
        });
49
    //]]>
50
    </script>
51
    <style type="text/css">
52
        /* Override staff-global.css which hides second, millisecond, and microsecond sliders */
53
        .ui_tpicker_second {
54
            display: block;
55
        }
56
        .test-success {
57
            /* same color as .text-success in Bootstrap 2.2.2 */
58
            color:#468847;
59
        }
60
    </style>
61
[% END %]
62
</head>
63
64
<body id="admin_saved_tasks" class="admin">
65
[% INCLUDE 'header.inc' %]
66
[% INCLUDE 'cat-search.inc' %]
67
<div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/admin/admin-home.pl">Administration</a> &rsaquo; Saved tasks</div>
68
69
<div id="doc3" class="yui-t2">
70
71
<div id="bd">
72
  <div id="yui-main">
73
    <div class="yui-b">
74
        [% IF ( op ) %]
75
            [% IF ( op == "new" ) || ( op == "edit" ) %]
76
                [%# If step is undefined, force it to be step one %]
77
                [% IF ( ! step ); step = "one"; END; %]
78
79
80
81
                [%# HEADING %]
82
                    [% IF ( op == "new" ) %]
83
                        <h1>New saved task</h1>
84
                    [% ELSIF ( op == "edit" ) %]
85
                        <h1>Modify saved task</h1>
86
                    [% END %]
87
                [%# /HEADING %]
88
89
                [%# TODO: Get this working properly... <div class="alert">Validation failed.</div> #]
90
91
                [%# FORM %]
92
                    <form action="/cgi-bin/koha/admin/[% filename %]" name="detail-form" method="post" id="saved-task-details" novalidate="novalidate">
93
                        [% IF ( op == "new" ) %]
94
                            <input type="hidden" name="op" value="create" />
95
                        [% ELSIF ( op == "edit" ) %]
96
                            <input type="hidden" name="op" value="update" />
97
                            <input type="hidden" name="saved_task_id" value="[% saved_task.task_id %]" />
98
                        [% END %]
99
                        <input type="hidden" name="step" value="[% step %]" />
100
                        <fieldset class="rows">
101
                            <ol>
102
                                [% IF ( op == "edit") && ( step == "one" ) && (! task_type_is_valid ) %]
103
                                <li>
104
                                    <label for="invalid_task_type">Current invalid task type:</label>
105
                                    <input id="invalid_task_type" type="text" disabled="disabled" value="[% saved_task.task_type %]" size="60" />
106
                                    <span class="error">Sorry! This task type is invalid. Please choose a new one from the following list.</span>
107
                                <li>
108
                                [% END %]
109
                                <li>
110
                                    <label for="task_type">Task type: </label>
111
                                    [% IF ( step == "one" ) %]
112
                                        [% IF ( available_plugins ) %]
113
                                        <select id="task_type" name="task_type">
114
                                            [% IF ( op == "new") %]
115
                                                [% FOREACH plugin IN available_plugins %]
116
                                                    <option value="[% plugin %]">[% plugin %]</option>
117
                                                [% END %]
118
                                            [% ELSIF ( op == "edit" ) %]
119
                                                [% FOREACH plugin IN available_plugins %]
120
                                                    [% IF ( saved_task.task_type == plugin ) %]
121
                                                        <option selected="selected" value="[% plugin %]">[% plugin %]</option>
122
                                                    [% ELSE %]
123
                                                        <option value="[% plugin %]">[% plugin %]</option>
124
                                                    [% END %]
125
                                                [% END %]
126
                                            [% END %]
127
                                        </select>
128
                                        [% END %]
129
130
                                    [% ELSIF ( step == "two" ) %]
131
                                        <input type="text" disabled="disabled" value="[% task_type %]" size="60" />
132
                                        <input type="hidden" name="task_type" value="[% task_type %]" />
133
                                    [% END %]
134
                                </li>
135
                            </ol>
136
                        </fieldset>
137
138
                        [% IF ( step == "one" ) %]
139
                            <fieldset class="action">
140
                                <input type="submit" value="Next">
141
                                <a class="cancel" href="/cgi-bin/koha/admin/[% filename %]">Cancel</a>
142
                            </fieldset>
143
                        [% ELSIF ( step == "two" ) %]
144
                            <fieldset class="rows">
145
                                <legend>Task:</legend>
146
                                <ol>
147
                                    <li>
148
                                        <label for="start_time">Start time: </label>
149
                                        <input type="text" id="start_time" class="datetime_local" name="start_time" value="[% saved_task.start_time %]" size="30" />
150
                                        <span class="help">This value will be treated as local server time, and times in the past will start immediately.</span>
151
                                    </li>
152
                                    <li>
153
                                        <label for="repeat_interval">Repeat interval: </label>
154
                                        <input type="text" id="repeat_interval" name="repeat_interval" value="[% saved_task.repeat_interval %]" size="4" />
155
                                        <span class="help">seconds</span>
156
                                        [% IF (errors.repeat_interval.not_numeric) %]<span class="error">[The repeat interval must be a purely numeric value.]</span>[% END %]
157
                                    </li>
158
                                </ol>
159
                            </fieldset>
160
                            [%# Try to include the template, but if it fails, fallback to a regular text view %]
161
                            [% TRY %]
162
                                [% INCLUDE $task_template %]
163
                            [% CATCH %]
164
                            <fieldset class="rows">
165
                                <legend>Plugin parameters:</legend>
166
                                <ol>
167
                                    <li>
168
                                        <label for="params">Params: </label>
169
                                        <textarea id="params" name="params" cols="60" rows="20">[% saved_task.params %]</textarea>
170
                                    </li>
171
                                </ol>
172
                            </fieldset>
173
                            [% END %]
174
                            <fieldset class="action">
175
                                <input type="submit" value="Save">
176
                                <a class="cancel" href="/cgi-bin/koha/admin/[% filename %]">Cancel</a>
177
                            </fieldset>
178
                        [% END %]
179
                    </form>
180
                [%# /FORM %]
181
            [% END #/edit or new %]
182
183
184
            [% IF ( op == "list" ) %]
185
                <div id="toolbar" class="btn-toolbar">
186
                    <a id="newserver" class="btn btn-small" href="/cgi-bin/koha/admin/[% filename %]?op=new"><i class="icon-plus"></i> New saved task</a>
187
                </div>
188
                <h1>Saved tasks</h1>
189
                [% IF ( saved_response ) %]
190
                    [% IF ( saved_response == 'delete_success' ) %]
191
                        <div class="alert">Deletion successful.</div>
192
                    [% ELSIF ( saved_response == 'delete_failure' ) %]
193
                        <div class="alert">Deletion failed.</div>
194
                    [% END %]
195
                [% END %]
196
                <table id="taskst">
197
                    <thead>
198
                        <tr>
199
                            <th>Start time</th>
200
                            <th>Repeat interval</th>
201
                            <th>Task type</th>
202
                            <th>Params</th>
203
                            <th></th>
204
                            <th></th>
205
                            <th></th>
206
                        </tr>
207
                    </thead>
208
                    <tbody>
209
                    [% FOREACH saved_task IN saved_tasks %]
210
                        <tr>
211
                            <td>[% IF ( saved_task.start_time ) != "0000-00-00 00:00:00"; saved_task.start_time; END; %]</td>
212
                            <td>[% saved_task.repeat_interval %]</td>
213
                            <td>[% saved_task.task_type %]</td>
214
                            <td>
215
                                <ul>
216
                                [% FOREACH pair IN saved_task.params_as_perl.pairs %]
217
                                   <li>[% pair.key %] => [% pair.value %]</li>
218
                                [% END %]
219
                                </ul>
220
                            </td>
221
                            <td><a href="/cgi-bin/koha/admin/[% filename %]?op=edit&saved_task_id=[% saved_task.task_id %]">Edit</a></td>
222
                            <td><a href="/cgi-bin/koha/admin/[% filename %]?op=send&saved_task_id=[% saved_task.task_id %]">Send to Icarus</a></td>
223
                            <td><a href="/cgi-bin/koha/admin/[% filename %]?op=delete&saved_task_id=[% saved_task.task_id %]">Delete</a></td>
224
                        </tr>
225
                    [% END %]
226
                    </tbody>
227
                </table>
228
                <div id="daemon_controls">
229
                    <h1>Icarus dashboard</h1>
230
                    <table>
231
                    <tr>
232
                        <th>Status</th>
233
                        <th></th>
234
                    </tr>
235
                    <tr>
236
                        <td>
237
238
                        [% IF ( daemon_status == 'Permission denied' ) #Apache doesn't have permission to write to socket
239
                            || ( daemon_status == 'Connection refused' ) #Socket exists, but server is down
240
                            || ( daemon_status == 'No such file or directory' ) #Socket doesn't exist at all
241
                        %]
242
                            <span id="icarus_status">Unable to contact</span>
243
                        [% ELSIF ( daemon_status == 'online' ) %]
244
                            <span id="icarus_status">Online</span>
245
                        [% ELSIF ( daemon_status == 'shutting down' ) %]
246
                            <span id="icarus_status">Shutting down</span>
247
                        [% ELSE %]
248
                            <span id="icarus_status">[% daemon_status %]</span>
249
                        [% END %]
250
                        </td>
251
                        [%# TODO: Also provide controls for starting/restarting Icarus? %]
252
                        <td><a href="/cgi-bin/koha/admin/[% filename %]?server_action=shutdown">Shutdown Icarus</a></td>
253
                    </tr>
254
                    </table>
255
                </div>
256
                <div id="tasks">
257
                    <h1>Active Icarus tasks</h1>
258
                    [% IF ( task_response ) %]
259
                        [% IF ( task_response.action == 'error' ) %]
260
                            [% IF ( task_response.error_message ) %]
261
                                [% IF ( task_response.error_message == 'No such process' ) %]
262
                                    <div class="alert">Task [% task_response.task_id %] does not exist.</div>
263
                                [% END %]
264
                            [% END %]
265
                        [% ELSIF ( task_response.action == 'pending' ) %]
266
                            <div class="alert">Initialising task [% task_response.task_id %].</div>
267
                        [% ELSIF ( task_response.action == 'already pending' ) %]
268
                            <div class="alert">Already initialised task [% task_response.task_id %].</div>
269
                        [% ELSIF ( task_response.action == 'already started' ) %]
270
                            <div class="alert">Already started task [% task_response.task_id %].</div>
271
                        [% ELSIF ( task_response.action == 'removed' ) %]
272
                            <div class="alert">Removing task [% task_response.task_id %].</div>
273
                        [% END %]
274
                    [% END %]
275
                    [% IF ( tasks ) %]
276
                        <table>
277
                            <thead>
278
                                <tr>
279
                                    <th>Task id</th>
280
                                    <th>Status</th>
281
                                    <th>Next start time (local server time)</th>
282
                                    <th>Repeat interval</th>
283
                                    <th>Task type</th>
284
                                    <th>Params</th>
285
                                    <th></th>
286
                                    <th></th>
287
                                </tr>
288
                            </thead>
289
                            <tbody>
290
                            [% FOREACH task IN tasks %]
291
                                <tr>
292
                                    <td>[% task.task_id %]</td>
293
                                    <td>
294
                                        [% SWITCH task.task.status %]
295
                                        [% CASE 'new' %]
296
                                        <span>New</span>
297
                                        [% CASE 'pending' %]
298
                                        <span>Pending</span>
299
                                        [% CASE 'started' %]
300
                                        <span>Started</span>
301
                                        [% CASE 'stopping' %]
302
                                        <span>Stopping</span>
303
                                        [% CASE %]
304
                                        <span>[% task.task.status %]</span>
305
                                        [% END %]
306
                                    </td>
307
                                    <td>[% task.task.start %]</td>
308
                                    <td>[% task.task.repeat_interval %]</td>
309
                                    <td>[% task.task.type %]</td>
310
                                    <td>
311
                                        <ul>
312
                                        [% FOREACH pair IN task.task.params.pairs %]
313
                                           <li>[% pair.key %] => [% pair.value %]</li>
314
                                        [% END %]
315
                                        </ul>
316
                                    </td>
317
                                    <td><a href="/cgi-bin/koha/admin/[% filename %]?server_action=start&server_task_id=[% task.task_id %]">Start</a></td>
318
                                    <td><a href="/cgi-bin/koha/admin/[% filename %]?server_action=remove&server_task_id=[% task.task_id %]">Remove</a></td>
319
                                </tr>
320
                            [% END %]
321
                            </tbody>
322
                        </table>
323
                    [% END %]
324
                </div>
325
            [% END #/list %]
326
        [% END #/op %]
327
    </div>
328
  </div>
329
  <div class="yui-b">
330
    [% INCLUDE 'admin-menu.inc' %]
331
  </div>
332
</div>
333
[% INCLUDE 'intranet-bottom.inc' %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl (+57 lines)
Line 0 Link Here
1
<?xml version="1.0" encoding="UTF-8"?>
2
<xsl:stylesheet version="1.0"
3
    xmlns:marc="http://www.loc.gov/MARC21/slim"
4
    xmlns:oai="http://www.openarchives.org/OAI/2.0/"
5
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
6
    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>
7
    <!-- NOTE: This XSLT strips the OAI-PMH wrapper from the metadata. -->
8
    <!-- NOTE: This XSLT also adds the OAI-PMH identifier back in as a MARC field -->
9
10
    <!-- Match the root oai:record element -->
11
    <xsl:template match="oai:record">
12
        <!-- Apply templates only to the child metadata element(s) -->
13
        <xsl:apply-templates select="oai:metadata" />
14
    </xsl:template>
15
16
    <!-- Matches an oai:metadata element -->
17
    <xsl:template match="oai:metadata">
18
        <!-- Do nothing but apply templates to child elements -->
19
        <xsl:apply-templates />
20
    </xsl:template>
21
22
    <!-- Identity transformation: this template copies attributes and nodes -->
23
    <xsl:template match="@* | node()">
24
        <!-- Create a copy of this attribute or node -->
25
        <xsl:copy>
26
            <!-- Recursively apply this template to the attributes and child nodes of this element -->
27
            <xsl:apply-templates select="@* | node()" />
28
        </xsl:copy>
29
    </xsl:template>
30
31
    <xsl:template match="marc:record">
32
        <xsl:copy>
33
            <!-- Apply all relevant templates for all attributes and elements -->
34
            <xsl:apply-templates select="@* | node()"/>
35
36
            <!-- Add new node (or whatever else you want to do after copying the existing record) -->
37
38
            <xsl:text>  </xsl:text><xsl:element name="datafield" xmlns="http://www.loc.gov/MARC21/slim">
39
                <xsl:attribute name="ind1"><xsl:text>7</xsl:text></xsl:attribute>
40
                <xsl:attribute name="ind2"><xsl:text> </xsl:text></xsl:attribute>
41
                <xsl:attribute name="tag">024</xsl:attribute>
42
43
                <xsl:element name="subfield">
44
                    <xsl:attribute name="code">a</xsl:attribute>
45
                    <xsl:value-of select="/oai:record/oai:header/oai:identifier"/>
46
                </xsl:element>
47
48
                <xsl:element name="subfield">
49
                    <xsl:attribute name="code">2</xsl:attribute>
50
                    <xsl:text>uri</xsl:text>
51
                </xsl:element>
52
            </xsl:element>
53
            <!-- Newline -->
54
            <xsl:text>&#xa;</xsl:text>
55
        </xsl:copy>
56
    </xsl:template>
57
</xsl:stylesheet>
(-)a/misc/bin/icarusd.pl (+156 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
#######################################################################
4
5
use Modern::Perl;
6
use POSIX; #For daemonizing
7
use Fcntl qw(:flock); #For pidfile
8
use Getopt::Long;
9
10
#Make the STDOUT filehandle hot, so that you can use shell re-direction. Otherwise, you'll suffer from buffering.
11
STDOUT->autoflush(1);
12
#Note that STDERR, by default, is already hot.
13
14
#######################################################################
15
#FIXME: Debugging signals
16
#BEGIN {
17
#    package POE::Kernel;
18
#    use constant TRACE_SIGNALS => 1;
19
#}
20
21
use POE;
22
use JSON; #For Listener messages
23
use XML::LibXML; #For configuration files
24
25
use Koha::Icarus::Listener;
26
27
#######################################################################
28
29
my ($filename,$daemon,$log);
30
GetOptions (
31
    "f|file|filename=s"     => \$filename, #/kohawebs/dev/dcook/koha-dev/etc/koha-conf.xml
32
    "l|log=s"               => \$log,
33
    "d|daemon"              => \$daemon,
34
) or die("Error in command line arguments\n");
35
36
#Declare the variable with file scope so the flock stays for the duration of the process's life
37
my $pid_filehandle;
38
39
#Read configuration file
40
my $config = read_config_file($filename);
41
42
my $SOCK_PATH = $config->{socket};
43
my $pid_file = $config->{pidfile};
44
my $max_tasks = $config->{max_tasks};
45
46
#Overwrite configuration file with command line options
47
if ($log){
48
    $config->{log} = $log;
49
}
50
51
#Go into daemon mode, if user has included flag
52
if ($daemon){
53
    daemonize();
54
}
55
56
if ($pid_file){
57
    #NOTE: The filehandle needs to have file scope, so that the flock is preserved.
58
    $pid_filehandle = make_pid_file($pid_file);
59
}
60
61
#FIXME: Do we want to log to file only in daemon mode? $config->{log} should be populated by either the config file or the l|log GetOpt...
62
if ($daemon && $config->{log}){
63
    log_to_file($config->{log});
64
}
65
66
67
#FIXME: 1) In daemon mode, SIGUSR1 or SIGHUP for reloading/restarting?
68
#######################################################################
69
70
#Creates Icarus Listener
71
Koha::Icarus::Listener->spawn({
72
    Socket => $SOCK_PATH,
73
    MaxTasks => $max_tasks,
74
    Verbosity => 1,
75
});
76
77
POE::Kernel->run();
78
79
exit;
80
81
sub read_config_file {
82
    my $filename = shift;
83
    my $config = {};
84
    if ( -e $filename ){
85
        eval {
86
            my $doc = XML::LibXML->load_xml(location => $filename);
87
            if ($doc){
88
                my $root = $doc->documentElement;
89
                my $icarus = $root->find('icarus')->shift;
90
                if ($icarus){
91
                    #Get all child nodes for the 'icarus' element
92
                    my @childnodes = $icarus->childNodes();
93
                    foreach my $node (@childnodes){
94
                        #Only consider nodes that are elements
95
                        if ($node->nodeType == XML_ELEMENT_NODE){
96
                            my $config_key = $node->nodeName;
97
                            my $first_child = $node->firstChild;
98
                            #Only consider nodes that have a text node as their first child
99
                            if ($first_child && $first_child->nodeType == XML_TEXT_NODE){
100
                                $config->{$config_key} = $first_child->nodeValue;
101
                            }
102
                        }
103
                    }
104
                }
105
            }
106
        };
107
    }
108
    return $config;
109
}
110
111
#######################################################################
112
#NOTE: On Debian, you can use the daemon binary to make a process into a daemon,
113
# the following subs are for systems that don't have the daemon binary.
114
115
sub daemonize {
116
    my $pid = fork;
117
    die "Couldn't fork: $!" unless defined($pid);
118
    if ($pid){
119
        exit; #Parent exit
120
    }
121
    POSIX::setsid() or die "Can't start a new session: $!";
122
}
123
124
sub log_to_file {
125
    my $logfile = shift;
126
    #Open a filehandle to append to a log file
127
    open(LOG, '>>', $logfile) or die "Unable to open a filehandle for $logfile: $!\n"; # --output
128
    LOG->autoflush(1); #Make filehandle hot (ie don't buffer)
129
    *STDOUT = *LOG; #Re-assign STDOUT to LOG | --stdout
130
    *STDERR = *STDOUT; #Re-assign STDERR to STDOUT | --stderr
131
}
132
133
sub make_pid_file {
134
    my $pidfile = shift;
135
    if ( ! -e $pidfile ){
136
        open(my $fh, '>', $pidfile) or die "Unable to write to $pidfile: $!\n";
137
        $fh->close;
138
    }
139
140
    open(my $pidfilehandle, '+<', $pidfile) or die "Unable to open a filehandle for $pidfile: $!\n";
141
    if (flock($pidfilehandle, LOCK_EX|LOCK_NB)){
142
        #Write pid to pidfile
143
        print "Acquiring lock on $pidfile\n";
144
        #Now that we've acquired a lock, let's truncate the file
145
        truncate($pidfilehandle, 0);
146
        print $pidfilehandle $$."\n" or die $!;
147
        #Flush the filehandle so you're not suffering from buffering
148
        $pidfilehandle->flush();
149
        return $pidfilehandle;
150
    } else {
151
        my $number = <$pidfilehandle>;
152
        chomp($number);
153
        warn "$0 is already running with pid $number. Exiting.\n";
154
        exit(1);
155
    }
156
}
(-)a/svc/import_oai (-1 / +197 lines)
Line 0 Link Here
0
- 
1
#!/usr/bin/perl
2
3
# Copyright 2012 CatalystIT Ltd
4
# Copyright 2016 Prosentient Systems
5
#
6
# This file is part of Koha.
7
#
8
# Koha is free software; you can redistribute it and/or modify it
9
# under the terms of the GNU General Public License as published by
10
# the Free Software Foundation; either version 3 of the License, or
11
# (at your option) any later version.
12
#
13
# Koha is distributed in the hope that it will be useful, but
14
# WITHOUT ANY WARRANTY; without even the implied warranty of
15
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16
# GNU General Public License for more details.
17
#
18
# You should have received a copy of the GNU General Public License
19
# along with Koha; if not, see <http://www.gnu.org/licenses>.
20
#
21
22
use Modern::Perl;
23
use XML::LibXML;
24
use URI;
25
use File::Basename;
26
27
use CGI qw ( -utf8 );
28
use C4::Auth qw/check_api_auth/;
29
use C4::Context;
30
use C4::ImportBatch;
31
use C4::Matcher;
32
use XML::Simple;
33
34
my $query = new CGI;
35
binmode STDOUT, ':encoding(UTF-8)';
36
37
my ($status, $cookie, $sessionID) = check_api_auth($query, { editcatalogue => 'edit_catalogue'} );
38
unless ($status eq "ok") {
39
    print $query->header(-type => 'text/xml', -status => '403 Forbidden');
40
    print XMLout({ auth_status => $status }, NoAttr => 1, RootName => 'response', XMLDecl => 1);
41
    exit 0;
42
}
43
44
my $xml;
45
if ($query->request_method eq "POST") {
46
    $xml = $query->param('xml');
47
}
48
if ($xml) {
49
    #TODO: You could probably use $query->Vars here instead...
50
    my %params = map { $_ => $query->param($_) } $query->param;
51
    my $result = import_oai($xml, \%params );
52
    print $query->header(-type => 'text/xml');
53
    print XMLout($result, NoAttr => 1, RootName => 'response', XMLDecl => 1);
54
} else {
55
    print $query->header(-type => 'text/xml', -status => '400 Bad Request');
56
}
57
58
exit 0;
59
60
sub import_oai {
61
    my ($inxml, $params) = @_;
62
63
    my $result = {};
64
65
    my $filter      = delete $params->{filter}      || '';
66
    my $import_mode = delete $params->{import_mode} || '';
67
    my $framework   = delete $params->{framework}   || '';
68
69
    if (my $matcher_code = delete $params->{match}) {
70
        $params->{matcher_id} = C4::Matcher::GetMatcherId($matcher_code);
71
    }
72
73
    my $batch_id = GetWebserviceBatchId($params);
74
    unless ($batch_id) {
75
        $result->{'status'} = "failed";
76
        $result->{'error'} = "Batch create error";
77
        return $result;
78
    }
79
80
    #Log it in the import_oai table here...
81
82
    #Parse the XML string into a XML::LibXML object
83
    my $doc = XML::LibXML->load_xml(string => $inxml);
84
85
    #Get the root element
86
    my $root = $doc->documentElement;
87
88
    #Register namespaces for searching purposes
89
    my $xpc = XML::LibXML::XPathContext->new();
90
    $xpc->registerNs('oai','http://www.openarchives.org/OAI/2.0/');
91
92
    my $xpath_identifier = XML::LibXML::XPathExpression->new("oai:header/oai:identifier");
93
    my $identifier = $xpc->findnodes($xpath_identifier,$root)->shift;
94
    my $identifier_string = $identifier->textContent;
95
96
    my $xpath_datestamp = XML::LibXML::XPathExpression->new("oai:header/oai:datestamp");
97
    my $datestamp = $xpc->findnodes($xpath_datestamp,$root)->shift;
98
    my $datestamp_string = $datestamp->textContent;
99
100
    my $status_string = "";
101
102
    #OAI-PMH Header = identifier, datestamp, status, setSpec?
103
    #OAI-PMH Metadata
104
105
    my $log_dbh = C4::Context->dbh;
106
    my $log_sql = "INSERT INTO import_oai (header_identifier, header_datestamp, header_status, metadata) VALUES (?, ?, ?, ?)";
107
    my $log_sth = $log_dbh->prepare($log_sql);
108
    $log_sth->execute($identifier_string,$datestamp_string,$status_string,$inxml);
109
110
111
112
    #Filter the OAI-PMH record into a MARCXML record
113
    my $metadata_xml;
114
115
    #Source a default XSLT
116
    my $htdocs  = C4::Context->config('intrahtdocs');
117
    my $theme   = C4::Context->preference("template");
118
    #FIXME: This doesn't work for UNIMARC!
119
    my $xslfilename = "$htdocs/$theme/en/xslt/OAI2MARC21slim.xsl";
120
121
    #FIXME: There's a better way to do these filters...
122
    if ($filter){
123
        my $filter_uri = URI->new($filter);
124
        if ($filter_uri){
125
            my $scheme = $filter_uri->scheme;
126
            if ($scheme && $scheme eq "file"){
127
                my $path = $filter_uri->path;
128
                #Filters may theoretically be .xsl or .pm files
129
                my($filename, $dirs, $suffix) = fileparse($path,(".xsl",".pm"));
130
                if ($suffix && $suffix eq ".xsl"){
131
                    #If this new path exists, change the filter XSLT to it
132
                    if ( -f $path ){
133
                        $xslfilename = $path;
134
                    }
135
                }
136
            }
137
        }
138
    }
139
140
    if ( -f $xslfilename ){
141
        #FIXME: Ideally, it would be good to use Koha::XSLT_Handler here... (especially for persistent environments...)
142
        my $xslt = XML::LibXSLT->new();
143
        my $style_doc = XML::LibXML->load_xml(location => $xslfilename);
144
        my $stylesheet = $xslt->parse_stylesheet($style_doc);
145
        if ($stylesheet){
146
            my $results = $stylesheet->transform($doc);
147
            $metadata_xml = $stylesheet->output_as_bytes($results);
148
        }
149
    } else {
150
        $result->{'status'} = "failed";
151
        $result->{'error'} = "Metadata filter unavailable";
152
        return $result;
153
    }
154
155
156
157
158
159
160
161
162
163
    #Import the MARCXML record into Koha
164
    my $marcflavour = C4::Context->preference('marcflavour') || 'MARC21';
165
    my $marc_record = eval {MARC::Record::new_from_xml( $metadata_xml, "utf8", $marcflavour)};
166
    if ($@) {
167
        $result->{'status'} = "failed";
168
        $result->{'error'} = $@;
169
        return $result;
170
    }
171
172
    my $import_record_id = AddBiblioToBatch($batch_id, 0, $marc_record, "utf8", int(rand(99999)));
173
    my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 'UPDATE COUNTS');
174
175
    my $matcher = C4::Matcher->new($params->{record_type} || 'biblio');
176
    $matcher = C4::Matcher->fetch($params->{matcher_id});
177
    my $number_of_matches =  BatchFindDuplicates($batch_id, $matcher);
178
179
    # XXX we are ignoring the result of this;
180
    BatchCommitRecords($batch_id, $framework) if lc($import_mode) eq 'direct';
181
182
    my $dbh = C4::Context->dbh();
183
    my $sth = $dbh->prepare("SELECT matched_biblionumber FROM import_biblios WHERE import_record_id =?");
184
    $sth->execute($import_record_id);
185
    my $biblionumber=$sth->fetchrow_arrayref->[0] || '';
186
    $sth = $dbh->prepare("SELECT overlay_status FROM import_records WHERE import_record_id =?");
187
    $sth->execute($import_record_id);
188
    my $match_status = $sth->fetchrow_arrayref->[0] || 'no_match';
189
    my $url = 'http://'. C4::Context->preference('staffClientBaseURL') .'/cgi-bin/koha/catalogue/detail.pl?biblionumber='. $biblionumber;
190
191
    $result->{'status'} = "ok";
192
    $result->{'import_batch_id'} = $batch_id;
193
    $result->{'match_status'} = $match_status;
194
    $result->{'biblionumber'} = $biblionumber;
195
    $result->{'url'} = $url;
196
    return $result;
197
}

Return to bug 10662