View | Details | Raw Unified | Return to bug 10662
Collapse All | Expand All

(-)a/C4/Installer/PerlDependencies.pm (+5 lines)
Lines 807-812 our $PERL_DEPS = { Link Here
807
        required => 1,
807
        required => 1,
808
        min_ver => '1.00',
808
        min_ver => '1.00',
809
    },
809
    },
810
    'POE' => {
811
        'usage'    => 'Icarus job server',
812
        'required' => '1',
813
        'min_ver'  => '1.35',
814
    },
810
};
815
};
811
816
812
1;
817
1;
(-)a/Koha/Icarus.pm (+177 lines)
Line 0 Link Here
1
package Koha::Icarus;
2
3
# Copyright 2016 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use IO::Socket::UNIX;
22
use IO::Select;
23
use URI;
24
use JSON;
25
26
sub new {
27
    my ($class, $args) = @_;
28
    $args = {} unless defined $args;
29
    return bless ($args, $class);
30
}
31
32
sub connected {
33
    my ($self) = @_;
34
    if ($self->{_connected}){
35
        return 1;
36
    }
37
}
38
39
sub connect {
40
    my ($self) =  @_;
41
    my $socket_uri = $self->{socket_uri};
42
    if ($socket_uri){
43
        my $uri = URI->new($socket_uri);
44
        if ($uri && $uri->scheme eq 'unix'){
45
            my $socket_path = $uri->path;
46
            my $socket = IO::Socket::UNIX->new(
47
                Type => IO::Socket::UNIX::SOCK_STREAM(),
48
                Peer => $socket_path,
49
            );
50
            if ($socket){
51
                my $socketio = new IO::Select();
52
                $socketio->add($socket);
53
                #FIXME: Should probably fix these return values...
54
                $self->{_socketio} = $socketio;
55
                $self->{_socket} = $socket;
56
                my $message = $self->_read();
57
                if ($message eq 'HELLO'){
58
                    $self->{_connected} = 1;
59
                    return 1;
60
                }
61
            }
62
        }
63
    }
64
    return 0;
65
}
66
67
sub add_task {
68
    my ($self, $args) = @_;
69
    my $task = $args->{task};
70
    if ($task && %$task){
71
        my $response = $self->command("add task", undef, $task);
72
        if ($response){
73
            return $response;
74
        }
75
    }
76
}
77
78
sub start_task {
79
    my ($self, $args) = @_;
80
    my $task_id = $args->{task_id};
81
    if ($task_id){
82
        my $response = $self->command("start task", $task_id);
83
        if ($response){
84
            return $response;
85
        }
86
    }
87
}
88
89
sub remove_task {
90
    my ($self, $args) = @_;
91
    my $task_id = $args->{task_id};
92
    if ($task_id){
93
        my $response = $self->command("remove task", $task_id);
94
        if ($response){
95
            return $response;
96
        }
97
    }
98
}
99
100
sub list_tasks {
101
   my ($self) = @_;
102
   my $response = $self->command("list tasks");
103
    if ($response){
104
        if (my $tasks = $response->{tasks}){
105
            return $tasks;
106
        }
107
    }
108
}
109
110
sub shutdown {
111
    my ($self) = @_;
112
    my $response = $self->command("shutdown");
113
    if ($response){
114
        return $response;
115
    }
116
}
117
118
119
120
121
122
sub command {
123
    my ($self, $command, $task_id, $task) = @_;
124
    my $serialized = $self->_serialize({ "command" => $command, "task_id" => $task_id, "task" => $task });
125
    if ($serialized){
126
        $self->_write({ serialized => $serialized });
127
        my $json = $self->_read();
128
        if ($json){
129
            my $response = from_json($json);
130
            if ($response){
131
                return $response;
132
            }
133
        }
134
    }
135
}
136
137
sub _serialize {
138
    my ($self, $output) = @_;
139
    my $serialized = to_json($output);
140
    return $serialized;
141
}
142
143
sub _write {
144
    my ($self, $args) = @_;
145
    my $socket = $self->{_socket};
146
    my $output = $args->{serialized};
147
    if ($output){
148
        if (my $socketio = $self->{_socketio}){
149
            if (my @filehandles = $socketio->can_write(5)){
150
                foreach my $filehandle (@filehandles){
151
                    #Localize output record separator as null
152
                    local $\ = "\x00";
153
                    print $socket $output;
154
                }
155
            }
156
        }
157
    }
158
}
159
160
sub _read {
161
    my ($self) = @_;
162
    if (my $socketio = $self->{_socketio}){
163
        if (my @filehandles = $socketio->can_read(5)){
164
            foreach my $filehandle (@filehandles){
165
                #Localize input record separator as null
166
                local $/ = "\x00";
167
                my $message = <$filehandle>;
168
                chomp($message) if $message;
169
                return $message;
170
            }
171
        }
172
    }
173
}
174
175
176
177
1;
(-)a/Koha/Icarus/Base.pm (+32 lines)
Line 0 Link Here
1
package Koha::Icarus::Base;
2
3
use Modern::Perl;
4
use DateTime;
5
6
use constant DEBUG => 9;
7
use constant SILENT => 0;
8
9
sub new {
10
    my ($class, $args) = @_;
11
    $args = {} unless defined $args;
12
    return bless ($args, $class);
13
}
14
15
sub debug {
16
    my ($self,$message) = @_;
17
    if ($self->{Verbosity} == DEBUG){
18
        $self->log($message);
19
    }
20
}
21
22
sub log {
23
    my ($self,$message) = @_;
24
    my $id = $self->{_id};
25
    my $component = $self->{_component} // "component";
26
    if ( ($self->{Verbosity}) && ($self->{Verbosity} > SILENT) ){
27
        my $now = DateTime->now(time_zone => "local");
28
        say "[$now] [$component $id] $message";
29
    }
30
}
31
32
1;
(-)a/Koha/Icarus/Listener.pm (+330 lines)
Line 0 Link Here
1
package Koha::Icarus::Listener;
2
3
use Modern::Perl;
4
use parent 'Koha::Icarus::Base';
5
6
use POE qw(Wheel::ReadWrite Wheel::SocketFactory Wheel::Run);
7
use IO::Socket qw(AF_UNIX);
8
use URI;
9
use Koha::Icarus::Task;
10
use JSON; #For "on_client_input"
11
12
my $null_filter = POE::Filter::Line->new(
13
     Literal => chr(0),
14
);
15
16
sub new {
17
    my ($class, $args) = @_;
18
    $args = {} unless defined $args;
19
    $args->{_component} = "server";
20
    $args->{_id} = "undefined";
21
    return bless ($args, $class);
22
}
23
24
#NOTE: "spawn" inspired by http://poe.perl.org/?POE_Cookbook/Object_Methods
25
sub spawn {
26
    my ($class, $args) = @_;
27
    my $self = $class->new($args);
28
    POE::Session->create(
29
        object_states => [
30
            $self => {
31
                _start => "on_server_start",
32
                shutdown => "shutdown",
33
                set_verbosity => "set_verbosity",
34
                _child => "on_task_event",
35
                got_list_tasks => "on_list_tasks",
36
                graceful_shutdown => "graceful_shutdown",
37
                got_client_accept => "on_client_accept",
38
                got_client_error => "on_client_error",
39
                got_server_error => "on_server_error",
40
                got_add_task => "on_add_task",
41
                got_client_input => "on_client_input",
42
            },
43
        ],
44
    );
45
}
46
47
#Methods for POE::Session
48
49
sub on_server_start {
50
    my ($self, $kernel,$heap,$session) = @_[OBJECT, KERNEL,HEAP,SESSION];
51
    my $server_id = $session->ID;
52
    $self->{_id} = $server_id; #Set internal id for logging purposes
53
54
55
    my $bind_address_uri = $self->{Socket};
56
    my $max_tasks = $self->{MaxTasks};
57
58
    $kernel->sig(INT => "graceful_shutdown");
59
    $kernel->sig(TERM => "graceful_shutdown");
60
61
    $heap->{max_tasks} = $max_tasks // 25; #Default maximum of 25 unless otherwise specified
62
63
    $self->log("Maximum number of tasks allowed: $heap->{max_tasks}");
64
    $self->log("Starting server...");
65
66
    my %server_params = (
67
        SuccessEvent => "got_client_accept",
68
        FailureEvent => "got_server_error",
69
    );
70
71
    #TODO: At this time, only "unix" sockets are supported. In future, perhaps TCP/IP sockets could also be supported.
72
    my $uri = URI->new($bind_address_uri);
73
    my $scheme = $uri->scheme;
74
75
    if ($scheme eq 'unix'){
76
        my $bind_address = $uri->path;
77
        $server_params{SocketDomain} = AF_UNIX;
78
        $server_params{BindAddress} = $bind_address;
79
        #When starting a unix socket server, you need to remove any existing references to that socket file.
80
        if ($bind_address && (-e $bind_address) ){
81
            $self->debug("Unlinking $bind_address");
82
            unlink $bind_address or warn "Could not unlink $bind_address: $!";
83
        }
84
    }
85
86
    $heap->{server} = POE::Wheel::SocketFactory->new(%server_params);
87
88
    if ($scheme eq 'unix'){
89
        #FIXME/DEBUGGING: This is a way to force a permission denied error...
90
        #chmod 0755, $uri->path;
91
        #Make the socket writeable to other users like Apache
92
        chmod 0666, $uri->path;
93
    }
94
95
}
96
97
sub shutdown {
98
    my ($self,$heap,$session,$kernel) = @_[OBJECT, HEAP,SESSION,KERNEL];
99
100
    if ($heap->{server}){
101
        $self->log("Shutting down server...");
102
        #Delete the server, so that you can't get any new connections
103
        delete $heap->{server} if $heap->{server};
104
    }
105
106
    if ($heap->{client}){
107
        $self->log("Shutting down any remaining clients...");
108
        #Delete the clients, so that you bring down the existing connections
109
        delete $heap->{client}; #http://www.perlmonks.org/?node_id=176971
110
    }
111
}
112
113
sub on_task_event {
114
    my ($self, $kernel, $heap,$session) = @_[OBJECT,KERNEL, HEAP,SESSION];
115
    my ($action,$child_session,$task) = @_[ARG0,ARG1,ARG2];
116
117
    my $child_id = $child_session->ID;
118
119
    $self->debug("$action child $child_id");
120
121
122
    if ($action eq 'create'){
123
        #NOTE: The $task variable is returned by the child POE session's _start event
124
        my $task_id = $child_session->ID;
125
        $heap->{tasks}->{$task_id}->{task} = $task;
126
127
    } elsif ($action eq 'lose'){
128
        my $task_id = $child_session->ID;
129
        delete $heap->{tasks}->{$task_id};
130
    }
131
}
132
133
#TODO: Put this in a parent class?
134
sub set_verbosity {
135
    my ($self,$session,$kernel,$new_verbosity) = @_[OBJECT,SESSION,KERNEL,ARG0];
136
    if (defined $new_verbosity){
137
        $self->{Verbosity} = $new_verbosity;
138
    }
139
}
140
141
sub on_list_tasks {
142
    my ($self, $kernel, $heap,$session) = @_[OBJECT, KERNEL, HEAP,SESSION];
143
144
    #DEBUG: You can access the POE::Kernel's sessions with "$POE::Kernel::poe_kernel->[POE::Kernel::KR_SESSIONS]".
145
    #While it's black magic you shouldn't touch, it can be helpful when debugging.
146
147
    my @tasks = ();
148
    foreach my $task_id (keys %{$heap->{tasks}} ){
149
        push(@tasks,{ task_id => $task_id, task => $heap->{tasks}->{$task_id}->{task} });
150
    }
151
    return \@tasks;
152
}
153
154
sub graceful_shutdown {
155
    my ($self, $heap,$session,$kernel,$signal) = @_[OBJECT, HEAP,SESSION,KERNEL,ARG0];
156
157
    #Tell the kernel that you're handling the signal sent to this session
158
    $kernel->sig_handled();
159
    $kernel->sig($signal);
160
161
    my $tasks = $kernel->call($session,"got_list_tasks");
162
163
164
    if ( $heap->{tasks} && %{$heap->{tasks}} ){
165
        $self->log("Waiting for tasks to finish...");
166
        foreach my $task_id (keys %{$heap->{tasks}}){
167
            $self->log("Task $task_id still exists...");
168
            $kernel->post($task_id,"got_task_stop");
169
        }
170
    } else {
171
        $self->log("All tasks have finished");
172
        $kernel->yield("shutdown");
173
        return;
174
    }
175
176
    $self->log("Attempting graceful shutdown in 1 second...");
177
    #NOTE: Basically, we just try another graceful shutdown on the next tick.
178
    $kernel->delay("graceful_shutdown" => 1);
179
}
180
181
#Accept client connection to listener
182
sub on_client_accept {
183
    my ($self, $client_socket, $server_wheel_id, $heap, $session) = @_[OBJECT, ARG0, ARG3, HEAP,SESSION];
184
185
    my $client_wheel = POE::Wheel::ReadWrite->new(
186
      Handle => $client_socket,
187
      InputEvent => "got_client_input",
188
      ErrorEvent => "got_client_error",
189
      InputFilter => $null_filter,
190
      OutputFilter => $null_filter,
191
    );
192
193
    $client_wheel->put("HELLO");
194
    $heap->{client}->{ $client_wheel->ID() } = $client_wheel;
195
196
    $self->debug("Connection ".$client_wheel->ID()." started.");
197
198
}
199
200
#Handle server error - shutdown server
201
sub on_server_error {
202
    my ($self, $operation, $errnum, $errstr, $heap, $session) = @_[OBJECT, ARG0, ARG1, ARG2,HEAP, SESSION];
203
    $self->log("Server $operation error $errnum: $errstr");
204
    delete $heap->{server};
205
}
206
207
#Handle client error - including disconnect
208
sub on_client_error {
209
    my ($self, $wheel_id,$heap,$session) = @_[OBJECT, ARG3,HEAP,SESSION];
210
211
    $self->debug("Connection $wheel_id failed or ended.");
212
213
    delete $heap->{client}->{$wheel_id};
214
215
}
216
217
sub on_add_task {
218
    my ($self, $message, $kernel, $heap, $session) = @_[OBJECT, ARG0, KERNEL, HEAP,SESSION];
219
220
    #Fetch a list of all tasks
221
    my @task_keys = keys %{$heap->{tasks}};
222
223
    #If the number in the list is less than the max, add a new task
224
    #else die.
225
    if (scalar @task_keys < $heap->{max_tasks}){
226
        my $server_id = $session->ID;
227
        my $task_session = Koha::Icarus::Task->spawn({ message => $message, server_id => $server_id, Verbosity => $self->{Verbosity}, SpoolDir => $self->{SpoolDir}, });
228
        return $task_session->ID;
229
    } else {
230
        #This die should be caught by the event caller...
231
        die "Maximum number of tasks already reached.\n";
232
    }
233
}
234
235
sub on_client_input {
236
    my ($self, $input, $wheel_id, $session, $kernel, $heap) = @_[OBJECT, ARG0, ARG1, SESSION, KERNEL, HEAP];
237
238
    #Store server id more explicitly
239
    my $server_id = $session->ID;
240
241
    #Server listener has received input from client
242
    my $client = $heap->{client}->{$wheel_id};
243
244
    #Parse input from client
245
    my $message = from_json($input);
246
247
    if ( ref $message eq 'HASH' ){
248
        #Read "command" from client
249
        if (my $command = $message->{command}){
250
            $self->log("Message received with command \"$command\".");
251
            if ($command eq 'add task'){
252
                my $output = {};
253
254
                #Create a task session
255
                eval {
256
                   #NOTE: The server automatically keeps track of its child tasks
257
                    my $task_id = $kernel->call($server_id,"got_add_task",$message);
258
259
                    $output->{action} = "added";
260
                    $output->{task_id} = $task_id;
261
                };
262
                if ($@){
263
                    $self->debug("$@");
264
                    chomp($@);
265
                    $output->{action} = "error";
266
                    $output->{error_message} = $@;
267
                }
268
                my $server_output = to_json($output);
269
                $client->put($server_output);
270
                return;
271
272
            } elsif ( ($command eq 'remove task') || ($command eq 'start task' ) ){
273
274
                my $task_id = $message->{task_id};
275
276
                my $output = {
277
                    task_id => $task_id,
278
                };
279
280
                if ($command eq 'remove task'){
281
                    $kernel->call($task_id,"got_task_stop");
282
                    $output->{action} = "removed";
283
                } elsif ($command eq 'start task'){
284
                    my $response = $kernel->call($task_id, "on_task_init");
285
                    $output->{action} = $response;
286
                }
287
288
                if ($!){
289
                    $output->{action} = "error";
290
                    $output->{error_message} = $!;
291
                }
292
293
                #FIXME: What do we actually want to send back to the client?
294
                my $server_output = to_json($output);
295
                $client->put($server_output);
296
                return;
297
298
            } elsif ($command eq 'list tasks'){
299
300
                #Get tasks from listener (ie self)
301
                my $tasks = $kernel->call($server_id, "got_list_tasks");
302
303
                #Prepare output for client
304
                my $server_output = to_json({tasks => $tasks}, {pretty => 1});
305
306
                #Send output to client
307
                $client->put($server_output);
308
                return;
309
310
            } elsif ($command eq 'shutdown'){
311
                $kernel->post($server_id, "graceful_shutdown");
312
                my $server_output = to_json({action => 'shutting down'});
313
                $client->put($server_output);
314
                return;
315
            } else {
316
                $self->log("The message contained an invalid command!");
317
                $client->put("Sorry! That is an invalid command!");
318
                return;
319
            }
320
        } else {
321
            $self->log("The message was missing a command!");
322
        }
323
    } else {
324
        $self->log("The message was malformed!");
325
    }
326
    $client->put("Sorry! That is an invalid message!");
327
    return;
328
}
329
330
1;
(-)a/Koha/Icarus/Task.pm (+324 lines)
Line 0 Link Here
1
package Koha::Icarus::Task;
2
3
use Modern::Perl;
4
use parent 'Koha::Icarus::Base';
5
6
use POE qw(Wheel::Run);
7
use DateTime;
8
use DateTime::Format::Strptime;
9
use JSON;
10
use Module::Load::Conditional qw/can_load/;
11
12
my $datetime_pattern = DateTime::Format::Strptime->new(
13
    pattern   => '%F %T',
14
    time_zone => 'local',
15
);
16
my $epoch_pattern = DateTime::Format::Strptime->new(
17
    pattern   => '%s',
18
);
19
20
sub new {
21
    my ($class, $args) = @_;
22
    $args = {} unless defined $args;
23
    $args->{_component} = "task";
24
    $args->{_id} = "undefined";
25
    return bless ($args, $class);
26
}
27
28
#NOTE: "spawn" inspired by http://poe.perl.org/?POE_Cookbook/Object_Methods
29
sub spawn {
30
    my ($class, $args) = @_;
31
    my $self = $class->new($args);
32
    my $task_session = POE::Session->create(
33
        object_states => [
34
            $self => {
35
                _start => "on_task_create",
36
                 "got_child_stdout" => "on_child_stdout",
37
                 "got_child_stderr" => "on_child_stderr",
38
                 "got_child_close"  => "on_child_close",
39
                 "got_child_signal" => "on_child_signal",
40
                 "got_terminal_signal" => "on_terminal_signal",
41
                 "child_process_success" => "child_process_success",
42
                 "child_process_failure" => "child_process_failure",
43
                 "got_task_stop" => "on_task_stop",
44
                 "on_task_init" => "on_task_init",
45
                 "on_task_start" => "on_task_start",
46
            },
47
        ],
48
    );
49
    return $task_session;
50
}
51
52
sub on_task_create {
53
    my ($self, $session, $kernel, $heap) = @_[OBJECT, SESSION, KERNEL, HEAP];
54
55
    #Trap terminal signals so that the task can stop gracefully.
56
    $kernel->sig(INT => "got_terminal_signal");
57
    $kernel->sig(TERM => "got_terminal_signal");
58
59
    my $task_id = $session->ID;
60
    if ($task_id){
61
        #Tell the kernel that this task is waiting for an external action (ie keepalive counter)
62
        $kernel->refcount_increment($task_id,"waiting task");
63
        $self->{_id} = $task_id; #Set internal id for logging purposes
64
    }
65
66
    my $server_id = $self->{server_id};
67
    if ($server_id){
68
        $heap->{server_id} = $server_id;
69
    }
70
71
    my $task = undef;
72
    my $message = $self->{message};
73
    if ($message){
74
        $task = $message->{task};
75
        if ($task){
76
            $task->{status} = 'new';
77
            $heap->{task} = $task;
78
        }
79
    }
80
    return $task; #This return value is used by the parent POE session's _child handler
81
}
82
83
#This sub is just to start it now, or set it to start in the future... if the time is now or in the past, it starts now... if it's in the future, it starts in the future...
84
sub on_task_init {
85
    my ($self, $session, $kernel, $heap) = @_[OBJECT, SESSION, KERNEL, HEAP];
86
    my $response = 'pending';
87
    my $task = $heap->{task};
88
    my $status = $task->{status};
89
    if ($status){
90
        if ($status eq 'started'){
91
            $response = 'already started';
92
        } elsif ($status eq 'pending'){
93
            $response = 'already pending';
94
        } else {
95
            $task->{status} = 'pending';
96
97
            my $start = $task->{start};
98
            my $start_message = $start;
99
100
101
            my $dt;
102
            if ( $dt = $datetime_pattern->parse_datetime($start) ){
103
                #e.g. 2016-04-06 00:00:00
104
            } elsif ( $dt = $epoch_pattern->parse_datetime($start) ){
105
                #e.g. 1459837498 or apparently 0000-00-00 00:00:00
106
            } else {
107
                #If we don't match the datetime_pattern or epoch_pattern, then we start right now.
108
                $dt = DateTime->now( time_zone => 'local', );
109
            }
110
            if ($dt){
111
                $start = $dt->epoch;
112
                $start_message = $dt;
113
            }
114
115
116
            $self->log("Start task at $start_message");
117
            #NOTE: $start must be in UNIX epoch time (ie number of seconds that have elapsed since 00:00:00 UTC Thursday 1 January 1970)
118
            $kernel->alarm("on_task_start",$start);
119
        }
120
    }
121
    return $response;
122
}
123
124
sub on_task_start {
125
    my ($self, $session, $kernel, $heap) = @_[OBJECT, SESSION, KERNEL, HEAP];
126
    my $task = $heap->{task};
127
    $task->{status} = 'started';
128
129
    if (my $repeat_interval = $task->{repeat_interval}){
130
        #NOTE: Reset the start time with a human readable timestamp
131
        my $dt = DateTime->now( time_zone => 'local', );
132
        $dt->add( seconds => $repeat_interval );
133
        $task->{start} = $dt->strftime("%F %T");
134
    }
135
    #FIXME: You need to impose child process limits here! How many child processes are allowed to be running at any given time? Well, you can only have one child process per task...
136
    #so it's really more of a limit on the number of tasks... you probably need to have an internal task queue... that's easy enough though.
137
    my $child = POE::Wheel::Run->new(
138
        ProgramArgs => [ $task, ],
139
        Program => sub {
140
            my ($task) = @_;
141
142
            #Perform some last minute POE calls before running the task module plugin
143
            my $session = $poe_kernel->get_active_session();
144
            if ($session){
145
                my $heap = $session->get_heap();
146
                $poe_kernel->call($heap->{server_id},"set_verbosity",0); #This turns off the server logging in this forked process, so the following call() doesn't mess up our logs
147
                $poe_kernel->call($heap->{server_id},"shutdown"); #Shutdown the socket listener on the child process, so there's zero chance of writing to or reading from the socket in the child process
148
            }
149
150
            #NOTE: I don't know if this really needs to be run, but it shouldn't hurt.
151
            $poe_kernel->stop();
152
153
            #Try to load the task type module.
154
            my $task_type = $task->{type};
155
            if ( can_load ( modules => { $task_type => undef, }, ) ){
156
                #Create the object
157
                my $task_object = $task_type->new({task => $task, Verbosity => $self->{Verbosity}, SpoolDir => $self->{SpoolDir}, });
158
                if ($task_object){
159
                    #Synchronous action: run the task module
160
                    $task_object->run;
161
                }
162
            } else {
163
                die "Couldn't load module $task_type: $Module::Load::Conditional::ERROR"
164
            }
165
        },
166
        StdoutEvent  => "got_child_stdout",
167
        StderrEvent  => "got_child_stderr",
168
        CloseEvent   => "got_child_close",
169
        NoSetPgrp => 1, #Keep child processes in same group as parent. This is especially useful when using Ctrl+C to kill the whole group.
170
    );
171
172
    $kernel->sig_child($child->PID, "got_child_signal");
173
    # Wheel events include the wheel's ID.
174
    $_[HEAP]{children_by_wid}{$child->ID} = $child;
175
    # Signal events include the process ID.
176
    $_[HEAP]{children_by_pid}{$child->PID} = $child;
177
178
    $self->debug("child pid ".$child->PID." started as wheel ".$child->ID);
179
}
180
181
sub on_task_stop {
182
    my ($self, $session, $kernel, $heap) = @_[OBJECT, SESSION, KERNEL, HEAP];
183
    my $task = $heap->{task};
184
    $task->{status} = 'stopping';
185
    my $task_id = $session->ID;
186
    my $server_id = $heap->{server_id};
187
188
    if ($heap->{stopping}){
189
        $self->debug("Task is already in the process of stopping...");
190
191
    } else {
192
193
        $self->log("Trying to stop task.");
194
195
196
        #Mark this task as stopping
197
        $heap->{stopping} = 1;
198
199
        #Stop the task from spawning new jobs
200
        $kernel->alarm("on_task_start");
201
202
        my $children_by_pid = $heap->{children_by_pid};
203
        if ($children_by_pid && %$children_by_pid){
204
205
            $self->debug("Child processes in progres...");
206
            my $child_processes = $heap->{children_by_pid};
207
            foreach my $child_pid (keys %$child_processes){
208
                my $child = $child_processes->{$child_pid};
209
                $self->debug("Telling child pid $child_pid to stop");
210
                $child->put("quit");
211
                #TODO: Perhaps it would be worthwhile having a kill switch too?
212
                # my $rv = $child->kill("TERM");
213
            }
214
        }
215
216
        $self->log("Removing task keepalive.");
217
218
        $kernel->refcount_decrement($task_id,"waiting task");
219
    }
220
}
221
222
sub on_terminal_signal {
223
    my ($self, $signal,$session,$kernel) = @_[OBJECT, ARG0,SESSION,KERNEL];
224
    $self->debug("Trapped SIGNAL: $signal.");
225
    #Gracefully stop the task
226
    $kernel->call($session, "got_task_stop");
227
}
228
229
sub child_process_failure {
230
    my ($self, $heap,$session,$kernel) = @_[OBJECT, HEAP,SESSION,KERNEL];
231
    my $task = $heap->{task};
232
    $task->{status} = "failed";
233
}
234
235
sub child_process_success {
236
    my ($self, $heap,$session,$kernel) = @_[OBJECT, HEAP,SESSION,KERNEL];
237
    my $task = $heap->{task};
238
    if (my $repeat_interval = $task->{repeat_interval}){
239
        if ($heap->{stopping}){
240
            $self->log("Will skip repeating the task, as task is stopping.");
241
        } else {
242
            $self->log("Will repeat the task");
243
            $task->{status} = "restarting";
244
            $kernel->yield("on_task_init");
245
        }
246
    } else {
247
        $self->debug("I'm going to stop this task");
248
        $kernel->yield("got_task_stop");
249
    }
250
}
251
252
#############################################################
253
#                                                           #
254
#      Methods for communicating with child processes       #
255
#                                                           #
256
#############################################################
257
# Originally inspired by the POE::Wheel::Run perldoc example
258
259
# Wheel event, including the wheel's ID
260
sub on_child_stdout {
261
    my ($self, $stdout_line, $wheel_id, $session) = @_[OBJECT, ARG0, ARG1, SESSION];
262
    my $child = $_[HEAP]{children_by_wid}{$wheel_id};
263
    #NOTE: Log everything child process sends to STDOUT
264
    $self->log("[pid ".$child->PID."] STDOUT: $stdout_line");
265
266
    #If the child outputs a line to STDOUT which starts with UPDATE_PARAMS=, we capture the data,
267
    #and update the task params.
268
    if ($stdout_line =~ /^UPDATE_PARAMS=(.*)$/){
269
        my $json_string = $1;
270
        my $json = from_json($json_string);
271
        my $task = $_[HEAP]->{task};
272
        my $params = $task->{params};
273
        foreach my $key (%$json){
274
            if (defined $params->{$key}){
275
                #FIXME: Don't just overwrite? Only update differences?
276
                $params->{$key} = $json->{$key};
277
            }
278
        }
279
        $_[HEAP]->{task} = $task;
280
    }
281
}
282
283
# Wheel event, including the wheel's ID.
284
sub on_child_stderr {
285
    my ($self, $stderr_line, $wheel_id, $session) = @_[OBJECT, ARG0, ARG1,SESSION];
286
    my $child = $_[HEAP]{children_by_wid}{$wheel_id};
287
    #NOTE: Log everything child process sends to STDERR
288
    $self->log("[pid ".$child->PID."] STDERR: $stderr_line");
289
}
290
291
# Wheel event, including the wheel's ID.
292
sub on_child_close {
293
    my ($self, $wheel_id,$session,$kernel) = @_[OBJECT, ARG0,SESSION,KERNEL];
294
295
    my $child = delete $_[HEAP]{children_by_wid}{$wheel_id};
296
297
    # May have been reaped by on_child_signal().
298
    unless (defined $child) {
299
        $self->debug("[wid $wheel_id] closed all pipes.");
300
        return;
301
    }
302
    $self->debug("[pid ".$child->PID."] closed all pipes.");
303
    delete $_[HEAP]{children_by_pid}{$child->PID};
304
}
305
306
sub on_child_signal {
307
    my ($self, $heap,$kernel,$pid,$exit_code,$session) = @_[OBJECT, HEAP,KERNEL,ARG1,ARG2,SESSION];
308
309
    #If the child's exit code is 0, handle this successful exit status
310
    if ($exit_code == 0){
311
        $kernel->yield("child_process_success");
312
    } else {
313
        $kernel->yield("child_process_failure");
314
    }
315
    $self->debug("pid $pid exited with status $exit_code.");
316
    my $child = delete $_[HEAP]{children_by_pid}{$pid};
317
318
    # May have been reaped by on_child_close().
319
    return unless defined $child;
320
321
    delete $_[HEAP]{children_by_wid}{$child->ID};
322
}
323
324
1;
(-)a/Koha/Icarus/Task/Base.pm (+24 lines)
Line 0 Link Here
1
package Koha::Icarus::Task::Base;
2
3
use Modern::Perl;
4
use IO::Select;
5
6
sub new {
7
    my ($class, $args) = @_;
8
    $args = {} unless defined $args;
9
    return bless ($args, $class);
10
}
11
12
sub listen_for_instruction {
13
    my ($self) = @_;
14
    my $select = $self->{_select} ||= IO::Select->new(\*STDIN);
15
    if (my @ready_FHs  = $select->can_read(0) ){
16
        foreach my $FH (@ready_FHs){
17
            my $line = $FH->getline();
18
            chomp($line);
19
            return $line;
20
        }
21
    }
22
}
23
24
1;
(-)a/Koha/Icarus/Task/Download/OAIPMH/Biblio.pm (+318 lines)
Line 0 Link Here
1
package Koha::Icarus::Task::Download::OAIPMH::Biblio;
2
3
use Modern::Perl;
4
use parent 'Koha::Icarus::Task::Base';
5
6
use DateTime;
7
use DateTime::Format::Strptime;
8
use HTTP::OAI;
9
use File::Path qw(make_path);
10
use Digest::MD5;
11
use JSON;
12
use URI;
13
14
my $strp = DateTime::Format::Strptime->new(
15
        pattern   => '%Y%m%dT%H%M%S.%NZ',
16
);
17
18
my $oai_second_granularity = DateTime::Format::Strptime->new(
19
        pattern   => '%Y-%m-%dT%H:%M:%SZ',
20
);
21
22
my $oai_day_granularity = DateTime::Format::Strptime->new(
23
        pattern   => '%Y-%m-%d',
24
);
25
26
sub validate_parameter_names {
27
28
}
29
sub validate_repeat_interval {
30
    my ($self,$repeat_interval) = @_;
31
    if (defined $repeat_interval && $repeat_interval =~ /^\d+$/){
32
        return undef;
33
    }
34
    $self->{invalid_data}++;
35
    return { not_numeric => 1, };
36
}
37
38
sub validate_url {
39
    my ($self,$url) = @_;
40
    my $response = {};
41
    if (my $url_obj = URI->new($url)){
42
        if ($url_obj->scheme ne "http"){
43
            $response->{not_http} = 1;
44
            $self->{invalid_data}++;
45
        }
46
        if ( ! $url_obj->path){
47
            $response->{no_path} = 1;
48
            $self->{invalid_data}++;
49
        }
50
    } else {
51
        $response->{not_a_url} = 1;
52
        $self->{invalid_data}++;
53
    }
54
55
    return $response;
56
}
57
58
sub validate {
59
    my ($self, $args) = @_;
60
    #Reset the invalid data counter...
61
    $self->{invalid_data} = 0;
62
    my $errors = { };
63
    my $task = $self->{task};
64
    my $tests = $args->{tests};
65
    if ($task){
66
        if ($tests && $tests eq 'all'){
67
            #warn "PARAMS = ".$task->{params};
68
        }
69
    }
70
    my $params = $task->{params};
71
72
    #validate_start_time
73
    $errors->{"repeat_interval"} = $self->validate_repeat_interval($task->{repeat_interval});
74
75
    $errors->{"url"} = $self->validate_url($params->{url});
76
77
    #NOTE: You don't need to validate these 3 HTTP Basic Auth parameters
78
    #validate_username
79
    #validate_password
80
    #validate_realm
81
82
    #OAI-PMH parameters
83
    #validate_verb
84
    #validate_sets
85
    #validate_marcxml
86
    #validate_from
87
    #validate_until
88
89
    #Download parameters
90
    #validate_queue
91
92
    return $errors;
93
}
94
95
sub new {
96
    my ($class, $args) = @_;
97
    $args = {} unless defined $args;
98
    $args->{invalid_data} = 0;
99
    return bless ($args, $class);
100
}
101
102
sub validate_queue {
103
    my ( $self ) = @_;
104
    my $task = $self->{task};
105
    my $spooldir = $self->{SpoolDir};
106
    if ( -d $spooldir ){
107
        chdir $spooldir or die "$!";
108
        
109
        my $named_queue = $task->{params}->{queue};
110
        my $queue = "queue/$named_queue";
111
        if ( ! -d $queue ){
112
            make_path($queue,{ mode => 0755 });
113
        }
114
        if ( -d $queue ){
115
            chdir $queue or die "$!";
116
        }
117
    } else {
118
        die "$!";
119
    }
120
}
121
122
sub run {
123
    my ( $self ) = @_;
124
    $self->validate_queue;
125
126
    my $task = $self->{task};
127
128
    #DEBUGGING/FIXME: Remove these lines
129
    if ($self->{Verbosity} && $self->{Verbosity} == 9){
130
        use Data::Dumper;
131
        warn Dumper($task);
132
    }
133
134
    my $params = $task->{params};
135
136
    my $now = DateTime->now(); #This is in UTC time, which is required by the OAI-PMH protocol.
137
    if ( $oai_second_granularity->parse_datetime($params->{from}) ){
138
        $now->set_formatter($oai_second_granularity);
139
    } else {
140
        $now->set_formatter($oai_day_granularity);
141
    }
142
143
    $params->{until}  = "$now" if $task->{repeat_interval};
144
145
    $self->{digester} = Digest::MD5->new();
146
    $self->create_harvester;
147
    my $sets = $self->prepare_sets;
148
149
    #Send a OAI-PMH request for each set
150
    foreach my $set (@{$sets}){
151
        my $response = $self->send_request({set => $set});
152
        $self->handle_response({ response => $response, set => $set,});
153
    }
154
155
    #FIXME: Do you want to update the task only when the task is finished, or
156
    #also after each resumption?
157
    #Update the task params in Icarus after the task is finished...
158
    #TODO: This really does make it seem like you should be handling the repeat_interval within the child process rather than the parent...
159
    if ($task->{repeat_interval}){
160
        $params->{from} = "$now";
161
        $params->{until} = "";
162
        my $json_update = to_json($params);
163
        say STDOUT "UPDATE_PARAMS=$json_update";
164
    }
165
166
}
167
168
#FIXME: I wonder if it would be faster to send your own HTTP requests and not use HTTP::OAI...
169
sub send_request {
170
    my ( $self, $args ) = @_;
171
172
    #NOTE: This is plugin specific as the plugins define when they stop to listen for instructions...
173
    #NOTE: Before sending a new request, check if Icarus has already asked us to quit.
174
    my $instruction = $self->listen_for_instruction();
175
    if ($instruction eq 'quit'){
176
        warn "I was asked to quit!";
177
        return;
178
    }
179
180
    my $set = $args->{set};
181
    my $resumptionToken = $args->{resumptionToken};
182
183
    my $response;
184
    my $task_params = $self->{task}->{params};
185
186
    my $harvester = $self->{harvester};
187
    my $verb = $task_params->{verb};
188
    if ($verb eq 'GetRecord'){
189
        $response = $harvester->GetRecord(
190
            metadataPrefix => $task_params->{metadataPrefix},
191
            identifier => $task_params->{identifier},
192
         );
193
    } elsif ($verb eq 'ListRecords'){
194
        $response = $harvester->ListRecords(
195
            metadataPrefix => $task_params->{metadataPrefix},
196
            from => $task_params->{from},
197
            until => $task_params->{until},
198
            set => $set,
199
            resumptionToken => $resumptionToken,
200
        );
201
    }
202
    return $response;
203
}
204
205
sub create_harvester {
206
    my ( $self ) = @_;
207
    my $task_params = $self->{task}->{params};
208
209
    #FIXME: DEBUGGING
210
    #use HTTP::OAI::Debug qw(+);
211
212
    #Create HTTP::OAI::Harvester object
213
    my $harvester = new HTTP::OAI::Harvester( baseURL => $task_params->{url} );
214
    if ($harvester){
215
        $harvester->timeout(5); #NOTE: the default timeout is 180
216
        #Set HTTP Basic Authentication Credentials
217
        my $uri = URI->new($task_params->{url});
218
        my $host = $uri->host;
219
        my $port = $uri->port;
220
        $harvester->credentials($host.":".$port, $task_params->{realm}, $task_params->{username}, $task_params->{password});
221
    }
222
    $self->{harvester} = $harvester;
223
}
224
225
sub prepare_sets {
226
    my ( $self ) = @_;
227
    my $task_params = $self->{task}->{params};
228
    my @sets = ();
229
    if ($task_params->{sets}){
230
        @sets = split(/\|/, $task_params->{sets});
231
    }
232
    #If no sets are defined, create a null element to force the foreach loop to run once
233
    if (!@sets){
234
        push(@sets,undef)
235
    }
236
    return \@sets;
237
}
238
239
sub handle_response {
240
    my ( $self, $args ) = @_;
241
    my $params = $self->{task}->{params};
242
    my $response = $args->{response};
243
    my $set = $args->{set};
244
    if ($response){
245
        #NOTE: We have options at this point
246
        #Option 1: Use $response->toDOM() to handle the XML response as a single document
247
        #Option 2: Use $response->next() to handle each record individually. You would need to create a new document using $rec->header->dom() and $rec->metadata->dom() anyway.
248
249
        #NOTE: I wonder which option would be the fastest. For now, we're going with Option 1:
250
        my $dom = $response->toDOM;
251
        my $root = $dom->documentElement;
252
253
        #FIXME: Provide these as arguments so you're not re-creating them for each response
254
        my $xpc = XML::LibXML::XPathContext->new();
255
        $xpc->registerNs('oai','http://www.openarchives.org/OAI/2.0/');
256
        my $xpath = XML::LibXML::XPathExpression->new("(oai:GetRecord|oai:ListRecords)/oai:record");
257
258
259
        my @records = $xpc->findnodes($xpath,$root);
260
        my $now_pretty = DateTime->now();
261
262
        $now_pretty->set_formatter($strp);
263
        print "Downloaded ".scalar @records." records at $now_pretty\n";
264
        foreach my $record (@records) {
265
266
            #FIXME: This is where you could put a filter to prevent certain records from being saved...
267
268
            #Create a new XML document from the XML fragment
269
            my $document = XML::LibXML::Document->new( "1.0", "UTF-8" );
270
            $document->setDocumentElement($record);
271
            my $record_string = $document->toString;
272
273
            #NOTE: We have options at this point.
274
            #Option 1: Write documents to disk, and have a separate importer upload the documents
275
            #Option 2: Use AnyEvent::HTTP or POE::Component::Client::HTTP to send to a HTTP API asynchronously
276
            #Option 3: Write records to a database, and have a separate importer upload the documents
277
            #Option 4: Shared memory, although that seems fragile if nothing else
278
            #Option 5: Write the records to a socket/pipe
279
280
            #NOTE: I wonder which option would be the fastest. For now, we're going to go with Option 1:
281
            $self->{digester}->add($record_string);
282
            my $digest = $self->{digester}->hexdigest;
283
            #FIXME: If a record appears more than once during the download signified by $now, you'll
284
            #overwrite the former with the latter. While this acts as a sort of heavy-handed de-duplication,
285
            #you need to take into account the importer daemon...
286
287
            require Time::HiRes;
288
            my $epoch = Time::HiRes::time();
289
            my $now = DateTime->from_epoch(epoch => $epoch);
290
            $now->set_formatter($strp);
291
292
            my $filename = "$now-$digest";
293
            #NOTE: Here is where we write the XML out to disk
294
            eval {
295
                my $state = $document->toFile($filename);
296
            };
297
            if ($@){
298
                die("Error while writing to disk: $@");
299
            }
300
        }
301
302
303
        #NOTE: Check if object has method due to bug in HTTP::OAI which causes fatal error on $response->resumptionToken if no real response is fetched from the OAI-PMH server
304
        if ($response->can("resumptionToken")){
305
            my $resumption_token = $response->resumptionToken->resumptionToken if $response->resumptionToken && $response->resumptionToken->resumptionToken;
306
            if ($resumption_token){
307
                #warn "Resumption Token = $resumption_token";
308
                my $resumed_response = $self->send_request({set => $set, resumptionToken => $resumption_token});
309
                $self->handle_response({ response => $resumed_response, set => $set,});
310
            }
311
        }
312
313
        #In theory $response->resume(resumptionToken => resumptionToken) should kick off another response...
314
        warn $response->message if $response->is_error;
315
    }
316
}
317
318
1;
(-)a/Koha/Icarus/Task/Upload/OAIPMH/Biblio.pm (+118 lines)
Line 0 Link Here
1
package Koha::Icarus::Task::Upload::OAIPMH::Biblio;
2
3
use Modern::Perl;
4
use parent 'Koha::Icarus::Task::Base';
5
#use URI;
6
use LWP::UserAgent;
7
use HTTP::Status qw(:constants);
8
9
my $ua = LWP::UserAgent->new;
10
11
#FIXME: If we store the cookie jar on disk, we can prevent unnecessary HTTP requests...
12
#We would need to make sure that it's stored on a private per-instance basis though...
13
$ua->cookie_jar({});
14
15
16
sub new {
17
    my ($class, $args) = @_;
18
    $args = {} unless defined $args;
19
    return bless ($args, $class);
20
}
21
22
sub run {
23
    my ( $self ) = @_;
24
25
    my $task = $self->{task};
26
    my $params = $task->{params};
27
    
28
    my $spooldir = $self->{SpoolDir};
29
    my $named_queue = $task->{params}->{queue};
30
    my $queue = "$spooldir/queue/$named_queue";
31
32
    if ($self->{Verbosity} && $self->{Verbosity} == 9){
33
        use Data::Dumper;
34
        warn Dumper($task);
35
    }
36
37
    if ( -d $queue){
38
        my $is_opened = opendir(my $dh, $queue);
39
        if ($is_opened){
40
            my @files = sort readdir($dh);
41
            foreach my $file (@files){
42
                #NOTE: This is plugin specific as the plugins define when they stop to listen for instructions...
43
                my $instruction = $self->listen_for_instruction();
44
                if ($instruction eq 'quit'){
45
                    warn "I was asked to quit!";
46
                    return;
47
                }
48
49
                next if $file =~ /^\.+$/;
50
                my $filepath = "$queue/$file";
51
                if ( -d $filepath ){
52
                    #Do nothing for directories
53
                } elsif ( -e $filepath ){
54
                    print "File: $file\n";
55
56
                    #Slurp mode
57
                    local $/;
58
                    #TODO: Check flock on $filepath first
59
                    open( my $fh, '<', $filepath );
60
                    my $data   = <$fh>;
61
62
                    #TODO: Improve this section...
63
                    #Send to Koha API... (we could speed this up using Asynchronous HTTP requests with AnyEvent::HTTP...)
64
                    my $resp = post_to_api($data,$params);
65
66
                    my $status = $resp->code;
67
68
                    if ($status == HTTP_UNAUTHORIZED || $status == HTTP_FORBIDDEN) {
69
                        $resp = remote_authenticate($params);
70
                        $resp = post_to_api($data,$params) if $resp->is_success;
71
                    }
72
73
                    if ($resp->code == HTTP_OK){
74
                        print "Success.\n";
75
                        print $resp->decoded_content;
76
                        print "\n";
77
                        unlink $filepath;
78
                    }
79
                }
80
            }
81
        } else {
82
            die "Couldn't open queue";
83
        }
84
    } else {
85
        die "Queue doesn't exist";
86
    }
87
}
88
89
sub post_to_api {
90
    my ($data, $params) = @_;
91
    print "Posting to API...\n";
92
    my $resp = $ua->post( $params->{target_uri},
93
                  {'nomatch_action' => $params->{nomatch_action},
94
                   'overlay_action' => $params->{overlay_action},
95
                   'match'          => $params->{match},
96
                   'framework'      => $params->{framework},
97
                   'filter'         => $params->{filter},
98
                   'record_type'    => "biblio",
99
                   'xml'            => $data}
100
    );
101
    return $resp;
102
}
103
104
sub remote_authenticate {
105
    my ($params) = @_;
106
    print "Authenticating...\n";
107
108
    my $auth_uri = $params->{auth_uri};
109
    my $user = $params->{auth_username};
110
    my $password = $params->{auth_password};
111
    my $resp = $ua->post( $auth_uri, { userid => $user, password => $password } );
112
    if ($resp->code == HTTP_OK){
113
        print "Authenticated.\n";
114
    }
115
    return $resp
116
}
117
118
1;
(-)a/Koha/SavedTask.pm (+86 lines)
Line 0 Link Here
1
package Koha::SavedTask;
2
3
# Copyright Prosentient Systems 2016
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use Carp;
23
24
use JSON;
25
26
use base qw(Koha::Object);
27
28
29
30
=head1 NAME
31
32
Koha::SavedTask -
33
34
=head1 API
35
36
=head2 Class Methods
37
38
=cut
39
40
41
42
=head3 _type
43
44
=cut
45
46
sub _type {
47
    return 'SavedTask';
48
}
49
50
sub params_as_perl {
51
    my ($self) = @_;
52
    my $perl = from_json($self->params);
53
    return $perl;
54
}
55
56
sub serialize {
57
    my ($self,$args) = @_;
58
    my $for = $args->{for};
59
    my $type = $args->{type};
60
    if ($for eq 'icarus'){
61
        my $json_params = $self->params;
62
        my $perl_params = from_json($json_params);
63
64
        my $icarus_task = {
65
            type => $self->task_type,
66
            start => $self->start_time,
67
            repeat_interval => $self->repeat_interval,
68
            params => $perl_params,
69
        };
70
        if ($type eq 'perl'){
71
            return  $icarus_task;
72
        } elsif ($type eq 'json'){
73
            my $json = to_json($icarus_task);
74
            return $json;
75
        }
76
    }
77
    return undef;
78
}
79
80
=head1 AUTHOR
81
82
David Cook <dcook@prosentient.com.au>
83
84
=cut
85
86
1;
(-)a/Koha/SavedTasks.pm (+62 lines)
Line 0 Link Here
1
package Koha::SavedTasks;
2
3
# Copyright Prosentient Systems 2016
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use Carp;
23
24
use Koha::Database;
25
26
use Koha::SavedTask;
27
28
use base qw(Koha::Objects);
29
30
=head1 NAME
31
32
Koha::SavedTasks -
33
34
=head1 API
35
36
=head2 Class Methods
37
38
=cut
39
40
=head3 _type
41
42
=cut
43
44
sub _type {
45
    return 'SavedTask';
46
}
47
48
=head3 object_class
49
50
=cut
51
52
sub object_class {
53
    return 'Koha::SavedTask';
54
}
55
56
=head1 AUTHOR
57
58
David Cook <dcook@prosentient.com.au>
59
60
=cut
61
62
1;
(-)a/Koha/Schema/Result/SavedTask.pm (+98 lines)
Line 0 Link Here
1
use utf8;
2
package Koha::Schema::Result::SavedTask;
3
4
# Created by DBIx::Class::Schema::Loader
5
# DO NOT MODIFY THE FIRST PART OF THIS FILE
6
7
=head1 NAME
8
9
Koha::Schema::Result::SavedTask
10
11
=cut
12
13
use strict;
14
use warnings;
15
16
use base 'DBIx::Class::Core';
17
18
=head1 TABLE: C<saved_tasks>
19
20
=cut
21
22
__PACKAGE__->table("saved_tasks");
23
24
=head1 ACCESSORS
25
26
=head2 task_id
27
28
  data_type: 'integer'
29
  extra: {unsigned => 1}
30
  is_auto_increment: 1
31
  is_nullable: 0
32
33
=head2 start_time
34
35
  data_type: 'datetime'
36
  datetime_undef_if_invalid: 1
37
  is_nullable: 0
38
39
=head2 repeat_interval
40
41
  data_type: 'integer'
42
  extra: {unsigned => 1}
43
  is_nullable: 0
44
45
=head2 task_type
46
47
  data_type: 'varchar'
48
  is_nullable: 0
49
  size: 255
50
51
=head2 params
52
53
  data_type: 'text'
54
  is_nullable: 0
55
56
=cut
57
58
__PACKAGE__->add_columns(
59
  "task_id",
60
  {
61
    data_type => "integer",
62
    extra => { unsigned => 1 },
63
    is_auto_increment => 1,
64
    is_nullable => 0,
65
  },
66
  "start_time",
67
  {
68
    data_type => "datetime",
69
    datetime_undef_if_invalid => 1,
70
    is_nullable => 0,
71
  },
72
  "repeat_interval",
73
  { data_type => "integer", extra => { unsigned => 1 }, is_nullable => 0 },
74
  "task_type",
75
  { data_type => "varchar", is_nullable => 0, size => 255 },
76
  "params",
77
  { data_type => "text", is_nullable => 0 },
78
);
79
80
=head1 PRIMARY KEY
81
82
=over 4
83
84
=item * L</task_id>
85
86
=back
87
88
=cut
89
90
__PACKAGE__->set_primary_key("task_id");
91
92
93
# Created by DBIx::Class::Schema::Loader v0.07042 @ 2016-01-27 13:35:22
94
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gnoi7I9fiXM3IfDysMTm+A
95
96
97
# You can replace this text with custom code or comments, and it will be preserved on regeneration
98
1;
(-)a/Makefile.PL (-1 / +26 lines)
Lines 198-203 Directory for Zebra's data files. Link Here
198
198
199
Directory for Zebra's UNIX-domain sockets.
199
Directory for Zebra's UNIX-domain sockets.
200
200
201
=item ICARUS_RUN_DIR
202
203
Directory for Icarus's UNIX-domain socket and pid file.
204
205
=item ICARUS_SPOOL_DIR
206
207
Directory for Icarus's temporary data like queues
208
201
=item MISC_DIR
209
=item MISC_DIR
202
210
203
Directory for for miscellaenous scripts, among other
211
Directory for for miscellaenous scripts, among other
Lines 317-322 my $target_map = { Link Here
317
  './skel/var/log/koha'         => { target => 'LOG_DIR', trimdir => -1 },
325
  './skel/var/log/koha'         => { target => 'LOG_DIR', trimdir => -1 },
318
  './skel/var/spool/koha'       => { target => 'BACKUP_DIR', trimdir => -1 },
326
  './skel/var/spool/koha'       => { target => 'BACKUP_DIR', trimdir => -1 },
319
  './skel/var/run/koha/zebradb' => { target => 'ZEBRA_RUN_DIR', trimdir => -1 },
327
  './skel/var/run/koha/zebradb' => { target => 'ZEBRA_RUN_DIR', trimdir => -1 },
328
  './skel/var/run/koha/icarus' => { target => 'ICARUS_RUN_DIR', trimdir => 6 },
329
  './skel/var/spool/koha/icarus' => { target => 'ICARUS_SPOOL_DIR', trimdir => 6 },
320
  './skel/var/lock/koha/zebradb/authorities' => { target => 'ZEBRA_LOCK_DIR', trimdir => 6 },
330
  './skel/var/lock/koha/zebradb/authorities' => { target => 'ZEBRA_LOCK_DIR', trimdir => 6 },
321
  './skel/var/lib/koha/zebradb/authorities/key'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
331
  './skel/var/lib/koha/zebradb/authorities/key'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
322
  './skel/var/lib/koha/zebradb/authorities/register'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
332
  './skel/var/lib/koha/zebradb/authorities/register'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
Lines 414-419 System user account that will own Koha's files. Link Here
414
424
415
System group that will own Koha's files.
425
System group that will own Koha's files.
416
426
427
=item ICARUS_MAX_TASKS
428
429
Maximum number of tasks allowed by Icarus.
430
417
=back
431
=back
418
432
419
=cut
433
=cut
Lines 448-454 my %config_defaults = ( Link Here
448
  'USE_MEMCACHED'     => 'no',
462
  'USE_MEMCACHED'     => 'no',
449
  'MEMCACHED_SERVERS' => '127.0.0.1:11211',
463
  'MEMCACHED_SERVERS' => '127.0.0.1:11211',
450
  'MEMCACHED_NAMESPACE' => 'KOHA',
464
  'MEMCACHED_NAMESPACE' => 'KOHA',
451
  'FONT_DIR'          => '/usr/share/fonts/truetype/ttf-dejavu'
465
  'FONT_DIR'          => '/usr/share/fonts/truetype/ttf-dejavu',
466
  'ICARUS_MAX_TASKS'    => '30',
452
);
467
);
453
468
454
# set some default configuration options based on OS
469
# set some default configuration options based on OS
Lines 1092-1097 Memcached namespace?); Link Here
1092
Path to DejaVu fonts?);
1107
Path to DejaVu fonts?);
1093
  $config{'FONT_DIR'} = _get_value('FONT_DIR', $msg, $defaults->{'FONT_DIR'}, $valid_values, $install_log_values);
1108
  $config{'FONT_DIR'} = _get_value('FONT_DIR', $msg, $defaults->{'FONT_DIR'}, $valid_values, $install_log_values);
1094
1109
1110
  $msg = q(
1111
Maximum number of tasks allowed by Icarus?);
1112
  $config{'ICARUS_MAX_TASKS'} = _get_value('ICARUS_MAX_TASKS', $msg, $defaults->{'ICARUS_MAX_TASKS'}, $valid_values, $install_log_values);
1113
1095
1114
1096
  $msg = q(
1115
  $msg = q(
1097
Would you like to run the database-dependent test suite?);
1116
Would you like to run the database-dependent test suite?);
Lines 1241-1246 sub get_target_directories { Link Here
1241
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'koha', 'plugins');
1260
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'koha', 'plugins');
1242
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1261
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1243
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
1262
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
1263
        $dirmap{'ICARUS_RUN_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'run', 'icarus');
1264
        $dirmap{'ICARUS_SPOOL_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'spool', 'icarus');
1244
    } elsif ($mode eq 'dev') {
1265
    } elsif ($mode eq 'dev') {
1245
        my $curdir = File::Spec->rel2abs(File::Spec->curdir());
1266
        my $curdir = File::Spec->rel2abs(File::Spec->curdir());
1246
        $dirmap{'API_CGI_DIR'} = File::Spec->catdir($curdir, 'api');
1267
        $dirmap{'API_CGI_DIR'} = File::Spec->catdir($curdir, 'api');
Lines 1276-1281 sub get_target_directories { Link Here
1276
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'plugins');
1297
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'plugins');
1277
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1298
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1278
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
1299
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
1300
        $dirmap{'ICARUS_RUN_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'run', 'icarus');
1301
        $dirmap{'ICARUS_SPOOL_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'spool', 'icarus');
1279
    } else {
1302
    } else {
1280
        # mode is standard, i.e., 'fhs'
1303
        # mode is standard, i.e., 'fhs'
1281
        $dirmap{'API_CGI_DIR'} = File::Spec->catdir(@basedir, $package, 'api');
1304
        $dirmap{'API_CGI_DIR'} = File::Spec->catdir(@basedir, $package, 'api');
Lines 1300-1305 sub get_target_directories { Link Here
1300
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'plugins');
1323
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'plugins');
1301
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'zebradb');
1324
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'zebradb');
1302
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'run', $package, 'zebradb');
1325
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'run', $package, 'zebradb');
1326
        $dirmap{'ICARUS_RUN_DIR'} = File::Spec->catdir(File::Spec->rootdir(), 'var', 'run', $package, 'icarus');
1327
        $dirmap{'ICARUS_SPOOL_DIR'} = File::Spec->catdir(File::Spec->rootdir(), 'var', 'spool', $package, 'icarus');
1303
    }
1328
    }
1304
1329
1305
    _get_env_overrides(\%dirmap);
1330
    _get_env_overrides(\%dirmap);
(-)a/admin/saved_tasks.pl (+379 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright Prosentient Systems 2016
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
=head1 NAME
21
22
saved_tasks.pl
23
24
=head1 DESCRIPTION
25
26
Admin page to manage saved tasks
27
28
=cut
29
30
use Modern::Perl;
31
use CGI qw ( -utf8 );
32
use C4::Auth;
33
use C4::Output;
34
use C4::Context;
35
36
use Koha::SavedTasks;
37
use Koha::Icarus;
38
use Module::Load::Conditional qw/can_load check_install/;
39
use JSON;
40
41
my $try_to_connect = 1;
42
my $input = new CGI;
43
my ($template, $loggedinuser, $cookie, $flags) = get_template_and_user( {
44
    template_name   => 'admin/saved_tasks.tt',
45
    query           => $input,
46
    type            => 'intranet',
47
    authnotrequired => 0,
48
    flagsrequired   => { 'parameters' => 'parameters_remaining_permissions' },
49
} );
50
51
my $filename = "saved_tasks.pl";
52
$template->param(
53
    filename => $filename,
54
);
55
56
my $context = C4::Context->new();
57
58
59
my $task_server = $input->param("task_server") // "icarus";
60
61
62
my $socket_uri = $context->{"icarus"}->{"socket"};
63
64
my @available_plugins = ();
65
my $task_plugins = $context->{"icarus"}->{"task_plugin"};
66
if ($task_plugins && ref $task_plugins eq 'ARRAY'){
67
    #FIXME: This should probably be a module method... validation that a plugin is installed...
68
    foreach my $task_plugin (@$task_plugins){
69
        #Check that plugin module is installed
70
        if ( check_install( module => $task_plugin ) ){
71
                push(@available_plugins,$task_plugin);
72
        }
73
    }
74
}
75
76
$template->param(
77
    available_plugins => \@available_plugins,
78
);
79
80
#Server action and task id
81
my $server_action = $input->param("server_action");
82
my $server_task_id = $input->param('server_task_id');
83
84
#Saved task op
85
my $op = $input->param('op');
86
my $step = $input->param('step');
87
88
#Saved task id
89
my $saved_task_id = $input->param('saved_task_id');
90
91
92
#Create Koha-Icarus interface object
93
my $icarus = Koha::Icarus->new({ socket_uri => $socket_uri });
94
my $daemon_status = "";
95
96
97
98
#NOTE: If you're having problems starting the server from the web ui,
99
#remember that Apache must be able to write to icarus.log, icarus.pid, icarus.sock and the directories containing them.
100
if ($server_action && $server_action eq "start_server" ){
101
    my $icarusd = $context->{icarus}->{bin};
102
    my $KOHA_CONF = $ENV{'KOHA_CONF'};
103
    if ( -f $icarusd && $KOHA_CONF){
104
        my $start_daemon = "$icarusd -f $KOHA_CONF --daemon -v 9";
105
            #NOTE: If the daemon didn't close STDOUT and STDERR itself, we'd need to redirect them to /dev/null them here to prevent Apache from looping/erroring
106
            if (system("$start_daemon") == 0){
107
                #This means the parent daemon process succeeded. However, it's still possible that the child daemon process has failed.
108
            } else {
109
                $try_to_connect = 0;
110
                $daemon_status = "Start failed";
111
            }
112
    }
113
}
114
115
if ($try_to_connect){
116
    #Connect to Icarus
117
    if ( $icarus->connect() ){
118
        $daemon_status = "online";
119
        if ($server_action){
120
            if ($server_action eq 'shutdown_server'){
121
                my $response = $icarus->shutdown;
122
                if ( $response && (my $action = $response->{action}) ){
123
                    $daemon_status = $action;
124
                }
125
            } elsif ($server_action eq 'start' && $server_task_id){
126
                my $response = $icarus->start_task({ task_id => $server_task_id });
127
                $template->param(
128
                    task_response => $response,
129
                );
130
            } elsif ($server_action eq 'remove' && $server_task_id){
131
                my $response = $icarus->remove_task({ task_id => $server_task_id });
132
                $template->param(
133
                    task_response => $response,
134
                );
135
            }
136
        }
137
    } else {
138
        warn "Daemon status: $!";
139
        $daemon_status = $!;
140
    }
141
}
142
143
$template->param(
144
    daemon_status => $daemon_status,
145
);
146
147
148
149
my $params = $input->param("params");
150
151
#NOTE: Parse the parameters manually, so that you can "name[]" style of parameter, which we use in the special plugin templates...
152
my $saved_params = {};
153
#Fetch the names of all the parameters passed to your script
154
my @parameter_names = $input->param;
155
#Iterate through these parameter names and look for "params[]"
156
foreach my $parameter_name (@parameter_names){
157
    if ($parameter_name =~ /^params\[(.*)\]$/){
158
        #Capture the hash key
159
        my $key = $1;
160
        #Fetch the actual individual value
161
        my $parameter_value = $input->param($parameter_name);
162
        if ($parameter_value){
163
            $saved_params->{$key} = $parameter_value;
164
        }
165
    }
166
}
167
if (%$saved_params){
168
    my $json = to_json($saved_params, { pretty => 1, });
169
    if ($json){
170
        $params = $json;
171
    }
172
}
173
174
my $start_time = $input->param("start_time");
175
my $repeat_interval = $input->param("repeat_interval");
176
my $task_type = $input->param("task_type");
177
if ($task_type){
178
    my $task_template = $task_type;
179
    #Create the template name by stripping the colons out of the task type text
180
    $task_template =~ s/://g;
181
    $template->param(
182
        task_template => "tasks/$task_template.inc",
183
    );
184
}
185
186
187
if ($op){
188
    #FIXME: This is cheating... you should create a TemplateToolkit plugin that does this and use it instead
189
    if ($op eq 'new' || $op eq 'update'){
190
        require Koha::Database;
191
        require Koha::BiblioFrameworks;
192
        my @frameworks = Koha::BiblioFrameworks->as_list();
193
        $template->{VARS}->{ frameworks } = \@frameworks;
194
    }
195
196
    if ($op eq 'new'){
197
198
    } elsif ($op eq 'create'){
199
200
        #Validate the $task here
201
        if ($step){
202
            if ($step eq "one"){
203
204
                $op = "new";
205
                $template->param(
206
                    step => "two",
207
                    task_type => $task_type,
208
                );
209
            } elsif ($step eq "two"){
210
                my $new_task = Koha::SavedTask->new({
211
                    start_time => $start_time,
212
                    repeat_interval => $repeat_interval,
213
                    task_type => $task_type,
214
                    params => $params,
215
                });
216
217
                #Serialize the data as an Icarus task
218
                my $icarus_task = $new_task->serialize({ for => "icarus", type => "perl", });
219
220
                my $valid = 1;
221
                #Load the plugin module, and create an object instance in order to validate user-entered data
222
                if ( can_load( modules => { $task_type => undef, }, ) ){
223
                    my $plugin = $task_type->new({ task => $icarus_task, });
224
                    if ($plugin->can("validate")){
225
                        my $errors = $plugin->validate({
226
                            "tests" => "all",
227
                        });
228
                        if (%$errors){
229
                            $template->param(
230
                                errors => $errors,
231
                            );
232
                        }
233
                        if ($plugin->{invalid_data} > 0){
234
                            $valid = 0;
235
                        }
236
                    }
237
                }
238
239
                if ($valid){
240
                    $new_task->store();
241
                    $op = "list";
242
                } else {
243
                    $op = "new";
244
                    #Create a Perl data structure from the JSON
245
                    my $editable_params = from_json($params);
246
                    $template->param(
247
                        step => "two",
248
                        task_type => $task_type,
249
                        saved_task => $new_task,
250
                        params => $editable_params,
251
                    );
252
                }
253
            }
254
        }
255
256
    } elsif ($op eq 'edit'){
257
        my $task = Koha::SavedTasks->find($saved_task_id);
258
        if ($task){
259
            #Check if the task's saved task type is actually available...
260
            #FIXME: This should be a Koha::Icarus method...
261
            my $task_type_is_valid = grep { $task->task_type eq $_ } @available_plugins;
262
            $template->param(
263
                task_type_is_valid => $task_type_is_valid,
264
                saved_task => $task,
265
            );
266
        }
267
    } elsif ($op eq 'update'){
268
        if ($step){
269
            my $task = Koha::SavedTasks->find($saved_task_id);
270
            if ($task){
271
                if ($step eq "one"){
272
                    #We've completed step one, which is choosing the task type,
273
                    #so now we're going to populate the form for editing the rest of the values
274
                    $op = "edit";
275
                    #This is the JSON string that we've saved in the database
276
                    my $current_params_string = $task->params;
277
                    my $editable_params = from_json($current_params_string);
278
279
                    $template->param(
280
                        step => "two",
281
                        task_type => $task_type,
282
                        saved_task => $task,
283
                        params => $editable_params,
284
285
                    );
286
                } elsif ($step eq "two"){
287
                    #We've completed step two, so we're storing the data now...
288
                    $task->set({
289
                        start_time => $start_time,
290
                        repeat_interval => $repeat_interval,
291
                        task_type => $task_type,
292
                        params => $params,
293
                    });
294
                    $task->store;
295
                    #FIXME: Validate the $task here...
296
                    if (my $valid = 1){
297
                        $op = "list";
298
                    } else {
299
                        $op = "edit";
300
                        $template->param(
301
                            step => "two",
302
                            task_type => $task_type,
303
                            saved_task => $task,
304
                        );
305
                    }
306
                }
307
            }
308
        }
309
    } elsif ($op eq 'send'){
310
        my $sent_response;
311
        if ($icarus->connected){
312
            if ($saved_task_id){
313
                #Look up task
314
                my $task = Koha::SavedTasks->find($saved_task_id);
315
                if ($task){
316
                    #Create a task for Icarus, and send it to Icarus
317
                    my $icarus_task = $task->serialize({ for => "icarus", type => "perl", });
318
                    if ($icarus_task){
319
                        $icarus->add_task({ task => $icarus_task, });
320
                        $op = "list";
321
                    }
322
                }
323
            }
324
        } else {
325
            $sent_response = "icarus_offline";
326
            $template->param(
327
                sent_response => $sent_response,
328
            );
329
            $op = "list";
330
        }
331
    } elsif ($op eq 'delete'){
332
        my $saved_response = "delete_failure";
333
        if ($saved_task_id){
334
            #Look up task
335
            my $task = Koha::SavedTasks->find($saved_task_id);
336
            if ($task){
337
                if (my $something = $task->delete){
338
                    $saved_response = "delete_success";
339
                }
340
            }
341
        }
342
        $template->param(
343
            saved_response => $saved_response,
344
        );
345
        $op = "list";
346
    } else {
347
        #Don't recognize $op, so fallback to list
348
        $op = "list";
349
    }
350
} else {
351
    #No $op, so fallback to list
352
    $op = "list";
353
}
354
355
if ($op eq 'list'){
356
    #Get active tasks from Icarus
357
    if ($icarus->connected){
358
        my $tasks = $icarus->list_tasks();
359
        if ($tasks && @$tasks){
360
            #Sort tasks that come from Icarus, since it returns an unsorted list of hashrefs
361
            my @sorted_tasks = sort { $a->{task_id} <=> $b->{task_id} } @$tasks;
362
            $template->param(
363
                tasks => \@sorted_tasks,
364
            );
365
        }
366
    }
367
368
    #Get saved tasks from Koha
369
    my @saved_tasks = Koha::SavedTasks->as_list();
370
    $template->param(
371
        saved_tasks => \@saved_tasks,
372
    );
373
}
374
375
$template->param(
376
    op => $op,
377
);
378
379
output_html_with_http_headers $input, $cookie, $template->output;
(-)a/docs/Icarus/README (+33 lines)
Line 0 Link Here
1
TODO:
2
3
_ICARUS_
4
- Improve error handling for Koha::Icarus::Task::Upload::*
5
    - When does it get a status of "Failed"?
6
- Data validation:
7
    "Koha::Icarus::Task::Upload::OAIPMH::Biblio":
8
        - Validate HTTP URLs and filepaths...
9
        - MAKE IT SO YOU HAVE TO USE A RECORD MATCHING RULE! To at the very least strip the OAI wrapper...
10
    - Add PLUGIN->validate("parameter_names")
11
    - Add PLUGIN->validate("parameter_values")
12
        - For the downloader, this would validate HTTP && OAI-PMH parameters...
13
14
#######
15
16
ICARUS:
17
- admin/saved_tasks.pl
18
    - Add a clone button to ease task creation
19
- Make the "Task type" prettier (and translateable) on saved_tasks.pl.
20
- Provide more options for the Icarus dashboard? (already have start/shutdown...)
21
- Add the ability to "edit" and "pause" active Icarus tasks
22
    - A pause function would make debugging much easier.
23
- Add help pages for WEB GUI
24
- Make "Koha::Icarus::Task::Upload::OAIPMH::Biblio" use asynchronous HTTP requests to speed up the import
25
- Instead of using file:///home/koha/koha-dev/var/spool/oaipmh, use something like file:///tmp/koha-instance/koha-dev/oaipmh
26
    - How is the user going to specify file:///tmp/koha-instance/koha-dev/oaipmh? Or do you put this in koha-conf.xml and then make a user-defined relative path?
27
- WEB UI:
28
    - Add `name` to saved_tasks?
29
- Move "Saved tasks" from Administration to Tools?
30
    - Look at existing bugs for schedulers:
31
        - https://bugs.koha-community.org/bugzilla3/show_bug.cgi?id=14712
32
        - https://bugs.koha-community.org/bugzilla3/show_bug.cgi?id=1993
33
- Handle datestamp granularity better for OAI-PMH download tasks?
(-)a/etc/koha-conf.xml (+10 lines)
Lines 137-140 __PAZPAR2_TOGGLE_XML_POST__ Link Here
137
 </ttf>
137
 </ttf>
138
138
139
</config>
139
</config>
140
<icarus>
141
    <bin>__SCRIPT_DIR__/icarusd.pl</bin>
142
    <socket>unix:__ICARUS_RUN_DIR__/icarus.sock</socket>
143
    <pidfile>__ICARUS_RUN_DIR__/icarus.pid</pidfile>
144
    <spooldir>__ICARUS_SPOOL_DIR__</spooldir>
145
    <log>__LOG_DIR__/icarus.log</log>
146
    <task_plugin>Koha::Icarus::Task::Download::OAIPMH::Biblio</task_plugin>
147
    <task_plugin>Koha::Icarus::Task::Upload::OAIPMH::Biblio</task_plugin>
148
    <max_tasks>__ICARUS_MAX_TASKS__</max_tasks>
149
</icarus>
140
</yazgfs>
150
</yazgfs>
(-)a/installer/data/mysql/atomicupdate/bug_10662-Icarus.sql (+9 lines)
Line 0 Link Here
1
DROP TABLE IF EXISTS saved_tasks;
2
CREATE TABLE  saved_tasks (
3
  task_id int(10) unsigned NOT NULL AUTO_INCREMENT,
4
  start_time datetime NOT NULL,
5
  repeat_interval int(10) unsigned NOT NULL,
6
  task_type varchar(255) CHARACTER SET utf8 NOT NULL,
7
  params text CHARACTER SET utf8 NOT NULL,
8
  PRIMARY KEY (task_id) USING BTREE
9
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc (+1 lines)
Lines 78-83 Link Here
78
    [% IF Koha.Preference('SMSSendDriver') == 'Email' %]
78
    [% IF Koha.Preference('SMSSendDriver') == 'Email' %]
79
        <li><a href="/cgi-bin/koha/admin/sms_providers.pl">SMS cellular providers</a></li>
79
        <li><a href="/cgi-bin/koha/admin/sms_providers.pl">SMS cellular providers</a></li>
80
    [% END %]
80
    [% END %]
81
    <li><a href="/cgi-bin/koha/admin/saved_tasks.pl">Saved tasks</a></li>
81
</ul>
82
</ul>
82
</div>
83
</div>
83
</div>
84
</div>
(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/tasks/KohaIcarusTaskDownloadOAIPMHBiblio.inc (+87 lines)
Line 0 Link Here
1
[%# USE CGI %]
2
[%# server_name = CGI.server_name; server_port = CGI.server_port; server = server_name _ ":" _ server_port; %]
3
4
<fieldset class="rows">
5
    <legend>HTTP parameters:</legend>
6
    <ol>
7
        <li>
8
            <label for="url">URL: </label>
9
            [% IF ( params.url ) %]
10
                <input type="text" id="url" name="params[url]" value="[% params.url %]" size="30" />
11
            [% ELSE %]
12
                <input type="text" id="url" name="params[url]" value="http://" size="30" />
13
            [% END %]
14
            [% IF (errors.url.no_path) %]<span class="error">[The URL must have a path after "http://" like "koha-community.org/cgi-bin/koha/oai.pl".]</span>[% END %]
15
            [% IF (errors.url.not_http) %]<span class="error">[The URL begin with a scheme of "http://" like "http://koha-community.org/cgi-bin/koha/oai.pl".]</span>[% END %]
16
            [% IF (errors.url.not_a_url) %]<span class="error">[The value of this field must be a URL like "http://koha-community.org/cgi-bin/koha/oai.pl".]</span>[% END %]
17
18
        </li>
19
    </ol>
20
    <span class="help">The following parameters are not required by all OAI-PMH repositories, so they may be optional for this task.</span>
21
    <ol>
22
        <li>
23
            <label for="username">Username: </label>
24
            <input type="text" id="username" name="params[username]" value="[% params.username %]" size="30" />
25
        </li>
26
        <li>
27
            <label for="password">Password: </label>
28
            <input type="text" id="password" name="params[password]" value="[% params.password %]" size="30" />
29
        </li>
30
        <li>
31
            <label for="realm">Realm: </label>
32
            <input type="text" id="realm" name="params[realm]" value="[% params.realm %]" size="30" />
33
        </li>
34
    </ol>
35
</fieldset>
36
<fieldset class="rows">
37
    <legend>OAI-PMH parameters:</legend>
38
    <ol>
39
        <li>
40
            <label for="verb">Verb: </label>
41
            <select id="verb" name="params[verb]">
42
            [% FOREACH verb IN [ 'GetRecord', 'ListRecords' ] %]
43
                [% IF ( params.verb ) && ( verb == params.verb ) %]
44
                    <option selected="selected" value="[% verb %]">[% verb %]</option>
45
                [% ELSE %]
46
                    <option value="[% verb %]">[% verb %]</option>
47
                [% END %]
48
            [% END %]
49
            </select>
50
        </li>
51
        <li>
52
            <label for="identifier">Identifier: </label>
53
            <input type="text" id="identifier" name="params[identifier]" value="[% params.identifier %]" size="30" />
54
            <span class="help">This identifier will only be used with the GetRecord verb.</span>
55
        </li>
56
        <li>
57
            <label for="sets">Sets: </label>
58
            <input type="text" id="sets" name="params[sets]" value="[% params.sets %]" size="30" /><span class="help">You may specify several sets by separating the sets with a pipe (e.g. set1|set2 )</span>
59
        </li>
60
        <li>
61
            <label for="metadataPrefix">Metadata Prefix: </label>
62
            <input type="text" id="metadataPrefix" name="params[metadataPrefix]" value="[% params.metadataPrefix %]" size="30" />
63
        </li>
64
        <li>
65
            <label for="opt_from">From: </label>
66
            <input type="text" class="datetime_utc" id="opt_from" name="params[from]" value="[% params.from %]" size="30" /><span class="help">This value will be treated as UTC time. Note that some repositories only support YYYY-MM-DD datestamps.</span>
67
        </li>
68
        <li>
69
            <label for="opt_until">Until: </label>
70
            <input type="text" class="datetime_utc" id="opt_until" name="params[until]" value="[% params.until %]" size="30" /><span class="help">This value will be treated as UTC time. Note that some repositories only support YYYY-MM-DD datestamps.</span>
71
        </li>
72
    </ol>
73
</fieldset>
74
<fieldset class="rows">
75
    <legend>Download parameters:</legend>
76
    <ol>
77
        <li>
78
            <label for="queue">Queue: </label>
79
            [% IF ( params.queue ) %]
80
                <input type="text" id="queue" name="params[queue]" value="[% params.queue %]" size="30" />
81
            [% ELSE %]
82
                <input type="text" id="queue" name="params[queue]" value="biblio-sync" size="30" />
83
            [% END %]
84
            <span class="help">Choose a unique name for the queue where your downloaded records will be stored prior to upload. (e.g. biblio-sync, unioncatalogue)</span>
85
        </li>
86
    </ol>
87
</fieldset>
(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/tasks/KohaIcarusTaskUploadOAIPMHBiblio.inc (+132 lines)
Line 0 Link Here
1
[% USE CGI %]
2
[% server = CGI.http('HTTP_ORIGIN') %]
3
[%# TODO: Test if this works with Plack and HTTPS... %]
4
[% default_auth_uri = server _ '/cgi-bin/koha/svc/authentication' %]
5
[% default_target_uri = server _ '/cgi-bin/koha/svc/import_oai' %]
6
<fieldset class="rows">
7
    <legend>Import source parameters:</legend>
8
    <ol>
9
        <li>
10
            <label for="queue">Queue: </label>
11
            [% IF ( params.queue ) %]
12
                <input type="text" id="queue" name="params[queue]" value="[% params.queue %]" size="30" />
13
            [% ELSE %]
14
                <input type="text" id="queue" name="params[queue]" value="biblio-sync" size="30" />
15
            [% END %]
16
            <span class="help">This is the name of a unique queue you already defined for downloaded records (e.g. biblio-sync, unioncatalogue)</span>
17
        </li>
18
    </ol>
19
</fieldset>
20
<fieldset class="rows">
21
    <legend>API authentication parameters:</legend>
22
    <ol>
23
        <li>
24
            <label for="auth_uri">URL: </label>
25
            [% IF ( params.auth_uri ) %]
26
                <input type="text" id="auth_uri" name="params[auth_uri]" value="[% params.auth_uri %]" size="30" />
27
            [% ELSE %]
28
                <input type="text" id="auth_uri" name="params[auth_uri]" value="[% default_auth_uri %]" size="30" />
29
            [% END %]
30
            [% IF (errors.auth_uri.no_path) %]<span class="error">[The URL must have a path after "http://" like "koha-community.org/cgi-bin/koha/svc/authentication".]</span>[% END %]
31
            [% IF (errors.auth_uri.not_http) %]<span class="error">[The URL begin with a scheme of "http://" like "http://koha-community.org/cgi-bin/koha/svc/authentication".]</span>[% END %]
32
            [% IF (errors.auth_uri.not_a_url) %]<span class="error">[The value of this field must be a URL like "http://koha-community.org/cgi-bin/koha/svc/authentication".]</span>[% END %]
33
            <span class="help">This is a Koha authentication URL. The default value </span>
34
        </li>
35
        <li>
36
            <label for="auth_username">Username: </label>
37
            <input type="text" id="auth_username" name="params[auth_username]" value="[% params.auth_username %]" size="30" />
38
            <span class="help">This user must have permission to edit the catalogue.</span>
39
        </li>
40
        <li>
41
            <label for="auth_password">Password: </label>
42
            <input type="text" id="auth_password" name="params[auth_password]" value="[% params.auth_password %]" size="30" />
43
        </li>
44
    </ol>
45
</fieldset>
46
<fieldset class="rows">
47
    <legend>Import target parameters:</legend>
48
    <ol>
49
        <li>
50
            <label for="target_uri">URL: </label>
51
            [% IF ( params.target_uri ) %]
52
                <input type="text" id="target_uri" name="params[target_uri]" value="[% params.target_uri %]" size="30" />
53
            [% ELSE %]
54
                <input type="text" id="target_uri" name="params[target_uri]" value="[% default_target_uri %]" size="30" />
55
            [% END %]
56
            [% IF (errors.target_uri.no_path) %]<span class="error">[The URL must have a path after "http://" like "koha-community.org/cgi-bin/koha/svc/import_oai".]</span>[% END %]
57
            [% IF (errors.target_uri.not_http) %]<span class="error">[The URL begin with a scheme of "http://" like "http://koha-community.org/cgi-bin/koha/svc/import_oai".]</span>[% END %]
58
            [% IF (errors.target_uri.not_a_url) %]<span class="error">[The value of this field must be a URL like "http://koha-community.org/cgi-bin/koha/svc/import_oai".]</span>[% END %]
59
        </li>
60
61
        <li>
62
            <label for="match">Record matching rule code</label>
63
            <input type="text" id="match" name="params[match]" value="[% params.match %]" size="30" />
64
            <span class="help">This code must exist in "Record matching rules" in Administration for record matching to work. (Example code: OAI)</span>
65
        </li>
66
        <li>
67
            [%# TODO: Ideally, I'd like to use 'tools-overlay-action.inc' but the logic doesn't work here. Perhaps it would be better as a TT plugin. %]
68
            <label for="overlay_action">Action if matching record found</label>
69
            <select name="params[overlay_action]" id="overlay_action">
70
            [% IF ( params.overlay_action == "replace" ) %]
71
                <option value="replace"  selected="selected">
72
            [% ELSE %]
73
                <option value="replace">
74
            [% END %]
75
                Replace existing record with incoming record</option>
76
            [% IF ( params.overlay_action == "create_new" ) %]
77
                <option value="create_new" selected="selected">
78
            [% ELSE %]
79
                <option value="create_new">
80
            [% END %]
81
                Add incoming record</option>
82
            [% IF ( params.overlay_action == "ignore" ) %]
83
                <option value="ignore" selected="selected">
84
            [% ELSE %]
85
                <option value="ignore">
86
            [% END %]
87
                Ignore incoming record</option>
88
            </select>
89
        </li>
90
        <li>
91
            [%# TODO: Ideally, I'd like to use 'tools-nomatch-action.inc' but the logic doesn't work here. Perhaps it would be better as a TT plugin. %]
92
            <label for="nomatch_action">Action if no match is found</label>
93
            <select name="params[nomatch_action]" id="nomatch_action">
94
            [% IF ( params.nomatch_action == "create_new" ) %]
95
                <option value="create_new" selected="selected">
96
            [% ELSE %]
97
                <option value="create_new">
98
            [% END %]
99
                Add incoming record</option>
100
            [% IF ( params.nomatch_action == "ignore" ) %]
101
                <option value="ignore" selected="selected">
102
            [% ELSE %]
103
                <option value="ignore">
104
            [% END %]
105
                Ignore incoming record</option>
106
            </select>
107
        </li>
108
        <li>
109
            <label>MARC Bibliographic Framework</label>
110
            <select name="params[framework]">
111
                <option value="">Default framework</option>
112
                [% FOREACH framework IN frameworks %]
113
                    [% IF ( params.framework == framework.frameworkcode ) %]
114
                        <option selected="selected" value="[% framework.frameworkcode %]">[% framework.frameworktext %]</option>
115
                    [% ELSE %]
116
                        <option value="[% framework.frameworkcode %]">[% framework.frameworktext %]</option>
117
                    [% END %]
118
                [% END %]
119
            </select>
120
            <span class="help">This framework is only used for new records. Koha will use the original record framework when replacing records.</span>
121
        </li>
122
        <li>
123
            <label for="filter">Filter</label>
124
            [% IF ( params.filter ) %]
125
                <input type="text" id="filter" name="params[filter]" value="[% params.filter %]" size="30" />
126
            [% ELSE %]
127
                <input type="text" id="filter" name="params[filter]" value="default" size="30" />
128
            [% END %]
129
            <span class="help">Using the keyword "default", Koha will use the default filter for converting OAI-PMH records to MARCXML records. Alternatively, you can provide a file URL like file:///etc/koha/sites/libraryname/OAI2MARC21slim.xsl or file:///usr/share/koha/intranet/htdocs/intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl to use a different filter.</span>
130
        </li>
131
    </ol>
132
</fieldset>
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt (+2 lines)
Lines 120-125 Link Here
120
                        <dt><a href="/cgi-bin/koha/admin/sms_providers.pl">SMS cellular providers</a></dt>
120
                        <dt><a href="/cgi-bin/koha/admin/sms_providers.pl">SMS cellular providers</a></dt>
121
                        <dd>Define a list of cellular providers for sending SMS messages via email.</dd>
121
                        <dd>Define a list of cellular providers for sending SMS messages via email.</dd>
122
                    [% END %]
122
                    [% END %]
123
                    <dt><a href="/cgi-bin/koha/admin/saved_tasks.pl">Saved tasks</a></dt>
124
                    <dd>Define tasks which may be run in the background</dd>
123
                </dl>
125
                </dl>
124
            </div>
126
            </div>
125
        </div>
127
        </div>
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/saved_tasks.tt (+345 lines)
Line 0 Link Here
1
[% INCLUDE 'doc-head-open.inc' %]
2
<title>Koha &rsaquo; Administration &rsaquo; Saved tasks</title>
3
[% INCLUDE 'doc-head-close.inc' %]
4
[% INCLUDE 'calendar.inc' %]
5
<script type="text/javascript" src="[% interface %]/lib/jquery/plugins/jquery-ui-timepicker-addon.min.js"></script>
6
[% INCLUDE 'timepicker.inc' %]
7
[% IF ( op == "list" ) %]
8
    <link rel="stylesheet" type="text/css" href="[% interface %]/[% theme %]/css/datatables.css" />
9
    [% INCLUDE 'datatables.inc' %]
10
    <script type="text/javascript">
11
    //<![CDATA[
12
        $(document).ready(function() {
13
            $("#taskst").dataTable($.extend(true, {}, dataTablesDefaults, {
14
                "aoColumnDefs": [
15
                    { "aTargets": [3,4,5,6], "bSortable": false },
16
                ],
17
                "sPaginationType": "four_button"
18
            }));
19
        });
20
    //]]>
21
    </script>
22
[% ELSIF ( op == "edit" ) || ( op == "new" ) %]
23
    <script type="text/javascript">
24
    //<![CDATA[
25
        $(document).ready(function() {
26
            [%# Ideally, it would be nice to record the timezone here too, but currently we use MySQL's DATETIME field which doesn't store ISO 8601 timezone designators... %]
27
            $(".datetime_local").datetimepicker({
28
                dateFormat: "yy-mm-dd",
29
                timeFormat: "HH:mm:ss",
30
                hour: 0,
31
                minute: 0,
32
                second: 0,
33
                showSecond: 1,
34
            });
35
            $(".datetime_utc").datetimepicker({
36
                separator: "T",
37
                timeSuffix: 'Z',
38
                dateFormat: "yy-mm-dd",
39
                timeFormat: "HH:mm:ss",
40
                hour: 0,
41
                minute: 0,
42
                second: 0,
43
                showSecond: 1,
44
                // timezone doesn't work with the "Now" button in v1.4.3 although it appears to in v1.6.1
45
                // timezone: +000,
46
            });
47
48
        });
49
    //]]>
50
    </script>
51
    <style type="text/css">
52
        /* Override staff-global.css which hides second, millisecond, and microsecond sliders */
53
        .ui_tpicker_second {
54
            display: block;
55
        }
56
        .test-success {
57
            /* same color as .text-success in Bootstrap 2.2.2 */
58
            color:#468847;
59
        }
60
    </style>
61
[% END %]
62
</head>
63
64
<body id="admin_saved_tasks" class="admin">
65
[% INCLUDE 'header.inc' %]
66
[% INCLUDE 'cat-search.inc' %]
67
<div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/admin/admin-home.pl">Administration</a> &rsaquo; Saved tasks</div>
68
69
<div id="doc3" class="yui-t2">
70
71
<div id="bd">
72
  <div id="yui-main">
73
    <div class="yui-b">
74
        [% IF ( op ) %]
75
            [% IF ( op == "new" ) || ( op == "edit" ) %]
76
                [%# If step is undefined, force it to be step one %]
77
                [% IF ( ! step ); step = "one"; END; %]
78
79
80
81
                [%# HEADING %]
82
                    [% IF ( op == "new" ) %]
83
                        <h1>New saved task</h1>
84
                    [% ELSIF ( op == "edit" ) %]
85
                        <h1>Modify saved task</h1>
86
                    [% END %]
87
                [%# /HEADING %]
88
89
                [%# TODO: Get this working properly... <div class="alert">Validation failed.</div> #]
90
91
                [%# FORM %]
92
                    <form action="/cgi-bin/koha/admin/[% filename %]" name="detail-form" method="post" id="saved-task-details" novalidate="novalidate">
93
                        [% IF ( op == "new" ) %]
94
                            <input type="hidden" name="op" value="create" />
95
                        [% ELSIF ( op == "edit" ) %]
96
                            <input type="hidden" name="op" value="update" />
97
                            <input type="hidden" name="saved_task_id" value="[% saved_task.task_id %]" />
98
                        [% END %]
99
                        <input type="hidden" name="step" value="[% step %]" />
100
                        <fieldset class="rows">
101
                            <ol>
102
                                [% IF ( op == "edit") && ( step == "one" ) && (! task_type_is_valid ) %]
103
                                <li>
104
                                    <label for="invalid_task_type">Current invalid task type:</label>
105
                                    <input id="invalid_task_type" type="text" disabled="disabled" value="[% saved_task.task_type %]" size="60" />
106
                                    <span class="error">Sorry! This task type is invalid. Please choose a new one from the following list.</span>
107
                                <li>
108
                                [% END %]
109
                                <li>
110
                                    <label for="task_type">Task type: </label>
111
                                    [% IF ( step == "one" ) %]
112
                                        [% IF ( available_plugins ) %]
113
                                        <select id="task_type" name="task_type">
114
                                            [% IF ( op == "new") %]
115
                                                [% FOREACH plugin IN available_plugins %]
116
                                                    <option value="[% plugin %]">[% plugin %]</option>
117
                                                [% END %]
118
                                            [% ELSIF ( op == "edit" ) %]
119
                                                [% FOREACH plugin IN available_plugins %]
120
                                                    [% IF ( saved_task.task_type == plugin ) %]
121
                                                        <option selected="selected" value="[% plugin %]">[% plugin %]</option>
122
                                                    [% ELSE %]
123
                                                        <option value="[% plugin %]">[% plugin %]</option>
124
                                                    [% END %]
125
                                                [% END %]
126
                                            [% END %]
127
                                        </select>
128
                                        [% END %]
129
130
                                    [% ELSIF ( step == "two" ) %]
131
                                        <input type="text" disabled="disabled" value="[% task_type %]" size="60" />
132
                                        <input type="hidden" name="task_type" value="[% task_type %]" />
133
                                    [% END %]
134
                                </li>
135
                            </ol>
136
                        </fieldset>
137
138
                        [% IF ( step == "one" ) %]
139
                            <fieldset class="action">
140
                                <input type="submit" value="Next">
141
                                <a class="cancel" href="/cgi-bin/koha/admin/[% filename %]">Cancel</a>
142
                            </fieldset>
143
                        [% ELSIF ( step == "two" ) %]
144
                            <fieldset class="rows">
145
                                <legend>Task:</legend>
146
                                <ol>
147
                                    <li>
148
                                        <label for="start_time">Start time: </label>
149
                                        <input type="text" id="start_time" class="datetime_local" name="start_time" value="[% saved_task.start_time %]" size="30" />
150
                                        <span class="help">This value will be treated as local server time, and times in the past will start immediately.</span>
151
                                    </li>
152
                                    <li>
153
                                        <label for="repeat_interval">Repeat interval: </label>
154
                                        <input type="text" id="repeat_interval" name="repeat_interval" value="[% saved_task.repeat_interval %]" size="4" />
155
                                        <span class="help">seconds</span>
156
                                        [% IF (errors.repeat_interval.not_numeric) %]<span class="error">[The repeat interval must be a purely numeric value.]</span>[% END %]
157
                                    </li>
158
                                </ol>
159
                            </fieldset>
160
                            [%# Try to include the template, but if it fails, fallback to a regular text view %]
161
                            [% TRY %]
162
                                [% INCLUDE $task_template %]
163
                            [% CATCH %]
164
                            <fieldset class="rows">
165
                                <legend>Plugin parameters:</legend>
166
                                <ol>
167
                                    <li>
168
                                        <label for="params">Params: </label>
169
                                        <textarea id="params" name="params" cols="60" rows="20">[% saved_task.params %]</textarea>
170
                                    </li>
171
                                </ol>
172
                            </fieldset>
173
                            [% END %]
174
                            <fieldset class="action">
175
                                <input type="submit" value="Save">
176
                                <a class="cancel" href="/cgi-bin/koha/admin/[% filename %]">Cancel</a>
177
                            </fieldset>
178
                        [% END %]
179
                    </form>
180
                [%# /FORM %]
181
            [% END #/edit or new %]
182
183
184
            [% IF ( op == "list" ) %]
185
                <div id="toolbar" class="btn-toolbar">
186
                    <a id="newserver" class="btn btn-small" href="/cgi-bin/koha/admin/[% filename %]?op=new"><i class="fa fa-plus"></i> New saved task</a>
187
                </div>
188
                <h1>Saved tasks</h1>
189
                [% IF ( saved_response ) %]
190
                    [% IF ( saved_response == 'delete_success' ) %]
191
                        <div class="alert">Deletion successful.</div>
192
                    [% ELSIF ( saved_response == 'delete_failure' ) %]
193
                        <div class="alert">Deletion failed.</div>
194
                    [% END %]
195
                [% END %]
196
                [% IF ( sent_response ) %]
197
                    [% IF ( sent_response == 'icarus_offline' ) %]
198
                        <div class="alert">Send failed. Icarus is currently offline.</div>
199
                    [% END %]
200
                [% END %]
201
                <table id="taskst">
202
                    <thead>
203
                        <tr>
204
                            <th>Start time</th>
205
                            <th>Repeat interval</th>
206
                            <th>Task type</th>
207
                            <th>Params</th>
208
                            <th></th>
209
                            <th></th>
210
                            <th></th>
211
                        </tr>
212
                    </thead>
213
                    <tbody>
214
                    [% FOREACH saved_task IN saved_tasks %]
215
                        <tr>
216
                            <td>[% IF ( saved_task.start_time ) != "0000-00-00 00:00:00"; saved_task.start_time; END; %]</td>
217
                            <td>[% saved_task.repeat_interval %]</td>
218
                            <td>[% saved_task.task_type %]</td>
219
                            <td>
220
                                <ul>
221
                                [% FOREACH pair IN saved_task.params_as_perl.pairs %]
222
                                   <li>[% pair.key %] => [% IF ( pair.key.match('.*password.*') ); '########'; ELSE; pair.value; END; %]</li>
223
                                [% END %]
224
                                </ul>
225
                            </td>
226
                            <td><a href="/cgi-bin/koha/admin/[% filename %]?op=edit&saved_task_id=[% saved_task.task_id %]">Edit</a></td>
227
                            <td><a href="/cgi-bin/koha/admin/[% filename %]?op=send&saved_task_id=[% saved_task.task_id %]">Send to Icarus</a></td>
228
                            <td><a href="/cgi-bin/koha/admin/[% filename %]?op=delete&saved_task_id=[% saved_task.task_id %]">Delete</a></td>
229
                        </tr>
230
                    [% END %]
231
                    </tbody>
232
                </table>
233
                <div id="daemon_controls">
234
                    <h1>Icarus dashboard</h1>
235
                    <table>
236
                    <tr>
237
                        <th>Status</th>
238
                        <th></th>
239
                        <th></th>
240
                    </tr>
241
                    <tr>
242
                        <td>
243
244
                        [% IF ( daemon_status == 'No such file or directory' ) #Socket doesn't exist at all %]
245
                            <span id="icarus_status">Unable to contact</span>
246
                        [% ELSIF ( daemon_status == 'Permission denied' ) #Apache doesn't have permission to write to socket %]
247
                            <span id="icarus_status">Permission denied</span>
248
                        [% ELSIF ( daemon_status == 'Connection refused' ) #Socket exists, but server is down %]
249
                            <span id="icarus_status">Connection refused</span>
250
                        [% ELSIF ( daemon_status == 'Start failed' ) %]
251
                            <span id="icarus_status">Start failed</span>
252
                        [% ELSIF ( daemon_status == 'online' ) %]
253
                            <span id="icarus_status">Online</span>
254
                        [% ELSIF ( daemon_status == 'shutting down' ) %]
255
                            <span id="icarus_status">Shutting down</span>
256
                        [% ELSE %]
257
                            <span id="icarus_status">[% daemon_status %]</span>
258
                        [% END %]
259
                        </td>
260
                        [%# TODO: Also provide controls for starting/restarting Icarus? %]
261
                        <td><a href="/cgi-bin/koha/admin/[% filename %]?server_action=start_server">Start Icarus</a></td>
262
                        <td><a href="/cgi-bin/koha/admin/[% filename %]?server_action=shutdown_server">Shutdown Icarus</a></td>
263
                    </tr>
264
                    </table>
265
                </div>
266
                <div id="tasks">
267
                    <h1>Active Icarus tasks</h1>
268
                    [% IF ( task_response ) %]
269
                        [% IF ( task_response.action == 'error' ) %]
270
                            [% IF ( task_response.error_message ) %]
271
                                [% IF ( task_response.error_message == 'No such process' ) %]
272
                                    <div class="alert">Task [% task_response.task_id %] does not exist.</div>
273
                                [% END %]
274
                            [% END %]
275
                        [% ELSIF ( task_response.action == 'pending' ) %]
276
                            <div class="alert">Initialising task [% task_response.task_id %].</div>
277
                        [% ELSIF ( task_response.action == 'already pending' ) %]
278
                            <div class="alert">Already initialised task [% task_response.task_id %].</div>
279
                        [% ELSIF ( task_response.action == 'already started' ) %]
280
                            <div class="alert">Already started task [% task_response.task_id %].</div>
281
                        [% ELSIF ( task_response.action == 'removed' ) %]
282
                            <div class="alert">Removing task [% task_response.task_id %].</div>
283
                        [% END %]
284
                    [% END %]
285
                    [% IF ( tasks ) %]
286
                        <table>
287
                            <thead>
288
                                <tr>
289
                                    <th>Task id</th>
290
                                    <th>Status</th>
291
                                    <th>Next start time (local server time)</th>
292
                                    <th>Repeat interval</th>
293
                                    <th>Task type</th>
294
                                    <th>Params</th>
295
                                    <th></th>
296
                                    <th></th>
297
                                </tr>
298
                            </thead>
299
                            <tbody>
300
                            [% FOREACH task IN tasks %]
301
                                <tr>
302
                                    <td>[% task.task_id %]</td>
303
                                    <td>
304
                                        [% SWITCH task.task.status %]
305
                                        [% CASE 'new' %]
306
                                        <span>New</span>
307
                                        [% CASE 'pending' %]
308
                                        <span>Pending</span>
309
                                        [% CASE 'started' %]
310
                                        <span>Started</span>
311
                                        [% CASE 'stopping' %]
312
                                        <span>Stopping</span>
313
                                        [% CASE 'failed' %]
314
                                        <span>Failed</span>
315
                                        [% CASE %]
316
                                        <span>[% task.task.status %]</span>
317
                                        [% END %]
318
                                    </td>
319
                                    <td>[% task.task.start %]</td>
320
                                    <td>[% task.task.repeat_interval %]</td>
321
                                    <td>[% task.task.type %]</td>
322
                                    <td>
323
                                        <ul>
324
                                        [% FOREACH pair IN task.task.params.pairs %]
325
                                           <li>[% pair.key %] => [% IF ( pair.key.match('.*password.*') ); '########'; ELSE; pair.value; END; %]</li>
326
                                        [% END %]
327
                                        </ul>
328
                                    </td>
329
                                    <td><a href="/cgi-bin/koha/admin/[% filename %]?server_action=start&server_task_id=[% task.task_id %]">Start</a></td>
330
                                    <td><a href="/cgi-bin/koha/admin/[% filename %]?server_action=remove&server_task_id=[% task.task_id %]">Remove</a></td>
331
                                </tr>
332
                            [% END %]
333
                            </tbody>
334
                        </table>
335
                    [% END %]
336
                </div>
337
            [% END #/list %]
338
        [% END #/op %]
339
    </div>
340
  </div>
341
  <div class="yui-b">
342
    [% INCLUDE 'admin-menu.inc' %]
343
  </div>
344
</div>
345
[% INCLUDE 'intranet-bottom.inc' %]
(-)a/misc/bin/icarusd.pl (+213 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
#######################################################################
4
5
use Modern::Perl;
6
use POSIX; #For daemonizing
7
use Fcntl qw(:flock); #For pidfile
8
use Getopt::Long;
9
use Pod::Usage;
10
11
#Make the STDOUT filehandle hot, so that you can use shell re-direction. Otherwise, you'll suffer from buffering.
12
STDOUT->autoflush(1);
13
#Note that STDERR, by default, is already hot.
14
15
#######################################################################
16
#FIXME: Debugging signals
17
=pod
18
BEGIN {
19
    package POE::Kernel;
20
    use constant TRACE_SIGNALS => 1;
21
    use constant ASSERT_USAGE => 1;
22
    use constant ASSERT_DATA => 1;
23
}
24
=cut
25
26
27
use POE;
28
use JSON; #For Listener messages
29
use XML::LibXML; #For configuration files
30
31
use Koha::Icarus::Listener;
32
33
#######################################################################
34
35
my ($filename,$daemon,$log,$help);
36
my $verbosity = 1;
37
GetOptions (
38
    "f|file|filename=s"     => \$filename, #/kohawebs/dev/dcook/koha-dev/etc/koha-conf.xml
39
    "l|log=s"               => \$log,
40
    "d|daemon"              => \$daemon,
41
    "v=i"                   => \$verbosity,
42
    "h|?"                   => \$help,
43
) or pod2usage(2);
44
pod2usage(1) if $help;
45
46
47
if ( ! $filename || ! -f $filename ){
48
    warn "Failed to start.\n";
49
    if ( ! $filename ){
50
        warn("You must provide a valid configuration file using the -f switch.\n");
51
        pod2usage(1);
52
    }
53
    if ( ! -f $filename ){
54
        die(qq{"$filename" is not a file.\n});
55
    }
56
}
57
58
#Declare the variable with file scope so the flock stays for the duration of the process's life
59
my $pid_filehandle;
60
61
#Read configuration file
62
my $config = read_config_file($filename);
63
64
my $SOCK_PATH = $config->{socket} || '';
65
my $spooldir = $config->{spooldir} || '';
66
my $pid_file = $config->{pidfile} || '';
67
my $max_tasks = $config->{max_tasks};
68
69
#Overwrite configuration file with command line options
70
if ($log){
71
    $config->{log} = $log;
72
}
73
74
#Test file permissions...
75
my @warnings = ();
76
foreach my $file_to_check ($pid_file, $config->{log}){
77
    local (*TMP);
78
    if ($file_to_check){
79
        utime(undef, undef, $file_to_check) || open(TMP, ">>$file_to_check") || push(@warnings,"couldn't touch $file_to_check: $!");
80
    }
81
}
82
if (@warnings){
83
    foreach my $warning (@warnings){
84
        warn $warning;
85
    }
86
    exit 1;
87
}
88
89
#Go into daemon mode, if user has included flag
90
if ($daemon){
91
    daemonize();
92
}
93
94
if ($pid_file){
95
    #NOTE: The filehandle needs to have file scope, so that the flock is preserved.
96
    $pid_filehandle = make_pid_file($pid_file);
97
}
98
99
#FIXME: Do we want to log to file only in daemon mode? $config->{log} should be populated by either the config file or the l|log GetOpt...
100
if ($daemon && $config->{log}){
101
    log_to_file($config->{log});
102
}
103
104
105
#FIXME: 1) In daemon mode, SIGUSR1 or SIGHUP for reloading/restarting?
106
#######################################################################
107
108
#Creates Icarus Listener
109
Koha::Icarus::Listener->spawn({
110
    Socket => $SOCK_PATH,
111
    MaxTasks => $max_tasks,
112
    Verbosity => $verbosity,
113
    SpoolDir => $spooldir,
114
});
115
116
POE::Kernel->run();
117
118
exit;
119
120
sub read_config_file {
121
    my $filename = shift;
122
    my $config = {};
123
    if ( -e $filename ){
124
        eval {
125
            my $doc = XML::LibXML->load_xml(location => $filename);
126
            if ($doc){
127
                my $root = $doc->documentElement;
128
                my $icarus = $root->find('icarus')->shift;
129
                if ($icarus){
130
                    #Get all child nodes for the 'icarus' element
131
                    my @childnodes = $icarus->childNodes();
132
                    foreach my $node (@childnodes){
133
                        #Only consider nodes that are elements
134
                        if ($node->nodeType == XML_ELEMENT_NODE){
135
                            my $config_key = $node->nodeName;
136
                            my $first_child = $node->firstChild;
137
                            #Only consider nodes that have a text node as their first child
138
                            if ($first_child && $first_child->nodeType == XML_TEXT_NODE){
139
                                $config->{$config_key} = $first_child->nodeValue;
140
                            }
141
                        }
142
                    }
143
                }
144
            }
145
        };
146
    }
147
    return $config;
148
}
149
150
#######################################################################
151
#NOTE: On Debian, you can use the daemon binary to make a process into a daemon,
152
# the following subs are for systems that don't have the daemon binary.
153
154
sub daemonize {
155
    my $pid = fork;
156
    die "Couldn't fork: $!" unless defined($pid);
157
    if ($pid){
158
        exit; #Parent exit
159
    }
160
    POSIX::setsid() or die "Can't start a new session: $!";
161
162
    #Change to known system directory
163
    chdir('/');
164
165
    #Close inherited file handles, so that you can truly run in the background.
166
    open STDIN,  '<', '/dev/null';
167
    open STDOUT, '>', '/dev/null';
168
    open STDERR, '>&STDOUT';
169
170
    #FIXME: You should probabl rset file creation mask here as well...
171
}
172
173
sub log_to_file {
174
    my $logfile = shift;
175
    #Open a filehandle to append to a log file
176
    open(LOG, '>>', $logfile) or die "Unable to open a filehandle for $logfile: $!\n"; # --output
177
    LOG->autoflush(1); #Make filehandle hot (ie don't buffer)
178
    *STDOUT = *LOG; #Re-assign STDOUT to LOG | --stdout
179
    *STDERR = *STDOUT; #Re-assign STDERR to STDOUT | --stderr
180
}
181
182
sub make_pid_file {
183
    my $pidfile = shift;
184
    if ( ! -e $pidfile ){
185
        open(my $fh, '>', $pidfile) or die "Unable to write to $pidfile: $!\n";
186
        $fh->close;
187
    }
188
189
    open(my $pidfilehandle, '+<', $pidfile) or die "Unable to open a filehandle for $pidfile: $!\n";
190
    if (flock($pidfilehandle, LOCK_EX|LOCK_NB)){
191
        #Write pid to pidfile
192
        warn "Acquiring lock on $pidfile\n";
193
        #Now that we've acquired a lock, let's truncate the file
194
        truncate($pidfilehandle, 0);
195
        print $pidfilehandle $$."\n" or die $!;
196
        #Flush the filehandle so you're not suffering from buffering
197
        $pidfilehandle->flush();
198
        return $pidfilehandle;
199
    } else {
200
        my $number = <$pidfilehandle>;
201
        chomp($number);
202
        warn "$0 is already running with pid $number. Exiting.\n";
203
        exit(1);
204
    }
205
}
206
207
__END__
208
209
=head1 SYNOPSIS
210
211
icarusd.pl -f koha-conf.xml [--log icarus.log] [--daemon] [ -v 0-9 ] [-h]
212
213
=cut
(-)a/rewrite-config.PL (+3 lines)
Lines 149-154 $prefix = $ENV{'INSTALL_BASE'} || "/usr"; Link Here
149
  "__MEMCACHED_SERVERS__" => "",
149
  "__MEMCACHED_SERVERS__" => "",
150
  "__MEMCACHED_NAMESPACE__" => "",
150
  "__MEMCACHED_NAMESPACE__" => "",
151
  "__FONT_DIR__" => "/usr/share/fonts/truetype/ttf-dejavu",
151
  "__FONT_DIR__" => "/usr/share/fonts/truetype/ttf-dejavu",
152
  "__ICARUS_RUN_DIR__" => "$prefix/var/run/icarus",
153
  "__ICARUS_SPOOL_DIR__" => "$prefix/var/spool/icarus",
154
  "__ICARUS_MAX_TASKS__" => "30",
152
);
155
);
153
156
154
# Override configuration from the environment
157
# Override configuration from the environment
(-)a/skel/var/run/koha/icarus/README (+1 lines)
Line 0 Link Here
1
icarus dir
(-)a/skel/var/spool/koha/icarus/README (-1 / +1 lines)
Line 0 Link Here
0
- 
1
icarus spool

Return to bug 10662