View | Details | Raw Unified | Return to bug 10662
Collapse All | Expand All

(-)a/Koha/Icarus.pm (+177 lines)
Line 0 Link Here
1
package Koha::Icarus;
2
3
# Copyright 2016 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use IO::Socket::UNIX;
22
use IO::Select;
23
use URI;
24
use JSON;
25
26
sub new {
27
    my ($class, $args) = @_;
28
    $args = {} unless defined $args;
29
    return bless ($args, $class);
30
}
31
32
sub connected {
33
    my ($self) = @_;
34
    if ($self->{_connected}){
35
        return 1;
36
    }
37
}
38
39
sub connect {
40
    my ($self) =  @_;
41
    my $socket_uri = $self->{socket_uri};
42
    if ($socket_uri){
43
        my $uri = URI->new($socket_uri);
44
        if ($uri && $uri->scheme eq 'unix'){
45
            my $socket_path = $uri->path;
46
            my $socket = IO::Socket::UNIX->new(
47
                Type => IO::Socket::UNIX::SOCK_STREAM(),
48
                Peer => $socket_path,
49
            );
50
            if ($socket){
51
                my $socketio = new IO::Select();
52
                $socketio->add($socket);
53
                #FIXME: Should probably fix these return values...
54
                $self->{_socketio} = $socketio;
55
                $self->{_socket} = $socket;
56
                my $message = $self->_read();
57
                if ($message eq 'HELLO'){
58
                    $self->{_connected} = 1;
59
                    return 1;
60
                }
61
            }
62
        }
63
    }
64
    return 0;
65
}
66
67
sub add_task {
68
    my ($self, $args) = @_;
69
    my $task = $args->{task};
70
    if ($task && %$task){
71
        my $response = $self->command("add task", undef, $task);
72
        if ($response){
73
            return $response;
74
        }
75
    }
76
}
77
78
sub start_task {
79
    my ($self, $args) = @_;
80
    my $task_id = $args->{task_id};
81
    if ($task_id){
82
        my $response = $self->command("start task", $task_id);
83
        if ($response){
84
            return $response;
85
        }
86
    }
87
}
88
89
sub remove_task {
90
    my ($self, $args) = @_;
91
    my $task_id = $args->{task_id};
92
    if ($task_id){
93
        my $response = $self->command("remove task", $task_id);
94
        if ($response){
95
            return $response;
96
        }
97
    }
98
}
99
100
sub list_tasks {
101
   my ($self) = @_;
102
   my $response = $self->command("list tasks");
103
    if ($response){
104
        if (my $tasks = $response->{tasks}){
105
            return $tasks;
106
        }
107
    }
108
}
109
110
sub shutdown {
111
    my ($self) = @_;
112
    my $response = $self->command("shutdown");
113
    if ($response){
114
        return $response;
115
    }
116
}
117
118
119
120
121
122
sub command {
123
    my ($self, $command, $task_id, $task) = @_;
124
    my $serialized = $self->_serialize({ "command" => $command, "task_id" => $task_id, "task" => $task });
125
    if ($serialized){
126
        $self->_write({ serialized => $serialized });
127
        my $json = $self->_read();
128
        if ($json){
129
            my $response = from_json($json);
130
            if ($response){
131
                return $response;
132
            }
133
        }
134
    }
135
}
136
137
sub _serialize {
138
    my ($self, $output) = @_;
139
    my $serialized = to_json($output);
140
    return $serialized;
141
}
142
143
sub _write {
144
    my ($self, $args) = @_;
145
    my $socket = $self->{_socket};
146
    my $output = $args->{serialized};
147
    if ($output){
148
        if (my $socketio = $self->{_socketio}){
149
            if (my @filehandles = $socketio->can_write(5)){
150
                foreach my $filehandle (@filehandles){
151
                    #Localize output record separator as null
152
                    local $\ = "\x00";
153
                    print $socket $output;
154
                }
155
            }
156
        }
157
    }
158
}
159
160
sub _read {
161
    my ($self) = @_;
162
    if (my $socketio = $self->{_socketio}){
163
        if (my @filehandles = $socketio->can_read(5)){
164
            foreach my $filehandle (@filehandles){
165
                #Localize input record separator as null
166
                local $/ = "\x00";
167
                my $message = <$filehandle>;
168
                chomp($message) if $message;
169
                return $message;
170
            }
171
        }
172
    }
173
}
174
175
176
177
1;
(-)a/Koha/Icarus/Base.pm (+32 lines)
Line 0 Link Here
1
package Koha::Icarus::Base;
2
3
use Modern::Perl;
4
use DateTime;
5
6
use constant DEBUG => 9;
7
use constant SILENT => 0;
8
9
sub new {
10
    my ($class, $args) = @_;
11
    $args = {} unless defined $args;
12
    return bless ($args, $class);
13
}
14
15
sub debug {
16
    my ($self,$message) = @_;
17
    if ($self->{Verbosity} == DEBUG){
18
        $self->log($message);
19
    }
20
}
21
22
sub log {
23
    my ($self,$message) = @_;
24
    my $id = $self->{_id};
25
    my $component = $self->{_component} // "component";
26
    if ( ($self->{Verbosity}) && ($self->{Verbosity} > SILENT) ){
27
        my $now = DateTime->now(time_zone => "local");
28
        say "[$now] [$component $id] $message";
29
    }
30
}
31
32
1;
(-)a/Koha/Icarus/Listener.pm (+328 lines)
Line 0 Link Here
1
package Koha::Icarus::Listener;
2
3
use Modern::Perl;
4
use parent 'Koha::Icarus::Base';
5
6
use POE qw(Wheel::ReadWrite Wheel::SocketFactory Wheel::Run);
7
use IO::Socket qw(AF_UNIX);
8
use URI;
9
use Koha::Icarus::Task;
10
use JSON; #For "on_client_input"
11
12
my $null_filter = POE::Filter::Line->new(
13
     Literal => chr(0),
14
);
15
16
sub new {
17
    my ($class, $args) = @_;
18
    $args = {} unless defined $args;
19
    $args->{_component} = "server";
20
    $args->{_id} = "undefined";
21
    return bless ($args, $class);
22
}
23
24
#NOTE: "spawn" inspired by http://poe.perl.org/?POE_Cookbook/Object_Methods
25
sub spawn {
26
    my ($class, $args) = @_;
27
    my $self = $class->new($args);
28
    POE::Session->create(
29
        object_states => [
30
            $self => {
31
                _start => "on_server_start",
32
                shutdown => "shutdown",
33
                set_verbosity => "set_verbosity",
34
                _child => "on_task_event",
35
                got_list_tasks => "on_list_tasks",
36
                graceful_shutdown => "graceful_shutdown",
37
                got_client_accept => "on_client_accept",
38
                got_client_error => "on_client_error",
39
                got_server_error => "on_server_error",
40
                got_add_task => "on_add_task",
41
                got_client_input => "on_client_input",
42
            },
43
        ],
44
    );
45
}
46
47
#Methods for POE::Session
48
49
sub on_server_start {
50
    my ($self, $kernel,$heap,$session) = @_[OBJECT, KERNEL,HEAP,SESSION];
51
    my $server_id = $session->ID;
52
    $self->{_id} = $server_id; #Set internal id for logging purposes
53
54
    my $bind_address_uri = $self->{Socket};
55
    my $max_tasks = $self->{MaxTasks};
56
57
    $kernel->sig(INT => "graceful_shutdown");
58
    $kernel->sig(TERM => "graceful_shutdown");
59
60
    $heap->{max_tasks} = $max_tasks // 25; #Default maximum of 25 unless otherwise specified
61
62
    $self->log("Maximum number of tasks allowed: $heap->{max_tasks}");
63
    $self->log("Starting server...");
64
65
    my %server_params = (
66
        SuccessEvent => "got_client_accept",
67
        FailureEvent => "got_server_error",
68
    );
69
70
    #TODO: At this time, only "unix" sockets are supported. In future, perhaps TCP/IP sockets could also be supported.
71
    my $uri = URI->new($bind_address_uri);
72
    my $scheme = $uri->scheme;
73
74
    if ($scheme eq 'unix'){
75
        my $bind_address = $uri->path;
76
        $server_params{SocketDomain} = AF_UNIX;
77
        $server_params{BindAddress} = $bind_address;
78
        #When starting a unix socket server, you need to remove any existing references to that socket file.
79
        if ($bind_address && (-e $bind_address) ){
80
            unlink $bind_address;
81
        }
82
    }
83
84
    $heap->{server} = POE::Wheel::SocketFactory->new(%server_params);
85
86
    if ($scheme eq 'unix'){
87
        #FIXME/DEBUGGING: This is a way to force a permission denied error...
88
        #chmod 0755, $uri->path;
89
        #Make the socket writeable to other users like Apache
90
        chmod 0666, $uri->path;
91
    }
92
93
}
94
95
sub shutdown {
96
    my ($self,$heap,$session,$kernel) = @_[OBJECT, HEAP,SESSION,KERNEL];
97
98
    if ($heap->{server}){
99
        $self->log("Shutting down server...");
100
        #Delete the server, so that you can't get any new connections
101
        delete $heap->{server} if $heap->{server};
102
    }
103
104
    if ($heap->{client}){
105
        $self->log("Shutting down any remaining clients...");
106
        #Delete the clients, so that you bring down the existing connections
107
        delete $heap->{client}; #http://www.perlmonks.org/?node_id=176971
108
    }
109
}
110
111
sub on_task_event {
112
    my ($self, $kernel, $heap,$session) = @_[OBJECT,KERNEL, HEAP,SESSION];
113
    my ($action,$child_session,$task) = @_[ARG0,ARG1,ARG2];
114
115
    my $child_id = $child_session->ID;
116
    
117
    $self->debug("$action child $child_id");
118
    
119
120
    if ($action eq 'create'){
121
        #NOTE: The $task variable is returned by the child POE session's _start event
122
        my $task_id = $child_session->ID;
123
        $heap->{tasks}->{$task_id}->{task} = $task;
124
125
    } elsif ($action eq 'lose'){
126
        my $task_id = $child_session->ID;
127
        delete $heap->{tasks}->{$task_id};
128
    }
129
}
130
131
#TODO: Put this in a parent class?
132
sub set_verbosity {
133
    my ($self,$session,$kernel,$new_verbosity) = @_[OBJECT,SESSION,KERNEL,ARG0];
134
    if (defined $new_verbosity){
135
        $self->{Verbosity} = $new_verbosity;
136
    }
137
}
138
139
sub on_list_tasks {
140
    my ($self, $kernel, $heap,$session) = @_[OBJECT, KERNEL, HEAP,SESSION];
141
142
    #DEBUG: You can access the POE::Kernel's sessions with "$POE::Kernel::poe_kernel->[POE::Kernel::KR_SESSIONS]".
143
    #While it's black magic you shouldn't touch, it can be helpful when debugging.
144
145
    my @tasks = ();
146
    foreach my $task_id (keys %{$heap->{tasks}} ){
147
        push(@tasks,{ task_id => $task_id, task => $heap->{tasks}->{$task_id}->{task} });
148
    }
149
    return \@tasks;
150
}
151
152
sub graceful_shutdown {
153
    my ($self, $heap,$session,$kernel,$signal) = @_[OBJECT, HEAP,SESSION,KERNEL,ARG0];
154
155
    #Tell the kernel that you're handling the signal sent to this session
156
    $kernel->sig_handled();
157
    $kernel->sig($signal);
158
159
    my $tasks = $kernel->call($session,"got_list_tasks");
160
161
162
    if ( $heap->{tasks} && %{$heap->{tasks}} ){
163
        $self->log("Waiting for tasks to finish...");
164
        foreach my $task_id (keys %{$heap->{tasks}}){
165
            $self->log("Task $task_id still exists...");
166
            $kernel->post($task_id,"got_task_stop");
167
        }
168
    } else {
169
        $self->log("All tasks have finished");
170
        $kernel->yield("shutdown");
171
        return;
172
    }
173
174
    $self->log("Attempting graceful shutdown in 1 second...");
175
    #NOTE: Basically, we just try another graceful shutdown on the next tick.
176
    $kernel->delay("graceful_shutdown" => 1);
177
}
178
179
#Accept client connection to listener
180
sub on_client_accept {
181
    my ($self, $client_socket, $server_wheel_id, $heap, $session) = @_[OBJECT, ARG0, ARG3, HEAP,SESSION];
182
183
    my $client_wheel = POE::Wheel::ReadWrite->new(
184
      Handle => $client_socket,
185
      InputEvent => "got_client_input",
186
      ErrorEvent => "got_client_error",
187
      InputFilter => $null_filter,
188
      OutputFilter => $null_filter,
189
    );
190
191
    $client_wheel->put("HELLO");
192
    $heap->{client}->{ $client_wheel->ID() } = $client_wheel;
193
    
194
    $self->debug("Connection ".$client_wheel->ID()." started.");
195
    
196
}
197
198
#Handle server error - shutdown server
199
sub on_server_error {
200
    my ($self, $operation, $errnum, $errstr, $heap, $session) = @_[OBJECT, ARG0, ARG1, ARG2,HEAP, SESSION];
201
    $self->debug("Server $operation error $errnum: $errstr\n");
202
    delete $heap->{server};
203
}
204
205
#Handle client error - including disconnect
206
sub on_client_error {
207
    my ($self, $wheel_id,$heap,$session) = @_[OBJECT, ARG3,HEAP,SESSION];
208
    
209
    $self->debug("Connection $wheel_id failed or ended.");
210
    
211
    delete $heap->{client}->{$wheel_id};
212
213
}
214
215
sub on_add_task {
216
    my ($self, $message, $kernel, $heap, $session) = @_[OBJECT, ARG0, KERNEL, HEAP,SESSION];
217
218
    #Fetch a list of all tasks
219
    my @task_keys = keys %{$heap->{tasks}};
220
221
    #If the number in the list is less than the max, add a new task
222
    #else die.
223
    if (scalar @task_keys < $heap->{max_tasks}){
224
        my $server_id = $session->ID;
225
        my $task_session = Koha::Icarus::Task->spawn({ message => $message, server_id => $server_id, Verbosity => $self->{Verbosity}, });
226
        return $task_session->ID;
227
    } else {
228
        #This die should be caught by the event caller...
229
        die "Maximum number of tasks already reached.\n";
230
    }
231
}
232
233
sub on_client_input {
234
    my ($self, $input, $wheel_id, $session, $kernel, $heap) = @_[OBJECT, ARG0, ARG1, SESSION, KERNEL, HEAP];
235
236
    #Store server id more explicitly
237
    my $server_id = $session->ID;
238
239
    #Server listener has received input from client
240
    my $client = $heap->{client}->{$wheel_id};
241
242
    #Parse input from client
243
    my $message = from_json($input);
244
245
    if ( ref $message eq 'HASH' ){
246
        #Read "command" from client
247
        if (my $command = $message->{command}){
248
            $self->log("Message received with command \"$command\".");
249
            if ($command eq 'add task'){
250
                my $output = {};
251
252
                #Create a task session
253
                eval {
254
                   #NOTE: The server automatically keeps track of its child tasks
255
                    my $task_id = $kernel->call($server_id,"got_add_task",$message);
256
257
                    $output->{action} = "added";
258
                    $output->{task_id} = $task_id;
259
                };
260
                if ($@){
261
                    $self->debug("$@");
262
                    chomp($@);
263
                    $output->{action} = "error";
264
                    $output->{error_message} = $@;
265
                }
266
                my $server_output = to_json($output);
267
                $client->put($server_output);
268
                return;
269
270
            } elsif ( ($command eq 'remove task') || ($command eq 'start task' ) ){
271
272
                my $task_id = $message->{task_id};
273
274
                my $output = {
275
                    task_id => $task_id,
276
                };
277
278
                if ($command eq 'remove task'){
279
                    $kernel->call($task_id,"got_task_stop");
280
                    $output->{action} = "removed";
281
                } elsif ($command eq 'start task'){
282
                    my $response = $kernel->call($task_id, "on_task_init");
283
                    $output->{action} = $response;
284
                }
285
286
                if ($!){
287
                    $output->{action} = "error";
288
                    $output->{error_message} = $!;
289
                }
290
291
                #FIXME: What do we actually want to send back to the client?
292
                my $server_output = to_json($output);
293
                $client->put($server_output);
294
                return;
295
296
            } elsif ($command eq 'list tasks'){
297
298
                #Get tasks from listener (ie self)
299
                my $tasks = $kernel->call($server_id, "got_list_tasks");
300
301
                #Prepare output for client
302
                my $server_output = to_json({tasks => $tasks}, {pretty => 1});
303
304
                #Send output to client
305
                $client->put($server_output);
306
                return;
307
308
            } elsif ($command eq 'shutdown'){
309
                $kernel->post($server_id, "graceful_shutdown");
310
                my $server_output = to_json({action => 'shutting down'});
311
                $client->put($server_output);
312
                return;
313
            } else {
314
                $self->log("The message contained an invalid command!");
315
                $client->put("Sorry! That is an invalid command!");
316
                return;
317
            }
318
        } else {
319
            $self->log("The message was missing a command!");
320
        }
321
    } else {
322
        $self->log("The message was malformed!");
323
    }
324
    $client->put("Sorry! That is an invalid message!");
325
    return;
326
}
327
328
1;
(-)a/Koha/Icarus/Task.pm (+315 lines)
Line 0 Link Here
1
package Koha::Icarus::Task;
2
3
use Modern::Perl;
4
use parent 'Koha::Icarus::Base';
5
6
use POE qw(Wheel::Run);
7
use DateTime;
8
use DateTime::Format::Strptime;
9
use JSON;
10
use Module::Load::Conditional qw/can_load/;
11
12
my $datetime_pattern = DateTime::Format::Strptime->new(
13
    pattern   => '%F %T',
14
    time_zone => 'local',
15
);
16
my $epoch_pattern = DateTime::Format::Strptime->new(
17
    pattern   => '%s',
18
);
19
20
sub new {
21
    my ($class, $args) = @_;
22
    $args = {} unless defined $args;
23
    $args->{_component} = "task";
24
    $args->{_id} = "undefined";
25
    return bless ($args, $class);
26
}
27
28
#NOTE: "spawn" inspired by http://poe.perl.org/?POE_Cookbook/Object_Methods
29
sub spawn {
30
    my ($class, $args) = @_;
31
    my $self = $class->new($args);
32
    my $task_session = POE::Session->create(
33
        object_states => [
34
            $self => {
35
                _start => "on_task_create",
36
                 "got_child_stdout" => "on_child_stdout",
37
                 "got_child_stderr" => "on_child_stderr",
38
                 "got_child_close"  => "on_child_close",
39
                 "got_child_signal" => "on_child_signal",
40
                 "got_terminal_signal" => "on_terminal_signal",
41
                 "child_process_success" => "child_process_success",
42
                 "got_task_stop" => "on_task_stop",
43
                 "on_task_init" => "on_task_init",
44
                 "on_task_start" => "on_task_start",
45
            },
46
        ],
47
    );
48
    return $task_session;
49
}
50
51
sub on_task_create {
52
    my ($self, $session, $kernel, $heap) = @_[OBJECT, SESSION, KERNEL, HEAP];
53
54
    #Trap terminal signals so that the task can stop gracefully.
55
    $kernel->sig(INT => "got_terminal_signal");
56
    $kernel->sig(TERM => "got_terminal_signal");
57
58
    my $task_id = $session->ID;
59
    if ($task_id){
60
        #Tell the kernel that this task is waiting for an external action (ie keepalive counter)
61
        $kernel->refcount_increment($task_id,"waiting task");
62
        $self->{_id} = $task_id; #Set internal id for logging purposes
63
    }
64
65
    my $server_id = $self->{server_id};
66
    if ($server_id){
67
        $heap->{server_id} = $server_id;
68
    }
69
70
    my $task = undef;
71
    my $message = $self->{message};
72
    if ($message){
73
        $task = $message->{task};
74
        if ($task){
75
            $task->{status} = 'new';
76
            $heap->{task} = $task;
77
        }
78
    }
79
    return $task; #This return value is used by the parent POE session's _child handler
80
}
81
82
#This sub is just to start it now, or set it to start in the future... if the time is now or in the past, it starts now... if it's in the future, it starts in the future...
83
sub on_task_init {
84
    my ($self, $session, $kernel, $heap) = @_[OBJECT, SESSION, KERNEL, HEAP];
85
    my $response = 'pending';
86
    my $task = $heap->{task};
87
    my $status = $task->{status};
88
    if ($status){
89
        if ($status eq 'started'){
90
            $response = 'already started';
91
        } elsif ($status eq 'pending'){
92
            $response = 'already pending';
93
        } else {
94
            $task->{status} = 'pending';
95
96
            my $start = $task->{start};
97
            my $start_message = $start;
98
            
99
            
100
            my $dt;
101
            if ( $dt = $datetime_pattern->parse_datetime($start) ){
102
                #e.g. 2016-04-06 00:00:00
103
            } elsif ( $dt = $epoch_pattern->parse_datetime($start) ){
104
                #e.g. 1459837498 or apparently 0000-00-00 00:00:00
105
            } else {
106
                #If we don't match the datetime_pattern or epoch_pattern, then we start right now.
107
                $dt = DateTime->now( time_zone => 'local', );
108
            }
109
            if ($dt){
110
                $start = $dt->epoch;
111
                $start_message = $dt;
112
            }
113
114
115
            $self->log("Start task at $start_message");
116
            #NOTE: $start must be in UNIX epoch time (ie number of seconds that have elapsed since 00:00:00 UTC Thursday 1 January 1970)
117
            $kernel->alarm("on_task_start",$start);
118
        }
119
    }
120
    return $response;
121
}
122
123
sub on_task_start {
124
    my ($self, $session, $kernel, $heap) = @_[OBJECT, SESSION, KERNEL, HEAP];
125
    my $task = $heap->{task};
126
    $task->{status} = 'started';
127
128
    if (my $repeat_interval = $task->{repeat_interval}){
129
        #NOTE: Reset the start time with a human readable timestamp
130
        my $dt = DateTime->now( time_zone => 'local', );
131
        $dt->add( seconds => $repeat_interval );
132
        $task->{start} = $dt->strftime("%F %T");
133
    }
134
    #FIXME: You need to impose child process limits here! How many child processes are allowed to be running at any given time? Well, you can only have one child process per task...
135
    #so it's really more of a limit on the number of tasks... you probably need to have an internal task queue... that's easy enough though.
136
    my $child = POE::Wheel::Run->new(
137
        ProgramArgs => [ $task, ],
138
        Program => sub {
139
            my ($task) = @_;
140
141
            #Perform some last minute POE calls before running the task module plugin
142
            my $session = $poe_kernel->get_active_session();
143
            if ($session){
144
                my $heap = $session->get_heap();
145
                $poe_kernel->call($heap->{server_id},"set_verbosity",0); #This turns off the server logging in this forked process, so the following call() doesn't mess up our logs
146
                $poe_kernel->call($heap->{server_id},"shutdown"); #Shutdown the socket listener on the child process, so there's zero chance of writing to or reading from the socket in the child process
147
            }
148
149
            #NOTE: I don't know if this really needs to be run, but it shouldn't hurt.
150
            $poe_kernel->stop();
151
152
            #Try to load the task type module.
153
            my $task_type = $task->{type};
154
            if ( can_load ( modules => { $task_type => undef, }, ) ){
155
                #Create the object
156
                my $task_object = $task_type->new({task => $task, Verbosity => $self->{Verbosity}, });
157
                if ($task_object){
158
                    #Synchronous action: run the task module
159
                    $task_object->run;
160
                }
161
            } else {
162
                die "Couldn't load module $task_type: $Module::Load::Conditional::ERROR"
163
            }
164
        },
165
        StdoutEvent  => "got_child_stdout",
166
        StderrEvent  => "got_child_stderr",
167
        CloseEvent   => "got_child_close",
168
        NoSetPgrp => 1, #Keep child processes in same group as parent. This is especially useful when using Ctrl+C to kill the whole group.
169
    );
170
171
    $kernel->sig_child($child->PID, "got_child_signal");
172
    # Wheel events include the wheel's ID.
173
    $_[HEAP]{children_by_wid}{$child->ID} = $child;
174
    # Signal events include the process ID.
175
    $_[HEAP]{children_by_pid}{$child->PID} = $child;
176
177
    $self->debug("child pid ".$child->PID." started as wheel ".$child->ID);
178
}
179
180
sub on_task_stop {
181
    my ($self, $session, $kernel, $heap) = @_[OBJECT, SESSION, KERNEL, HEAP];
182
    my $task = $heap->{task};
183
    $task->{status} = 'stopping';
184
    my $task_id = $session->ID;
185
    my $server_id = $heap->{server_id};
186
187
    if ($heap->{stopping}){
188
        $self->debug("Task is already in the process of stopping...");
189
190
    } else {
191
        
192
        $self->log("Trying to stop task.");
193
        
194
195
        #Mark this task as stopping
196
        $heap->{stopping} = 1;
197
198
        #Stop the task from spawning new jobs
199
        $kernel->alarm("on_task_start");
200
201
        my $children_by_pid = $heap->{children_by_pid};
202
        if ($children_by_pid && %$children_by_pid){
203
204
            $self->debug("Child processes in progres...");
205
            my $child_processes = $heap->{children_by_pid};
206
            foreach my $child_pid (keys %$child_processes){
207
                my $child = $child_processes->{$child_pid};
208
                $self->debug("Telling child pid $child_pid to stop");
209
                $child->put("quit");
210
                #TODO: Perhaps it would be worthwhile having a kill switch too?
211
                # my $rv = $child->kill("TERM");
212
            }
213
        }
214
        
215
        $self->log("Removing task keepalive.");
216
        
217
        $kernel->refcount_decrement($task_id,"waiting task");
218
    }
219
}
220
221
sub on_terminal_signal {
222
    my ($self, $signal,$session,$kernel) = @_[OBJECT, ARG0,SESSION,KERNEL];
223
    $self->debug("Trapped SIGNAL: $signal.");
224
    #Gracefully stop the task
225
    $kernel->call($session, "got_task_stop");
226
}
227
228
sub child_process_success {
229
    my ($self, $heap,$session,$kernel) = @_[OBJECT, HEAP,SESSION,KERNEL];
230
    my $task = $heap->{task};
231
    if (my $repeat_interval = $task->{repeat_interval}){
232
        if ($heap->{stopping}){
233
            $self->log("Will skip repeating the task, as task is stopping.");
234
        } else {
235
            $self->log("Will repeat the task");
236
            $task->{status} = "restarting";
237
            $kernel->yield("on_task_init");
238
        }
239
    } else {
240
        $self->debug("I'm going to stop this task");
241
        $kernel->yield("got_task_stop");
242
    }
243
}
244
245
#############################################################
246
#                                                           #
247
#      Methods for communicating with child processes       #
248
#                                                           #
249
#############################################################
250
# Originally inspired by the POE::Wheel::Run perldoc example
251
252
# Wheel event, including the wheel's ID
253
sub on_child_stdout {
254
    my ($self, $stdout_line, $wheel_id, $session) = @_[OBJECT, ARG0, ARG1, SESSION];
255
    my $child = $_[HEAP]{children_by_wid}{$wheel_id};
256
    #NOTE: Log everything child process sends to STDOUT
257
    $self->log("[pid ".$child->PID."] STDOUT: $stdout_line");
258
259
    #If the child outputs a line to STDOUT which starts with UPDATE_PARAMS=, we capture the data,
260
    #and update the task params.
261
    if ($stdout_line =~ /^UPDATE_PARAMS=(.*)$/){
262
        my $json_string = $1;
263
        my $json = from_json($json_string);
264
        my $task = $_[HEAP]->{task};
265
        my $params = $task->{params};
266
        foreach my $key (%$json){
267
            if (defined $params->{$key}){
268
                #FIXME: Don't just overwrite? Only update differences?
269
                $params->{$key} = $json->{$key};
270
            }
271
        }
272
        $_[HEAP]->{task} = $task;
273
    }
274
}
275
276
# Wheel event, including the wheel's ID.
277
sub on_child_stderr {
278
    my ($self, $stderr_line, $wheel_id, $session) = @_[OBJECT, ARG0, ARG1,SESSION];
279
    my $child = $_[HEAP]{children_by_wid}{$wheel_id};
280
    #NOTE: Log everything child process sends to STDERR
281
    $self->log("[pid ".$child->PID."] STDERR: $stderr_line");
282
}
283
284
# Wheel event, including the wheel's ID.
285
sub on_child_close {
286
    my ($self, $wheel_id,$session,$kernel) = @_[OBJECT, ARG0,SESSION,KERNEL];
287
288
    my $child = delete $_[HEAP]{children_by_wid}{$wheel_id};
289
290
    # May have been reaped by on_child_signal().
291
    unless (defined $child) {
292
        $self->debug("[wid $wheel_id] closed all pipes.");
293
        return;
294
    }
295
    $self->debug("[pid ".$child->PID."] closed all pipes.");
296
    delete $_[HEAP]{children_by_pid}{$child->PID};
297
}
298
299
sub on_child_signal {
300
    my ($self, $heap,$kernel,$pid,$exit_code,$session) = @_[OBJECT, HEAP,KERNEL,ARG1,ARG2,SESSION];
301
302
    #If the child's exit code is 0, handle this successful exit status
303
    if ($exit_code == 0){
304
        $kernel->yield("child_process_success");
305
    }
306
    $self->debug("pid $pid exited with status $exit_code.");
307
    my $child = delete $_[HEAP]{children_by_pid}{$pid};
308
309
    # May have been reaped by on_child_close().
310
    return unless defined $child;
311
312
    delete $_[HEAP]{children_by_wid}{$child->ID};
313
}
314
315
1;
(-)a/Koha/Icarus/Task/Base.pm (+24 lines)
Line 0 Link Here
1
package Koha::Icarus::Task::Base;
2
3
use Modern::Perl;
4
use IO::Select;
5
6
sub new {
7
    my ($class, $args) = @_;
8
    $args = {} unless defined $args;
9
    return bless ($args, $class);
10
}
11
12
sub listen_for_instruction {
13
    my ($self) = @_;
14
    my $select = $self->{_select} ||= IO::Select->new(\*STDIN);
15
    if (my @ready_FHs  = $select->can_read(0) ){
16
        foreach my $FH (@ready_FHs){
17
            my $line = $FH->getline();
18
            chomp($line);
19
            return $line;
20
        }
21
    }
22
}
23
24
1;
(-)a/Koha/Icarus/Task/Download/OAIPMH/Biblio.pm (+316 lines)
Line 0 Link Here
1
package Koha::Icarus::Task::Download::OAIPMH::Biblio;
2
3
use Modern::Perl;
4
use parent 'Koha::Icarus::Task::Base';
5
6
use DateTime;
7
use DateTime::Format::Strptime;
8
use HTTP::OAI;
9
use File::Path qw(make_path);
10
use Digest::MD5;
11
use JSON;
12
use URI;
13
14
my $strp = DateTime::Format::Strptime->new(
15
        pattern   => '%Y%m%dT%H%M%S.%NZ',
16
);
17
18
my $oai_second_granularity = DateTime::Format::Strptime->new(
19
        pattern   => '%Y-%m-%dT%H:%M:%SZ',
20
);
21
22
my $oai_day_granularity = DateTime::Format::Strptime->new(
23
        pattern   => '%Y-%m-%d',
24
);
25
26
sub validate_parameter_names {
27
28
}
29
sub validate_repeat_interval {
30
    my ($self,$repeat_interval) = @_;
31
    if (defined $repeat_interval && $repeat_interval =~ /^\d+$/){
32
        return undef;
33
    }
34
    $self->{invalid_data}++;
35
    return { not_numeric => 1, };
36
}
37
38
sub validate_url {
39
    my ($self,$url) = @_;
40
    my $response = {};
41
    if (my $url_obj = URI->new($url)){
42
        if ($url_obj->scheme ne "http"){
43
            $response->{not_http} = 1;
44
            $self->{invalid_data}++;
45
        }
46
        if ( ! $url_obj->path){
47
            $response->{no_path} = 1;
48
            $self->{invalid_data}++;
49
        }
50
    } else {
51
        $response->{not_a_url} = 1;
52
        $self->{invalid_data}++;
53
    }
54
55
    return $response;
56
}
57
58
sub validate {
59
    my ($self, $args) = @_;
60
    #Reset the invalid data counter...
61
    $self->{invalid_data} = 0;
62
    my $errors = { };
63
    my $task = $self->{task};
64
    my $tests = $args->{tests};
65
    if ($task){
66
        if ($tests && $tests eq 'all'){
67
            #warn "PARAMS = ".$task->{params};
68
        }
69
    }
70
    my $params = $task->{params};
71
72
    #validate_start_time
73
    $errors->{"repeat_interval"} = $self->validate_repeat_interval($task->{repeat_interval});
74
75
    $errors->{"url"} = $self->validate_url($params->{url});
76
77
    #NOTE: You don't need to validate these 3 HTTP Basic Auth parameters
78
    #validate_username
79
    #validate_password
80
    #validate_realm
81
82
    #OAI-PMH parameters
83
    #validate_verb
84
    #validate_sets
85
    #validate_marcxml
86
    #validate_from
87
    #validate_until
88
89
    #Download parameters
90
    #validate_queue
91
92
    return $errors;
93
}
94
95
sub new {
96
    my ($class, $args) = @_;
97
    $args = {} unless defined $args;
98
    $args->{invalid_data} = 0;
99
    return bless ($args, $class);
100
}
101
102
sub validate_queue {
103
    my ( $self ) = @_;
104
    my $task = $self->{task};
105
    if (my $queue = $task->{params}->{queue}){
106
107
        my $queue_uri = URI->new($queue);
108
        #TODO: In theory, you could even use a DBI DSN like DBI:mysql:database=koha;host=koha.db;port=3306.
109
        #Then you could provide the table, username, and password in the params as well...
110
111
        #NOTE: If the queue directory doesn't exist on the filesystem, we try to make it and change to it.
112
        if ($queue_uri->scheme eq 'file'){
113
            my $filepath = $queue_uri->file;
114
            if ( ! -d $filepath ){
115
                make_path($filepath,{ mode => 0755 });
116
            }
117
            if ( -d $filepath ){
118
                chdir $filepath or die "$!";
119
            }
120
        }
121
122
    }
123
}
124
125
sub run {
126
    my ( $self ) = @_;
127
    $self->validate_queue;
128
129
    my $task = $self->{task};
130
131
    #DEBUGGING/FIXME: Remove these lines
132
    if ($self->{Verbosity} && $self->{Verbosity} == 9){
133
        use Data::Dumper;
134
        warn Dumper($task);
135
    }
136
137
    my $params = $task->{params};
138
139
    my $now = DateTime->now(); #This is in UTC time, which is required by the OAI-PMH protocol.
140
    if ( $oai_second_granularity->parse_datetime($params->{from}) ){
141
        $now->set_formatter($oai_second_granularity);
142
    } else {
143
        $now->set_formatter($oai_day_granularity);
144
    }
145
146
    $params->{until}  = "$now" if $task->{repeat_interval};
147
148
    $self->{digester} = Digest::MD5->new();
149
    $self->create_harvester;
150
    my $sets = $self->prepare_sets;
151
152
    #Send a OAI-PMH request for each set
153
    foreach my $set (@{$sets}){
154
        my $response = $self->send_request({set => $set});
155
        $self->handle_response({ response => $response, set => $set,});
156
    }
157
158
    #FIXME: Do you want to update the task only when the task is finished, or
159
    #also after each resumption?
160
    #Update the task params in Icarus after the task is finished...
161
    #TODO: This really does make it seem like you should be handling the repeat_interval within the child process rather than the parent...
162
    if ($task->{repeat_interval}){
163
        $params->{from} = "$now";
164
        $params->{until} = "";
165
        my $json_update = to_json($params);
166
        say STDOUT "UPDATE_PARAMS=$json_update";
167
    }
168
169
}
170
171
#FIXME: I wonder if it would be faster to send your own HTTP requests and not use HTTP::OAI...
172
sub send_request {
173
    my ( $self, $args ) = @_;
174
175
    #NOTE: This is plugin specific as the plugins define when they stop to listen for instructions...
176
    #NOTE: Before sending a new request, check if Icarus has already asked us to quit.
177
    my $instruction = $self->listen_for_instruction();
178
    if ($instruction eq 'quit'){
179
        warn "I was asked to quit!";
180
        return;
181
    }
182
183
    my $set = $args->{set};
184
    my $resumptionToken = $args->{resumptionToken};
185
186
    my $response;
187
    my $task_params = $self->{task}->{params};
188
189
    my $harvester = $self->{harvester};
190
    my $verb = $task_params->{verb};
191
    if ($verb eq 'GetRecord'){
192
        $response = $harvester->GetRecord(
193
            metadataPrefix => $task_params->{metadataPrefix},
194
            identifier => $task_params->{identifier},
195
         );
196
    } elsif ($verb eq 'ListRecords'){
197
        $response = $harvester->ListRecords(
198
            metadataPrefix => $task_params->{metadataPrefix},
199
            from => $task_params->{from},
200
            until => $task_params->{until},
201
            set => $set,
202
            resumptionToken => $resumptionToken,
203
        );
204
    }
205
    return $response;
206
}
207
208
sub create_harvester {
209
    my ( $self ) = @_;
210
    my $task_params = $self->{task}->{params};
211
212
    #FIXME: DEBUGGING
213
    #use HTTP::OAI::Debug qw(+);
214
215
    #Create HTTP::OAI::Harvester object
216
    my $harvester = new HTTP::OAI::Harvester( baseURL => $task_params->{url} );
217
    if ($harvester){
218
        $harvester->timeout(5); #NOTE: the default timeout is 180
219
        #Set HTTP Basic Authentication Credentials
220
        my $uri = URI->new($task_params->{url});
221
        my $host = $uri->host;
222
        my $port = $uri->port;
223
        $harvester->credentials($host.":".$port, $task_params->{realm}, $task_params->{username}, $task_params->{password});
224
    }
225
    $self->{harvester} = $harvester;
226
}
227
228
sub prepare_sets {
229
    my ( $self ) = @_;
230
    my $task_params = $self->{task}->{params};
231
    my @sets = ();
232
    if ($task_params->{sets}){
233
        @sets = split(/\|/, $task_params->{sets});
234
    }
235
    #If no sets are defined, create a null element to force the foreach loop to run once
236
    if (!@sets){
237
        push(@sets,undef)
238
    }
239
    return \@sets;
240
}
241
242
sub handle_response {
243
    my ( $self, $args ) = @_;
244
    my $params = $self->{task}->{params};
245
    my $response = $args->{response};
246
    my $set = $args->{set};
247
    if ($response){
248
        #NOTE: We have options at this point
249
        #Option 1: Use $response->toDOM() to handle the XML response as a single document
250
        #Option 2: Use $response->next() to handle each record individually. You would need to create a new document using $rec->header->dom() and $rec->metadata->dom() anyway.
251
252
        #NOTE: I wonder which option would be the fastest. For now, we're going with Option 1:
253
        my $dom = $response->toDOM;
254
        my $root = $dom->documentElement;
255
256
        #FIXME: Provide these as arguments so you're not re-creating them for each response
257
        my $xpc = XML::LibXML::XPathContext->new();
258
        $xpc->registerNs('oai','http://www.openarchives.org/OAI/2.0/');
259
        my $xpath = XML::LibXML::XPathExpression->new("(oai:GetRecord|oai:ListRecords)/oai:record");
260
261
262
        my @records = $xpc->findnodes($xpath,$root);
263
        my $now_pretty = DateTime->now();
264
265
        $now_pretty->set_formatter($strp);
266
        print "Downloaded ".scalar @records." records at $now_pretty\n";
267
        foreach my $record (@records) {
268
269
            #FIXME: This is where you could put a filter to prevent certain records from being saved...
270
271
            #Create a new XML document from the XML fragment
272
            my $document = XML::LibXML::Document->new( "1.0", "UTF-8" );
273
            $document->setDocumentElement($record);
274
            my $record_string = $document->toString;
275
276
            #NOTE: We have options at this point.
277
            #Option 1: Write documents to disk, and have a separate importer upload the documents
278
            #Option 2: Use AnyEvent::HTTP or POE::Component::Client::HTTP to send to a HTTP API asynchronously
279
            #Option 3: Write records to a database, and have a separate importer upload the documents
280
            #Option 4: Shared memory, although that seems fragile if nothing else
281
            #Option 5: Write the records to a socket/pipe
282
283
            #NOTE: I wonder which option would be the fastest. For now, we're going to go with Option 1:
284
            $self->{digester}->add($record_string);
285
            my $digest = $self->{digester}->hexdigest;
286
            #FIXME: If a record appears more than once during the download signified by $now, you'll
287
            #overwrite the former with the latter. While this acts as a sort of heavy-handed de-duplication,
288
            #you need to take into account the importer daemon...
289
290
            require Time::HiRes;
291
            my $epoch = Time::HiRes::time();
292
            my $now = DateTime->from_epoch(epoch => $epoch);
293
            $now->set_formatter($strp);
294
295
            my $filename = "$now-$digest";
296
            #NOTE: Here is where we write the XML out to disk
297
            my $state = $document->toFile($filename);
298
        }
299
300
301
        #NOTE: Check if object has method due to bug in HTTP::OAI which causes fatal error on $response->resumptionToken if no real response is fetched from the OAI-PMH server
302
        if ($response->can("resumptionToken")){
303
            my $resumption_token = $response->resumptionToken->resumptionToken if $response->resumptionToken && $response->resumptionToken->resumptionToken;
304
            if ($resumption_token){
305
                #warn "Resumption Token = $resumption_token";
306
                my $resumed_response = $self->send_request({set => $set, resumptionToken => $resumption_token});
307
                $self->handle_response({ response => $resumed_response, set => $set,});
308
            }
309
        }
310
311
        #In theory $response->resume(resumptionToken => resumptionToken) should kick off another response...
312
        warn $response->message if $response->is_error;
313
    }
314
}
315
316
1;
(-)a/Koha/Icarus/Task/Upload/OAIPMH/Biblio.pm (+118 lines)
Line 0 Link Here
1
package Koha::Icarus::Task::Upload::OAIPMH::Biblio;
2
3
use Modern::Perl;
4
use parent 'Koha::Icarus::Task::Base';
5
use URI;
6
use LWP::UserAgent;
7
use HTTP::Status qw(:constants);
8
9
my $ua = LWP::UserAgent->new;
10
11
#FIXME: If we store the cookie jar on disk, we can prevent unnecessary HTTP requests...
12
#We would need to make sure that it's stored on a private per-instance basis though...
13
$ua->cookie_jar({});
14
15
16
sub new {
17
    my ($class, $args) = @_;
18
    $args = {} unless defined $args;
19
    return bless ($args, $class);
20
}
21
22
sub run {
23
    my ( $self ) = @_;
24
25
    my $task = $self->{task};
26
27
    if ($self->{Verbosity} && $self->{Verbosity} == 9){
28
        use Data::Dumper;
29
        warn Dumper($task);
30
    }
31
32
    my $params = $task->{params};
33
34
    
35
    
36
37
    my $queue = $params->{queue};
38
    my $queue_uri = URI->new($queue);
39
40
    if ($queue_uri->scheme eq 'file'){
41
42
        my $path = $queue_uri->path;
43
        opendir(my $dh, $path);
44
        my @files = sort readdir($dh);
45
        foreach my $file (@files){
46
            #NOTE: This is plugin specific as the plugins define when they stop to listen for instructions...
47
            my $instruction = $self->listen_for_instruction();
48
            if ($instruction eq 'quit'){
49
                warn "I was asked to quit!";
50
                return;
51
            }
52
53
            next if $file =~ /^\.+$/;
54
            my $filepath = "$path/$file";
55
            if ( -d $filepath ){
56
                #Do nothing for directories
57
            } elsif ( -e $filepath ){
58
                print "File: $file\n";
59
60
                #Slurp mode
61
                local $/;
62
                #TODO: Check flock on $filepath first
63
                open( my $fh, '<', $filepath );
64
                my $data   = <$fh>;
65
66
                #TODO: Improve this section...
67
                #Send to Koha API... (we could speed this up using Asynchronous HTTP requests with AnyEvent::HTTP...)
68
                my $resp = post_to_api($data,$params);
69
70
                my $status = $resp->code;
71
72
                if ($status == HTTP_UNAUTHORIZED || $status == HTTP_FORBIDDEN) {
73
                    $resp = remote_authenticate($params);
74
                    $resp = post_to_api($data,$params) if $resp->is_success;     
75
                }
76
                
77
                if ($resp->code == HTTP_OK){
78
                    print "Success.\n";
79
                    print $resp->decoded_content;
80
                    print "\n";
81
                    unlink $filepath;
82
                }
83
            }
84
        }
85
    }
86
}
87
88
sub post_to_api {
89
    my ($data, $params) = @_;
90
    print "Posting to API...\n";
91
    my $resp = $ua->post( $params->{target_uri},
92
                  {'nomatch_action' => $params->{nomatch_action},
93
                   'overlay_action' => $params->{overlay_action},
94
                   'match'          => $params->{match},
95
                   'import_mode'    => $params->{import_mode},
96
                   'framework'      => $params->{framework},
97
                   'item_action'    => $params->{item_action},
98
                   'filter'         => $params->{filter},
99
                   'xml'            => $data}
100
    );
101
    return $resp;
102
}
103
104
sub remote_authenticate {
105
    my ($params) = @_;
106
    print "Authenticating...\n";
107
    
108
    my $auth_uri = $params->{auth_uri};
109
    my $user = $params->{auth_username};
110
    my $password = $params->{auth_password};
111
    my $resp = $ua->post( $auth_uri, { userid => $user, password => $password } );
112
    if ($resp->code == HTTP_OK){
113
        print "Authenticated.\n";
114
    }
115
    return $resp
116
}
117
118
1;
(-)a/Koha/OAI/Client/Record.pm (+249 lines)
Line 0 Link Here
1
package Koha::OAI::Client::Record;
2
3
# Copyright 2016 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
use XML::LibXML;
23
use XML::LibXSLT;
24
use MARC::Record;
25
26
use C4::Context;
27
use C4::Biblio;
28
use C4::ImportBatch;
29
use C4::Matcher;
30
31
use constant MAX_MATCHES => 99999; #NOTE: This is an arbitrary value. We want to get all matches. 
32
33
sub new {
34
    my ($class, $args) = @_;
35
    $args = {} unless defined $args;
36
    
37
    if (my $inxml = $args->{xml_string}){
38
        
39
        #Parse the XML string into a XML::LibXML object
40
        my $doc = XML::LibXML->load_xml(string => $inxml, { no_blanks => 1 }); 
41
        $args->{doc} = $doc;
42
        #NOTE: Don't load blank nodes...
43
44
        #Get the root element
45
        my $root = $doc->documentElement;
46
47
        #Register namespaces for searching purposes
48
        my $xpc = XML::LibXML::XPathContext->new();
49
        $xpc->registerNs('oai','http://www.openarchives.org/OAI/2.0/');
50
51
        my $xpath_identifier = XML::LibXML::XPathExpression->new("oai:header/oai:identifier");
52
        my $identifier = $xpc->findnodes($xpath_identifier,$root)->shift;
53
        #my $identifier_string = $identifier->textContent;
54
        $args->{header_identifier} = $identifier->textContent;
55
56
        my $xpath_datestamp = XML::LibXML::XPathExpression->new("oai:header/oai:datestamp");
57
        my $datestamp = $xpc->findnodes($xpath_datestamp,$root)->shift;
58
        #my $datestamp_string = $datestamp->textContent;
59
        $args->{header_datestamp} = $datestamp->textContent;
60
61
        my $xpath_status = XML::LibXML::XPathExpression->new(q{oai:header/@status});
62
        my $status_node = $xpc->findnodes($xpath_status,$root)->shift;
63
        #my $status_string = $status_node ? $status_node->textContent : "";
64
        $args->{header_status} = $status_node ? $status_node->textContent : "";
65
    }
66
    
67
    return bless ($args, $class);
68
}
69
70
sub is_deleted_upstream {
71
    my ($self, $args) = @_;
72
    if ($self->{header_status}){
73
        if ($self->{header_status} eq "deleted"){
74
            return 1;
75
        }
76
    }
77
    return 0;
78
}
79
80
sub filter {
81
    my ($self, $args) = @_;
82
    my $doc = $self->{doc};
83
    my $filter = $args->{filter};
84
    $self->{filter} = $filter; #FIXME
85
    #FIXME: Check that it's an XSLT here...
86
    if ( -f $filter ){
87
        #Filter is a valid filepath
88
89
        #FIXME: Ideally, it would be good to use Koha::XSLT_Handler here... (especially for persistent environments...)
90
        my $xslt = XML::LibXSLT->new();
91
        my $style_doc = XML::LibXML->load_xml(location => $filter);
92
        my $stylesheet = $xslt->parse_stylesheet($style_doc);
93
        if ($stylesheet){
94
            my $results = $stylesheet->transform($doc);
95
            my $metadata_xml = $stylesheet->output_as_bytes($results);
96
            #If the XSLT outputs nothing, then we don't meet the following condition, and we'll return 0 instead.
97
            if ($metadata_xml){
98
                $self->{filtered_record} = $metadata_xml;
99
                return 1;
100
            }
101
        }
102
    }
103
    return 0;
104
}
105
106
107
108
109
110
111
sub import_record {
112
    my ($self, $args) = @_;
113
    my $koha_record_numbers = "";
114
    my $errors = [];
115
    my $import_status = "error";
116
    my $match_status = "no_match";
117
    
118
    my $batch_id = $args->{import_batch_id};
119
    $self->{import_batch_id} = $batch_id; #FIXME
120
    my $matcher = $args->{matcher};
121
    my $framework = $args->{framework};
122
    my $import_mode = $args->{import_mode};
123
    
124
    my $metadata_xml = $self->{filtered_record};
125
    
126
    if ($metadata_xml){
127
        #Convert MARCXML into MARC::Record object
128
        my $marcflavour = C4::Context->preference('marcflavour') || 'MARC21';
129
        my $marc_record = eval {MARC::Record::new_from_xml( $metadata_xml, "utf8", $marcflavour)};
130
        if ($@) {
131
            warn "Error converting OAI-PMH filtered metadata into MARC::Record object: $@";
132
            #FIXME: Improve error handling
133
        }
134
135
        if ($self->is_deleted_upstream){
136
        
137
=pod
138
            my @matches = $matcher->get_matches($marc_record, MAX_MATCHES);
139
            if (@matches){
140
                $match_status = "matched";
141
            }
142
            my $delete_error;
143
            foreach my $match (@matches){
144
                    if (my $record_id = $match->{record_id}){
145
                        #FIXME: This is biblio specific... what about authority records?
146
                        my $error = C4::Biblio::DelBiblio($record_id);
147
                        if ($error){
148
                            $delete_error++;
149
                            $koha_record_numbers = [];
150
                            push(@$koha_record_numbers,$record_id);
151
                            
152
                            #FIXME: Find a better way of sending the errors in a predictable way...
153
                            push(@$errors,{"record_id" => $record_id, "error" => $error, });
154
                        }
155
                    }
156
                    
157
            }
158
            
159
            #If there are no delete errors, then the import was ok
160
            if ( ! $delete_error){
161
                $import_status = "ok";
162
            }
163
            #Deleted records will never actually have an records in them, so always mark them as cleaned so that other imports don't try to pick up the same batch.
164
            C4::ImportBatch::SetImportBatchStatus($batch_id, 'cleaned');
165
=cut
166
            my $import_record_id = AddBiblioToBatch($batch_id, 0, $marc_record, "utf8", int(rand(99999)));
167
            my $number_of_matches =  BatchFindDuplicates($batch_id, $matcher, MAX_MATCHES);
168
            if ($number_of_matches > 0){
169
                $match_status = "auto_match"; #See `import_records` table for other options... but this should be the right one.
170
            }
171
            my $results = GetImportRecordMatches($import_record_id); #Only works for biblio...
172
            my $delete_error;
173
            foreach my $result (@$results){
174
                if (my $record_id = $result->{biblionumber}){
175
                    #FIXME: This is biblio specific... what about authority records?
176
                    my $error = C4::Biblio::DelBiblio($record_id);
177
                    if ($error){
178
                        $delete_error++;
179
                        
180
                        $koha_record_numbers = [];
181
                        push(@$koha_record_numbers,$record_id);
182
                        
183
                        #FIXME: Find a better way of sending the errors in a predictable way...
184
                        push(@$errors,{"record_id" => $record_id, "error_msg" => $error, });
185
                    }
186
                }
187
            }
188
            
189
            if ($delete_error){
190
                $import_status = "error";
191
                C4::ImportBatch::SetImportBatchStatus($batch_id, 'importing');
192
            } else {
193
                $import_status = "ok";
194
                #Ideally, it would be nice to say what records were deleted, but Koha doesn't have that capacity at the moment, so just clean the batch.
195
                CleanBatch($batch_id);
196
            }
197
            
198
            
199
            
200
            
201
        } else {
202
            #Import the MARCXML record into Koha
203
            my $import_record_id = AddBiblioToBatch($batch_id, 0, $marc_record, "utf8", int(rand(99999)));
204
            #FIXME: Don't allow item imports do to the nature of OAI-PMH records updating over time...
205
            #my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 'UPDATE COUNTS');
206
            my $number_of_matches =  BatchFindDuplicates($batch_id, $matcher);
207
208
            # XXX we are ignoring the result of this;
209
            BatchCommitRecords($batch_id, $framework) if lc($import_mode) eq 'direct';
210
211
            my $dbh = C4::Context->dbh();
212
            my $sth = $dbh->prepare("SELECT matched_biblionumber FROM import_biblios WHERE import_record_id =?");
213
            $sth->execute($import_record_id);
214
            $koha_record_numbers = $sth->fetchrow_arrayref->[0] || '';
215
            $sth = $dbh->prepare("SELECT overlay_status FROM import_records WHERE import_record_id =?");
216
            $sth->execute($import_record_id);
217
            
218
            $match_status = $sth->fetchrow_arrayref->[0] || 'no_match';
219
            $import_status = "ok";
220
        }
221
    } else {
222
        #There's no filtered metadata...
223
        #Clean the batch, so future imports don't use the same batch.
224
        CleanBatch($batch_id);
225
    }
226
    $self->{status} = $import_status; #FIXME
227
    #$self->save_to_database();
228
    return ($import_status,$match_status,$koha_record_numbers, $errors);
229
}
230
231
sub save_to_database {
232
    my ($self,$args) = @_;
233
234
    my $header_identifier = $self->{header_identifier};
235
    my $header_datestamp = $self->{header_datestamp};
236
    my $header_status = $self->{header_status};
237
    my $metadata = $self->{doc}->toString(1);
238
    my $import_batch_id = $self->{import_batch_id};
239
    my $filter = $self->{filter};
240
    my $status = $self->{status};
241
242
    my $dbh = C4::Context->dbh;
243
    my $sql = "INSERT INTO import_oai (header_identifier, header_datestamp, header_status, metadata, import_batch_id, filter, status) VALUES (?, ?, ?, ?, ?, ?, ?)";
244
    my $sth = $dbh->prepare($sql);
245
    $sth->execute($header_identifier,$header_datestamp,$header_status,$metadata, $import_batch_id, $filter, $status);
246
}
247
248
249
1;
(-)a/Koha/SavedTask.pm (+86 lines)
Line 0 Link Here
1
package Koha::SavedTask;
2
3
# Copyright Prosentient Systems 2016
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use Carp;
23
24
use JSON;
25
26
use base qw(Koha::Object);
27
28
29
30
=head1 NAME
31
32
Koha::SavedTask -
33
34
=head1 API
35
36
=head2 Class Methods
37
38
=cut
39
40
41
42
=head3 _type
43
44
=cut
45
46
sub _type {
47
    return 'SavedTask';
48
}
49
50
sub params_as_perl {
51
    my ($self) = @_;
52
    my $perl = from_json($self->params);
53
    return $perl;
54
}
55
56
sub serialize {
57
    my ($self,$args) = @_;
58
    my $for = $args->{for};
59
    my $type = $args->{type};
60
    if ($for eq 'icarus'){
61
        my $json_params = $self->params;
62
        my $perl_params = from_json($json_params);
63
64
        my $icarus_task = {
65
            type => $self->task_type,
66
            start => $self->start_time,
67
            repeat_interval => $self->repeat_interval,
68
            params => $perl_params,
69
        };
70
        if ($type eq 'perl'){
71
            return  $icarus_task;
72
        } elsif ($type eq 'json'){
73
            my $json = to_json($icarus_task);
74
            return $json;
75
        }
76
    }
77
    return undef;
78
}
79
80
=head1 AUTHOR
81
82
David Cook <dcook@prosentient.com.au>
83
84
=cut
85
86
1;
(-)a/Koha/SavedTasks.pm (+62 lines)
Line 0 Link Here
1
package Koha::SavedTasks;
2
3
# Copyright Prosentient Systems 2016
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use Carp;
23
24
use Koha::Database;
25
26
use Koha::SavedTask;
27
28
use base qw(Koha::Objects);
29
30
=head1 NAME
31
32
Koha::SavedTasks -
33
34
=head1 API
35
36
=head2 Class Methods
37
38
=cut
39
40
=head3 _type
41
42
=cut
43
44
sub _type {
45
    return 'SavedTask';
46
}
47
48
=head3 object_class
49
50
=cut
51
52
sub object_class {
53
    return 'Koha::SavedTask';
54
}
55
56
=head1 AUTHOR
57
58
David Cook <dcook@prosentient.com.au>
59
60
=cut
61
62
1;
(-)a/Koha/Schema/Result/ImportOai.pm (+152 lines)
Line 0 Link Here
1
use utf8;
2
package Koha::Schema::Result::ImportOai;
3
4
# Created by DBIx::Class::Schema::Loader
5
# DO NOT MODIFY THE FIRST PART OF THIS FILE
6
7
=head1 NAME
8
9
Koha::Schema::Result::ImportOai
10
11
=cut
12
13
use strict;
14
use warnings;
15
16
use base 'DBIx::Class::Core';
17
18
=head1 TABLE: C<import_oai>
19
20
=cut
21
22
__PACKAGE__->table("import_oai");
23
24
=head1 ACCESSORS
25
26
=head2 import_oai_id
27
28
  data_type: 'integer'
29
  extra: {unsigned => 1}
30
  is_auto_increment: 1
31
  is_nullable: 0
32
33
=head2 header_identifier
34
35
  data_type: 'varchar'
36
  is_nullable: 0
37
  size: 45
38
39
=head2 header_datestamp
40
41
  data_type: 'datetime'
42
  datetime_undef_if_invalid: 1
43
  is_nullable: 0
44
45
=head2 header_status
46
47
  data_type: 'varchar'
48
  is_nullable: 1
49
  size: 45
50
51
=head2 metadata
52
53
  data_type: 'longtext'
54
  is_nullable: 0
55
56
=head2 last_modified
57
58
  data_type: 'timestamp'
59
  datetime_undef_if_invalid: 1
60
  default_value: current_timestamp
61
  is_nullable: 0
62
63
=head2 status
64
65
  data_type: 'varchar'
66
  is_nullable: 0
67
  size: 45
68
69
=head2 import_batch_id
70
71
  data_type: 'integer'
72
  is_foreign_key: 1
73
  is_nullable: 0
74
75
=head2 filter
76
77
  data_type: 'text'
78
  is_nullable: 0
79
80
=cut
81
82
__PACKAGE__->add_columns(
83
  "import_oai_id",
84
  {
85
    data_type => "integer",
86
    extra => { unsigned => 1 },
87
    is_auto_increment => 1,
88
    is_nullable => 0,
89
  },
90
  "header_identifier",
91
  { data_type => "varchar", is_nullable => 0, size => 45 },
92
  "header_datestamp",
93
  {
94
    data_type => "datetime",
95
    datetime_undef_if_invalid => 1,
96
    is_nullable => 0,
97
  },
98
  "header_status",
99
  { data_type => "varchar", is_nullable => 1, size => 45 },
100
  "metadata",
101
  { data_type => "longtext", is_nullable => 0 },
102
  "last_modified",
103
  {
104
    data_type => "timestamp",
105
    datetime_undef_if_invalid => 1,
106
    default_value => \"current_timestamp",
107
    is_nullable => 0,
108
  },
109
  "status",
110
  { data_type => "varchar", is_nullable => 0, size => 45 },
111
  "import_batch_id",
112
  { data_type => "integer", is_foreign_key => 1, is_nullable => 0 },
113
  "filter",
114
  { data_type => "text", is_nullable => 0 },
115
);
116
117
=head1 PRIMARY KEY
118
119
=over 4
120
121
=item * L</import_oai_id>
122
123
=back
124
125
=cut
126
127
__PACKAGE__->set_primary_key("import_oai_id");
128
129
=head1 RELATIONS
130
131
=head2 import_batch
132
133
Type: belongs_to
134
135
Related object: L<Koha::Schema::Result::ImportBatch>
136
137
=cut
138
139
__PACKAGE__->belongs_to(
140
  "import_batch",
141
  "Koha::Schema::Result::ImportBatch",
142
  { import_batch_id => "import_batch_id" },
143
  { is_deferrable => 1, on_delete => "RESTRICT", on_update => "RESTRICT" },
144
);
145
146
147
# Created by DBIx::Class::Schema::Loader v0.07042 @ 2016-04-12 11:02:33
148
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:QmCetOjXql0gsAi+wZ74Ng
149
150
151
# You can replace this text with custom code or comments, and it will be preserved on regeneration
152
1;
(-)a/Koha/Schema/Result/SavedTask.pm (+98 lines)
Line 0 Link Here
1
use utf8;
2
package Koha::Schema::Result::SavedTask;
3
4
# Created by DBIx::Class::Schema::Loader
5
# DO NOT MODIFY THE FIRST PART OF THIS FILE
6
7
=head1 NAME
8
9
Koha::Schema::Result::SavedTask
10
11
=cut
12
13
use strict;
14
use warnings;
15
16
use base 'DBIx::Class::Core';
17
18
=head1 TABLE: C<saved_tasks>
19
20
=cut
21
22
__PACKAGE__->table("saved_tasks");
23
24
=head1 ACCESSORS
25
26
=head2 task_id
27
28
  data_type: 'integer'
29
  extra: {unsigned => 1}
30
  is_auto_increment: 1
31
  is_nullable: 0
32
33
=head2 start_time
34
35
  data_type: 'datetime'
36
  datetime_undef_if_invalid: 1
37
  is_nullable: 0
38
39
=head2 repeat_interval
40
41
  data_type: 'integer'
42
  extra: {unsigned => 1}
43
  is_nullable: 0
44
45
=head2 task_type
46
47
  data_type: 'varchar'
48
  is_nullable: 0
49
  size: 255
50
51
=head2 params
52
53
  data_type: 'text'
54
  is_nullable: 0
55
56
=cut
57
58
__PACKAGE__->add_columns(
59
  "task_id",
60
  {
61
    data_type => "integer",
62
    extra => { unsigned => 1 },
63
    is_auto_increment => 1,
64
    is_nullable => 0,
65
  },
66
  "start_time",
67
  {
68
    data_type => "datetime",
69
    datetime_undef_if_invalid => 1,
70
    is_nullable => 0,
71
  },
72
  "repeat_interval",
73
  { data_type => "integer", extra => { unsigned => 1 }, is_nullable => 0 },
74
  "task_type",
75
  { data_type => "varchar", is_nullable => 0, size => 255 },
76
  "params",
77
  { data_type => "text", is_nullable => 0 },
78
);
79
80
=head1 PRIMARY KEY
81
82
=over 4
83
84
=item * L</task_id>
85
86
=back
87
88
=cut
89
90
__PACKAGE__->set_primary_key("task_id");
91
92
93
# Created by DBIx::Class::Schema::Loader v0.07042 @ 2016-01-27 13:35:22
94
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gnoi7I9fiXM3IfDysMTm+A
95
96
97
# You can replace this text with custom code or comments, and it will be preserved on regeneration
98
1;
(-)a/Makefile.PL (-1 / +18 lines)
Lines 198-203 Directory for Zebra's data files. Link Here
198
198
199
Directory for Zebra's UNIX-domain sockets.
199
Directory for Zebra's UNIX-domain sockets.
200
200
201
=item ICARUS_RUN_DIR
202
203
Directory for Icarus's UNIX-domain socket and pid file.
204
201
=item MISC_DIR
205
=item MISC_DIR
202
206
203
Directory for for miscellaenous scripts, among other
207
Directory for for miscellaenous scripts, among other
Lines 316-321 my $target_map = { Link Here
316
  './skel/var/log/koha'         => { target => 'LOG_DIR', trimdir => -1 },
320
  './skel/var/log/koha'         => { target => 'LOG_DIR', trimdir => -1 },
317
  './skel/var/spool/koha'       => { target => 'BACKUP_DIR', trimdir => -1 },
321
  './skel/var/spool/koha'       => { target => 'BACKUP_DIR', trimdir => -1 },
318
  './skel/var/run/koha/zebradb' => { target => 'ZEBRA_RUN_DIR', trimdir => -1 },
322
  './skel/var/run/koha/zebradb' => { target => 'ZEBRA_RUN_DIR', trimdir => -1 },
323
  './skel/var/run/koha/icarus' => { target => 'ICARUS_RUN_DIR', trimdir => 6 },
319
  './skel/var/lock/koha/zebradb/authorities' => { target => 'ZEBRA_LOCK_DIR', trimdir => 6 },
324
  './skel/var/lock/koha/zebradb/authorities' => { target => 'ZEBRA_LOCK_DIR', trimdir => 6 },
320
  './skel/var/lib/koha/zebradb/authorities/key'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
325
  './skel/var/lib/koha/zebradb/authorities/key'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
321
  './skel/var/lib/koha/zebradb/authorities/register'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
326
  './skel/var/lib/koha/zebradb/authorities/register'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
Lines 413-418 System user account that will own Koha's files. Link Here
413
418
414
System group that will own Koha's files.
419
System group that will own Koha's files.
415
420
421
=item ICARUS_MAX_TASKS
422
423
Maximum number of tasks allowed by Icarus.
424
416
=back
425
=back
417
426
418
=cut
427
=cut
Lines 447-453 my %config_defaults = ( Link Here
447
  'USE_MEMCACHED'     => 'no',
456
  'USE_MEMCACHED'     => 'no',
448
  'MEMCACHED_SERVERS' => '127.0.0.1:11211',
457
  'MEMCACHED_SERVERS' => '127.0.0.1:11211',
449
  'MEMCACHED_NAMESPACE' => 'KOHA',
458
  'MEMCACHED_NAMESPACE' => 'KOHA',
450
  'FONT_DIR'          => '/usr/share/fonts/truetype/ttf-dejavu'
459
  'FONT_DIR'          => '/usr/share/fonts/truetype/ttf-dejavu',
460
  'ICARUS_MAX_TASKS'    => '30',
451
);
461
);
452
462
453
# set some default configuration options based on OS
463
# set some default configuration options based on OS
Lines 1091-1096 Memcached namespace?); Link Here
1091
Path to DejaVu fonts?);
1101
Path to DejaVu fonts?);
1092
  $config{'FONT_DIR'} = _get_value('FONT_DIR', $msg, $defaults->{'FONT_DIR'}, $valid_values, $install_log_values);
1102
  $config{'FONT_DIR'} = _get_value('FONT_DIR', $msg, $defaults->{'FONT_DIR'}, $valid_values, $install_log_values);
1093
1103
1104
  $msg = q(
1105
Maximum number of tasks allowed by Icarus?);
1106
  $config{'ICARUS_MAX_TASKS'} = _get_value('ICARUS_MAX_TASKS', $msg, $defaults->{'ICARUS_MAX_TASKS'}, $valid_values, $install_log_values);
1107
1094
1108
1095
  $msg = q(
1109
  $msg = q(
1096
Would you like to run the database-dependent test suite?);
1110
Would you like to run the database-dependent test suite?);
Lines 1239-1244 sub get_target_directories { Link Here
1239
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'koha', 'plugins');
1253
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'koha', 'plugins');
1240
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1254
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1241
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
1255
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
1256
        $dirmap{'ICARUS_RUN_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'run', 'icarus');
1242
    } elsif ($mode eq 'dev') {
1257
    } elsif ($mode eq 'dev') {
1243
        my $curdir = File::Spec->rel2abs(File::Spec->curdir());
1258
        my $curdir = File::Spec->rel2abs(File::Spec->curdir());
1244
        $dirmap{'INTRANET_CGI_DIR'} = File::Spec->catdir($curdir);
1259
        $dirmap{'INTRANET_CGI_DIR'} = File::Spec->catdir($curdir);
Lines 1272-1277 sub get_target_directories { Link Here
1272
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'plugins');
1287
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'plugins');
1273
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1288
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'lib', 'zebradb');
1274
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
1289
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(@basedir, $package, 'var', 'run', 'zebradb');
1290
        $dirmap{'ICARUS_RUN_DIR'} = File::Spec->catdir(@basedir, $package, 'var', 'run', 'icarus');
1275
    } else {
1291
    } else {
1276
        # mode is standard, i.e., 'fhs'
1292
        # mode is standard, i.e., 'fhs'
1277
        $dirmap{'INTRANET_CGI_DIR'} = File::Spec->catdir(@basedir, $package, 'intranet', 'cgi-bin');
1293
        $dirmap{'INTRANET_CGI_DIR'} = File::Spec->catdir(@basedir, $package, 'intranet', 'cgi-bin');
Lines 1295-1300 sub get_target_directories { Link Here
1295
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'plugins');
1311
        $dirmap{'PLUGINS_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'plugins');
1296
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'zebradb');
1312
        $dirmap{'ZEBRA_DATA_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'lib', $package, 'zebradb');
1297
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'run', $package, 'zebradb');
1313
        $dirmap{'ZEBRA_RUN_DIR'} =  File::Spec->catdir(File::Spec->rootdir(), 'var', 'run', $package, 'zebradb');
1314
        $dirmap{'ICARUS_RUN_DIR'} = File::Spec->catdir(File::Spec->rootdir(), 'var', 'run', $package, 'icarus');
1298
    }
1315
    }
1299
1316
1300
    _get_env_overrides(\%dirmap);
1317
    _get_env_overrides(\%dirmap);
(-)a/admin/saved_tasks.pl (+347 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright Prosentient Systems 2016
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
=head1 NAME
21
22
saved_tasks.pl
23
24
=head1 DESCRIPTION
25
26
Admin page to manage saved tasks
27
28
=cut
29
30
use Modern::Perl;
31
use CGI qw ( -utf8 );
32
use C4::Auth;
33
use C4::Output;
34
use C4::Context;
35
36
use Koha::SavedTasks;
37
use Koha::Icarus;
38
use Module::Load::Conditional qw/can_load check_install/;
39
use JSON;
40
41
my $input = new CGI;
42
my ($template, $loggedinuser, $cookie, $flags) = get_template_and_user( {
43
    template_name   => 'admin/saved_tasks.tt',
44
    query           => $input,
45
    type            => 'intranet',
46
    authnotrequired => 0,
47
    flagsrequired   => { 'parameters' => 'parameters_remaining_permissions' },
48
} );
49
50
my $filename = "saved_tasks.pl";
51
$template->param(
52
    filename => $filename,
53
);
54
55
my $context = C4::Context->new();
56
57
58
my $task_server = $input->param("task_server") // "icarus";
59
60
61
my $socket_uri = $context->{"icarus"}->{"socket"};
62
63
my @available_plugins = ();
64
my $task_plugins = $context->{"icarus"}->{"task_plugin"};
65
if ($task_plugins && ref $task_plugins eq 'ARRAY'){
66
    #FIXME: This should probably be a module method... validation that a plugin is installed...
67
    foreach my $task_plugin (@$task_plugins){
68
        #Check that plugin module is installed
69
        if ( check_install( module => $task_plugin ) ){
70
                push(@available_plugins,$task_plugin);
71
        }
72
    }
73
}
74
75
$template->param(
76
    available_plugins => \@available_plugins,
77
);
78
79
#Server action and task id
80
my $server_action = $input->param("server_action");
81
my $server_task_id = $input->param('server_task_id');
82
83
#Saved task op
84
my $op = $input->param('op');
85
my $step = $input->param('step');
86
87
#Saved task id
88
my $saved_task_id = $input->param('saved_task_id');
89
90
91
#Create Koha-Icarus interface object
92
my $icarus = Koha::Icarus->new({ socket_uri => $socket_uri });
93
my $daemon_status = "";
94
95
#Connect to Icarus
96
if ( $icarus->connect() ){
97
    $daemon_status = "online";
98
    if ($server_action){
99
        if ($server_action eq 'shutdown'){
100
            my $response = $icarus->shutdown;
101
            if ( $response && (my $action = $response->{action}) ){
102
                $daemon_status = $action;
103
            }
104
        } elsif ($server_action eq 'start' && $server_task_id){
105
            my $response = $icarus->start_task({ task_id => $server_task_id });
106
            $template->param(
107
                task_response => $response,
108
            );
109
        } elsif ($server_action eq 'remove' && $server_task_id){
110
            my $response = $icarus->remove_task({ task_id => $server_task_id });
111
            $template->param(
112
                task_response => $response,
113
            );
114
        }
115
    }
116
} else {
117
    $daemon_status = $!;
118
}
119
$template->param(
120
    daemon_status => $daemon_status,
121
);
122
123
124
125
my $params = $input->param("params");
126
127
#NOTE: Parse the parameters manually, so that you can "name[]" style of parameter, which we use in the special plugin templates...
128
my $saved_params = {};
129
#Fetch the names of all the parameters passed to your script
130
my @parameter_names = $input->param;
131
#Iterate through these parameter names and look for "params[]"
132
foreach my $parameter_name (@parameter_names){
133
    if ($parameter_name =~ /^params\[(.*)\]$/){
134
        #Capture the hash key
135
        my $key = $1;
136
        #Fetch the actual individual value
137
        my $parameter_value = $input->param($parameter_name);
138
        if ($parameter_value){
139
            $saved_params->{$key} = $parameter_value;
140
        }
141
    }
142
}
143
if (%$saved_params){
144
    my $json = to_json($saved_params, { pretty => 1, });
145
    if ($json){
146
        $params = $json;
147
    }
148
}
149
150
my $start_time = $input->param("start_time");
151
my $repeat_interval = $input->param("repeat_interval");
152
my $task_type = $input->param("task_type");
153
if ($task_type){
154
    my $task_template = $task_type;
155
    #Create the template name by stripping the colons out of the task type text
156
    $task_template =~ s/://g;
157
    $template->param(
158
        task_template => "tasks/$task_template.inc",
159
    );
160
}
161
162
163
if ($op){
164
    if ($op eq 'new'){
165
166
    } elsif ($op eq 'create'){
167
168
        #Validate the $task here
169
        if ($step){
170
            if ($step eq "one"){
171
172
                $op = "new";
173
                $template->param(
174
                    step => "two",
175
                    task_type => $task_type,
176
                );
177
            } elsif ($step eq "two"){
178
                my $new_task = Koha::SavedTask->new({
179
                    start_time => $start_time,
180
                    repeat_interval => $repeat_interval,
181
                    task_type => $task_type,
182
                    params => $params,
183
                });
184
185
                #Serialize the data as an Icarus task
186
                my $icarus_task = $new_task->serialize({ for => "icarus", type => "perl", });
187
188
                my $valid = 1;
189
                #Load the plugin module, and create an object instance in order to validate user-entered data
190
                if ( can_load( modules => { $task_type => undef, }, ) ){
191
                    my $plugin = $task_type->new({ task => $icarus_task, });
192
                    if ($plugin->can("validate")){
193
                        my $errors = $plugin->validate({
194
                            "tests" => "all",
195
                        });
196
                        if (%$errors){
197
                            $template->param(
198
                                errors => $errors,
199
                            );
200
                        }
201
                        if ($plugin->{invalid_data} > 0){
202
                            $valid = 0;
203
                        }
204
                    }
205
                }
206
207
                if ($valid){
208
                    $new_task->store();
209
                    $op = "list";
210
                } else {
211
                    $op = "new";
212
                    #Create a Perl data structure from the JSON
213
                    my $editable_params = from_json($params);
214
                    $template->param(
215
                        step => "two",
216
                        task_type => $task_type,
217
                        saved_task => $new_task,
218
                        params => $editable_params,
219
                    );
220
                }
221
            }
222
        }
223
224
    } elsif ($op eq 'edit'){
225
        my $task = Koha::SavedTasks->find($saved_task_id);
226
        if ($task){
227
            #Check if the task's saved task type is actually available...
228
            #FIXME: This should be a Koha::Icarus method...
229
            my $task_type_is_valid = grep { $task->task_type eq $_ } @available_plugins;
230
            $template->param(
231
                task_type_is_valid => $task_type_is_valid,
232
                saved_task => $task,
233
            );
234
        }
235
    } elsif ($op eq 'update'){
236
        if ($step){
237
            my $task = Koha::SavedTasks->find($saved_task_id);
238
            if ($task){
239
                if ($step eq "one"){
240
                    #We've completed step one, which is choosing the task type,
241
                    #so now we're going to populate the form for editing the rest of the values
242
                    $op = "edit";
243
                    #This is the JSON string that we've saved in the database
244
                    my $current_params_string = $task->params;
245
                    my $editable_params = from_json($current_params_string);
246
247
                    $template->param(
248
                        step => "two",
249
                        task_type => $task_type,
250
                        saved_task => $task,
251
                        params => $editable_params,
252
253
                    );
254
                } elsif ($step eq "two"){
255
                    #We've completed step two, so we're storing the data now...
256
                    $task->set({
257
                        start_time => $start_time,
258
                        repeat_interval => $repeat_interval,
259
                        task_type => $task_type,
260
                        params => $params,
261
                    });
262
                    $task->store;
263
                    #FIXME: Validate the $task here...
264
                    if (my $valid = 1){
265
                        $op = "list";
266
                    } else {
267
                        $op = "edit";
268
                        $template->param(
269
                            step => "two",
270
                            task_type => $task_type,
271
                            saved_task => $task,
272
                        );
273
                    }
274
                }
275
            }
276
        }
277
    } elsif ($op eq 'send'){
278
        my $sent_response;
279
        if ($icarus->connected){
280
            if ($saved_task_id){
281
                #Look up task
282
                my $task = Koha::SavedTasks->find($saved_task_id);
283
                if ($task){
284
                    #Create a task for Icarus, and send it to Icarus
285
                    my $icarus_task = $task->serialize({ for => "icarus", type => "perl", });
286
                    if ($icarus_task){
287
                        $icarus->add_task({ task => $icarus_task, });
288
                        $op = "list";
289
                    }
290
                }
291
            }
292
        } else {
293
            $sent_response = "icarus_offline";
294
            $template->param(
295
                sent_response => $sent_response,
296
            );
297
            $op = "list";
298
        }
299
    } elsif ($op eq 'delete'){
300
        my $saved_response = "delete_failure";
301
        if ($saved_task_id){
302
            #Look up task
303
            my $task = Koha::SavedTasks->find($saved_task_id);
304
            if ($task){
305
                if (my $something = $task->delete){
306
                    $saved_response = "delete_success";
307
                }
308
            }
309
        }
310
        $template->param(
311
            saved_response => $saved_response,
312
        );
313
        $op = "list";
314
    } else {
315
        #Don't recognize $op, so fallback to list
316
        $op = "list";
317
    }
318
} else {
319
    #No $op, so fallback to list
320
    $op = "list";
321
}
322
323
if ($op eq 'list'){
324
    #Get active tasks from Icarus
325
    if ($icarus->connected){
326
        my $tasks = $icarus->list_tasks();
327
        if ($tasks && @$tasks){
328
            #Sort tasks that come from Icarus, since it returns an unsorted list of hashrefs
329
            my @sorted_tasks = sort { $a->{task_id} <=> $b->{task_id} } @$tasks;
330
            $template->param(
331
                tasks => \@sorted_tasks,
332
            );
333
        }
334
    }
335
336
    #Get saved tasks from Koha
337
    my @saved_tasks = Koha::SavedTasks->as_list();
338
    $template->param(
339
        saved_tasks => \@saved_tasks,
340
    );
341
}
342
343
$template->param(
344
    op => $op,
345
);
346
347
output_html_with_http_headers $input, $cookie, $template->output;
(-)a/docs/Icarus/README (+64 lines)
Line 0 Link Here
1
TODO: 
2
- Add paging to tools/manage-oai-imports
3
    - Do a simple version... most recent first and "more/less" buttons...
4
5
- Data validation:
6
    "Koha::Icarus::Task::Upload::OAIPMH::Biblio":
7
        - Validate HTTP URLs and filepaths...
8
        - MAKE IT SO YOU HAVE TO USE A RECORD MATCHING RULE! To at the very least strip the OAI wrapper...
9
    - Add PLUGIN->validate("parameter_names")
10
    - Add PLUGIN->validate("parameter_values")
11
        - For the downloader, this would validate HTTP && OAI-PMH parameters...
12
13
- tools/manage-oai-import
14
    - Improve error resolution
15
        - For deleted records, just try to re-run the import
16
        - For normal records, perhaps the ability to change filters and try again, or delete the OAI record all together?
17
        - NOTE: there's a problem with re-adding records to the batch... will have to write some SQL and maybe even just a import_record("retry") or a retry_import().
18
        
19
- Update kohastructure.sql in accordance with installer/data/mysql/atomicupdate/bug_10662-Build_import_oai_table.sql
20
21
22
23
24
25
26
27
28
29
30
- admin/saved_tasks.pl
31
    - Add a clone button to ease task creation
32
- Make the "Task type" prettier (and translateable) on saved_tasks.pl.
33
- Provide more options for the Icarus dashboard (start, restart, etc)
34
- Add the ability to "edit" and "pause" active Icarus tasks
35
    - A pause function would make debugging much easier.
36
37
- Add help pages for WEB GUI
38
- Add documentation to all code...
39
- Add unit tests
40
    
41
- Add a "file_name" to make the batch more obvious which makes it useful
42
- Add default OAI record matching rule
43
    - I thought about adding a SQL atomic update 'bug_10662-Add_oai_record_matching_rule.sql', but adding matching rules seems complex. This needs to be done in Perl.
44
    - Should the field include other fields like 022, 020, 245 rather than just 001 and 024a?
45
- Add entry to Cleanupdatabase.pl cronjob
46
    - You could remove all import_oai rows older than a certain age?
47
48
- Make "Koha::Icarus::Task::Upload::OAIPMH::Biblio" use asynchronous HTTP requests to speed up the import
49
- Add support for authority records and possibly holdings records (add record type to svc/import_oai parameters)
50
- Instead of using file:///home/koha/koha-dev/var/spool/oaipmh, use something like file:///tmp/koha-instance/koha-dev/oaipmh
51
    - How is the user going to specify file:///tmp/koha-instance/koha-dev/oaipmh? Or do you put this in koha-conf.xml and then make a user-defined relative path?
52
    
53
- WEB UI:
54
    - Add `name` to saved_tasks?
55
- Move "Saved tasks" from Administration to Tools?
56
    - Look at existing bugs for schedulers:
57
        - https://bugs.koha-community.org/bugzilla3/show_bug.cgi?id=14712
58
        - https://bugs.koha-community.org/bugzilla3/show_bug.cgi?id=1993
59
- Handle datestamp granularity better for OAI-PMH download tasks?
60
- Change `import_oai` database table?
61
    - import_oai's "metadata" should actually be "oai_record"... so it's not so confusing... it's NOT the metadata element... but rather the whole OAI record.
62
- Resolve all TODO/FIXME comments in the code
63
- Clean up the code
64
(-)a/etc/koha-conf.xml (+8 lines)
Lines 137-140 __PAZPAR2_TOGGLE_XML_POST__ Link Here
137
 </ttf>
137
 </ttf>
138
138
139
</config>
139
</config>
140
<icarus>
141
    <socket>unix:__ICARUS_RUN_DIR__/icarus.sock</socket>
142
    <pidfile>__ICARUS_RUN_DIR__/icarus.pid</pidfile>
143
    <log>__LOG_DIR__/icarus.log</log>
144
    <task_plugin>Koha::Icarus::Task::Download::OAIPMH::Biblio</task_plugin>
145
    <task_plugin>Koha::Icarus::Task::Upload::OAIPMH::Biblio</task_plugin>
146
    <max_tasks>__ICARUS_MAX_TASKS__</max_tasks>
147
</icarus>
140
</yazgfs>
148
</yazgfs>
(-)a/installer/data/mysql/atomicupdate/bug_10662-Build_import_oai_table.sql (+25 lines)
Line 0 Link Here
1
DROP TABLE IF EXISTS import_oai;
2
CREATE TABLE  import_oai (
3
  import_oai_id int(10) unsigned NOT NULL AUTO_INCREMENT,
4
  header_identifier varchar(45) CHARACTER SET utf8 NOT NULL,
5
  header_datestamp datetime NOT NULL,
6
  header_status varchar(45) CHARACTER SET utf8 DEFAULT NULL,
7
  metadata longtext CHARACTER SET utf8 NOT NULL,
8
  last_modified timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
9
  status varchar(45) CHARACTER SET utf8 NOT NULL,
10
  import_batch_id int(11) NOT NULL,
11
  filter text COLLATE utf8_unicode_ci NOT NULL,
12
  PRIMARY KEY (import_oai_id)
13
  KEY FK_import_oai_1 (import_batch_id),
14
  CONSTRAINT FK_import_oai_1 FOREIGN KEY (import_batch_id) REFERENCES import_batches (import_batch_id)
15
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
16
17
DROP TABLE IF EXISTS saved_tasks;
18
CREATE TABLE  saved_tasks (
19
  task_id int(10) unsigned NOT NULL AUTO_INCREMENT,
20
  start_time datetime NOT NULL,
21
  repeat_interval int(10) unsigned NOT NULL,
22
  task_type varchar(255) CHARACTER SET utf8 NOT NULL,
23
  params text CHARACTER SET utf8 NOT NULL,
24
  PRIMARY KEY (task_id) USING BTREE
25
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
(-)a/installer/data/mysql/kohastructure.sql (+31 lines)
Lines 3723-3728 CREATE TABLE IF NOT EXISTS edifact_ean ( Link Here
3723
  CONSTRAINT efk_branchcode FOREIGN KEY ( branchcode ) REFERENCES branches ( branchcode )
3723
  CONSTRAINT efk_branchcode FOREIGN KEY ( branchcode ) REFERENCES branches ( branchcode )
3724
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
3724
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
3725
3725
3726
--
3727
-- Table structure for table 'import_oai'
3728
--
3729
3730
DROP TABLE IF EXISTS import_oai;
3731
CREATE TABLE  import_oai (
3732
  import_oai_id int(10) unsigned NOT NULL AUTO_INCREMENT,
3733
  header_identifier varchar(45) CHARACTER SET utf8 NOT NULL,
3734
  header_datestamp datetime NOT NULL,
3735
  header_status varchar(45) CHARACTER SET utf8 DEFAULT NULL,
3736
  metadata longtext CHARACTER SET utf8 NOT NULL,
3737
  last_modified timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
3738
  status varchar(45) CHARACTER SET utf8 NOT NULL,
3739
  PRIMARY KEY (import_oai_id)
3740
) ENGINE=InnoDB AUTO_INCREMENT=297 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
3741
3742
--
3743
-- Table structure for table 'saved_tasks'
3744
--
3745
3746
DROP TABLE IF EXISTS saved_tasks;
3747
CREATE TABLE  saved_tasks (
3748
  task_id int(10) unsigned NOT NULL AUTO_INCREMENT,
3749
  start_time datetime NOT NULL,
3750
  repeat_interval int(10) unsigned NOT NULL,
3751
  task_type varchar(255) CHARACTER SET utf8 NOT NULL,
3752
  params text CHARACTER SET utf8 NOT NULL,
3753
  PRIMARY KEY (task_id) USING BTREE
3754
) ENGINE=InnoDB AUTO_INCREMENT=13 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
3755
3756
3726
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
3757
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
3727
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
3758
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
3728
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
3759
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc (+1 lines)
Lines 78-83 Link Here
78
    [% IF Koha.Preference('SMSSendDriver') == 'Email' %]
78
    [% IF Koha.Preference('SMSSendDriver') == 'Email' %]
79
        <li><a href="/cgi-bin/koha/admin/sms_providers.pl">SMS cellular providers</a></li>
79
        <li><a href="/cgi-bin/koha/admin/sms_providers.pl">SMS cellular providers</a></li>
80
    [% END %]
80
    [% END %]
81
    <li><a href="/cgi-bin/koha/admin/saved_tasks.pl">Saved tasks</a></li>
81
</ul>
82
</ul>
82
</div>
83
</div>
83
</div>
84
</div>
(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/tasks/KohaIcarusTaskDownloadOAIPMHBiblio.inc (+87 lines)
Line 0 Link Here
1
[%# USE CGI %]
2
[%# server_name = CGI.server_name; server_port = CGI.server_port; server = server_name _ ":" _ server_port; %]
3
4
<fieldset class="rows">
5
    <legend>HTTP parameters:</legend>
6
    <ol>
7
        <li>
8
            <label for="url">URL: </label>
9
            [% IF ( params.url ) %]
10
                <input type="text" id="url" name="params[url]" value="[% params.url %]" size="30" />
11
            [% ELSE %]
12
                <input type="text" id="url" name="params[url]" value="http://" size="30" />
13
            [% END %]
14
            [% IF (errors.url.no_path) %]<span class="error">[The URL must have a path after "http://" like "koha-community.org/cgi-bin/koha/oai.pl".]</span>[% END %]
15
            [% IF (errors.url.not_http) %]<span class="error">[The URL begin with a scheme of "http://" like "http://koha-community.org/cgi-bin/koha/oai.pl".]</span>[% END %]
16
            [% IF (errors.url.not_a_url) %]<span class="error">[The value of this field must be a URL like "http://koha-community.org/cgi-bin/koha/oai.pl".]</span>[% END %]
17
18
        </li>
19
    </ol>
20
    <span class="help">The following parameters are not required by all OAI-PMH repositories, so they may be optional for this task.</span>
21
    <ol>
22
        <li>
23
            <label for="username">Username: </label>
24
            <input type="text" id="username" name="params[username]" value="[% params.username %]" size="30" />
25
        </li>
26
        <li>
27
            <label for="password">Password: </label>
28
            <input type="text" id="password" name="params[password]" value="[% params.password %]" size="30" />
29
        </li>
30
        <li>
31
            <label for="realm">Realm: </label>
32
            <input type="text" id="realm" name="params[realm]" value="[% params.realm %]" size="30" />
33
        </li>
34
    </ol>
35
</fieldset>
36
<fieldset class="rows">
37
    <legend>OAI-PMH parameters:</legend>
38
    <ol>
39
        <li>
40
            <label for="verb">Verb: </label>
41
            <select id="verb" name="params[verb]">
42
            [% FOREACH verb IN [ 'GetRecord', 'ListRecords' ] %]
43
                [% IF ( params.verb ) && ( verb == params.verb ) %]
44
                    <option selected="selected" value="[% verb %]">[% verb %]</option>
45
                [% ELSE %]
46
                    <option value="[% verb %]">[% verb %]</option>
47
                [% END %]
48
            [% END %]
49
            </select>
50
        </li>
51
        <li>
52
            <label for="identifier">Identifier: </label>
53
            <input type="text" id="identifier" name="params[identifier]" value="[% params.identifier %]" size="30" />
54
            <span class="help">This identifier will only be used with the GetRecord verb.</span>
55
        </li>
56
        <li>
57
            <label for="sets">Sets: </label>
58
            <input type="text" id="sets" name="params[sets]" value="[% params.sets %]" size="30" /><span class="help">You may specify several sets by separating the sets with a pipe (e.g. set1|set2 )</span>
59
        </li>
60
        <li>
61
            <label for="metadataPrefix">Metadata Prefix: </label>
62
            <input type="text" id="metadataPrefix" name="params[metadataPrefix]" value="[% params.metadataPrefix %]" size="30" />
63
        </li>
64
        <li>
65
            <label for="opt_from">From: </label>
66
            <input type="text" class="datetime_utc" id="opt_from" name="params[from]" value="[% params.from %]" size="30" /><span class="help">This value will be treated as UTC time. Note that some repositories only support YYYY-MM-DD datestamps.</span>
67
        </li>
68
        <li>
69
            <label for="opt_until">Until: </label>
70
            <input type="text" class="datetime_utc" id="opt_until" name="params[until]" value="[% params.until %]" size="30" /><span class="help">This value will be treated as UTC time. Note that some repositories only support YYYY-MM-DD datestamps.</span>
71
        </li>
72
    </ol>
73
</fieldset>
74
<fieldset class="rows">
75
    <legend>Download parameters:</legend>
76
    <ol>
77
        <li>
78
            <label for="queue">Queue: </label>
79
            [% IF ( params.queue ) %]
80
                <input type="text" id="queue" name="params[queue]" value="[% params.queue %]" size="30" />
81
            [% ELSE %]
82
                <input type="text" id="queue" name="params[queue]" value="file://" size="30" />
83
            [% END %]
84
            <span class="help">This is a filepath on your system like file:///var/spool/koha/libraryname/oaipmh</span>
85
        </li>
86
    </ol>
87
</fieldset>
(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/tasks/KohaIcarusTaskUploadOAIPMHBiblio.inc (+143 lines)
Line 0 Link Here
1
[%# Use CGI plugin to create a default target URI %]
2
[%# TODO: Test if this works with Plack... %]
3
[% USE CGI %]
4
[% server = CGI.virtual_host %]
5
[% IF ( server_port = CGI.virtual_port ) %]
6
        [% IF ( server_port != '80' ) && ( server_port != '443' ) %]
7
                [% server = server _ ':' _ server_port %]
8
        [% END %]
9
[% END %]
10
[% default_auth_uri = 'http://' _ server _ '/cgi-bin/koha/svc/authentication' %]
11
[% default_target_uri = 'http://' _ server _ '/cgi-bin/koha/svc/import_oai' %]
12
<fieldset class="rows">
13
    <legend>Import source parameters:</legend>
14
    <ol>
15
        <li>
16
            <label for="queue">Queue: </label>
17
            [% IF ( params.queue ) %]
18
                <input type="text" id="queue" name="params[queue]" value="[% params.queue %]" size="30" />
19
            [% ELSE %]
20
                <input type="text" id="queue" name="params[queue]" value="file://" size="30" />
21
            [% END %]
22
            <span class="help">This is a filepath on your system like file:///var/spool/koha/libraryname/oaipmh</span>
23
        </li>
24
    </ol>
25
</fieldset>
26
<fieldset class="rows">
27
    <legend>API authentication parameters:</legend>
28
    <ol>
29
        <li>
30
            <label for="auth_uri">URL: </label>
31
            [% IF ( params.auth_uri ) %]
32
                <input type="text" id="auth_uri" name="params[auth_uri]" value="[% params.auth_uri %]" size="30" />
33
            [% ELSE %]
34
                <input type="text" id="auth_uri" name="params[auth_uri]" value="[% default_auth_uri %]" size="30" />
35
            [% END %]
36
            [% IF (errors.auth_uri.no_path) %]<span class="error">[The URL must have a path after "http://" like "koha-community.org/cgi-bin/koha/svc/authentication".]</span>[% END %]
37
            [% IF (errors.auth_uri.not_http) %]<span class="error">[The URL begin with a scheme of "http://" like "http://koha-community.org/cgi-bin/koha/svc/authentication".]</span>[% END %]
38
            [% IF (errors.auth_uri.not_a_url) %]<span class="error">[The value of this field must be a URL like "http://koha-community.org/cgi-bin/koha/svc/authentication".]</span>[% END %]
39
            <span class="help">This is a Koha authentication URL. The default value </span>
40
        </li>
41
        <li>
42
            <label for="auth_username">Username: </label>
43
            <input type="text" id="auth_username" name="params[auth_username]" value="[% params.auth_username %]" size="30" />
44
            <span class="help">This user must have permission to edit the catalogue.</span>
45
        </li>
46
        <li>
47
            <label for="auth_password">Password: </label>
48
            <input type="text" id="auth_password" name="params[auth_password]" value="[% params.auth_password %]" size="30" />
49
        </li>
50
    </ol>
51
</fieldset>
52
<fieldset class="rows">
53
    <legend>Import target parameters:</legend>
54
    <ol>
55
        <li>
56
            <label for="target_uri">URL: </label>
57
            [% IF ( params.target_uri ) %]
58
                <input type="text" id="target_uri" name="params[target_uri]" value="[% params.target_uri %]" size="30" />
59
            [% ELSE %]
60
                <input type="text" id="target_uri" name="params[target_uri]" value="[% default_target_uri %]" size="30" />
61
            [% END %]
62
            [% IF (errors.target_uri.no_path) %]<span class="error">[The URL must have a path after "http://" like "koha-community.org/cgi-bin/koha/svc/import_oai".]</span>[% END %]
63
            [% IF (errors.target_uri.not_http) %]<span class="error">[The URL begin with a scheme of "http://" like "http://koha-community.org/cgi-bin/koha/svc/import_oai".]</span>[% END %]
64
            [% IF (errors.target_uri.not_a_url) %]<span class="error">[The value of this field must be a URL like "http://koha-community.org/cgi-bin/koha/svc/import_oai".]</span>[% END %]
65
        </li>
66
67
        <li>
68
            <label for="match">Record matching rule code</label>
69
            <input type="text" id="match" name="params[match]" value="[% params.match %]" size="30" />
70
            <span class="help">This code must exist in "Record matching rules" in Administration for record matching to work. (Example code: OAI)</span>
71
        </li>
72
        <li>
73
            [%# TODO: Ideally, I'd like to use 'tools-overlay-action.inc' but the logic doesn't work here. Perhaps it would be better as a TT plugin. %]
74
            <label for="overlay_action">Action if matching record found</label>
75
            <select name="params[overlay_action]" id="overlay_action">
76
            [% IF ( params.overlay_action == "replace" ) %]
77
                <option value="replace"  selected="selected">
78
            [% ELSE %]
79
                <option value="replace">
80
            [% END %]
81
                Replace existing record with incoming record</option>
82
            [% IF ( params.overlay_action == "create_new" ) %]
83
                <option value="create_new" selected="selected">
84
            [% ELSE %]
85
                <option value="create_new">
86
            [% END %]
87
                Add incoming record</option>
88
            [% IF ( params.overlay_action == "ignore" ) %]
89
                <option value="ignore" selected="selected">
90
            [% ELSE %]
91
                <option value="ignore">
92
            [% END %]
93
                Ignore incoming record</option>
94
            </select>
95
        </li>
96
        <li>
97
            [%# TODO: Ideally, I'd like to use 'tools-nomatch-action.inc' but the logic doesn't work here. Perhaps it would be better as a TT plugin. %]
98
            <label for="nomatch_action">Action if no match is found</label>
99
            <select name="params[nomatch_action]" id="nomatch_action">
100
            [% IF ( params.nomatch_action == "create_new" ) %]
101
                <option value="create_new" selected="selected">
102
            [% ELSE %]
103
                <option value="create_new">
104
            [% END %]
105
                Add incoming record</option>
106
            [% IF ( params.nomatch_action == "ignore" ) %]
107
                <option value="ignore" selected="selected">
108
            [% ELSE %]
109
                <option value="ignore">
110
            [% END %]
111
                Ignore incoming record</option>
112
            </select>
113
        </li>
114
        <li>
115
            <label for="item_action">Item action</label>
116
            [%# TODO: Will you ever have a different mode than ignore? %]
117
            <input type="text" id="item_action"  value="ignore" size="30" disabled="disabled"/>
118
            <input type="hidden" name="params[item_action]" value="ignore" />
119
        </li>
120
        <li>
121
            <label for="import_mode">Import mode: </label>
122
            [%# TODO: Will you ever have a different mode than direct? %]
123
            <input type="text" id="import_mode" value="direct" size="30" disabled="disabled"/>
124
            <input type="hidden" name="params[import_mode]" value="direct" />
125
        </li>
126
        <li>
127
            <label>Framework</label>
128
        </li>
129
        <li>
130
            <label for="filter">Filter</label>
131
            [% IF ( params.filter ) %]
132
                <input type="text" id="filter" name="params[filter]" value="[% params.filter %]" size="30" />
133
            [% ELSE %]
134
                <input type="text" id="filter" name="params[filter]" value="file://" size="30" />
135
            [% END %]
136
            <span class="help">This is a filepath on your system like file:///etc/koha/sites/libraryname/OAI2MARC21slim.xsl or file:///usr/share/koha/intranet/htdocs/intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl</span>
137
        </li>
138
        </li>
139
        <li>
140
            <label>Record type</label>
141
        </li>
142
    </ol>
143
</fieldset>
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt (+2 lines)
Lines 120-125 Link Here
120
                        <dt><a href="/cgi-bin/koha/admin/sms_providers.pl">SMS cellular providers</a></dt>
120
                        <dt><a href="/cgi-bin/koha/admin/sms_providers.pl">SMS cellular providers</a></dt>
121
                        <dd>Define a list of cellular providers for sending SMS messages via email.</dd>
121
                        <dd>Define a list of cellular providers for sending SMS messages via email.</dd>
122
                    [% END %]
122
                    [% END %]
123
                    <dt><a href="/cgi-bin/koha/admin/saved_tasks.pl">Saved tasks</a></dt>
124
                    <dd>Define tasks which may be run in the background</dd>
123
                </dl>
125
                </dl>
124
            </div>
126
            </div>
125
        </div>
127
        </div>
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/saved_tasks.tt (+338 lines)
Line 0 Link Here
1
[% INCLUDE 'doc-head-open.inc' %]
2
<title>Koha &rsaquo; Administration &rsaquo; Saved tasks</title>
3
[% INCLUDE 'doc-head-close.inc' %]
4
[% INCLUDE 'calendar.inc' %]
5
<script type="text/javascript" src="[% interface %]/lib/jquery/plugins/jquery-ui-timepicker-addon.min.js"></script>
6
[% INCLUDE 'timepicker.inc' %]
7
[% IF ( op == "list" ) %]
8
    <link rel="stylesheet" type="text/css" href="[% themelang %]/css/datatables.css" />
9
    [% INCLUDE 'datatables.inc' %]
10
    <script type="text/javascript">
11
    //<![CDATA[
12
        $(document).ready(function() {
13
            $("#taskst").dataTable($.extend(true, {}, dataTablesDefaults, {
14
                "aoColumnDefs": [
15
                    { "aTargets": [3,4,5,6], "bSortable": false },
16
                ],
17
                "sPaginationType": "four_button"
18
            }));
19
        });
20
    //]]>
21
    </script>
22
[% ELSIF ( op == "edit" ) || ( op == "new" ) %]
23
    <script type="text/javascript">
24
    //<![CDATA[
25
        $(document).ready(function() {
26
            [%# Ideally, it would be nice to record the timezone here too, but currently we use MySQL's DATETIME field which doesn't store ISO 8601 timezone designators... %]
27
            $(".datetime_local").datetimepicker({
28
                dateFormat: "yy-mm-dd",
29
                timeFormat: "HH:mm:ss",
30
                hour: 0,
31
                minute: 0,
32
                second: 0,
33
                showSecond: 1,
34
            });
35
            $(".datetime_utc").datetimepicker({
36
                separator: "T",
37
                timeSuffix: 'Z',
38
                dateFormat: "yy-mm-dd",
39
                timeFormat: "HH:mm:ss",
40
                hour: 0,
41
                minute: 0,
42
                second: 0,
43
                showSecond: 1,
44
                // timezone doesn't work with the "Now" button in v1.4.3 although it appears to in v1.6.1
45
                // timezone: +000,
46
            });
47
48
        });
49
    //]]>
50
    </script>
51
    <style type="text/css">
52
        /* Override staff-global.css which hides second, millisecond, and microsecond sliders */
53
        .ui_tpicker_second {
54
            display: block;
55
        }
56
        .test-success {
57
            /* same color as .text-success in Bootstrap 2.2.2 */
58
            color:#468847;
59
        }
60
    </style>
61
[% END %]
62
</head>
63
64
<body id="admin_saved_tasks" class="admin">
65
[% INCLUDE 'header.inc' %]
66
[% INCLUDE 'cat-search.inc' %]
67
<div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/admin/admin-home.pl">Administration</a> &rsaquo; Saved tasks</div>
68
69
<div id="doc3" class="yui-t2">
70
71
<div id="bd">
72
  <div id="yui-main">
73
    <div class="yui-b">
74
        [% IF ( op ) %]
75
            [% IF ( op == "new" ) || ( op == "edit" ) %]
76
                [%# If step is undefined, force it to be step one %]
77
                [% IF ( ! step ); step = "one"; END; %]
78
79
80
81
                [%# HEADING %]
82
                    [% IF ( op == "new" ) %]
83
                        <h1>New saved task</h1>
84
                    [% ELSIF ( op == "edit" ) %]
85
                        <h1>Modify saved task</h1>
86
                    [% END %]
87
                [%# /HEADING %]
88
89
                [%# TODO: Get this working properly... <div class="alert">Validation failed.</div> #]
90
91
                [%# FORM %]
92
                    <form action="/cgi-bin/koha/admin/[% filename %]" name="detail-form" method="post" id="saved-task-details" novalidate="novalidate">
93
                        [% IF ( op == "new" ) %]
94
                            <input type="hidden" name="op" value="create" />
95
                        [% ELSIF ( op == "edit" ) %]
96
                            <input type="hidden" name="op" value="update" />
97
                            <input type="hidden" name="saved_task_id" value="[% saved_task.task_id %]" />
98
                        [% END %]
99
                        <input type="hidden" name="step" value="[% step %]" />
100
                        <fieldset class="rows">
101
                            <ol>
102
                                [% IF ( op == "edit") && ( step == "one" ) && (! task_type_is_valid ) %]
103
                                <li>
104
                                    <label for="invalid_task_type">Current invalid task type:</label>
105
                                    <input id="invalid_task_type" type="text" disabled="disabled" value="[% saved_task.task_type %]" size="60" />
106
                                    <span class="error">Sorry! This task type is invalid. Please choose a new one from the following list.</span>
107
                                <li>
108
                                [% END %]
109
                                <li>
110
                                    <label for="task_type">Task type: </label>
111
                                    [% IF ( step == "one" ) %]
112
                                        [% IF ( available_plugins ) %]
113
                                        <select id="task_type" name="task_type">
114
                                            [% IF ( op == "new") %]
115
                                                [% FOREACH plugin IN available_plugins %]
116
                                                    <option value="[% plugin %]">[% plugin %]</option>
117
                                                [% END %]
118
                                            [% ELSIF ( op == "edit" ) %]
119
                                                [% FOREACH plugin IN available_plugins %]
120
                                                    [% IF ( saved_task.task_type == plugin ) %]
121
                                                        <option selected="selected" value="[% plugin %]">[% plugin %]</option>
122
                                                    [% ELSE %]
123
                                                        <option value="[% plugin %]">[% plugin %]</option>
124
                                                    [% END %]
125
                                                [% END %]
126
                                            [% END %]
127
                                        </select>
128
                                        [% END %]
129
130
                                    [% ELSIF ( step == "two" ) %]
131
                                        <input type="text" disabled="disabled" value="[% task_type %]" size="60" />
132
                                        <input type="hidden" name="task_type" value="[% task_type %]" />
133
                                    [% END %]
134
                                </li>
135
                            </ol>
136
                        </fieldset>
137
138
                        [% IF ( step == "one" ) %]
139
                            <fieldset class="action">
140
                                <input type="submit" value="Next">
141
                                <a class="cancel" href="/cgi-bin/koha/admin/[% filename %]">Cancel</a>
142
                            </fieldset>
143
                        [% ELSIF ( step == "two" ) %]
144
                            <fieldset class="rows">
145
                                <legend>Task:</legend>
146
                                <ol>
147
                                    <li>
148
                                        <label for="start_time">Start time: </label>
149
                                        <input type="text" id="start_time" class="datetime_local" name="start_time" value="[% saved_task.start_time %]" size="30" />
150
                                        <span class="help">This value will be treated as local server time, and times in the past will start immediately.</span>
151
                                    </li>
152
                                    <li>
153
                                        <label for="repeat_interval">Repeat interval: </label>
154
                                        <input type="text" id="repeat_interval" name="repeat_interval" value="[% saved_task.repeat_interval %]" size="4" />
155
                                        <span class="help">seconds</span>
156
                                        [% IF (errors.repeat_interval.not_numeric) %]<span class="error">[The repeat interval must be a purely numeric value.]</span>[% END %]
157
                                    </li>
158
                                </ol>
159
                            </fieldset>
160
                            [%# Try to include the template, but if it fails, fallback to a regular text view %]
161
                            [% TRY %]
162
                                [% INCLUDE $task_template %]
163
                            [% CATCH %]
164
                            <fieldset class="rows">
165
                                <legend>Plugin parameters:</legend>
166
                                <ol>
167
                                    <li>
168
                                        <label for="params">Params: </label>
169
                                        <textarea id="params" name="params" cols="60" rows="20">[% saved_task.params %]</textarea>
170
                                    </li>
171
                                </ol>
172
                            </fieldset>
173
                            [% END %]
174
                            <fieldset class="action">
175
                                <input type="submit" value="Save">
176
                                <a class="cancel" href="/cgi-bin/koha/admin/[% filename %]">Cancel</a>
177
                            </fieldset>
178
                        [% END %]
179
                    </form>
180
                [%# /FORM %]
181
            [% END #/edit or new %]
182
183
184
            [% IF ( op == "list" ) %]
185
                <div id="toolbar" class="btn-toolbar">
186
                    <a id="newserver" class="btn btn-small" href="/cgi-bin/koha/admin/[% filename %]?op=new"><i class="icon-plus"></i> New saved task</a>
187
                </div>
188
                <h1>Saved tasks</h1>
189
                [% IF ( saved_response ) %]
190
                    [% IF ( saved_response == 'delete_success' ) %]
191
                        <div class="alert">Deletion successful.</div>
192
                    [% ELSIF ( saved_response == 'delete_failure' ) %]
193
                        <div class="alert">Deletion failed.</div>
194
                    [% END %]
195
                [% END %]
196
                [% IF ( sent_response ) %]
197
                    [% IF ( sent_response == 'icarus_offline' ) %]
198
                        <div class="alert">Send failed. Icarus is currently offline.</div>
199
                    [% END %]
200
                [% END %]
201
                <table id="taskst">
202
                    <thead>
203
                        <tr>
204
                            <th>Start time</th>
205
                            <th>Repeat interval</th>
206
                            <th>Task type</th>
207
                            <th>Params</th>
208
                            <th></th>
209
                            <th></th>
210
                            <th></th>
211
                        </tr>
212
                    </thead>
213
                    <tbody>
214
                    [% FOREACH saved_task IN saved_tasks %]
215
                        <tr>
216
                            <td>[% IF ( saved_task.start_time ) != "0000-00-00 00:00:00"; saved_task.start_time; END; %]</td>
217
                            <td>[% saved_task.repeat_interval %]</td>
218
                            <td>[% saved_task.task_type %]</td>
219
                            <td>
220
                                <ul>
221
                                [% FOREACH pair IN saved_task.params_as_perl.pairs %]
222
                                   <li>[% pair.key %] => [% pair.value %]</li>
223
                                [% END %]
224
                                </ul>
225
                            </td>
226
                            <td><a href="/cgi-bin/koha/admin/[% filename %]?op=edit&saved_task_id=[% saved_task.task_id %]">Edit</a></td>
227
                            <td><a href="/cgi-bin/koha/admin/[% filename %]?op=send&saved_task_id=[% saved_task.task_id %]">Send to Icarus</a></td>
228
                            <td><a href="/cgi-bin/koha/admin/[% filename %]?op=delete&saved_task_id=[% saved_task.task_id %]">Delete</a></td>
229
                        </tr>
230
                    [% END %]
231
                    </tbody>
232
                </table>
233
                <div id="daemon_controls">
234
                    <h1>Icarus dashboard</h1>
235
                    <table>
236
                    <tr>
237
                        <th>Status</th>
238
                        <th></th>
239
                    </tr>
240
                    <tr>
241
                        <td>
242
243
                        [% IF ( daemon_status == 'Permission denied' ) #Apache doesn't have permission to write to socket
244
                            || ( daemon_status == 'Connection refused' ) #Socket exists, but server is down
245
                            || ( daemon_status == 'No such file or directory' ) #Socket doesn't exist at all
246
                        %]
247
                            <span id="icarus_status">Unable to contact</span>
248
                        [% ELSIF ( daemon_status == 'online' ) %]
249
                            <span id="icarus_status">Online</span>
250
                        [% ELSIF ( daemon_status == 'shutting down' ) %]
251
                            <span id="icarus_status">Shutting down</span>
252
                        [% ELSE %]
253
                            <span id="icarus_status">[% daemon_status %]</span>
254
                        [% END %]
255
                        </td>
256
                        [%# TODO: Also provide controls for starting/restarting Icarus? %]
257
                        <td><a href="/cgi-bin/koha/admin/[% filename %]?server_action=shutdown">Shutdown Icarus</a></td>
258
                    </tr>
259
                    </table>
260
                </div>
261
                <div id="tasks">
262
                    <h1>Active Icarus tasks</h1>
263
                    [% IF ( task_response ) %]
264
                        [% IF ( task_response.action == 'error' ) %]
265
                            [% IF ( task_response.error_message ) %]
266
                                [% IF ( task_response.error_message == 'No such process' ) %]
267
                                    <div class="alert">Task [% task_response.task_id %] does not exist.</div>
268
                                [% END %]
269
                            [% END %]
270
                        [% ELSIF ( task_response.action == 'pending' ) %]
271
                            <div class="alert">Initialising task [% task_response.task_id %].</div>
272
                        [% ELSIF ( task_response.action == 'already pending' ) %]
273
                            <div class="alert">Already initialised task [% task_response.task_id %].</div>
274
                        [% ELSIF ( task_response.action == 'already started' ) %]
275
                            <div class="alert">Already started task [% task_response.task_id %].</div>
276
                        [% ELSIF ( task_response.action == 'removed' ) %]
277
                            <div class="alert">Removing task [% task_response.task_id %].</div>
278
                        [% END %]
279
                    [% END %]
280
                    [% IF ( tasks ) %]
281
                        <table>
282
                            <thead>
283
                                <tr>
284
                                    <th>Task id</th>
285
                                    <th>Status</th>
286
                                    <th>Next start time (local server time)</th>
287
                                    <th>Repeat interval</th>
288
                                    <th>Task type</th>
289
                                    <th>Params</th>
290
                                    <th></th>
291
                                    <th></th>
292
                                </tr>
293
                            </thead>
294
                            <tbody>
295
                            [% FOREACH task IN tasks %]
296
                                <tr>
297
                                    <td>[% task.task_id %]</td>
298
                                    <td>
299
                                        [% SWITCH task.task.status %]
300
                                        [% CASE 'new' %]
301
                                        <span>New</span>
302
                                        [% CASE 'pending' %]
303
                                        <span>Pending</span>
304
                                        [% CASE 'started' %]
305
                                        <span>Started</span>
306
                                        [% CASE 'stopping' %]
307
                                        <span>Stopping</span>
308
                                        [% CASE %]
309
                                        <span>[% task.task.status %]</span>
310
                                        [% END %]
311
                                    </td>
312
                                    <td>[% task.task.start %]</td>
313
                                    <td>[% task.task.repeat_interval %]</td>
314
                                    <td>[% task.task.type %]</td>
315
                                    <td>
316
                                        <ul>
317
                                        [% FOREACH pair IN task.task.params.pairs %]
318
                                           <li>[% pair.key %] => [% pair.value %]</li>
319
                                        [% END %]
320
                                        </ul>
321
                                    </td>
322
                                    <td><a href="/cgi-bin/koha/admin/[% filename %]?server_action=start&server_task_id=[% task.task_id %]">Start</a></td>
323
                                    <td><a href="/cgi-bin/koha/admin/[% filename %]?server_action=remove&server_task_id=[% task.task_id %]">Remove</a></td>
324
                                </tr>
325
                            [% END %]
326
                            </tbody>
327
                        </table>
328
                    [% END %]
329
                </div>
330
            [% END #/list %]
331
        [% END #/op %]
332
    </div>
333
  </div>
334
  <div class="yui-b">
335
    [% INCLUDE 'admin-menu.inc' %]
336
  </div>
337
</div>
338
[% INCLUDE 'intranet-bottom.inc' %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-oai-import.tt (+122 lines)
Line 0 Link Here
1
[% INCLUDE 'doc-head-open.inc' %]
2
<title>Koha &rsaquo; Tools &rsaquo; Manage OAI-PMH record imports
3
[% IF ( import_oai_id ) %]
4
 &rsaquo; Record [% import_oai_id %]
5
[% END %]
6
</title>
7
[% INCLUDE 'doc-head-close.inc' %]
8
<link rel="stylesheet" type="text/css" href="[% themelang %]/css/datatables.css" />
9
[% INCLUDE 'datatables.inc' %]
10
</head>
11
12
<body id="tools_manage-oai-import" class="tools">
13
[% INCLUDE 'header.inc' %]
14
[% INCLUDE 'cat-search.inc' %]
15
16
    <div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> 
17
    [% IF ( import_oai_id ) %]
18
     &rsaquo;
19
     <a href="[% script_name %]">Manage OAI-PMH record imports</a>
20
     &rsaquo; Record [% import_oai_id %]
21
    [% ELSE %]
22
     &rsaquo; Manage OAI-PMH record imports
23
    [% END %]
24
    </div>
25
26
    <div id="doc3" class="yui-t2">
27
        <div id="bd">
28
            <div id="yui-main">
29
                <div class="yui-b">
30
                    [% IF ( import_oai_id ) %]
31
                        [% IF ( view_record ) %]
32
                            <h1>Record [% import_oai_id %]</h1>
33
                            [% IF ( oai_record.metadata ) %]
34
                                <div style="white-space:pre">[% oai_record.metadata | xml %]</div>
35
                            [% END %]
36
                        [% ELSIF ( retry ) %]
37
                            <fieldset class="rows">
38
                                <ol>
39
                                    <li>
40
                                        <span class="label">Import status:</span>
41
                                        [% IF ( import_status ) %]
42
                                            [% IF ( import_status == "ok" ) %]
43
                                            OK
44
                                            [% ELSIF ( import_status == "error" ) %]
45
                                            ERROR
46
                                            [% END %]
47
                                        [% END %]
48
                                    </li>
49
                                    [% IF ( errors ) %]
50
                                        [% FOREACH error IN errors %]
51
                                            <li>
52
                                                <span class="label">Error:</span>
53
                                                [%# FIXME: These English messages come straight from C4::Biblio... %]
54
                                                [% error.error_msg %]
55
                                                [% IF ( record_type ) && ( record_type == "biblio" ) %]
56
                                                    <a title="View biblio record" href="/cgi-bin/koha/catalogue/detail.pl?biblionumber=[% error.record_id %]">(View biblio record)</a>
57
                                                [% END %]
58
                                            </li>
59
                                        [% END %]
60
                                    [% END %]
61
                                </ol>
62
                            </fieldset>
63
                        [% END %]
64
                    [% ELSE %]
65
                        <h1>Manage OAI-PMH record imports</h1>
66
                        <table>
67
                            <thead>
68
                                <tr>
69
                                    <th>Record identifier</th>
70
                                    <th>Record datestamp</th>
71
                                    <th>Provider status</th>
72
                                    <th>Import status</th>
73
                                    <th>Import batch</th>
74
                                    <th>OAI-PMH record</th>
75
                                    [%# <th>Filter</th> %]
76
                                </tr>
77
                            </thead>
78
                            <tbody>
79
                                [% WHILE (oai_record = oai_records.next) %]
80
                                <tr>
81
                                    <td>[% oai_record.header_identifier %]</td>
82
                                    <td>[% oai_record.header_datestamp %]</td>
83
                                    <td>
84
                                        [% IF ( oai_record.header_status ) %]
85
                                            [% IF ( oai_record.header_status == "deleted" ) %]
86
                                                DELETED
87
                                            [% END %]
88
                                        [% END %]
89
                                    </td>
90
                                    <td>
91
                                        [% IF ( oai_record.status ) %]
92
                                            [% IF ( oai_record.status == "ok" ) %]
93
                                                OK
94
                                            [% ELSIF ( oai_record.status == "error" ) %]
95
                                                <a title="Retry import" href="[% script_name %]?op=retry&import_oai_id=[% oai_record.import_oai_id %]">ERROR - Click to retry</a>
96
                                            [% END %]
97
                                            
98
                                        [% ELSE %]
99
                                            Unknown
100
                                        [% END %]
101
                                    </td>
102
                                    <td>
103
                                        [% IF ( oai_record.import_batch_id ) %]
104
                                            <a title="View import batch" href="/cgi-bin/koha/tools/manage-marc-import.pl?import_batch_id=[% oai_record.import_batch_id %]">View batch [% oai_record.import_batch_id %]</a>
105
                                        [% END %]
106
                                    </td>
107
                                    [%# oai_record.filter %]
108
                                    <td><a title="View OAI-PMH record" href="[% script_name %]?op=view_record&import_oai_id=[% oai_record.import_oai_id %]">View record [% oai_record.import_oai_id %]</a></td>
109
                                </tr>
110
                                [% END %]
111
                                
112
                            </tbody>
113
                        </table>
114
                    [% END %]
115
                </div>
116
            </div>
117
            <div class="yui-b">
118
                [% INCLUDE 'tools-menu.inc' %]
119
            </div>
120
        </div>
121
    </div>
122
[% INCLUDE 'intranet-bottom.inc' %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl (+74 lines)
Line 0 Link Here
1
<?xml version="1.0" encoding="UTF-8"?>
2
<xsl:stylesheet version="1.0"
3
    xmlns:marc="http://www.loc.gov/MARC21/slim"
4
    xmlns:oai="http://www.openarchives.org/OAI/2.0/"
5
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
6
    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>
7
    <!-- NOTE: This XSLT strips the OAI-PMH wrapper from the metadata. -->
8
    <!-- NOTE: This XSLT also adds the OAI-PMH identifier back in as a MARC field -->
9
10
    <!-- Match the root oai:record element -->
11
    <xsl:template match="oai:record">
12
        <!-- Apply templates only when the oai record is for a deleted item -->
13
        <xsl:apply-templates select="oai:header[@status='deleted']" />
14
        <!-- Apply templates only to the child metadata element(s) -->
15
        <xsl:apply-templates select="oai:metadata" />
16
    </xsl:template>
17
18
    <!-- Matches an oai:metadata element -->
19
    <xsl:template match="oai:metadata">
20
        <!-- Only apply further templates to marc:record elements -->
21
        <!-- This prevents the identity transformation from outputting other non-MARC metadata formats -->
22
        <xsl:apply-templates select="//marc:record"/>
23
    </xsl:template>
24
25
    <!-- We need to create a MARCXML record from OAI records marked "deleted" to handle OAI deletions correctly in Koha -->
26
    <xsl:template match="oai:header[@status='deleted']">
27
        <xsl:element name="record" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
28
            xmlns="http://www.loc.gov/MARC21/slim">
29
            <xsl:attribute name="xsi:schemaLocation">http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd</xsl:attribute>
30
            <xsl:call-template name="add_oai"/>
31
        </xsl:element>
32
    </xsl:template>
33
34
    <!-- Identity transformation: this template copies attributes and nodes -->
35
    <xsl:template match="@* | node()">
36
        <!-- Create a copy of this attribute or node -->
37
        <xsl:copy>
38
            <!-- Recursively apply this template to the attributes and child nodes of this element -->
39
            <xsl:apply-templates select="@* | node()" />
40
        </xsl:copy>
41
    </xsl:template>
42
43
44
    <xsl:template match="marc:record">
45
        <xsl:copy>
46
            <!-- Apply all relevant templates for all attributes and elements -->
47
            <xsl:apply-templates select="@* | node()"/>
48
49
            <!-- Add new node (or whatever else you want to do after copying the existing record) -->
50
            <xsl:call-template name="add_oai"/>
51
52
            <!-- Newline -->
53
            <xsl:text>&#xa;</xsl:text>
54
        </xsl:copy>
55
    </xsl:template>
56
57
    <!-- Template for adding the OAI-PMH identifier as 024$a -->
58
    <xsl:template name="add_oai">
59
        <xsl:element name="datafield" xmlns="http://www.loc.gov/MARC21/slim">
60
            <xsl:attribute name="ind1"><xsl:text>7</xsl:text></xsl:attribute>
61
            <xsl:attribute name="ind2"><xsl:text> </xsl:text></xsl:attribute>
62
            <xsl:attribute name="tag">024</xsl:attribute>
63
            <xsl:element name="subfield">
64
                <xsl:attribute name="code">a</xsl:attribute>
65
                <xsl:value-of select="/oai:record/oai:header/oai:identifier"/>
66
            </xsl:element>
67
            <xsl:element name="subfield">
68
                <xsl:attribute name="code">2</xsl:attribute>
69
                <xsl:text>uri</xsl:text>
70
            </xsl:element>
71
         </xsl:element>
72
    </xsl:template>
73
74
</xsl:stylesheet>
(-)a/misc/bin/icarusd.pl (+181 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
#######################################################################
4
5
use Modern::Perl;
6
use POSIX; #For daemonizing
7
use Fcntl qw(:flock); #For pidfile
8
use Getopt::Long;
9
use Pod::Usage;
10
11
#Make the STDOUT filehandle hot, so that you can use shell re-direction. Otherwise, you'll suffer from buffering.
12
STDOUT->autoflush(1);
13
#Note that STDERR, by default, is already hot.
14
15
#######################################################################
16
#FIXME: Debugging signals
17
#BEGIN {
18
#    package POE::Kernel;
19
#    use constant TRACE_SIGNALS => 1;
20
#}
21
22
use POE;
23
use JSON; #For Listener messages
24
use XML::LibXML; #For configuration files
25
26
use Koha::Icarus::Listener;
27
28
#######################################################################
29
30
my ($filename,$daemon,$log,$help);
31
my $verbosity = 1;
32
GetOptions (
33
    "f|file|filename=s"     => \$filename, #/kohawebs/dev/dcook/koha-dev/etc/koha-conf.xml
34
    "l|log=s"               => \$log,
35
    "d|daemon"              => \$daemon,
36
    "v=i"                   => \$verbosity,
37
    "h|?"                   => \$help,
38
) or pod2usage(2);
39
pod2usage(1) if $help;
40
41
42
if ( ! $filename || ! -f $filename ){
43
    print "Failed to start.\n";
44
    if ( ! $filename ){
45
        print("You must provide a valid configuration file using the -f switch.\n");
46
        pod2usage(1);
47
    }
48
    if ( ! -f $filename ){
49
        die(qq{"$filename" is not a file.\n});
50
    }
51
}
52
53
#Declare the variable with file scope so the flock stays for the duration of the process's life
54
my $pid_filehandle;
55
56
#Read configuration file
57
my $config = read_config_file($filename);
58
59
my $SOCK_PATH = $config->{socket};
60
my $pid_file = $config->{pidfile};
61
my $max_tasks = $config->{max_tasks};
62
63
#Overwrite configuration file with command line options
64
if ($log){
65
    $config->{log} = $log;
66
}
67
68
#Go into daemon mode, if user has included flag
69
if ($daemon){
70
    daemonize();
71
}
72
73
if ($pid_file){
74
    #NOTE: The filehandle needs to have file scope, so that the flock is preserved.
75
    $pid_filehandle = make_pid_file($pid_file);
76
}
77
78
#FIXME: Do we want to log to file only in daemon mode? $config->{log} should be populated by either the config file or the l|log GetOpt...
79
if ($daemon && $config->{log}){
80
    log_to_file($config->{log});
81
}
82
83
84
#FIXME: 1) In daemon mode, SIGUSR1 or SIGHUP for reloading/restarting?
85
#######################################################################
86
87
#Creates Icarus Listener
88
Koha::Icarus::Listener->spawn({
89
    Socket => $SOCK_PATH,
90
    MaxTasks => $max_tasks,
91
    Verbosity => $verbosity,
92
});
93
94
POE::Kernel->run();
95
96
exit;
97
98
sub read_config_file {
99
    my $filename = shift;
100
    my $config = {};
101
    if ( -e $filename ){
102
        eval {
103
            my $doc = XML::LibXML->load_xml(location => $filename);
104
            if ($doc){
105
                my $root = $doc->documentElement;
106
                my $icarus = $root->find('icarus')->shift;
107
                if ($icarus){
108
                    #Get all child nodes for the 'icarus' element
109
                    my @childnodes = $icarus->childNodes();
110
                    foreach my $node (@childnodes){
111
                        #Only consider nodes that are elements
112
                        if ($node->nodeType == XML_ELEMENT_NODE){
113
                            my $config_key = $node->nodeName;
114
                            my $first_child = $node->firstChild;
115
                            #Only consider nodes that have a text node as their first child
116
                            if ($first_child && $first_child->nodeType == XML_TEXT_NODE){
117
                                $config->{$config_key} = $first_child->nodeValue;
118
                            }
119
                        }
120
                    }
121
                }
122
            }
123
        };
124
    }
125
    return $config;
126
}
127
128
#######################################################################
129
#NOTE: On Debian, you can use the daemon binary to make a process into a daemon,
130
# the following subs are for systems that don't have the daemon binary.
131
132
sub daemonize {
133
    my $pid = fork;
134
    die "Couldn't fork: $!" unless defined($pid);
135
    if ($pid){
136
        exit; #Parent exit
137
    }
138
    POSIX::setsid() or die "Can't start a new session: $!";
139
}
140
141
sub log_to_file {
142
    my $logfile = shift;
143
    #Open a filehandle to append to a log file
144
    open(LOG, '>>', $logfile) or die "Unable to open a filehandle for $logfile: $!\n"; # --output
145
    LOG->autoflush(1); #Make filehandle hot (ie don't buffer)
146
    *STDOUT = *LOG; #Re-assign STDOUT to LOG | --stdout
147
    *STDERR = *STDOUT; #Re-assign STDERR to STDOUT | --stderr
148
}
149
150
sub make_pid_file {
151
    my $pidfile = shift;
152
    if ( ! -e $pidfile ){
153
        open(my $fh, '>', $pidfile) or die "Unable to write to $pidfile: $!\n";
154
        $fh->close;
155
    }
156
157
    open(my $pidfilehandle, '+<', $pidfile) or die "Unable to open a filehandle for $pidfile: $!\n";
158
    if (flock($pidfilehandle, LOCK_EX|LOCK_NB)){
159
        #Write pid to pidfile
160
        print "Acquiring lock on $pidfile\n";
161
        #Now that we've acquired a lock, let's truncate the file
162
        truncate($pidfilehandle, 0);
163
        print $pidfilehandle $$."\n" or die $!;
164
        #Flush the filehandle so you're not suffering from buffering
165
        $pidfilehandle->flush();
166
        return $pidfilehandle;
167
    } else {
168
        my $number = <$pidfilehandle>;
169
        chomp($number);
170
        warn "$0 is already running with pid $number. Exiting.\n";
171
        exit(1);
172
    }
173
}
174
175
__END__
176
177
=head1 SYNOPSIS
178
179
icarusd.pl -f koha-conf.xml [--log icarus.log] [--daemon] [ -v 0-9 ] [-h]
180
181
=cut
(-)a/rewrite-config.PL (+2 lines)
Lines 148-153 $prefix = $ENV{'INSTALL_BASE'} || "/usr"; Link Here
148
  "__MEMCACHED_SERVERS__" => "",
148
  "__MEMCACHED_SERVERS__" => "",
149
  "__MEMCACHED_NAMESPACE__" => "",
149
  "__MEMCACHED_NAMESPACE__" => "",
150
  "__FONT_DIR__" => "/usr/share/fonts/truetype/ttf-dejavu",
150
  "__FONT_DIR__" => "/usr/share/fonts/truetype/ttf-dejavu",
151
  "__ICARUS_RUN_DIR__" => "$prefix/var/run/icarus",
152
  "__ICARUS_MAX_TASKS__" => "30",
151
);
153
);
152
154
153
# Override configuration from the environment
155
# Override configuration from the environment
(-)a/skel/var/run/koha/icarus/README (+1 lines)
Line 0 Link Here
1
icarus dir
(-)a/svc/import_oai (+143 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright 2016 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
use XML::LibXML;
23
use URI;
24
use File::Basename;
25
26
use CGI qw ( -utf8 );
27
use C4::Auth qw/check_api_auth/;
28
use C4::Context;
29
use C4::ImportBatch;
30
use C4::Matcher;
31
use XML::Simple;
32
use C4::Biblio;
33
34
use Koha::OAI::Client::Record;
35
36
my $query = new CGI;
37
binmode STDOUT, ':encoding(UTF-8)';
38
39
my ($status, $cookie, $sessionID) = check_api_auth($query, { editcatalogue => 'edit_catalogue'} );
40
unless ($status eq "ok") {
41
    print $query->header(-type => 'text/xml', -status => '403 Forbidden');
42
    print XMLout({ auth_status => $status }, NoAttr => 1, RootName => 'response', XMLDecl => 1);
43
    exit 0;
44
}
45
46
my $xml;
47
if ($query->request_method eq "POST") {
48
    $xml = $query->param('xml');
49
}
50
if ($xml) {
51
    #TODO: You could probably use $query->Vars here instead...
52
    my %params = map { $_ => $query->param($_) } $query->param;
53
    my $result = import_oai($xml, \%params );
54
    print $query->header(-type => 'text/xml');
55
    print XMLout($result, NoAttr => 1, RootName => 'response', XMLDecl => 1);
56
} else {
57
    print $query->header(-type => 'text/xml', -status => '400 Bad Request');
58
}
59
60
exit 0;
61
62
sub import_oai {
63
    my ($inxml, $params) = @_;
64
65
    my $result = {};
66
    my $status = "error";
67
68
    my $filter      = delete $params->{filter}      || '';
69
    my $import_mode = delete $params->{import_mode} || '';
70
    my $framework   = delete $params->{framework}   || '';
71
72
    if (my $matcher_code = delete $params->{match}) {
73
        $params->{matcher_id} = C4::Matcher::GetMatcherId($matcher_code);
74
    }
75
76
    my $batch_id = GetWebserviceBatchId($params);
77
    #FIXME: Use the batch_id to create a more useful filename in the import_batches table...
78
    unless ($batch_id) {
79
        $result->{'status'} = "failed";
80
        $result->{'error'} = "Batch create error";
81
        return $result;
82
    }
83
84
    #Source a default XSLT to use for filtering
85
    my $htdocs  = C4::Context->config('intrahtdocs');
86
    my $theme   = C4::Context->preference("template");
87
    #FIXME: This doesn't work for UNIMARC!
88
    my $xslfilename = "$htdocs/$theme/en/xslt/OAI2MARC21slim.xsl";
89
90
    #FIXME: There's a better way to do these filters...
91
    if ($filter){
92
        my $filter_uri = URI->new($filter);
93
        if ($filter_uri){
94
            my $scheme = $filter_uri->scheme;
95
            if ($scheme && $scheme eq "file"){
96
                my $path = $filter_uri->path;
97
                #Filters may theoretically be .xsl or .pm files
98
                my($filename, $dirs, $suffix) = fileparse($path,(".xsl",".pm"));
99
                if ($suffix && $suffix eq ".xsl"){
100
                    #If this new path exists, change the filter XSLT to it
101
                    if ( -f $path ){
102
                        $xslfilename = $path;
103
                    }
104
                }
105
            }
106
        }
107
    }
108
    
109
    #Get matching rule matcher
110
    my $matcher = C4::Matcher->new($params->{record_type} || 'biblio');
111
    $matcher = C4::Matcher->fetch($params->{matcher_id});
112
    
113
    
114
    my $oai_record = Koha::OAI::Client::Record->new({ 
115
        xml_string => $inxml,
116
    });
117
118
    $oai_record->filter({
119
        filter => $xslfilename,
120
    });
121
    
122
    my ($import_status, $match_status, $koha_record_numbers, $errors) = $oai_record->import_record({
123
        matcher => $matcher,
124
        import_batch_id => $batch_id,
125
        import_mode => $import_mode,
126
        framework => $framework,
127
    });
128
    
129
    $oai_record->save_to_database();
130
    
131
    $result->{'match_status'} = $match_status;
132
    $result->{'import_batch_id'} = $batch_id;
133
    $result->{'koha_record_numbers'} = $koha_record_numbers; 
134
    
135
    if ($import_status && $import_status eq "ok"){
136
        $result->{'status'} = "ok";
137
    } else {
138
        $result->{'status'} = "failed";
139
        $result->{'errors'} = $errors;
140
    }
141
142
    return $result;
143
}
(-)a/tools/manage-oai-import.pl (-1 / +128 lines)
Line 0 Link Here
0
- 
1
#!/usr/bin/perl
2
3
# Copyright 2016 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
use Modern::Perl;
21
22
use Koha::Database;
23
24
use C4::Auth;
25
use C4::Output;
26
use C4::Koha;
27
use C4::Context;
28
use C4::Matcher;
29
use Koha::OAI::Client::Record;
30
31
my $script_name = "/cgi-bin/koha/tools/manage-oai-import.pl";
32
33
my $input = new CGI;
34
my $op = $input->param('op') || '';
35
36
my $import_oai_id = $input->param('import_oai_id');
37
#my $results_per_page = $input->param('results_per_page') || 25; 
38
39
40
41
my ($template, $loggedinuser, $cookie) = 
42
    get_template_and_user({template_name => "tools/manage-oai-import.tt",
43
        query => $input,
44
        type => "intranet",
45
        authnotrequired => 0,
46
        flagsrequired => {tools => 'manage_staged_marc'},
47
        debug => 1,
48
    });
49
50
my $schema = Koha::Database->new()->schema();
51
my $resultset = $schema->resultset('ImportOai');
52
my $oai_records = $resultset->search;
53
54
if ($import_oai_id){
55
    my $import_oai_record = $resultset->find($import_oai_id);
56
    $template->param(
57
        oai_record => $import_oai_record,
58
    );
59
    
60
    if ($op eq "view_record" && $import_oai_id){
61
        $template->param(
62
            view_record => 1,
63
            import_oai_id => $import_oai_id,
64
        );
65
    }
66
    
67
    if ($op eq "retry" && $import_oai_record){
68
        my $oai_record = Koha::OAI::Client::Record->new({ 
69
            xml_string => $import_oai_record->metadata,
70
        });
71
72
        $oai_record->filter({
73
            filter => $import_oai_record->filter,
74
        });
75
        my $import_batch_id = $import_oai_record->import_batch_id;
76
        if ($import_batch_id){
77
            my $import_batch_rs = $schema->resultset('ImportBatch');
78
            my $import_batch = $import_batch_rs->find($import_batch_id);
79
            my $matcher_id = $import_batch->matcher_id;
80
            
81
            my $record_type = $import_batch->record_type;
82
            $template->param(
83
                record_type => $record_type,
84
            );
85
            
86
            
87
            #my $matcher = C4::Matcher->new($record_type || 'biblio');
88
            my $matcher = C4::Matcher->fetch($matcher_id);
89
            
90
            
91
            #FIXME
92
            my $import_mode = "direct";
93
            #FIXME
94
            my $framework = "";
95
            
96
            my ($import_status, $match_status, $koha_record_numbers, $errors) = $oai_record->import_record({
97
                matcher => $matcher,
98
                import_batch_id => $import_batch_id,
99
                import_mode => $import_mode,
100
                framework => $framework,
101
            });
102
            
103
            if ($import_status){
104
                if ($import_status eq 'ok'){
105
                    $import_oai_record->status("ok");
106
                    $import_oai_record->update();
107
                } else {
108
                    $template->param(
109
                        import_status => $import_status,
110
                        errors => $errors,
111
                        retry => 1,
112
                        import_oai_id => $import_oai_id,
113
                    );
114
                }
115
            }
116
        }
117
    }
118
}
119
    
120
$template->param( 
121
    script_name => $script_name,
122
    oai_records => $oai_records,
123
);
124
125
    
126
127
128
output_html_with_http_headers $input, $cookie, $template->output;

Return to bug 10662