|
Lines 25-38
background_jobs_worker.pl - Worker script that will process background jobs
Link Here
|
| 25 |
|
25 |
|
| 26 |
=head1 DESCRIPTION |
26 |
=head1 DESCRIPTION |
| 27 |
|
27 |
|
| 28 |
This script will connect to the Stomp server (RabbitMQ) and subscribe to the queues passed in parameter (or the 'default' queue), |
28 |
This script will launch a worker for the specified queues. It will poll the database |
| 29 |
or if a Stomp server is not active it will poll the database every 10s for new jobs in the passed queue. |
29 |
every 10s for new jobs in the passed queue. |
| 30 |
|
30 |
|
| 31 |
You can specify some queues only (using --queue, which is repeatable) if you want to run several workers that will handle their own jobs. |
31 |
You can specify some queues only (using --queue, which is repeatable) if you want to |
|
|
32 |
run several workers that will handle their own jobs. |
| 32 |
|
33 |
|
| 33 |
--m --max-processes specifies how many jobs to process simultaneously |
34 |
--m --max-processes specifies how many jobs to process simultaneously |
| 34 |
|
35 |
|
| 35 |
Max processes will be set from the command line option, the environment variable MAX_PROCESSES, or the koha-conf file, in that order of precedence. |
36 |
Max processes will be set from the command line option, the environment variable MAX_PROCESSES, |
|
|
37 |
or the koha-conf file, in that order of precedence. |
| 38 |
|
| 36 |
By default the script will only run one job at a time. |
39 |
By default the script will only run one job at a time. |
| 37 |
|
40 |
|
| 38 |
=head1 OPTIONS |
41 |
=head1 OPTIONS |
|
Lines 53-69
The different values available are:
Link Here
|
| 53 |
=cut |
56 |
=cut |
| 54 |
|
57 |
|
| 55 |
use Modern::Perl; |
58 |
use Modern::Perl; |
| 56 |
use JSON qw( decode_json ); |
59 |
|
| 57 |
use Try::Tiny; |
60 |
use Try::Tiny; |
| 58 |
use Pod::Usage; |
61 |
use Pod::Usage; |
| 59 |
use Getopt::Long; |
62 |
use Getopt::Long; |
| 60 |
use Parallel::ForkManager; |
|
|
| 61 |
use Time::HiRes; |
| 62 |
|
63 |
|
| 63 |
use C4::Context; |
64 |
use Koha::Worker; |
| 64 |
use Koha::Logger; |
|
|
| 65 |
use Koha::BackgroundJobs; |
| 66 |
use C4::Context; |
| 67 |
|
65 |
|
| 68 |
$SIG{'PIPE'} = 'IGNORE'; # See BZ 35111; added to ignore PIPE error when connection lost on Ubuntu. |
66 |
$SIG{'PIPE'} = 'IGNORE'; # See BZ 35111; added to ignore PIPE error when connection lost on Ubuntu. |
| 69 |
|
67 |
|
|
Lines 73-82
my $max_processes = $ENV{MAX_PROCESSES};
Link Here
|
| 73 |
$max_processes ||= C4::Context->config('background_jobs_worker')->{max_processes} |
71 |
$max_processes ||= C4::Context->config('background_jobs_worker')->{max_processes} |
| 74 |
if C4::Context->config('background_jobs_worker'); |
72 |
if C4::Context->config('background_jobs_worker'); |
| 75 |
$max_processes ||= 1; |
73 |
$max_processes ||= 1; |
| 76 |
my $mq_timeout = $ENV{MQ_TIMEOUT} // 10; |
|
|
| 77 |
|
| 78 |
my $not_found_retries = {}; |
| 79 |
my $max_retries = $ENV{MAX_RETRIES} || 10; |
| 80 |
|
74 |
|
| 81 |
GetOptions( |
75 |
GetOptions( |
| 82 |
'm|max-processes=i' => \$max_processes, |
76 |
'm|max-processes=i' => \$max_processes, |
|
Lines 90-214
unless (@queues) {
Link Here
|
| 90 |
push @queues, 'default'; |
84 |
push @queues, 'default'; |
| 91 |
} |
85 |
} |
| 92 |
|
86 |
|
| 93 |
my $conn; |
87 |
my $worker = Koha::Worker->new( { max_processes => $max_processes, queues => \@queues } ); |
| 94 |
try { |
88 |
|
| 95 |
$conn = Koha::BackgroundJob->connect; |
89 |
# main loop |
| 96 |
} catch { |
90 |
$worker->run(); |
| 97 |
warn sprintf "Cannot connect to the message broker, the jobs will be processed anyway (%s)", $_; |
91 |
|
| 98 |
}; |
92 |
1; |
| 99 |
|
|
|
| 100 |
my $pm = Parallel::ForkManager->new($max_processes); |
| 101 |
|
| 102 |
if ($conn) { |
| 103 |
|
| 104 |
# FIXME cf note in Koha::BackgroundJob about $namespace |
| 105 |
my $namespace = C4::Context->config('memcached_namespace'); |
| 106 |
for my $queue (@queues) { |
| 107 |
$conn->subscribe( |
| 108 |
{ |
| 109 |
destination => sprintf( "/queue/%s-%s", $namespace, $queue ), |
| 110 |
ack => 'client', |
| 111 |
'prefetch-count' => 1, |
| 112 |
} |
| 113 |
); |
| 114 |
} |
| 115 |
} |
| 116 |
while (1) { |
| 117 |
if ($conn) { |
| 118 |
my $frame = $conn->receive_frame( { timeout => $mq_timeout } ); |
| 119 |
if ( !defined $frame ) { |
| 120 |
|
| 121 |
# timeout or connection issue? |
| 122 |
$pm->reap_finished_children; |
| 123 |
next; # will reconnect automatically |
| 124 |
} |
| 125 |
|
| 126 |
my $args = try { |
| 127 |
my $body = $frame->body; |
| 128 |
decode_json($body); # TODO Should this be from_json? Check utf8 flag. |
| 129 |
} catch { |
| 130 |
Koha::Logger->get( { interface => 'worker' } )->warn( sprintf "Frame not processed - %s", $_ ); |
| 131 |
return; |
| 132 |
}; |
| 133 |
|
| 134 |
unless ($args) { |
| 135 |
Koha::Logger->get( { interface => 'worker' } ) |
| 136 |
->warn( sprintf "Frame does not have correct args, ignoring it" ); |
| 137 |
$conn->nack( { frame => $frame, requeue => 'false' } ); |
| 138 |
next; |
| 139 |
} |
| 140 |
|
| 141 |
my $job = Koha::BackgroundJobs->find( $args->{job_id} ); |
| 142 |
|
| 143 |
if ( $job && $job->status ne 'new' ) { |
| 144 |
Koha::Logger->get( { interface => 'worker' } ) |
| 145 |
->warn( sprintf "Job %s has wrong status %s", $args->{job_id}, $job->status ); |
| 146 |
|
| 147 |
# nack without requeue, we do not want to process this frame again |
| 148 |
$conn->nack( { frame => $frame, requeue => 'false' } ); |
| 149 |
next; |
| 150 |
} |
| 151 |
|
| 152 |
unless ($job) { |
| 153 |
$not_found_retries->{ $args->{job_id} } //= 0; |
| 154 |
if ( ++$not_found_retries->{ $args->{job_id} } >= $max_retries ) { |
| 155 |
Koha::Logger->get( { interface => 'worker' } ) |
| 156 |
->warn( sprintf "Job %s not found, no more retry", $args->{job_id} ); |
| 157 |
|
| 158 |
# nack without requeue, we do not want to process this frame again |
| 159 |
$conn->nack( { frame => $frame, requeue => 'false' } ); |
| 160 |
next; |
| 161 |
} |
| 162 |
|
| 163 |
Koha::Logger->get( { interface => 'worker' } ) |
| 164 |
->debug( sprintf "Job %s not found, will retry later", $args->{job_id} ); |
| 165 |
|
| 166 |
# nack to force requeue |
| 167 |
$conn->nack( { frame => $frame, requeue => 'true' } ); |
| 168 |
Time::HiRes::sleep(0.5); |
| 169 |
next; |
| 170 |
} |
| 171 |
$conn->ack( { frame => $frame } ); |
| 172 |
|
| 173 |
$pm->start and next; |
| 174 |
srand(); # ensure each child process begins with a new seed |
| 175 |
process_job( $job, $args ); |
| 176 |
$pm->finish; |
| 177 |
|
| 178 |
} else { |
| 179 |
my $jobs = Koha::BackgroundJobs->search( { status => 'new', queue => \@queues } ); |
| 180 |
while ( my $job = $jobs->next ) { |
| 181 |
my $args = try { |
| 182 |
$job->json->decode( $job->data ); |
| 183 |
} catch { |
| 184 |
Koha::Logger->get( { interface => 'worker' } ) |
| 185 |
->warn( sprintf "Cannot decode data for job id=%s", $job->id ); |
| 186 |
$job->status('failed')->store; |
| 187 |
return; |
| 188 |
}; |
| 189 |
|
| 190 |
next unless $args; |
| 191 |
|
| 192 |
$pm->start and next; |
| 193 |
srand(); # ensure each child process begins with a new seed |
| 194 |
process_job( $job, { job_id => $job->id, %$args } ); |
| 195 |
$pm->finish; |
| 196 |
|
| 197 |
} |
| 198 |
$pm->reap_finished_children; |
| 199 |
sleep 10; |
| 200 |
} |
| 201 |
} |
| 202 |
$conn->disconnect; |
| 203 |
$pm->wait_all_children; |
| 204 |
|
| 205 |
sub process_job { |
| 206 |
my ( $job, $args ) = @_; |
| 207 |
try { |
| 208 |
$job->process($args); |
| 209 |
} catch { |
| 210 |
Koha::Logger->get( { interface => 'worker' } ) |
| 211 |
->warn( sprintf "Uncaught exception processing job id=%s: %s", $job->id, $_ ); |
| 212 |
$job->status('failed')->store; |
| 213 |
}; |
| 214 |
} |
| 215 |
- |
|
|