Line 0
Link Here
|
|
|
1 |
package Koha::BackgroundWorker; |
2 |
|
3 |
# Copyright 2024 Rijksmuseum |
4 |
# |
5 |
# This file is part of Koha. |
6 |
# |
7 |
# Koha is free software; you can redistribute it and/or modify it |
8 |
# under the terms of the GNU General Public License as published by |
9 |
# the Free Software Foundation; either version 3 of the License, or |
10 |
# (at your option) any later version. |
11 |
# |
12 |
# Koha is distributed in the hope that it will be useful, but |
13 |
# WITHOUT ANY WARRANTY; without even the implied warranty of |
14 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
15 |
# GNU General Public License for more details. |
16 |
# |
17 |
# You should have received a copy of the GNU General Public License |
18 |
# along with Koha; if not, see <http://www.gnu.org/licenses>. |
19 |
|
20 |
use Modern::Perl; |
21 |
use JSON qw( decode_json ); |
22 |
use Time::HiRes; |
23 |
use Parallel::ForkManager; |
24 |
use Try::Tiny qw(try catch); |
25 |
|
26 |
use C4::Context; |
27 |
use Koha::BackgroundJobs; |
28 |
use Koha::Logger; |
29 |
|
30 |
=head1 NAME |
31 |
|
32 |
Koha::BackgroundWorker - Centralized code for background worker scripts |
33 |
|
34 |
=head1 SYNOPSIS |
35 |
|
36 |
use Koha::BackgroundWorker; |
37 |
Koha::BackgroundWorker->run($params); |
38 |
|
39 |
=head1 DESCRIPTION |
40 |
|
41 |
This module allows you to use centralized worker code reading |
42 |
from Rabbit queues and specifying your own callback. |
43 |
|
44 |
|
45 |
=head1 METHODS |
46 |
|
47 |
=head2 run |
48 |
|
49 |
Koha::BackgroundWorker->run($params); |
50 |
|
51 |
Main method to start the worker. |
52 |
Available parameters: |
53 |
- callback |
54 |
- queues |
55 |
- max_processes |
56 |
- max_retries |
57 |
- mq_timeout |
58 |
|
59 |
=cut |
60 |
|
61 |
sub run { |
62 |
my ( $class, $params ) = @_; |
63 |
my $self = bless $params // {}, $class; |
64 |
$self->_init; |
65 |
my @queues = @{ $self->{queues} }; |
66 |
my $callback = $self->{callback}; |
67 |
|
68 |
my $pm = Parallel::ForkManager->new( $self->{max_processes} ); |
69 |
my $not_found_retries = {}; |
70 |
|
71 |
my $conn; |
72 |
try { |
73 |
$conn = Koha::BackgroundJob->connect; |
74 |
} catch { |
75 |
warn sprintf "Cannot connect to the message broker, the jobs will be processed anyway (%s)", $_; |
76 |
}; |
77 |
if ($conn) { |
78 |
|
79 |
# FIXME cf note in Koha::BackgroundJob about $namespace |
80 |
my $namespace = C4::Context->config('memcached_namespace'); |
81 |
for my $queue (@queues) { |
82 |
$conn->subscribe( |
83 |
{ |
84 |
destination => sprintf( "/queue/%s-%s", $namespace, $queue ), |
85 |
ack => 'client', |
86 |
'prefetch-count' => 1, |
87 |
} |
88 |
); |
89 |
} |
90 |
} |
91 |
|
92 |
### The main loop |
93 |
while (1) { |
94 |
if ($conn) { |
95 |
my $frame = $conn->receive_frame( { timeout => $self->{mq_timeout} } ); |
96 |
if ( !defined $frame ) { |
97 |
|
98 |
# timeout or connection issue? |
99 |
$pm->reap_finished_children; |
100 |
next; # will reconnect automatically |
101 |
} |
102 |
|
103 |
my $args = try { |
104 |
my $body = $frame->body; |
105 |
decode_json($body); # TODO Should this be from_json? Check utf8 flag. |
106 |
} catch { |
107 |
Koha::Logger->get( { interface => 'worker' } )->warn( sprintf "Frame not processed - %s", $_ ); |
108 |
return; |
109 |
}; |
110 |
|
111 |
unless ($args) { |
112 |
Koha::Logger->get( { interface => 'worker' } ) |
113 |
->warn( sprintf "Frame does not have correct args, ignoring it" ); |
114 |
$conn->nack( { frame => $frame, requeue => 'false' } ); |
115 |
next; |
116 |
} |
117 |
|
118 |
my $job = Koha::BackgroundJobs->find( $args->{job_id} ); |
119 |
|
120 |
if ( $job && $job->status ne 'new' ) { |
121 |
Koha::Logger->get( { interface => 'worker' } ) |
122 |
->warn( sprintf "Job %s has wrong status %s", $args->{job_id}, $job->status ); |
123 |
|
124 |
# nack without requeue, we do not want to process this frame again |
125 |
$conn->nack( { frame => $frame, requeue => 'false' } ); |
126 |
next; |
127 |
} |
128 |
|
129 |
unless ($job) { |
130 |
$not_found_retries->{ $args->{job_id} } //= 0; |
131 |
if ( ++$not_found_retries->{ $args->{job_id} } >= $self->{max_retries} ) { |
132 |
Koha::Logger->get( { interface => 'worker' } ) |
133 |
->warn( sprintf "Job %s not found, no more retry", $args->{job_id} ); |
134 |
|
135 |
# nack without requeue, we do not want to process this frame again |
136 |
$conn->nack( { frame => $frame, requeue => 'false' } ); |
137 |
next; |
138 |
} |
139 |
|
140 |
Koha::Logger->get( { interface => 'worker' } ) |
141 |
->debug( sprintf "Job %s not found, will retry later", $args->{job_id} ); |
142 |
|
143 |
# nack to force requeue |
144 |
$conn->nack( { frame => $frame, requeue => 'true' } ); |
145 |
Time::HiRes::sleep(0.5); |
146 |
next; |
147 |
} |
148 |
$conn->ack( { frame => $frame } ); |
149 |
|
150 |
$pm->start and next; |
151 |
srand(); # ensure each child process begins with a new seed |
152 |
&$callback( $job, $args ); |
153 |
$pm->finish; |
154 |
|
155 |
} else { |
156 |
my $jobs = Koha::BackgroundJobs->search( { status => 'new', queue => \@queues } ); |
157 |
while ( my $job = $jobs->next ) { |
158 |
my $args = try { |
159 |
$job->json->decode( $job->data ); |
160 |
} catch { |
161 |
Koha::Logger->get( { interface => 'worker' } ) |
162 |
->warn( sprintf "Cannot decode data for job id=%s", $job->id ); |
163 |
$job->status('failed')->store; |
164 |
return; |
165 |
}; |
166 |
|
167 |
next unless $args; |
168 |
|
169 |
$pm->start and next; |
170 |
srand(); # ensure each child process begins with a new seed |
171 |
&$callback( $job, { job_id => $job->id, %$args } ); |
172 |
$pm->finish; |
173 |
|
174 |
} |
175 |
sleep 10; |
176 |
} |
177 |
} |
178 |
|
179 |
# Work is done |
180 |
$conn->disconnect; |
181 |
$pm->wait_all_children; |
182 |
} |
183 |
|
184 |
sub _init { |
185 |
my ($self) = @_; |
186 |
$self->{max_processes} ||= C4::Context->config('background_jobs_worker')->{max_processes} |
187 |
if C4::Context->config('background_jobs_worker'); |
188 |
$self->{max_processes} ||= 1; |
189 |
$self->{mq_timeout} //= 10; |
190 |
$self->{max_retries} //= 10; |
191 |
} |
192 |
|
193 |
1; |