From 1a980eb2a78f013e45c20520b3dec84ec54c5b6e Mon Sep 17 00:00:00 2001 From: Kyle M Hall <kyle@bywatersolutions.com> Date: Tue, 3 Jan 2023 15:35:42 +0000 Subject: [PATCH] Bug 32558: Add ability for background_jobs_worker.pl to process multiple jobs simultaneously up to a limit Content-Type: text/plain; charset=utf-8 Right now background_jobs_worker.pl only processes jobs in serial. It would make sense to handle jobs in parallel up to a user definable limit. Test Plan: 1) Apply this patch 2) Stop background_jobs_worker.pl 3) Generate some background jobs by editing records, placing holds, etc 4) Watch processes in a new terminal: watch -n 0.1 'ps aux | grep background_jobs_worker.pl' 5) Run background_jobs_worker.pl with parameter -m 3 or some other number of max processes 6) Note the multiple forked processes in the ps output Test notes - also tested the following on KTD: 1. Stop background_jobs_worker.pl 2. Edit /etc/koha/sites/kohadev/koha-conf.xml - set max_processes to 10 3. Generate some background jobs 4. Watch processes in a new terminal: watch -n 0.1 'ps aux | grep background_jobs_worker.pl' 5. Restart all 6. Confirm multiple forked processes in the ps output Both methods work as expected and generate multiple forked processes based on the value set for max processes. Signed-off-by: emlam <emily.lamancusa@montgomerycountymd.gov> Signed-off-by: Marcel de Rooy <m.de.rooy@rijksmuseum.nl> --- debian/templates/koha-conf-site.xml.in | 5 +++++ etc/koha-conf.xml | 5 +++++ misc/background_jobs_worker.pl | 22 +++++++++++++++++++++- 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/debian/templates/koha-conf-site.xml.in b/debian/templates/koha-conf-site.xml.in index bdb994c517..187150fa6a 100644 --- a/debian/templates/koha-conf-site.xml.in +++ b/debian/templates/koha-conf-site.xml.in @@ -457,6 +457,11 @@ __END_SRU_PUBLICSERVER__ <vhost>__MESSAGE_BROKER_VHOST__</vhost> </message_broker> + <background_jobs_worker> + <!-- Max simultaneous processes per worker --> + <max_processes>1</max_processes> + </background_jobs_worker> + <do_not_remove_cookie>__KEEP_COOKIE__</do_not_remove_cookie> <do_not_remove_cookie>catalogue_editor_\d+</do_not_remove_cookie> <!-- Uncomment lines like hereunder to not clear cookies at logout: diff --git a/etc/koha-conf.xml b/etc/koha-conf.xml index 3e5623cb96..beb5ccd80c 100644 --- a/etc/koha-conf.xml +++ b/etc/koha-conf.xml @@ -274,6 +274,11 @@ <vhost></vhost> </message_broker> + <background_jobs_worker> + <!-- Max simultaneous processes per worker --> + <max_processes>1</max_processes> + </background_jobs_worker> + <do_not_remove_cookie>catalogue_editor_\d+</do_not_remove_cookie> <!-- Uncomment lines like hereunder to not clear cookies at logout: The cookie name is case sensitive. diff --git a/misc/background_jobs_worker.pl b/misc/background_jobs_worker.pl index b98d5f135f..add96711c2 100755 --- a/misc/background_jobs_worker.pl +++ b/misc/background_jobs_worker.pl @@ -21,7 +21,7 @@ background_jobs_worker.pl - Worker script that will process background jobs =head1 SYNOPSIS -./background_jobs_worker.pl [--queue QUEUE] +./background_jobs_worker.pl [--queue QUEUE] [-m|--max-processes MAX_PROCESSES] =head1 DESCRIPTION @@ -30,6 +30,11 @@ or if a Stomp server is not active it will poll the database every 10s for new j You can specify some queues only (using --queue, which is repeatable) if you want to run several workers that will handle their own jobs. +--m --max-processes specifies how many jobs to process simultaneously + +Max processes will be set from the command line option, the environment variable MAX_PROCESSES, or the koha-conf file, in that order of precedence. +By default the script will only run one job at a time. + =head1 OPTIONS =over @@ -52,16 +57,25 @@ use JSON qw( decode_json ); use Try::Tiny; use Pod::Usage; use Getopt::Long; +use Parallel::ForkManager; use Koha::Logger; use Koha::BackgroundJobs; +use C4::Context; my ( $help, @queues ); + +my $max_processes = $ENV{MAX_PROCESSES}; +$max_processes ||= C4::Context->config('background_jobs_worker')->{max_processes} if C4::Context->config('background_jobs_worker'); +$max_processes ||= 1; + GetOptions( + 'm|max-processes=i' => \$max_processes, 'h|help' => \$help, 'queue=s' => \@queues, ) || pod2usage(1); + pod2usage(0) if $help; unless (@queues) { @@ -75,6 +89,8 @@ try { warn sprintf "Cannot connect to the message broker, the jobs will be processed anyway (%s)", $_; }; +my $pm = Parallel::ForkManager->new($max_processes); + if ( $conn ) { # FIXME cf note in Koha::BackgroundJob about $namespace my $namespace = C4::Context->config('memcached_namespace'); @@ -115,7 +131,9 @@ while (1) { next; } + $pm->start and next; process_job( $job, $args ); + $pm->finish; } else { my $jobs = Koha::BackgroundJobs->search({ status => 'new', queue => \@queues }); @@ -130,7 +148,9 @@ while (1) { next unless $args; + $pm->start and next; process_job( $job, { job_id => $job->id, %$args } ); + $pm->finish; } sleep 10; -- 2.30.2