From 87ff68a507a5424130e0d97deef658a1f5b796e0 Mon Sep 17 00:00:00 2001 From: John Doe Date: Thu, 13 Nov 2025 13:10:51 +0000 Subject: [PATCH] Bug 41245: Add ability to limit background workers by job type In practice, some job types can cause other job types to get "backed up". It wouuld be good to be able to specify background job workers for specific job types. Test plan: 1) Apply this patch 2) Stop all background jobs 3) Ensure there are new jobs for at least 2 job types 4) Run background_jobs_worker.pl with --type 5) Verify the type of job specified is the only one not processed 6) Stop the worker 7) Qeueue some new jobs of that type 8) Run background_jobs_worker.pl with --not-type 9) Verify the type of job specified is the only one *not* processed --- debian/docs/koha-worker.xml | 16 +++++ debian/scripts/koha-worker | 82 ++++++++++++++++++++------ misc/workers/background_jobs_worker.pl | 29 ++++++++- 3 files changed, 106 insertions(+), 21 deletions(-) diff --git a/debian/docs/koha-worker.xml b/debian/docs/koha-worker.xml index 56a5f0cb82f..6c8c376b1e9 100644 --- a/debian/docs/koha-worker.xml +++ b/debian/docs/koha-worker.xml @@ -27,6 +27,8 @@ || + + | | @@ -67,6 +69,20 @@ Note: There used to be a queue called elastic_index, but since the introduction of koha-es-indexer this queue should not be active. + + + + Allows specifying the background job type this worker. + Current types are: pseudonymize_statistic, marc_import_commit_batch, update_elastic_index, update_holds_queue_for_biblios, batch_item_record_modification + + + + + + Allows specifying the background job type this worker to not process. + This option operates as the inverse of --type. + + | diff --git a/debian/scripts/koha-worker b/debian/scripts/koha-worker index aa51c90c13f..77c7d1c0f4b 100755 --- a/debian/scripts/koha-worker +++ b/debian/scripts/koha-worker @@ -40,7 +40,7 @@ $scriptname This script lets you manage the worker daemon for your Koha instances. Usage: -$scriptname [--start|--stop|--restart] [--queue queue_name] [--quiet|-q] instancename1 [instancename2...] +$scriptname [--start|--stop|--restart] [--queue queue_name] [--type job_type] [--not-type job_type] [--quiet|-q] instancename1 [instancename2...] $scriptname --status instancename1 [instancename2...] $scriptname -h|--help @@ -49,6 +49,8 @@ $scriptname -h|--help --restart Restart the worker daemon for the specified instances --queue Specify the queue/worker to restart - 'default' is used if not specified current queues are: default, long_tasks + --type Specify the job type for this worker to process + --not-type Specify the job type for this worker to skip processing --status Show the status of the worker for the specified instances --quiet|-q Make the script quiet about non existent instance names (useful for calling from another scripts). @@ -64,10 +66,12 @@ start_worker() { local name=$1 local queue=$2 + local job_type=$3 + local not_job_type=$4 - if ! is_worker_running $name $queue; then + if ! is_worker_running $name $queue $job_type $not_job_type; then - worker_name=`get_worker_name $name $queue` + worker_name=`get_worker_name $name $queue $job_type $notjob__type` export KOHA_CONF="/etc/koha/sites/$name/koha-conf.xml" @@ -78,15 +82,22 @@ start_worker() --verbose=1 --respawn --delay=30 \ --user=$name-koha.$name-koha" - log_daemon_msg "Starting Koha worker daemon for $name ($queue)" + local job_limits="" + if $job_type; then + job_limits="--job-type $job_type" + elif $not_job_type; then + job_limits="--not-job-type $not_job_type" + fi + + log_daemon_msg "Starting Koha worker daemon for $name --queue $queue $job_limits" - if daemon $DAEMONOPTS -- $worker_DAEMON --queue ${queue}; then + if daemon $DAEMONOPTS -- $worker_DAEMON --queue ${queue} $job_limits; then log_end_msg 0 else log_end_msg 1 fi else - log_daemon_msg "Error: worker already running for $name ($queue)" + log_daemon_msg "Error: worker already running for --queue $queue $job_limits" log_end_msg 1 fi } @@ -95,6 +106,8 @@ stop_worker() { local name=$1 local queue=$2 + local job_type=$3 + local not_job_type=$4 if is_worker_running $name $queue; then export KOHA_CONF="/etc/koha/sites/$name/koha-conf.xml" @@ -108,15 +121,22 @@ stop_worker() --verbose=1 --respawn --delay=30 \ --user=$name-koha.$name-koha" - log_daemon_msg "Stopping Koha worker daemon for $name ($queue)" + local job_limits="" + if $job_type; then + job_limits="--job-type $job_type" + elif $not_job_type; then + job_limits="--not-job-type $not_job_type" + fi + + log_daemon_msg "Stopping Koha worker daemon for --queue $queue $job_limits" - if daemon $DAEMONOPTS --stop -- $worker_DAEMON --queue ${queue}; then + if daemon $DAEMONOPTS --stop -- $worker_DAEMON --queue ${queue} $job_limits; then log_end_msg 0 else log_end_msg 1 fi else - log_daemon_msg "Error: worker not running for $name ($queue)" + log_daemon_msg "Error: worker not running for --queue $queue $job_limits" log_end_msg 1 fi } @@ -125,6 +145,8 @@ restart_worker() { local name=$1 local queue=$2 + local job_type=$3 + local not_job_type=$4 if is_worker_running $name $queue; then export KOHA_CONF="/etc/koha/sites/$name/koha-conf.xml" @@ -138,15 +160,22 @@ restart_worker() --verbose=1 --respawn --delay=30 \ --user=$name-koha.$name-koha" - log_daemon_msg "Restarting Koha worker daemon for $name ($queue)" + local job_limits="" + if $job_type; then + job_limits="--job-type $job_type" + elif $not_job_type; then + job_limits="--not-job-type $not_job_type" + fi + + log_daemon_msg "Restarting Koha worker daemon for --queue $queue $job_limits" - if daemon $DAEMONOPTS --restart -- $worker_DAEMON --queue ${queue}; then + if daemon $DAEMONOPTS --restart -- $worker_DAEMON --queue ${queue} $job_limits; then log_end_msg 0 else log_end_msg 1 fi else - log_warning_msg "Worker not running for $name ($queue)." + log_warning_msg "Worker not running for --queue $queue $job_limits." start_worker $name $queue fi } @@ -155,12 +184,21 @@ worker_status() { local name=$1 local queue=$2 + local job_type=$3 + local not_job_type=$4 + + local job_limits="" + if $job_type; then + job_limits="--job-type $job_type" + elif $not_job_type; then + job_limits="--not-job-type $not_job_type" + fi if is_worker_running ${name} ${queue}; then - log_daemon_msg "worker running for ${name} ($queue)" + log_daemon_msg "worker running for ${name} --queue $queue $job_limits" log_end_msg 0 else - log_daemon_msg "worker not running for ${name} ($queue)" + log_daemon_msg "worker not running for ${name} --queue $queue $job_limits" log_end_msg 3 fi } @@ -177,6 +215,8 @@ set_action() op="" queue="default" quiet="no" +job_type="" +not_job_type="" # Read command line parameters while [ $# -gt 0 ]; do @@ -202,6 +242,12 @@ while [ $# -gt 0 ]; do --queue) queue="$2" shift 2 ;; + --type) + type="$2" + shift 2 ;; + --not-type) + type="$2" + shift 2 ;; -*) die "Error: invalid option switch ($1)" ;; *) @@ -231,16 +277,16 @@ if [ $# -gt 0 ]; then case $op in "start") - start_worker $name $queue + start_worker $name $queue $job_type $not_job_type ;; "stop") - stop_worker $name $queue + stop_worker $name $queue $job_type $not_job_type ;; "restart") - restart_worker $name $queue + restart_worker $name $queue $job_type $not_job_type ;; "status") - worker_status $name $queue + worker_status $name $queue $job_type $not_job_type esac else diff --git a/misc/workers/background_jobs_worker.pl b/misc/workers/background_jobs_worker.pl index afb5f603e54..8e470bdcba9 100755 --- a/misc/workers/background_jobs_worker.pl +++ b/misc/workers/background_jobs_worker.pl @@ -21,7 +21,7 @@ background_jobs_worker.pl - Worker script that will process background jobs =head1 SYNOPSIS -./background_jobs_worker.pl [--queue QUEUE] [-m|--max-processes MAX_PROCESSES] +./background_jobs_worker.pl [--queue QUEUE] [-m|--max-processes MAX_PROCESSES] [-t JOB_TYPE] [-n NOT_JOB_TYPE] =head1 DESCRIPTION @@ -32,6 +32,10 @@ You can specify some queues only (using --queue, which is repeatable) if you wan --m --max-processes specifies how many jobs to process simultaneously +-t --type specifies what job type to process, repeatable + +-n --not-type specifies what job type to skip processing, repeatable + Max processes will be set from the command line option, the environment variable MAX_PROCESSES, or the koha-conf file, in that order of precedence. By default the script will only run one job at a time. @@ -67,7 +71,7 @@ use C4::Context; $SIG{'PIPE'} = 'IGNORE'; # See BZ 35111; added to ignore PIPE error when connection lost on Ubuntu. -my ( $help, @queues ); +my ( $help, @queues, @job_types, @not_job_types ); my $max_processes = $ENV{MAX_PROCESSES}; $max_processes ||= C4::Context->config('background_jobs_worker')->{max_processes} @@ -82,6 +86,8 @@ GetOptions( 'm|max-processes=i' => \$max_processes, 'h|help' => \$help, 'queue=s' => \@queues, + 't|type=s' => \@job_types, + 'n|not-type=s' => \@not_job_types, ) || pod2usage(1); pod2usage(0) if $help; @@ -174,6 +180,15 @@ while (1) { next; } + if ( $job && @job_types && !grep { $job->type } @job_types ) { + $conn->nack( { frame => $frame, requeue => 'true' } ); + next; + } + if ( $job && @not_job_types && grep { $job->type } @not_job_types ) { + $conn->nack( { frame => $frame, requeue => 'true' } ); + next; + } + unless ($job) { $not_found_retries->{ $args->{job_id} } //= 0; if ( ++$not_found_retries->{ $args->{job_id} } >= $max_retries ) { @@ -201,7 +216,15 @@ while (1) { $pm->finish; } else { - my $jobs = Koha::BackgroundJobs->search( { status => 'new', queue => \@queues } ); + my $params = { status => 'new', queue => \@queues }; + + if (@job_types) { + $params->{type} = { '-in' => \@job_types }; + } elsif (@not_job_types) { + $params->{type} = { '-not_in' => \@job_types }; + } + + my $jobs = Koha::BackgroundJobs->search($params); while ( my $job = $jobs->next ) { my $args = try { $job->json->decode( $job->data ); -- 2.50.1 (Apple Git-155)