Attachment 161326 Details for Bug 29507 – Bug 29507: Speed up auto renew cronjob via parallel processing

[patch] Bug 29507: Speed up auto renew cronjob via parallel processing

Bug-29507-Speed-up-auto-renew-cronjob-via-parallel.patch (text/plain), 4.68 KB, created by Matt Blenkinsop on 2024-01-24 11:33:52 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Matt Blenkinsop

Created: 2024-01-24 11:33:52 UTC

Size: 4.68 KB

patch

obsolete

>From 362dd6cec4a9a2b0d7bf1721178188d815b6ebbc Mon Sep 17 00:00:00 2001
>From: Nick Clemens <nick@bywatersolutions.com>
>Date: Tue, 23 Jan 2024 18:47:13 +0000
>Subject: [PATCH] Bug 29507: Speed up auto renew cronjob via parallel
> processing
>
>The cron can take a very long time to run on systems with many issues.
>For example, a partner with ~250k auto_renew issues is taking about 9 hours to run.
>
>If we run that same number of issues in 5 parallel chunks
>( splitting the number of issues as evenly as possible ), it could take under 2 hours.
>
>Test Plan:
>1) Generate a number of issues marked for auto_renew
>2) Run the automatic_renewals.pl, use the `time` utility to track how much time it took to run
>3) Set parallel_loops to 10 in auto_renew_cronjob section of config in koha-conf
>4) Repeat step 2, note the improvement in speed
>5) Experiment with other values
>
>Signed-off-by: Matt Blenkinsop <matt.blenkinsop@ptfs-europe.com>
>---
> debian/templates/koha-conf-site.xml.in |  4 ++
> etc/koha-conf.xml                      |  4 ++
> misc/cronjobs/automatic_renewals.pl    | 60 ++++++++++++++++++++++----
> 3 files changed, 60 insertions(+), 8 deletions(-)
>
>diff --git a/debian/templates/koha-conf-site.xml.in b/debian/templates/koha-conf-site.xml.in
>index c35b6d84325..32ceb66d420 100644
>--- a/debian/templates/koha-conf-site.xml.in
>+++ b/debian/templates/koha-conf-site.xml.in
>@@ -483,5 +483,9 @@ __END_SRU_PUBLICSERVER__
> 
>  <mfa_range>1</mfa_range>
> 
>+ <auto_renew_cronjob>
>+   <parallel_loops_count>1</parallel_loops_count>
>+ </auto_renew_cronjob>
>+
> </config>
> </yazgfs>
>diff --git a/etc/koha-conf.xml b/etc/koha-conf.xml
>index f7cdd2f8d00..2c02ae263f8 100644
>--- a/etc/koha-conf.xml
>+++ b/etc/koha-conf.xml
>@@ -294,5 +294,9 @@
> 
>  <mfa_range>1</mfa_range>
> 
>+ <auto_renew_cronjob>
>+   <parallel_loops_count>1</parallel_loops_count>
>+ </auto_renew_cronjob>
>+
> </config>
> </yazgfs>
>diff --git a/misc/cronjobs/automatic_renewals.pl b/misc/cronjobs/automatic_renewals.pl
>index 8661d4f47bf..cdb2e1167cc 100755
>--- a/misc/cronjobs/automatic_renewals.pl
>+++ b/misc/cronjobs/automatic_renewals.pl
>@@ -75,7 +75,8 @@ chosen 'Digests only' on the advance messages.
> =cut
> 
> use Modern::Perl;
>-use Pod::Usage qw( pod2usage );
>+use Parallel::ForkManager;
>+use Pod::Usage   qw( pod2usage );
> use Getopt::Long qw( GetOptions );
> 
> use Koha::Script -cron;
>@@ -135,7 +136,7 @@ $verbose = 1 unless $verbose or $confirm;
> print "Test run only\n" unless $confirm;
> 
> print "getting auto renewals\n" if $verbose;
>-my $auto_renews = Koha::Checkouts->search(
>+my @auto_renews = Koha::Checkouts->search(
>     {
>         auto_renew                   => 1,
>         'patron.autorenew_checkouts' => 1,
>@@ -143,13 +144,58 @@ my $auto_renews = Koha::Checkouts->search(
>     {
>         join => ['patron','item']
>     }
>-);
>-print "found " . $auto_renews->count . " auto renewals\n" if $verbose;
>+)->as_list;
>+print "found " . scalar @auto_renews . " auto renewals\n" if $verbose;
>+
>+my $cron_options = C4::Context->config('auto_renew_cronjob');
>+my $loops = $cron_options ? $cron_options->{parallel_loops_count} // 1 : 1;
>+
>+# Split the list of issues into chunks to run in parallel
>+my @chunks;
>+if ( $loops > 1 ) {
>+    my $i              = 0;
>+    my $borrowernumber = 0;
>+    while (@auto_renews) {
>+        my $auto_renew = pop(@auto_renews);
>+        if ( $borrowernumber != $auto_renew->borrowernumber ) {
>+            $i++ if $borrowernumber;
>+            $borrowernumber = $auto_renew->borrowernumber;
>+        }
>+        $i = 0 if $i >= $loops;
>+        push( @{ $chunks[$i] }, $auto_renew );
>+    }
>+    my $pm = Parallel::ForkManager->new($loops);
>+  DATA_LOOP:
>+    foreach my $chunk (@chunks) {
>+        my $pid = $pm->start and next DATA_LOOP;
>+        _ProcessRenewals($chunk);
>+        $pm->finish;
>+    }
>+    $pm->wait_all_children;
>+}
>+else {
>+    _ProcessRenewals( \@auto_renews );
>+}
>+
>+cronlogaction({ action => 'End', info => "COMPLETED" });
>+
>+
>+=head1 METHODS
>+
>+=head2 _ProcessRenewals
>+
>+    Internal method to process the queue in chunks
>+
>+=cut
>+
>+sub _ProcessRenewals {
>+    my $auto_renew_issues = shift;
> 
> my $renew_digest = {};
> my %report;
> my @item_renewal_ids;
>-while ( my $auto_renew = $auto_renews->next ) {
>+
>+    foreach my $auto_renew (@$auto_renew_issues) {
>     print "examining item '" . $auto_renew->itemnumber . "' to auto renew\n" if $verbose;
> 
>     my ( $borrower_preferences, $wants_messages, $wants_digest ) = ( undef, 0, 0 );
>@@ -295,9 +341,7 @@ if ( $send_notices && $confirm ) {
>     }
> }
> 
>-cronlogaction({ action => 'End', info => "COMPLETED" });
>-
>-=head1 METHODS
>+}
> 
> =head2 send_digests
> 
>-- 
>2.37.1 (Apple Git-137.1)

Actions: View | Diff | Splinter Review

Attachments on bug 29507: 127743 | 127744 | 132056 | 150692 | 157772 | 161314 | 161315 | 161326 | 161327 | 165091 | 165092 | 168362 | 168363 | 168381 | 168382