From b953c57944c6253b21c206f82305610fc7adad52 Mon Sep 17 00:00:00 2001
From: Kyle M Hall <kyle@bywatersolutions.com>
Date: Mon, 9 Aug 2021 13:46:00 -0400
Subject: [PATCH] Bug 28833: Speed up holds queue builder via parallel
 processing

The holds queue builder can take a very long time to run on systems with many holds. For example, a partner with 124,784 unfilled ( not found ) holds, is taking about 64 minutes to run. If we run that same number of holds in 5 parallel chunks ( splitting the number of records as evenly as possible, but *not* taking into account the holds per bib ), it takes 21.5 minutes. If we use 10 loops, it takes less then 14 minutes.
---
 C4/HoldsQueue.pm | 63 ++++++++++++++++++++++++++++++------------------
 1 file changed, 39 insertions(+), 24 deletions(-)

diff --git a/C4/HoldsQueue.pm b/C4/HoldsQueue.pm
index 3774214527..9c3212f589 100644
--- a/C4/HoldsQueue.pm
+++ b/C4/HoldsQueue.pm
@@ -33,9 +33,11 @@ use Koha::Items;
 use Koha::Patrons;
 use Koha::Libraries;
 
-use List::Util qw(shuffle);
-use List::MoreUtils qw(any);
 use Data::Dumper;
+use List::MoreUtils qw(any);
+use List::Util qw(shuffle);
+use POSIX qw(ceil);
+use Parallel::Loops;
 
 use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
 BEGIN {
@@ -202,29 +204,42 @@ sub CreateQueue {
 
     my $bibs_with_pending_requests = GetBibsWithPendingHoldRequests();
 
-    foreach my $biblionumber (@$bibs_with_pending_requests) {
-        $total_bibs++;
-        my $hold_requests   = GetPendingHoldRequestsForBib($biblionumber);
-        my $available_items = GetItemsAvailableToFillHoldRequestsForBib($biblionumber, $branches_to_use);
-        $total_requests        += scalar(@$hold_requests);
-        $total_available_items += scalar(@$available_items);
-
-        my $item_map = MapItemsToHoldRequests($hold_requests, $available_items, $branches_to_use, $transport_cost_matrix);
-        $item_map  or next;
-        my $item_map_size = scalar(keys %$item_map)
-          or next;
-
-        $num_items_mapped += $item_map_size;
-        CreatePicklistFromItemMap($item_map);
-        AddToHoldTargetMap($item_map);
-        if (($item_map_size < scalar(@$hold_requests  )) and
-            ($item_map_size < scalar(@$available_items))) {
-            # DOUBLE CHECK, but this is probably OK - unfilled item-level requests
-            # FIXME
-            #warn "unfilled requests for $biblionumber";
-            #warn Dumper($hold_requests), Dumper($available_items), Dumper($item_map);
+    # Split the list of bibs into groups to run in parallel
+    my $chunks_count = 10;
+    my $bibs_per_chunk = ceil( scalar @$bibs_with_pending_requests / $chunks_count );
+    my @chunks;
+
+    push( @chunks, [ splice @$bibs_with_pending_requests, 0, $bibs_per_chunk ] ) while @$bibs_with_pending_requests;
+    push( @{$chunks[0]}, @$bibs_with_pending_requests ); # Add any remainders to the first parallel process
+
+    my $pl = Parallel::Loops->new($chunks_count);
+    $pl->foreach( \@chunks, sub {
+        $bibs_with_pending_requests = $_;
+
+        foreach my $biblionumber (@$bibs_with_pending_requests) {
+            $total_bibs++;
+            my $hold_requests   = GetPendingHoldRequestsForBib($biblionumber);
+            my $available_items = GetItemsAvailableToFillHoldRequestsForBib($biblionumber, $branches_to_use);
+            $total_requests        += scalar(@$hold_requests);
+            $total_available_items += scalar(@$available_items);
+
+            my $item_map = MapItemsToHoldRequests($hold_requests, $available_items, $branches_to_use, $transport_cost_matrix);
+            $item_map  or next;
+            my $item_map_size = scalar(keys %$item_map)
+              or next;
+
+            $num_items_mapped += $item_map_size;
+            CreatePicklistFromItemMap($item_map);
+            AddToHoldTargetMap($item_map);
+            if (($item_map_size < scalar(@$hold_requests  )) and
+                ($item_map_size < scalar(@$available_items))) {
+                # DOUBLE CHECK, but this is probably OK - unfilled item-level requests
+                # FIXME
+                #warn "unfilled requests for $biblionumber";
+                #warn Dumper($hold_requests), Dumper($available_items), Dumper($item_map);
+            }
         }
-    }
+    });
 }
 
 =head2 GetBibsWithPendingHoldRequests
-- 
2.30.1 (Apple Git-130)