From f825b0e019b3bf10f213341f13eeac078661d747 Mon Sep 17 00:00:00 2001 From: Kyle M Hall Date: Mon, 9 Aug 2021 13:46:00 -0400 Subject: [PATCH] Bug 28833: Speed up holds queue builder via parallel processing The holds queue builder can take a very long time to run on systems with many holds. For example, a partner with 124,784 unfilled ( not found ) holds, is taking about 64 minutes to run. If we run that same number of holds in 5 parallel chunks ( splitting the number of records as evenly as possible, but *not* taking into account the holds per bib ), it takes 21.5 minutes. If we use 10 loops, it takes less then 14 minutes. --- C4/HoldsQueue.pm | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/C4/HoldsQueue.pm b/C4/HoldsQueue.pm index 3774214527..f5db972015 100644 --- a/C4/HoldsQueue.pm +++ b/C4/HoldsQueue.pm @@ -33,9 +33,11 @@ use Koha::Items; use Koha::Patrons; use Koha::Libraries; -use List::Util qw(shuffle); -use List::MoreUtils qw(any); use Data::Dumper; +use List::MoreUtils qw(any); +use List::Util qw(shuffle); +use POSIX qw(ceil); +use Parallel::Loops; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS); BEGIN { @@ -176,15 +178,13 @@ Top level function that turns reserves into tmp_holdsqueue and hold_fill_targets =cut sub CreateQueue { + my $loops = shift || 1; + my $dbh = C4::Context->dbh; $dbh->do("DELETE FROM tmp_holdsqueue"); # clear the old table for new info $dbh->do("DELETE FROM hold_fill_targets"); - my $total_bibs = 0; - my $total_requests = 0; - my $total_available_items = 0; - my $num_items_mapped = 0; my $branches_to_use; my $transport_cost_matrix; @@ -202,19 +202,41 @@ sub CreateQueue { my $bibs_with_pending_requests = GetBibsWithPendingHoldRequests(); + # Split the list of bibs into groups to run in parallel + if ( $loops > 1 ) { + my $bibs_per_chunk = ceil( scalar @$bibs_with_pending_requests / $loops ); + my @chunks; + + push( @chunks, [ splice @$bibs_with_pending_requests, 0, $bibs_per_chunk ] ) while @$bibs_with_pending_requests; + push( @{$chunks[0]}, @$bibs_with_pending_requests ); # Add any remainders to the first parallel process + + my $pl = Parallel::Loops->new($loops); + $pl->foreach( \@chunks, sub { + _ProcessBiblios($_); + }); + } else { + _ProcessBiblios($bibs_with_pending_requests, $branches_to_use, $transport_cost_matrix); + } +} + +=head2 _ProcessBiblios + +=cut + +sub _ProcessBiblios { + my $bibs_with_pending_requests = shift; + my $branches_to_use = shift; + my $transport_cost_matrix = shift; + foreach my $biblionumber (@$bibs_with_pending_requests) { - $total_bibs++; my $hold_requests = GetPendingHoldRequestsForBib($biblionumber); my $available_items = GetItemsAvailableToFillHoldRequestsForBib($biblionumber, $branches_to_use); - $total_requests += scalar(@$hold_requests); - $total_available_items += scalar(@$available_items); my $item_map = MapItemsToHoldRequests($hold_requests, $available_items, $branches_to_use, $transport_cost_matrix); $item_map or next; my $item_map_size = scalar(keys %$item_map) or next; - $num_items_mapped += $item_map_size; CreatePicklistFromItemMap($item_map); AddToHoldTargetMap($item_map); if (($item_map_size < scalar(@$hold_requests )) and -- 2.30.1 (Apple Git-130)