From 9140165dcb406b2f0203b0bd7a8eef76c47b7778 Mon Sep 17 00:00:00 2001 From: Kyle M Hall Date: Mon, 8 Apr 2013 14:20:53 -0400 Subject: [PATCH] Bug 10004 - Delay batch record count updates until the end of the z39.50 search [3.8.x] [3.10.x] Right now when a z39.50 search is run, _update_batch_record_counts is run for each record. If multiple duplicate bibs are in the search, it will be run for each of them. For Koha installations with very large numbers of batches, this can cause significant slowdowns. It would be better to delay this update until the very end so we can update each batch only once. In addition, we can fork this process, as it is not neccessary to wait for it before displaying results. Test Plan: 1) Apply patch 2) Perform a z39.50 search --- C4/Breeding.pm | 7 ++++--- C4/ImportBatch.pm | 6 ++++++ acqui/z3950_search.pl | 32 ++++++++++++++++++++++++++++++-- cataloguing/z3950_search.pl | 30 +++++++++++++++++++++++++++++- 4 files changed, 69 insertions(+), 6 deletions(-) diff --git a/C4/Breeding.pm b/C4/Breeding.pm index 0653d50..1e2f9e7 100644 --- a/C4/Breeding.pm +++ b/C4/Breeding.pm @@ -25,6 +25,7 @@ use C4::Koha; use C4::Charset; use MARC::File::USMARC; use C4::ImportBatch; +use List::MoreUtils qw(uniq); use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS); @@ -71,7 +72,7 @@ C4::Breeding : module to add biblios to import_records via =cut sub ImportBreeding { - my ($marcrecords,$overwrite_biblio,$filename,$encoding,$z3950random,$batch_type) = @_; + my ( $marcrecords, $overwrite_biblio, $filename, $encoding, $z3950random, $batch_type, $update_counts ) = @_; my @marcarray = split /\x1D/, $marcrecords; my $dbh = C4::Context->dbh; @@ -137,7 +138,7 @@ sub ImportBreeding { if ($breedingid && $overwrite_biblio eq '1') { ModBiblioInBatch($breedingid, $marcrecord); } else { - my $import_id = AddBiblioToBatch($batch_id, $imported, $marcrecord, $encoding, $z3950random); + my $import_id = AddBiblioToBatch($batch_id, $imported, $marcrecord, $encoding, $z3950random, $update_counts); $breedingid = $import_id; } $imported++; @@ -145,7 +146,7 @@ sub ImportBreeding { } } } - return ($notmarcrecord,$alreadyindb,$alreadyinfarm,$imported,$breedingid); + return ( $notmarcrecord, $alreadyindb, $alreadyinfarm, $imported, $breedingid, $batch_id ); } diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm index 386bac9..1c17784 100644 --- a/C4/ImportBatch.pm +++ b/C4/ImportBatch.pm @@ -73,6 +73,8 @@ BEGIN { SetImportRecordStatus GetImportRecordMatches SetImportRecordMatches + + UpdateBatchRecordCounts ); } @@ -1310,6 +1312,10 @@ sub _update_batch_record_counts { $sth->finish(); } +sub UpdateBatchRecordCounts { + return _update_batch_record_counts( @_ ); +} + sub _get_commit_action { my ($overlay_action, $nomatch_action, $item_action, $overlay_status, $import_record_id) = @_; diff --git a/acqui/z3950_search.pl b/acqui/z3950_search.pl index 9dc8b6e..ebc7c1e 100755 --- a/acqui/z3950_search.pl +++ b/acqui/z3950_search.pl @@ -32,6 +32,7 @@ use C4::Koha; use C4::Charset; use C4::Bookseller qw/ GetBookSellerFromId /; use ZOOM; +use List::MoreUtils qw(uniq); my $input = new CGI; my ( $template, $loggedinuser, $cookie ) = get_template_and_user( @@ -230,6 +231,7 @@ warn "query ".$query if $DEBUG; warn "# nremaining = $nremaining\n" if $DEBUG; + my @batch_id; while ( $nremaining-- ) { my $k; @@ -289,9 +291,13 @@ warn "query ".$query if $DEBUG; $oldbiblio->{issn} =~ s/\(/ \(/g if $oldbiblio->{issn}; my ( $notmarcrecord, $alreadyindb, $alreadyinfarm, - $imported, $breedingid + $imported, $breedingid, $batch_id ) - = ImportBreeding( $marcdata, 2, $serverhost[$k], $encoding[$k], $random, 'z3950' ); + = ImportBreeding( $marcdata, 2, $serverhost[$k], $encoding[$k], $random, 'z3950', 0 ); + + # Delay the updating of batch record counts until the very end + push( @batch_id, $batch_id ); + my %row_data; $row_data{server} = $servername[$k]; $row_data{isbn} = $oldbiblio->{isbn}; @@ -320,4 +326,26 @@ breeding_loop => \@breeding_loop, numberpending => $nremaining > 0 ? $nremaining : 0, errconn => \@errconn ); + + # Process the delayed batch record count updates +my $pid; +if ( $pid = fork ) { + # Parent, continue on as usual + + # prevent parent exiting from + # destroying the kid's database handle + # FIXME: according to DBI doc, this may not work for Oracle + $dbh->{InactiveDestroy} = 1; +} else { + close STDOUT; + close STDERR; + + @batch_id = uniq(@batch_id); + foreach my $bid (@batch_id) { + C4::ImportBatch::UpdateBatchRecordCounts::UpdateBatchRecordCounts($bid); + } + + exit; +} + output_html_with_http_headers $input, $cookie, $template->output; diff --git a/cataloguing/z3950_search.pl b/cataloguing/z3950_search.pl index 29057de..41a449c 100755 --- a/cataloguing/z3950_search.pl +++ b/cataloguing/z3950_search.pl @@ -30,6 +30,8 @@ use C4::Context; use C4::Breeding; use C4::Koha; use C4::Charset; +use List::MoreUtils qw(uniq); + use ZOOM; my $input = new CGI; @@ -70,6 +72,7 @@ my @serverloop = (); my @serverhost; my @servername; my @breeding_loop = (); +my @batch_id; my ( $start_run, $end_run ); @@ -287,10 +290,14 @@ else { $oldbiblio->{issn} =~ s/\(/ \(/g if $oldbiblio->{issn}; my ( $notmarcrecord, $alreadyindb, $alreadyinfarm, - $imported, $breedingid + $imported, $breedingid, $batch_id, ) = ImportBreeding( $marcdata, 2, $serverhost[$k], $encoding[$k], $random, 'z3950' ); + + # Delay the updating of batch record counts until the very end + push( @batch_id, $batch_id ); + my %row_data; $row_data{server} = $servername[$k]; $row_data{isbn} = $oldbiblio->{isbn}; @@ -329,6 +336,27 @@ else { errconn => \@errconn ); + # Process the delayed batch record count updates + my $pid; + if ( $pid = fork ) { + # Parent, continue on as usual + + # prevent parent exiting from + # destroying the kid's database handle + # FIXME: according to DBI doc, this may not work for Oracle + $dbh->{InactiveDestroy} = 1; + } else { + close STDOUT; + close STDERR; + + @batch_id = uniq(@batch_id); + foreach my $bid (@batch_id) { + C4::ImportBatch::UpdateBatchRecordCounts($bid); + } + + exit; + } + if ( $numberpending == 0 ) { $end_run = time(); $template->param( run_time => $end_run - $start_run ); -- 1.7.2.5