From 4d516152b3b862be957f7089d4c25375087331ca Mon Sep 17 00:00:00 2001 From: Paul Hoffman Date: Fri, 27 Sep 2019 08:20:14 -0400 Subject: [PATCH] Limit memory consumption when exporting, too --- installer/data/mysql/atomicupdate/bug_23626.perl | 2 +- installer/data/mysql/sysprefs.sql | 2 +- koha-tmpl/intranet-tmpl/prog/en/includes/chart.inc | 4 +-- .../en/modules/admin/preferences/web_services.pref | 4 +-- reports/guided_reports.pl | 30 ++++++++++++++-------- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/installer/data/mysql/atomicupdate/bug_23626.perl b/installer/data/mysql/atomicupdate/bug_23626.perl index 38b0cd4..01981f8 100644 --- a/installer/data/mysql/atomicupdate/bug_23626.perl +++ b/installer/data/mysql/atomicupdate/bug_23626.perl @@ -1,7 +1,7 @@ $DBversion = 'XXX'; # will be replaced by the RM if( CheckVersion( $DBversion ) ) { $dbh->do(q{ - INSERT IGNORE INTO systempreferences ( `variable`, `value`, `options`, `explanation`, `type` ) VALUES ('ReportsMaxRowsInChart','',NULL,'If greater than 0, sets the maximum number of rows that will be used when charting results of guided reports','Integer') + INSERT IGNORE INTO systempreferences ( `variable`, `value`, `options`, `explanation`, `type` ) VALUES ('ReportsMaxRowsReturned','',NULL,'If greater than 0, sets the maximum number of rows that will be used when charting results of guided reports','Integer') }); SetVersion( $DBversion ); diff --git a/installer/data/mysql/sysprefs.sql b/installer/data/mysql/sysprefs.sql index cebb9cb..25f8dea 100644 --- a/installer/data/mysql/sysprefs.sql +++ b/installer/data/mysql/sysprefs.sql @@ -508,7 +508,7 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` ('RentalsInNoissuesCharge','1',NULL,'Rental charges block checkouts (added to noissuescharge).','YesNo'), ('ReplyToDefault','',NULL,'Use this email address as the replyto in emails','Free'), ('ReportsLog','0',NULL,'If ON, log information about reports.','YesNo'), -('ReportsMaxRowsInChart','0',NULL,'If greater than 0, sets the maximum number of rows that will be used when charting results of guided reports','Integer'), +('ReportsMaxRowsReturned','0',NULL,'If greater than 0, sets the maximum number of rows that will be included in the results of guided reports','Integer'), ('RequestOnOpac','1',NULL,'If ON, globally enables patron holds on OPAC','YesNo'), ('RequireStrongPassword','1','','Require a strong login password for staff and patrons','YesNo'), ('ReservesControlBranch','PatronLibrary','ItemHomeLibrary|PatronLibrary','Branch checked for members reservations rights','Choice'), diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/chart.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/chart.inc index d092b69..eb82af5 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/chart.inc +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/chart.inc @@ -35,8 +35,8 @@
  • - [% IF Koha.Preference("ReportsMaxRowsInChart") %] - + [% IF Koha.Preference("ReportsMaxRowsReturned") %] + [% ELSE %] [% END %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref index 9ad663a..ed228a5 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref @@ -99,9 +99,9 @@ Web services: - rows of a report requested via the reports web service. - - Allow at most - - pref: ReportsMaxRowsInChart + - pref: ReportsMaxRowsReturned class: integer - - rows of a report to be used when charting results of guided reports + - rows of a report to be returned in the results of guided reports IdRef: - - pref: IdRef diff --git a/reports/guided_reports.pl b/reports/guided_reports.pl index 277e1e5..af87025 100755 --- a/reports/guided_reports.pl +++ b/reports/guided_reports.pl @@ -829,7 +829,7 @@ elsif ($phase eq 'Run this report'){ $template->param(header_types => $header_types); my ( $sth, $errors ) = execute_query( $sql, $offset, $limit, undef, $report_id ); my ($sth2, $errors2) = execute_query($sql); - my $max_rows = C4::Context->preference('ReportsMaxRowsInChart'); + my $max_rows = C4::Context->preference('ReportsMaxRowsReturned'); my $total = nb_rows($sql) || 0; unless ($sth) { die "execute_query failed to return sth for report $report_id: $sql"; @@ -890,6 +890,7 @@ elsif ($phase eq 'Export'){ my $format = $input->param('format'); my $reportname = $input->param('reportname'); my $reportfilename = $reportname ? "$reportname-reportresults.$format" : "reportresults.$format" ; + my $max_rows = C4::Context->preference('ReportsMaxRowsReturned'); $sql = get_prepped_report( $sql, \@param_names, \@sql_params ); my ($sth, $q_errors) = execute_query($sql); @@ -899,9 +900,13 @@ elsif ($phase eq 'Export'){ $type = 'application/octet-stream'; $content .= join("\t", header_cell_values($sth)) . "\n"; $content = Encode::decode('UTF-8', $content); + my $n = 0; while (my $row = $sth->fetchrow_arrayref()) { + $n++; $content .= join("\t", @$row) . "\n"; + last if $max_rows && $n >= $max_rows; } + $sth->finish(); } else { my $delimiter = C4::Context->preference('delimiter') || ','; if ( $format eq 'csv' ) { @@ -914,36 +919,39 @@ elsif ($phase eq 'Export'){ } else { push @$q_errors, { combine => 'HEADER ROW: ' . $csv->error_diag() } ; } + my $n = 0; while (my $row = $sth->fetchrow_arrayref()) { + $n++; if ($csv->combine(@$row)) { $content .= $csv->string() . "\n"; + last if $max_rows && $n >= $max_rows; } else { push @$q_errors, { combine => $csv->error_diag() } ; } } + $sth->finish(); } elsif ( $format eq 'ods' ) { $type = 'application/vnd.oasis.opendocument.spreadsheet'; my $ods_fh = File::Temp->new( UNLINK => 0 ); my $ods_filepath = $ods_fh->filename; - my $ods_content; + my @ods_content; # First line is headers my @headers = header_cell_values($sth); - push @$ods_content, \@headers; + push @ods_content, \@headers; # Other line in Unicode - my $sql_rows = $sth->fetchall_arrayref(); - foreach my $sql_row ( @$sql_rows ) { - my @content_row; - foreach my $sql_cell ( @$sql_row ) { - push @content_row, Encode::encode( 'UTF8', $sql_cell ); - } - push @$ods_content, \@content_row; + my $n = 0; + while (my $row = $sth->fetchrow_arrayref()) { + $n++; + push @ods_content, [ map { Encode::encode( 'UTF8', $_ ) } @$row ]; + last if $max_rows && $n >= $max_rows; } + $sth->finish(); # Process - generate_ods($ods_filepath, $ods_content); + generate_ods($ods_filepath, \@ods_content); # Output binmode(STDOUT); -- 2.1.4