From 756866cf6884577437af48c51e2b5f9f8b6d47f6 Mon Sep 17 00:00:00 2001 From: Kyle M Hall Date: Thu, 26 Sep 2013 12:47:13 -0400 Subject: [PATCH] [PASSED QA] Bug 10955 - Add ability to skip deletions in zebraqueue It seems that record deletions can cause extreme slowdowns for Koha installations with extremely large numbers of records. It would be helpful to be able to skip record deletions when processing the zebraqueue with rebuild_zebra.pl so the deletions can be processed with a lower frequency. Test Plan: 1) Disable any zebra indexing cronjobs you may have 2) Delete a record 3) Note the operation recordDelete in the zebraqueue table having done = 0 4) Run misc/migration_tools/rebuild_zebra.pl -b -z --skip-deletes 5) Note the delete still has done = 0 6) Run misc/migration_tools/rebuild_zebra.pl -b -z 7) Note the delete now has done = 1 Signed-off-by: Jonathan Druart Signed-off-by: Katrin Fischer Passes all tests and QA script. Also tested for authorities, no problems found. --- misc/migration_tools/rebuild_zebra.pl | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/misc/migration_tools/rebuild_zebra.pl b/misc/migration_tools/rebuild_zebra.pl index 04cf454..4465698 100755 --- a/misc/migration_tools/rebuild_zebra.pl +++ b/misc/migration_tools/rebuild_zebra.pl @@ -36,6 +36,7 @@ my $noshadow; my $want_help; my $as_xml; my $process_zebraqueue; +my $process_zebraqueue_skip_deletes; my $do_not_clear_zebraqueue; my $length; my $where; @@ -62,11 +63,12 @@ my $result = GetOptions( 'x' => \$as_xml, 'y' => \$do_not_clear_zebraqueue, 'z' => \$process_zebraqueue, - 'where:s' => \$where, - 'length:i' => \$length, + 'skip-deletes' => \$process_zebraqueue_skip_deletes, + 'where:s' => \$where, + 'length:i' => \$length, 'offset:i' => \$offset, - 'v+' => \$verbose_logging, - 'run-as-root' => \$run_as_root, + 'v+' => \$verbose_logging, + 'run-as-root' => \$run_as_root, ); if (not $result or $want_help) { @@ -258,7 +260,7 @@ sub index_records { my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_; my $num_records_exported = 0; - my $records_deleted; + my $records_deleted = {}; my $need_reset = check_zebra_dirs($server_dir); if ($need_reset) { print "$0: found broken zebra server directories: forcing a rebuild\n"; @@ -277,15 +279,20 @@ sub index_records { mkdir "$directory" unless (-d $directory); mkdir "$directory/$record_type" unless (-d "$directory/$record_type"); if ($process_zebraqueue) { - my $entries = select_zebraqueue_records($record_type, 'deleted'); - mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type"); - $records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_xml); - mark_zebraqueue_batch_done($entries); + my $entries; + + unless ( $process_zebraqueue_skip_deletes ) { + $entries = select_zebraqueue_records($record_type, 'deleted'); + mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type"); + $records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_xml); + mark_zebraqueue_batch_done($entries); + } + $entries = select_zebraqueue_records($record_type, 'updated'); mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type"); - $num_records_exported = export_marc_records_from_list($record_type, - $entries, "$directory/upd_$record_type", $as_xml, $noxml, $records_deleted); + $num_records_exported = export_marc_records_from_list($record_type,$entries, "$directory/upd_$record_type", $as_xml, $noxml, $records_deleted); mark_zebraqueue_batch_done($entries); + } else { my $sth = select_all_records($record_type); $num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_xml, $noxml, $nosanitize); @@ -759,6 +766,10 @@ Parameters: table. Cannot be used with -r or -s. + --skip-deletes select only updated records marked + in the zebraqueue table, not deletes. + Only effective with -z. + -r clear Zebra index before adding records to index. Implies -w. -- 1.8.3.2