From e9fcb5a350a65081e7674c50c2ce346d531e0ff1 Mon Sep 17 00:00:00 2001
From: Kyle M Hall <kyle@bywatersolutions.com>
Date: Thu, 26 Sep 2013 12:47:13 -0400
Subject: [PATCH] Bug 10955 - Add ability to skip deletions in zebraqueue

It seems that record deletions can cause extreme slowdowns for Koha
installations with extremely large numbers of records. It would be
helpful to be able to skip record deletions when processing the
zebraqueue with rebuild_zebra.pl so the deletions can be processed with
a lower frequency.

Test Plan:
1) Disable any zebra indexing cronjobs you may have
2) Delete a record
3) Note the operation recordDelete in the zebraqueue table having done = 0
4) Run misc/migration_tools/rebuild_zebra.pl -b -z --skip-deletes
5) Note the delete still has done = 0
6) Run misc/migration_tools/rebuild_zebra.pl -b -z
7) Note the delete now has done = 1
---
 misc/migration_tools/rebuild_zebra.pl |   65 +++++++++++++++++++--------------
 1 files changed, 38 insertions(+), 27 deletions(-)

diff --git a/misc/migration_tools/rebuild_zebra.pl b/misc/migration_tools/rebuild_zebra.pl
index a46526d..41fbcf0 100755
--- a/misc/migration_tools/rebuild_zebra.pl
+++ b/misc/migration_tools/rebuild_zebra.pl
@@ -35,6 +35,7 @@ my $do_munge;
 my $want_help;
 my $as_xml;
 my $process_zebraqueue;
+my $process_zebraqueue_skip_deletes;
 my $do_not_clear_zebraqueue;
 my $length;
 my $where;
@@ -45,26 +46,27 @@ my $run_user = (getpwuid($<))[0];
 my $verbose_logging = 0;
 my $zebraidx_log_opt = " -v none,fatal,warn ";
 my $result = GetOptions(
-    'd:s'           => \$directory,
-    'r|reset'       => \$reset,
-    's'             => \$skip_export,
-    'k'             => \$keep_export,
-    'I|skip-index'  => \$skip_index,
-    'nosanitize'    => \$nosanitize,
-    'b'             => \$biblios,
-    'noxml'         => \$noxml,
-    'w'             => \$noshadow,
-    'munge-config'  => \$do_munge,
-    'a'             => \$authorities,
-    'h|help'        => \$want_help,
-    'x'             => \$as_xml,
-    'y'             => \$do_not_clear_zebraqueue,
-    'z'             => \$process_zebraqueue,
-    'where:s'        => \$where,
-    'length:i'        => \$length,
-    'offset:i'      => \$offset,
-    'v+'             => \$verbose_logging,
-    'run-as-root'    => \$run_as_root,
+    'd:s'          => \$directory,
+    'r|reset'      => \$reset,
+    's'            => \$skip_export,
+    'k'            => \$keep_export,
+    'I|skip-index' => \$skip_index,
+    'nosanitize'   => \$nosanitize,
+    'b'            => \$biblios,
+    'noxml'        => \$noxml,
+    'w'            => \$noshadow,
+    'munge-config' => \$do_munge,
+    'a'            => \$authorities,
+    'h|help'       => \$want_help,
+    'x'            => \$as_xml,
+    'y'            => \$do_not_clear_zebraqueue,
+    'z'            => \$process_zebraqueue,
+    'skip-deletes' => \$process_zebraqueue_skip_deletes,
+    'where:s'      => \$where,
+    'length:i'     => \$length,
+    'offset:i'     => \$offset,
+    'v+'           => \$verbose_logging,
+    'run-as-root'  => \$run_as_root,
 );
 
 if (not $result or $want_help) {
@@ -213,7 +215,7 @@ sub index_records {
     my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
 
     my $num_records_exported = 0;
-    my $records_deleted;
+    my $records_deleted = {};
     my $need_reset = check_zebra_dirs($server_dir);
     if ($need_reset) {
         print "$0: found broken zebra server directories: forcing a rebuild\n";
@@ -232,15 +234,20 @@ sub index_records {
         mkdir "$directory" unless (-d $directory);
         mkdir "$directory/$record_type" unless (-d "$directory/$record_type");
         if ($process_zebraqueue) {
-            my $entries = select_zebraqueue_records($record_type, 'deleted');
-            mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type");
-            $records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_xml);
-            mark_zebraqueue_batch_done($entries);
+            my $entries;
+
+            unless ( $process_zebraqueue_skip_deletes ) {
+                $entries = select_zebraqueue_records($record_type, 'deleted');
+                mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type");
+                $records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_xml);
+                mark_zebraqueue_batch_done($entries);
+            }
+
             $entries = select_zebraqueue_records($record_type, 'updated');
             mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type");
-            $num_records_exported = export_marc_records_from_list($record_type,
-                                                                  $entries, "$directory/upd_$record_type", $as_xml, $noxml, $records_deleted);
+            $num_records_exported = export_marc_records_from_list($record_type,$entries, "$directory/upd_$record_type", $as_xml, $noxml, $records_deleted);
             mark_zebraqueue_batch_done($entries);
+
         } else {
             my $sth = select_all_records($record_type);
             $num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_xml, $noxml, $nosanitize);
@@ -697,6 +704,10 @@ Parameters:
                             table.  Cannot be used with -r
                             or -s.
 
+    --skip-deletes          select only updated records marked
+                            in the zebraqueue table, not deletes.
+                            Only effective with -z.
+
     -r                      clear Zebra index before
                             adding records to index. Implies -w.
 
-- 
1.7.2.5