Bugzilla – Attachment 25330 Details for
Bug 11078
rebuild_zebra.pl can lose updates due to race condition during full rebuilds
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 11078 Address February QA concerns
0003-Bug-11078-Address-February-QA-concerns.patch (text/plain), 5.96 KB, created by
Doug Kingston
on 2014-02-17 02:44:40 UTC
(
hide
)
Description:
Bug 11078 Address February QA concerns
Filename:
MIME Type:
Creator:
Doug Kingston
Created:
2014-02-17 02:44:40 UTC
Size:
5.96 KB
patch
obsolete
>From fa03f82ac60b9e2767c1460eeac5dc4bef78b538 Mon Sep 17 00:00:00 2001 >From: Doug Kingston <dpk@randomnotes.org> >Date: Sun, 16 Feb 2014 17:42:54 -0800 >Subject: [PATCH 3/3] Bug 11078 Address February QA concerns > >1. Add code to check if flock is available and ignore locking if >its missing (from M. de Rooy) > >2. Change default for adhoc invocations to abort if they cannot >obtain the lock. Added option -wait-for-lock if the user prefers >to wait until the lock is free, and then continue processing. > >3. added missing entry to t/db_dependent/zebra_config.pl > >4. added a fallback locking directory of /tmp >--- > misc/migration_tools/rebuild_zebra.pl | 68 +++++++++++++++++++++++++++------ > t/db_dependent/zebra_config.pl | 1 + > 2 files changed, 57 insertions(+), 12 deletions(-) > >diff --git a/misc/migration_tools/rebuild_zebra.pl b/misc/migration_tools/rebuild_zebra.pl >index 6e713ff..e202316 100755 >--- a/misc/migration_tools/rebuild_zebra.pl >+++ b/misc/migration_tools/rebuild_zebra.pl >@@ -44,6 +44,8 @@ my $where; > my $offset; > my $run_as_root; > my $run_user = (getpwuid($<))[0]; >+my $wait_for_lock = 0; >+my $use_flock; > > my $verbose_logging = 0; > my $zebraidx_log_opt = " -v none,fatal,warn "; >@@ -65,11 +67,12 @@ my $result = GetOptions( > 'x' => \$as_xml, > 'y' => \$do_not_clear_zebraqueue, > 'z' => \$process_zebraqueue, >- 'where:s' => \$where, >- 'length:i' => \$length, >+ 'where:s' => \$where, >+ 'length:i' => \$length, > 'offset:i' => \$offset, >- 'v+' => \$verbose_logging, >- 'run-as-root' => \$run_as_root, >+ 'v+' => \$verbose_logging, >+ 'run-as-root' => \$run_as_root, >+ 'wait-for-lock' => \$wait_for_lock, > ); > > if (not $result or $want_help) { >@@ -156,11 +159,16 @@ my ($biblioitemnumbertagfield,$biblioitemnumbertagsubfield) = &GetMarcFromKohaFi > > # Protect again simultaneous update of the zebra index by using a lock file. > # Create our own lock directory if its missing. This shouild be created >-# by koha-zebra-ctl.sh. >+# by koha-zebra-ctl.sh or at system installation. If the desired directory >+# does not exist and cannot be created, we fall back on /tmp - which will >+# always work. > > my $lockdir = C4::Context->config("zebra_lockdir") // "/var/lock"; > $lockdir .= "/rebuild"; >-mkpath($lockdir, 0, oct(755)) unless (-d $lockdir); >+unless (-d $lockdir) { >+ eval { mkpath($lockdir, 0, oct(755)) }; >+ $lockdir = "/tmp" if ($@); >+} > my $lockfile = $lockdir . "/rebuild..LCK"; > > if ( $verbose_logging ) { >@@ -187,21 +195,31 @@ my $tester = XML::LibXML->new(); > # record being overwritten by a rebuild if the update is applied after the export > # by the rebuild and before the rebuild finishes (more likely to effect large > # catalogs). >+# >+# We have chosen to exit immediately by default if we cannot obtain the lock >+# to prevent the potential for a infinite backlog from cron invocations, but an >+# option (wait-for-lock) is provided to let the program wait for the lock. >+# See http://bugs.koha-community.org/bugzilla3/show_bug.cgi?id=11078 for details. > open my $LockFH, q{>}, $lockfile or die "$lockfile: $!"; > if ($daemon_mode) { > while (1) { > # For incremental updates, skip the update if the updates are locked >- if (flock($LockFH, LOCK_EX|LOCK_NB)) { >+ if (_flock($LockFH, LOCK_EX|LOCK_NB)) { > do_one_pass() if ( zebraqueue_not_empty() ); >- flock($LockFH, LOCK_UN); >+ _flock($LockFH, LOCK_UN); > } > sleep $daemon_sleep; > } > } else { >- # all one-off invocations, wait for the lock to free >- flock($LockFH, LOCK_EX); >- do_one_pass(); >- flock($LockFH, LOCK_UN); >+ # all one-off invocations >+ my $lock_mode = ($wait_for_lock) ? LOCK_EX : LOCK_EX|LOCK_NB; >+ if (_flock($LockFH, $lock_mode)) { >+ do_one_pass(); >+ _flock($LockFH, LOCK_UN); >+ } else { >+ # Can't die() here because we have files to dlean up. >+ print "Aborting rebuild. Unable to flock $lockfile: $!\n"; >+ } > } > > >@@ -755,6 +773,26 @@ sub do_indexing { > > } > >+sub _flock { >+# test if flock is present; if so, use it; if not, return true >+# op refers to the official flock operations incl LOCK_EX, LOCK_UN, etc. >+# combining LOCK_EX with LOCK_NB returns immediately >+ my ($fh, $op)= @_; >+ if( !defined($use_flock) ) { >+ #check if flock is present; if not, you will have a fatal error >+ my $i=eval { flock($fh, $op) }; >+ #assuming that $fh and $op are fine(..), an undef i means no flock >+ $use_flock= defined($i)? 1: 0; >+ print "Warning: flock could not be used!\n" if $verbose_logging && !$use_flock; >+ return 1 if !$use_flock; >+ return $i; >+ } >+ else { >+ return 1 if !$use_flock; >+ return flock($fh, $op); >+ } >+} >+ > sub print_usage { > print <<_USAGE_; > $0: reindex MARC bibs and/or authorities in Zebra. >@@ -843,6 +881,12 @@ Parameters: > > --run-as-root explicitily allow script to run as 'root' user > >+ --wait-for-lock when not running in daemon mode, the default >+ behavior is to abort a rebuild if the rebuild >+ lock is busy. This option will cause the program >+ to wait for the lock to free and then continue >+ processing the rebuild request, >+ > --help or -h show this message. > _USAGE_ > } >diff --git a/t/db_dependent/zebra_config.pl b/t/db_dependent/zebra_config.pl >index 1395508..ceb9c0d 100755 >--- a/t/db_dependent/zebra_config.pl >+++ b/t/db_dependent/zebra_config.pl >@@ -28,6 +28,7 @@ if ($indexing_mode eq 'dom') { > make_path("$destination/var/lock/zebradb"); > make_path("$destination/var/lock/zebradb/biblios"); > make_path("$destination/var/lock/zebradb/authorities"); >+make_path("$destination/var/lock/zebradb/rebuild"); > make_path("$destination/var/lib/zebradb"); > make_path("$destination/var/lib/zebradb/biblios"); > make_path("$destination/var/lib/zebradb/biblios/key"); >-- >1.7.9.5 >
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 11078
:
22044
|
22045
|
22205
|
22458
|
22506
|
22508
|
22532
|
22985
|
22992
|
23220
|
23221
|
23573
|
23574
|
25114
|
25172
|
25330
|
25412
|
25446
|
25447