From 6e76c7f1bb2e606663db1c7a9a6a764d7fedb1c4 Mon Sep 17 00:00:00 2001 From: Doug Kingston Date: Thu, 17 Oct 2013 14:39:24 -0700 Subject: [PATCH] Bug 6345 Add locking to rebuild_zebra This patch adds flock based locking for rebuild_zebra.pl on a per-instance basis. This prevents races between full rebuilds and background incremental updates from the zebraqueue table in the database. The race condition exists whether you are doing incremental updates with a periodic cronjob or with the new daemon mode. Suppose you start a full rebuild at time T0 which will take until T20 to extract the records. Suppose also at T10, a biblio or auth is updated and processed through the zebraqueue by T15. In this situation the updated record in zebra will be overwritten when the full rebuild records are uploaded to zebra after T20. We prevent this by only allowing one rebuild_zebra per koha instance to be running at one time. When running in daemon mode, incremental updates will be skipped while a full rebuild is running, and resume afterwards. A full rebuild or other adhoc request will wait for any previous lock to clear. Tested by flocking the lock file while invoking rebuild_zebra.pl in various modes (daemon, adhoc zebraqueue task, and full rebuild) using flock program I will attach to bug. http://bugs.koha-community.org/show_bug.cgi?id=11078 Signed-off-by: Martin Renvoize http://bugs.koha-community.org/show_bug.cgi?id=6435 --- misc/migration_tools/rebuild_zebra.pl | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/misc/migration_tools/rebuild_zebra.pl b/misc/migration_tools/rebuild_zebra.pl index a91c681..7ba199c 100755 --- a/misc/migration_tools/rebuild_zebra.pl +++ b/misc/migration_tools/rebuild_zebra.pl @@ -5,6 +5,7 @@ use strict; use C4::Context; use Getopt::Long; +use Fcntl qw(:flock); use File::Temp qw/ tempdir /; use File::Path; use C4::Biblio; @@ -139,12 +140,19 @@ my $dbh = C4::Context->dbh; my ($biblionumbertagfield,$biblionumbertagsubfield) = &GetMarcFromKohaField("biblio.biblionumber",""); my ($biblioitemnumbertagfield,$biblioitemnumbertagsubfield) = &GetMarcFromKohaField("biblioitems.biblioitemnumber",""); +# We need to create our own lock directory which incorporates the database instance +# we are indexing to facilitate multiple instances on the same machine. +my $lockdir = "/var/lock/koha_rebuild_zebra_" . C4::Context->config("database"); +mkpath($lockdir, 0, 0755) unless (-d $lockdir); +my $lockfile = $lockdir . "/lock"; + if ( $verbose_logging ) { print "Zebra configuration information\n"; print "================================\n"; print "Zebra biblio directory = $biblioserverdir\n"; print "Zebra authorities directory = $authorityserverdir\n"; print "Koha directory = $kohadir\n"; + print "Lockfile = $lockfile\n"; print "BIBLIONUMBER in : $biblionumbertagfield\$$biblionumbertagsubfield\n"; print "BIBLIOITEMNUMBER in : $biblioitemnumbertagfield\$$biblioitemnumbertagsubfield\n"; print "================================\n"; @@ -156,13 +164,27 @@ if ($do_munge) { my $tester = XML::LibXML->new(); +# The main work is done here by calling do_one_pass(). We have added locking +# avoid race conditions between Full rebuilds and incremental updates either from +# daemon mode or periodic invocation from cron. The race can lead to an updated +# record being overwritten by a rebuild if the update is applied after the export +# by the rebuild and before the rebuild finishes (more likely to effect large +# catalogs). +open(LockFH, ">$lockfile") or die "$lockfile: $!"; if ($daemon_mode) { while (1) { - do_one_pass() if ( zebraqueue_not_empty() ); + # For incremental updates, skip the update if the updates are locked + if (flock(LockFH, LOCK_EX|LOCK_NB)) { + do_one_pass() if ( zebraqueue_not_empty() ); + flock(LockFH, LOCK_UN); + } sleep $daemon_sleep; } } else { + # all one-off invocations, wait for the lock to free + flock(LockFH, LOCK_EX); do_one_pass(); + flock(LockFH, LOCK_UN); } -- 1.7.10.4