Attachment 25114 Details for Bug 11078 – Bug 11078 Add locking to rebuild_zebra.pl

[patch] Bug 11078 Add locking to rebuild_zebra.pl

Bug-11078-Add-locking-to-rebuildzebrapl.patch (text/plain), 6.98 KB, created by Marcel de Rooy on 2014-02-07 09:44:32 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Marcel de Rooy

Created: 2014-02-07 09:44:32 UTC

Size: 6.98 KB

patch

obsolete

>From 2bc2020b97070bc7f7792604d7dee325f8a0a26c Mon Sep 17 00:00:00 2001
>From: Doug Kingston <dpk@randomnotes.org>
>Date: Thu, 28 Nov 2013 23:12:33 -0800
>Subject: [PATCH] Bug 11078 Add locking to rebuild_zebra.pl
>Content-Type: text/plain; charset=utf-8
>
>This patch adds locking to rebuild_zebra.pl to ensure that simultaneous
>changes are prevented (as one is likely to overwrite the other).
>Incremental updates in daemon mode will skipped if the lock is busy
>and they will be picked up on the next pass.  Non-daemon mode
>invocations will wait for the lock to clear and then proceed.
>Supporting changes made to Makefile.PL and templates for the new
>locking directory (paralleling the other zebra lock directories).
>We stash the zebra_lockdir in koha-conf.xml so rebuild_zebra.pl
>can find it.
>
>Signed-off-by: Martin Renvoize <martin.renvoize@ptfs-europe.com>
>
>Signed-off-by: Marcel de Rooy <m.de.rooy@rijksmuseum.nl>
>Amended the patch: just added the /rebuild from the follow-up.
>Works as advertised, but I post some comments on Bugzilla.
>---
> Makefile.PL                               |    5 ++++-
> debian/templates/koha-conf-site.xml.in    |    1 +
> etc/koha-conf.xml                         |    1 +
> misc/bin/koha-zebra-ctl.sh                |    1 +
> misc/migration_tools/rebuild_zebra.pl     |   27 ++++++++++++++++++++++++++-
> skel/var/lock/koha/zebradb/rebuild/README |    1 +
> 6 files changed, 34 insertions(+), 2 deletions(-)
> create mode 100644 skel/var/lock/koha/zebradb/rebuild/README
>
>diff --git a/Makefile.PL b/Makefile.PL
>index 66f9a0f..7d5eb4a 100644
>--- a/Makefile.PL
>+++ b/Makefile.PL
>@@ -186,7 +186,9 @@ Directory for Zebra configuration files.
> 
> =item ZEBRA_LOCK_DIR
> 
>-Directory for Zebra's lock files.
>+Directory for Zebra's lock files.  This includes subdirs for authorities,
>+biblios, and the zebra rebuild function.  Any activity to reindex
>+zebra from koha should interlock here with rebuild_zebra.pl.
> 
> =item ZEBRA_DATA_DIR
> 
>@@ -318,6 +320,7 @@ my $target_map = {
>   './skel/var/lib/koha/zebradb/biblios/register'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
>   './skel/var/lib/koha/zebradb/biblios/shadow'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
>   './skel/var/lib/koha/zebradb/biblios/tmp'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
>+  './skel/var/lock/koha/zebradb/rebuild' => { target => 'ZEBRA_LOCK_DIR', trimdir => 6 },
>   './skel/var/lib/koha/plugins' => { target => 'PLUGINS_DIR', trimdir => 6 },
>   './sms'                       => 'INTRANET_CGI_DIR',
>   './suggestion'                => 'INTRANET_CGI_DIR',
>diff --git a/debian/templates/koha-conf-site.xml.in b/debian/templates/koha-conf-site.xml.in
>index 9bf090b..371b1d2 100644
>--- a/debian/templates/koha-conf-site.xml.in
>+++ b/debian/templates/koha-conf-site.xml.in
>@@ -280,6 +280,7 @@
>  <memcached_namespace>__MEMCACHED_NAMESPACE__</memcached_namespace>
>  <zebra_bib_index_mode>__BIBLIOS_INDEXING_MODE__</zebra_bib_index_mode>
>  <zebra_auth_index_mode>__AUTHORITIES_INDEXING_MODE__</zebra_auth_index_mode>
>+ <zebra_lockdir>/var/lock/koha/__KOHASITE__</zebra_lockdir>
>  <queryparser_config>/etc/koha/searchengine/queryparser.yaml</queryparser_config>
> </config>
> 
>diff --git a/etc/koha-conf.xml b/etc/koha-conf.xml
>index 1da5378..72da2de 100644
>--- a/etc/koha-conf.xml
>+++ b/etc/koha-conf.xml
>@@ -297,6 +297,7 @@ __PAZPAR2_TOGGLE_XML_POST__
>  <useldapserver>0</useldapserver>
>  <zebra_bib_index_mode>__BIB_INDEX_MODE__</zebra_bib_index_mode>
>  <zebra_auth_index_mode>__AUTH_INDEX_MODE__</zebra_auth_index_mode>
>+ <zebra_lockdir>__ZEBRA_LOCK_DIR__</zebra_lockdir>
>  <queryparser_config>__KOHA_CONF_DIR__/searchengine/queryparser.yaml</queryparser_config>
> </config>
> </yazgfs>
>diff --git a/misc/bin/koha-zebra-ctl.sh b/misc/bin/koha-zebra-ctl.sh
>index 397eb19..82b9162 100755
>--- a/misc/bin/koha-zebra-ctl.sh
>+++ b/misc/bin/koha-zebra-ctl.sh
>@@ -65,6 +65,7 @@ case "$1" in
>         mkdir -p $LOCKDIR
>         mkdir -p $LOCKDIR/biblios
>         mkdir -p $LOCKDIR/authorities
>+        mkdir -p $LOCKDIR/rebuild
>         if [[ $EUID -eq 0 ]]; then
>             chown -R $USER:$GROUP $LOCKDIR
>         fi
>diff --git a/misc/migration_tools/rebuild_zebra.pl b/misc/migration_tools/rebuild_zebra.pl
>index 04cf454..28b858c 100755
>--- a/misc/migration_tools/rebuild_zebra.pl
>+++ b/misc/migration_tools/rebuild_zebra.pl
>@@ -5,6 +5,7 @@ use strict;
> 
> use C4::Context;
> use Getopt::Long;
>+use Fcntl qw(:flock);
> use File::Temp qw/ tempdir /;
> use File::Path;
> use C4::Biblio;
>@@ -151,12 +152,22 @@ my $dbh = C4::Context->dbh;
> my ($biblionumbertagfield,$biblionumbertagsubfield) = &GetMarcFromKohaField("biblio.biblionumber","");
> my ($biblioitemnumbertagfield,$biblioitemnumbertagsubfield) = &GetMarcFromKohaField("biblioitems.biblioitemnumber","");
> 
>+# Protect against simultaneous update of the zebra index by using a lock file.
>+# Create our own lock directory if its missing.  This shouild be created
>+# by koha-zebra-ctl.sh.
>+
>+my $lockdir = C4::Context->config("zebra_lockdir") // "/var/lock";
>+$lockdir .= '/rebuild';
>+mkpath($lockdir, 0, oct(755)) unless (-d $lockdir);
>+my $lockfile = $lockdir . "/rebuild..LCK";
>+
> if ( $verbose_logging ) {
>     print "Zebra configuration information\n";
>     print "================================\n";
>     print "Zebra biblio directory      = $biblioserverdir\n";
>     print "Zebra authorities directory = $authorityserverdir\n";
>     print "Koha directory              = $kohadir\n";
>+    print "Lockfile                    = $lockfile\n";
>     print "BIBLIONUMBER in :     $biblionumbertagfield\$$biblionumbertagsubfield\n";
>     print "BIBLIOITEMNUMBER in : $biblioitemnumbertagfield\$$biblioitemnumbertagsubfield\n";
>     print "================================\n";
>@@ -164,13 +175,27 @@ if ( $verbose_logging ) {
> 
> my $tester = XML::LibXML->new();
> 
>+# The main work is done here by calling do_one_pass().  We have added locking
>+# avoid race conditions between Full rebuilds and incremental updates either from
>+# daemon mode or periodic invocation from cron.  The race can lead to an updated
>+# record being overwritten by a rebuild if the update is applied after the export
>+# by the rebuild and before the rebuild finishes (more likely to effect large
>+# catalogs).
>+open my $LockFH, q{>}, $lockfile or die "$lockfile: $!";
> if ($daemon_mode) {
>     while (1) {
>-        do_one_pass() if ( zebraqueue_not_empty() );
>+        # For incremental updates, skip the update if the updates are locked
>+        if (flock($LockFH, LOCK_EX|LOCK_NB)) {
>+            do_one_pass() if ( zebraqueue_not_empty() );
>+            flock($LockFH, LOCK_UN);
>+        }
>         sleep $daemon_sleep;
>     }
> } else {
>+    # all one-off invocations, wait for the lock to free
>+    flock($LockFH, LOCK_EX);
>     do_one_pass();
>+    flock($LockFH, LOCK_UN);
> }
> 
> 
>diff --git a/skel/var/lock/koha/zebradb/rebuild/README b/skel/var/lock/koha/zebradb/rebuild/README
>new file mode 100644
>index 0000000..98b4c5b
>--- /dev/null
>+++ b/skel/var/lock/koha/zebradb/rebuild/README
>@@ -0,0 +1 @@
>+Zebra rebuild lock dir
>-- 
>1.7.7.6

Actions: View | Diff | Splinter Review

Attachments on bug 11078: 22044 | 22045 | 22205 | 22458 | 22506 | 22508 | 22532 | 22985 | 22992 | 23220 | 23221 | 23573 | 23574 | 25114 | 25172 | 25330 | 25412 | 25446 | 25447