@@ -, +, @@ --- Makefile.PL | 5 ++++- debian/templates/koha-conf-site.xml.in | 1 + etc/koha-conf.xml | 1 + misc/bin/koha-zebra-ctl.sh | 1 + misc/migration_tools/rebuild_zebra.pl | 27 ++++++++++++++++++++++++++- skel/var/lock/koha/zebradb/rebuild/README | 1 + 6 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 skel/var/lock/koha/zebradb/rebuild/README --- a/Makefile.PL +++ a/Makefile.PL @@ -186,7 +186,9 @@ Directory for Zebra configuration files. =item ZEBRA_LOCK_DIR -Directory for Zebra's lock files. +Directory for Zebra's lock files. This includes subdirs for authorities, +biblios, and the zebra rebuild function. Any activity to reindex +zebra from koha should interlock here with rebuild_zebra.pl. =item ZEBRA_DATA_DIR @@ -318,6 +320,7 @@ my $target_map = { './skel/var/lib/koha/zebradb/biblios/register' => { target => 'ZEBRA_DATA_DIR', trimdir => 6 }, './skel/var/lib/koha/zebradb/biblios/shadow' => { target => 'ZEBRA_DATA_DIR', trimdir => 6 }, './skel/var/lib/koha/zebradb/biblios/tmp' => { target => 'ZEBRA_DATA_DIR', trimdir => 6 }, + './skel/var/lock/koha/zebradb/rebuild' => { target => 'ZEBRA_LOCK_DIR', trimdir => 6 }, './skel/var/lib/koha/plugins' => { target => 'PLUGINS_DIR', trimdir => 6 }, './sms' => 'INTRANET_CGI_DIR', './suggestion' => 'INTRANET_CGI_DIR', --- a/debian/templates/koha-conf-site.xml.in +++ a/debian/templates/koha-conf-site.xml.in @@ -280,6 +280,7 @@ __MEMCACHED_NAMESPACE__ __BIBLIOS_INDEXING_MODE__ __AUTHORITIES_INDEXING_MODE__ + /var/lock/koha/__KOHASITE__ /etc/koha/searchengine/queryparser.yaml --- a/etc/koha-conf.xml +++ a/etc/koha-conf.xml @@ -297,6 +297,7 @@ __PAZPAR2_TOGGLE_XML_POST__ 0 __BIB_INDEX_MODE__ __AUTH_INDEX_MODE__ + __ZEBRA_LOCK_DIR__ __KOHA_CONF_DIR__/searchengine/queryparser.yaml --- a/misc/bin/koha-zebra-ctl.sh +++ a/misc/bin/koha-zebra-ctl.sh @@ -65,6 +65,7 @@ case "$1" in mkdir -p $LOCKDIR mkdir -p $LOCKDIR/biblios mkdir -p $LOCKDIR/authorities + mkdir -p $LOCKDIR/rebuild if [[ $EUID -eq 0 ]]; then chown -R $USER:$GROUP $LOCKDIR fi --- a/misc/migration_tools/rebuild_zebra.pl +++ a/misc/migration_tools/rebuild_zebra.pl @@ -5,6 +5,7 @@ use strict; use C4::Context; use Getopt::Long; +use Fcntl qw(:flock); use File::Temp qw/ tempdir /; use File::Path; use C4::Biblio; @@ -151,12 +152,22 @@ my $dbh = C4::Context->dbh; my ($biblionumbertagfield,$biblionumbertagsubfield) = &GetMarcFromKohaField("biblio.biblionumber",""); my ($biblioitemnumbertagfield,$biblioitemnumbertagsubfield) = &GetMarcFromKohaField("biblioitems.biblioitemnumber",""); +# Protect against simultaneous update of the zebra index by using a lock file. +# Create our own lock directory if its missing. This shouild be created +# by koha-zebra-ctl.sh. + +my $lockdir = C4::Context->config("zebra_lockdir") // "/var/lock"; +$lockdir .= '/rebuild'; +mkpath($lockdir, 0, oct(755)) unless (-d $lockdir); +my $lockfile = $lockdir . "/rebuild..LCK"; + if ( $verbose_logging ) { print "Zebra configuration information\n"; print "================================\n"; print "Zebra biblio directory = $biblioserverdir\n"; print "Zebra authorities directory = $authorityserverdir\n"; print "Koha directory = $kohadir\n"; + print "Lockfile = $lockfile\n"; print "BIBLIONUMBER in : $biblionumbertagfield\$$biblionumbertagsubfield\n"; print "BIBLIOITEMNUMBER in : $biblioitemnumbertagfield\$$biblioitemnumbertagsubfield\n"; print "================================\n"; @@ -164,13 +175,27 @@ if ( $verbose_logging ) { my $tester = XML::LibXML->new(); +# The main work is done here by calling do_one_pass(). We have added locking +# avoid race conditions between Full rebuilds and incremental updates either from +# daemon mode or periodic invocation from cron. The race can lead to an updated +# record being overwritten by a rebuild if the update is applied after the export +# by the rebuild and before the rebuild finishes (more likely to effect large +# catalogs). +open my $LockFH, q{>}, $lockfile or die "$lockfile: $!"; if ($daemon_mode) { while (1) { - do_one_pass() if ( zebraqueue_not_empty() ); + # For incremental updates, skip the update if the updates are locked + if (flock($LockFH, LOCK_EX|LOCK_NB)) { + do_one_pass() if ( zebraqueue_not_empty() ); + flock($LockFH, LOCK_UN); + } sleep $daemon_sleep; } } else { + # all one-off invocations, wait for the lock to free + flock($LockFH, LOCK_EX); do_one_pass(); + flock($LockFH, LOCK_UN); } --- a/skel/var/lock/koha/zebradb/rebuild/README +++ a/skel/var/lock/koha/zebradb/rebuild/README @@ -0,0 +1, @@ +Zebra rebuild lock dir --