View | Details | Raw Unified | Return to bug 11078
Collapse All | Expand All

(-)a/Makefile.PL (-1 / +4 lines)
Lines 186-192 Directory for Zebra configuration files. Link Here
186
186
187
=item ZEBRA_LOCK_DIR
187
=item ZEBRA_LOCK_DIR
188
188
189
Directory for Zebra's lock files.
189
Directory for Zebra's lock files.  This includes subdirs for authorities,
190
biblios, and the zebra rebuild function.  Any activity to reindex
191
zebra from koha should interlock here with rebuild_zebra.pl.
190
192
191
=item ZEBRA_DATA_DIR
193
=item ZEBRA_DATA_DIR
192
194
Lines 318-323 my $target_map = { Link Here
318
  './skel/var/lib/koha/zebradb/biblios/register'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
320
  './skel/var/lib/koha/zebradb/biblios/register'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
319
  './skel/var/lib/koha/zebradb/biblios/shadow'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
321
  './skel/var/lib/koha/zebradb/biblios/shadow'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
320
  './skel/var/lib/koha/zebradb/biblios/tmp'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
322
  './skel/var/lib/koha/zebradb/biblios/tmp'  => { target => 'ZEBRA_DATA_DIR', trimdir => 6 },
323
  './skel/var/lock/koha/zebradb/rebuild' => { target => 'ZEBRA_LOCK_DIR', trimdir => 6 },
321
  './skel/var/lib/koha/plugins' => { target => 'PLUGINS_DIR', trimdir => 6 },
324
  './skel/var/lib/koha/plugins' => { target => 'PLUGINS_DIR', trimdir => 6 },
322
  './sms'                       => 'INTRANET_CGI_DIR',
325
  './sms'                       => 'INTRANET_CGI_DIR',
323
  './suggestion'                => 'INTRANET_CGI_DIR',
326
  './suggestion'                => 'INTRANET_CGI_DIR',
(-)a/debian/templates/koha-conf-site.xml.in (+1 lines)
Lines 280-285 Link Here
280
 <memcached_namespace>__MEMCACHED_NAMESPACE__</memcached_namespace>
280
 <memcached_namespace>__MEMCACHED_NAMESPACE__</memcached_namespace>
281
 <zebra_bib_index_mode>__BIBLIOS_INDEXING_MODE__</zebra_bib_index_mode>
281
 <zebra_bib_index_mode>__BIBLIOS_INDEXING_MODE__</zebra_bib_index_mode>
282
 <zebra_auth_index_mode>__AUTHORITIES_INDEXING_MODE__</zebra_auth_index_mode>
282
 <zebra_auth_index_mode>__AUTHORITIES_INDEXING_MODE__</zebra_auth_index_mode>
283
 <zebra_lockdir>/var/lock/koha/__KOHASITE__</zebra_lockdir>
283
 <queryparser_config>/etc/koha/searchengine/queryparser.yaml</queryparser_config>
284
 <queryparser_config>/etc/koha/searchengine/queryparser.yaml</queryparser_config>
284
</config>
285
</config>
285
286
(-)a/etc/koha-conf.xml (+1 lines)
Lines 297-302 __PAZPAR2_TOGGLE_XML_POST__ Link Here
297
 <useldapserver>0</useldapserver><!-- see C4::Auth_with_ldap for extra configs you must add if you want to turn this on -->
297
 <useldapserver>0</useldapserver><!-- see C4::Auth_with_ldap for extra configs you must add if you want to turn this on -->
298
 <zebra_bib_index_mode>__BIB_INDEX_MODE__</zebra_bib_index_mode>
298
 <zebra_bib_index_mode>__BIB_INDEX_MODE__</zebra_bib_index_mode>
299
 <zebra_auth_index_mode>__AUTH_INDEX_MODE__</zebra_auth_index_mode>
299
 <zebra_auth_index_mode>__AUTH_INDEX_MODE__</zebra_auth_index_mode>
300
 <zebra_lockdir>__ZEBRA_LOCK_DIR__</zebra_lockdir>
300
 <queryparser_config>__KOHA_CONF_DIR__/searchengine/queryparser.yaml</queryparser_config>
301
 <queryparser_config>__KOHA_CONF_DIR__/searchengine/queryparser.yaml</queryparser_config>
301
</config>
302
</config>
302
</yazgfs>
303
</yazgfs>
(-)a/misc/bin/koha-zebra-ctl.sh (+1 lines)
Lines 65-70 case "$1" in Link Here
65
        mkdir -p $LOCKDIR
65
        mkdir -p $LOCKDIR
66
        mkdir -p $LOCKDIR/biblios
66
        mkdir -p $LOCKDIR/biblios
67
        mkdir -p $LOCKDIR/authorities
67
        mkdir -p $LOCKDIR/authorities
68
        mkdir -p $LOCKDIR/rebuild
68
        if [[ $EUID -eq 0 ]]; then
69
        if [[ $EUID -eq 0 ]]; then
69
            chown -R $USER:$GROUP $LOCKDIR
70
            chown -R $USER:$GROUP $LOCKDIR
70
        fi
71
        fi
(-)a/misc/migration_tools/rebuild_zebra.pl (-7 / +76 lines)
Lines 5-10 use strict; Link Here
5
5
6
use C4::Context;
6
use C4::Context;
7
use Getopt::Long;
7
use Getopt::Long;
8
use Fcntl qw(:flock);
8
use File::Temp qw/ tempdir /;
9
use File::Temp qw/ tempdir /;
9
use File::Path;
10
use File::Path;
10
use C4::Biblio;
11
use C4::Biblio;
Lines 42-47 my $where; Link Here
42
my $offset;
43
my $offset;
43
my $run_as_root;
44
my $run_as_root;
44
my $run_user = (getpwuid($<))[0];
45
my $run_user = (getpwuid($<))[0];
46
my $wait_for_lock = 0;
47
my $use_flock;
45
48
46
my $verbose_logging = 0;
49
my $verbose_logging = 0;
47
my $zebraidx_log_opt = " -v none,fatal,warn ";
50
my $zebraidx_log_opt = " -v none,fatal,warn ";
Lines 62-72 my $result = GetOptions( Link Here
62
    'x'             => \$as_xml,
65
    'x'             => \$as_xml,
63
    'y'             => \$do_not_clear_zebraqueue,
66
    'y'             => \$do_not_clear_zebraqueue,
64
    'z'             => \$process_zebraqueue,
67
    'z'             => \$process_zebraqueue,
65
    'where:s'        => \$where,
68
    'where:s'       => \$where,
66
    'length:i'        => \$length,
69
    'length:i'      => \$length,
67
    'offset:i'      => \$offset,
70
    'offset:i'      => \$offset,
68
    'v+'             => \$verbose_logging,
71
    'v+'            => \$verbose_logging,
69
    'run-as-root'    => \$run_as_root,
72
    'run-as-root'   => \$run_as_root,
73
    'wait-for-lock' => \$wait_for_lock,
70
);
74
);
71
75
72
if (not $result or $want_help) {
76
if (not $result or $want_help) {
Lines 151-162 my $dbh = C4::Context->dbh; Link Here
151
my ($biblionumbertagfield,$biblionumbertagsubfield) = &GetMarcFromKohaField("biblio.biblionumber","");
155
my ($biblionumbertagfield,$biblionumbertagsubfield) = &GetMarcFromKohaField("biblio.biblionumber","");
152
my ($biblioitemnumbertagfield,$biblioitemnumbertagsubfield) = &GetMarcFromKohaField("biblioitems.biblioitemnumber","");
156
my ($biblioitemnumbertagfield,$biblioitemnumbertagsubfield) = &GetMarcFromKohaField("biblioitems.biblioitemnumber","");
153
157
158
# Protect again simultaneous update of the zebra index by using a lock file.
159
# Create our own lock directory if its missing.  This shouild be created
160
# by koha-zebra-ctl.sh or at system installation.  If the desired directory
161
# does not exist and cannot be created, we fall back on /tmp - which will
162
# always work.
163
164
my $lockdir = C4::Context->config("zebra_lockdir") // "/var/lock";
165
$lockdir .= "/rebuild";
166
unless (-d $lockdir) {
167
    eval { mkpath($lockdir, 0, oct(755)) };
168
    $lockdir = "/tmp" if ($@);
169
}
170
my $lockfile = $lockdir . "/rebuild..LCK";
171
154
if ( $verbose_logging ) {
172
if ( $verbose_logging ) {
155
    print "Zebra configuration information\n";
173
    print "Zebra configuration information\n";
156
    print "================================\n";
174
    print "================================\n";
157
    print "Zebra biblio directory      = $biblioserverdir\n";
175
    print "Zebra biblio directory      = $biblioserverdir\n";
158
    print "Zebra authorities directory = $authorityserverdir\n";
176
    print "Zebra authorities directory = $authorityserverdir\n";
159
    print "Koha directory              = $kohadir\n";
177
    print "Koha directory              = $kohadir\n";
178
    print "Lockfile                    = $lockfile\n";
160
    print "BIBLIONUMBER in :     $biblionumbertagfield\$$biblionumbertagsubfield\n";
179
    print "BIBLIONUMBER in :     $biblionumbertagfield\$$biblionumbertagsubfield\n";
161
    print "BIBLIOITEMNUMBER in : $biblioitemnumbertagfield\$$biblioitemnumbertagsubfield\n";
180
    print "BIBLIOITEMNUMBER in : $biblioitemnumbertagfield\$$biblioitemnumbertagsubfield\n";
162
    print "================================\n";
181
    print "================================\n";
Lines 164-176 if ( $verbose_logging ) { Link Here
164
183
165
my $tester = XML::LibXML->new();
184
my $tester = XML::LibXML->new();
166
185
186
# The main work is done here by calling do_one_pass().  We have added locking
187
# avoid race conditions between Full rebuilds and incremental updates either from
188
# daemon mode or periodic invocation from cron.  The race can lead to an updated
189
# record being overwritten by a rebuild if the update is applied after the export
190
# by the rebuild and before the rebuild finishes (more likely to effect large
191
# catalogs).
192
#
193
# We have chosen to exit immediately by default if we cannot obtain the lock
194
# to prevent the potential for a infinite backlog from cron invocations, but an
195
# option (wait-for-lock) is provided to let the program wait for the lock.
196
# See http://bugs.koha-community.org/bugzilla3/show_bug.cgi?id=11078 for details.
197
open my $LockFH, q{>}, $lockfile or die "$lockfile: $!";
167
if ($daemon_mode) {
198
if ($daemon_mode) {
168
    while (1) {
199
    while (1) {
169
        do_one_pass() if ( zebraqueue_not_empty() );
200
        # For incremental updates, skip the update if the updates are locked
201
        if (_flock($LockFH, LOCK_EX|LOCK_NB)) {
202
            do_one_pass() if ( zebraqueue_not_empty() );
203
            _flock($LockFH, LOCK_UN);
204
        }
170
        sleep $daemon_sleep;
205
        sleep $daemon_sleep;
171
    }
206
    }
172
} else {
207
} else {
173
    do_one_pass();
208
    # all one-off invocations
209
    my $lock_mode = ($wait_for_lock) ? LOCK_EX : LOCK_EX|LOCK_NB;
210
    if (_flock($LockFH, $lock_mode)) {
211
        do_one_pass();
212
        _flock($LockFH, LOCK_UN);
213
    } else {
214
        # Can't die() here because we have files to dlean up.
215
        print "Aborting rebuild.  Unable to flock $lockfile: $!\n";
216
    }
174
}
217
}
175
218
176
219
Lines 228-234 sub zebraqueue_not_empty { Link Here
228
        $where_str = 'server = "authorityserver" AND done = 0;';
271
        $where_str = 'server = "authorityserver" AND done = 0;';
229
    }
272
    }
230
    my $query =
273
    my $query =
231
      $dbh->prepare( 'SELECT COUNT(*) FROM zebraqueue WHERE ' . $where_str );
274
        $dbh->prepare('SELECT COUNT(*) FROM zebraqueue WHERE ' . $where_str );
232
275
233
    $query->execute;
276
    $query->execute;
234
    my $count = $query->fetchrow_arrayref->[0];
277
    my $count = $query->fetchrow_arrayref->[0];
Lines 724-729 sub do_indexing { Link Here
724
767
725
}
768
}
726
769
770
sub _flock {
771
# test if flock is present; if so, use it; if not, return true
772
# op refers to the official flock operations incl LOCK_EX, LOCK_UN, etc.
773
# combining LOCK_EX with LOCK_NB returns immediately
774
    my ($fh, $op)= @_;
775
    if( !defined($use_flock) ) {
776
        #check if flock is present; if not, you will have a fatal error
777
        my $i=eval { flock($fh, $op) };
778
        #assuming that $fh and $op are fine(..), an undef i means no flock
779
        $use_flock= defined($i)? 1: 0;
780
        print "Warning: flock could not be used!\n" if $verbose_logging && !$use_flock;
781
        return 1 if !$use_flock;
782
        return $i;
783
    }
784
    else {
785
        return 1 if !$use_flock;
786
        return flock($fh, $op);
787
    }
788
}
789
727
sub print_usage {
790
sub print_usage {
728
    print <<_USAGE_;
791
    print <<_USAGE_;
729
$0: reindex MARC bibs and/or authorities in Zebra.
792
$0: reindex MARC bibs and/or authorities in Zebra.
Lines 809-814 Parameters: Link Here
809
872
810
    --run-as-root           explicitily allow script to run as 'root' user
873
    --run-as-root           explicitily allow script to run as 'root' user
811
874
875
    --wait-for-lock         when not running in daemon mode, the default
876
                            behavior is to abort a rebuild if the rebuild
877
                            lock is busy.  This option will cause the program
878
                            to wait for the lock to free and then continue
879
                            processing the rebuild request,
880
812
    --help or -h            show this message.
881
    --help or -h            show this message.
813
_USAGE_
882
_USAGE_
814
}
883
}
(-)a/skel/var/lock/koha/zebradb/rebuild/README (+1 lines)
Line 0 Link Here
1
Zebra rebuild lock dir
(-)a/t/db_dependent/zebra_config.pl (-1 / +1 lines)
Lines 31-36 if ($indexing_mode eq 'dom') { Link Here
31
make_path("$destination/var/lock/zebradb");
31
make_path("$destination/var/lock/zebradb");
32
make_path("$destination/var/lock/zebradb/biblios");
32
make_path("$destination/var/lock/zebradb/biblios");
33
make_path("$destination/var/lock/zebradb/authorities");
33
make_path("$destination/var/lock/zebradb/authorities");
34
make_path("$destination/var/lock/zebradb/rebuild");
34
make_path("$destination/var/lib/zebradb");
35
make_path("$destination/var/lib/zebradb");
35
make_path("$destination/var/lib/zebradb/biblios");
36
make_path("$destination/var/lib/zebradb/biblios");
36
make_path("$destination/var/lib/zebradb/biblios/key");
37
make_path("$destination/var/lib/zebradb/biblios/key");
37
- 

Return to bug 11078