View | Details | Raw Unified | Return to bug 7286
Collapse All | Expand All

(-)a/misc/migration_tools/rebuild_zebra_sliced.zsh (-1 / +174 lines)
Line 0 Link Here
0
- 
1
#!/usr/bin/zsh
2
3
# Copyright 2011 BibLibre SARL
4
# This file is part of Koha.
5
#
6
# Koha is free software; you can redistribute it and/or modify it under the
7
# terms of the GNU General Public License as published by the Free Software
8
# Foundation; either version 2 of the License, or (at your option) any later
9
# version.
10
#
11
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
12
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
14
#
15
# You should have received a copy of the GNU General Public License along
16
# with Koha; if not, write to the Free Software Foundation, Inc.,
17
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
18
19
20
INCREMENT=$1
21
BIBLIOSTART=$2
22
BIBLIOEND=$3
23
TEMPDIRECTORY=$4
24
RMLOGS=$5
25
26
#echo " | $INCREMENT , $BIBLIOSTART , $BIBLIOEND , $TEMPDIRECTORY , $RMLOGS | ";
27
# this script rebuild the zebra index recursively
28
# INCREMENT specify how many records we must try at once
29
# BIBLIOSTART is the record number to BIBLIOSTART on
30
# BIBLIOEND is the record number to BIBLIOEND on, if it's greater than the last biblionumber it will be redefine
31
# TEMPDIRECTORY define a temporary directory to rebuild_zebra
32
# RMLOGS is used to remove files into logs/, it's use and define by script itself so you have not to define it
33
# At the BIBLIOEND of each "chunk", this script checks if the indexing process has been OK
34
# if it hasn't, the slice is splitted in 10, and the reindexing is called again on each smaller chunk
35
# if the increment goes to 1, it means we tried to reindex 1 by 1, and the failing indexing concern wrong records
36
37
# the logs are stored in a directory called logs/ that must be a subdirectory of reindex.zsh
38
39
# at the BIBLIOEND of the script, just type :
40
#grep -l "previous transaction" `ls rebuild1.*.err`
41
# the result contains all the biblios that have not been indexed
42
# WARNING : the numbers are not the biblionumber but the record number, they can be reached by :
43
# SELECT biblionumber FROM biblio LIMIT YourNumberHere,1;
44
45
# EXAMPLE to run the script on a 800 000 biblios database :
46
# ./reindex.zsh 50000 0 800000
47
# will reindex the DB, BIBLIOSTARTing with chunks of 50k biblios
48
#if { grep -E "previous transaction" `dirname $0`/logs/rebuild$INCREMENT.$i.err } ; then
49
50
lastbiblionumber=`perl -e '#!/usr/bin/perl
51
use C4::Context;
52
my $dbh = C4::Context->dbh;
53
my $querylastbiblionumber = "SELECT max(biblionumber) FROM biblio;";
54
my $sthlastbiblionumber   = $dbh->prepare($querylastbiblionumber);
55
$sthlastbiblionumber->execute();
56
my ( $lastbiblionumber ) = $sthlastbiblionumber->fetchrow_array; print $lastbiblionumber;'`
57
#echo $lastbiblionumber;
58
let "maxbiblionumber = $lastbiblionumber + 1"
59
if [ $# = 4 ]
60
then
61
    RMLOGS=yes
62
elif [ $# = 3 ]
63
then
64
    RMLOGS=yes
65
    TEMPDIRECTORY=/tmp/rebuild
66
elif [ $# = 2 ]
67
then
68
    RMLOGS=yes
69
    TEMPDIRECTORY=/tmp/rebuild
70
    BIBLIOEND=$lastbiblionumber
71
elif [ $# = 1 ]
72
then
73
    RMLOGS=yes
74
    TEMPDIRECTORY=/tmp/rebuild
75
    BIBLIOSTART=0
76
    BIBLIOEND=$lastbiblionumber
77
elif [ $# = 0 ]
78
then
79
    RMLOGS=yes
80
    TEMPDIRECTORY=/tmp/rebuild
81
    INCREMENT=10000
82
    BIBLIOSTART=0
83
    BIBLIOEND=$lastbiblionumber
84
fi
85
if [[ $INCREMENT =~ ^10*$ ]]
86
then
87
else
88
    echo "The first argument (INCREMENT) must be 1 or a multiple of 10"
89
    exit 2
90
fi
91
if [[ $BIBLIOSTART =~ ^[0-9]*$ ]]
92
then
93
else
94
    echo "The second argument (BIBLIOSTART) must be an integer"
95
    exit 2
96
fi
97
if [[ $BIBLIOEND =~ ^[0-9]*$ ]]
98
then
99
else
100
    echo "The third argument (BIBLIOEND) must be an integer"
101
    exit 2
102
fi
103
if [ $BIBLIOSTART -ge $BIBLIOEND ]
104
then
105
    echo "The second argument (BIBLIOSTART) must be lower than the third argument (BIBLIOEND)"
106
    exit 2
107
fi
108
if [[ $maxbiblionumber =~ ^[0-9]*$ ]]
109
then
110
else
111
    echo "error : last biblionumber is not an integer"
112
    exit 2
113
fi
114
if [ $BIBLIOEND -gt $lastbiblionumber ];
115
then
116
    BIBLIOEND=$lastbiblionumber
117
fi
118
ls `dirname $0`/logs/ >/dev/null 2>&1
119
if [ $? != 0 ]
120
then
121
    mkdir `dirname $0`/logs
122
else
123
    if [ $RMLOGS = "yes" ]
124
    then
125
        rm `dirname $0`/logs/*
126
    fi
127
fi
128
ls $TEMPDIRECTORY >/dev/null 2>&1
129
if [ $? != 0 ]
130
then
131
    mkdir $TEMPDIRECTORY
132
    if [ $? != 0 ]
133
    then
134
        echo "can't create dir $TEMPDIRECTORY"
135
        exit 2
136
    fi
137
fi
138
#/home/koha/src/misc/migration_tools/rebuild_zebra.pl -r -b -v -x -nosanitize -length 1 -offset 1
139
for ((i=$BIBLIOSTART ; i<=$BIBLIOEND ; i=i+$INCREMENT))
140
do
141
    echo "I = " $i "with increment " $INCREMENT
142
    #echo "BIBLIOSTART $BIBLIOSTART INCREMENT $INCREMENT BIBLIOEND $BIBLIOEND i $i"
143
    #echo "`dirname $0`/rebuild_zebra.pl -b -v -x -nosanitize -d $TEMPDIRECTORY -k -length $INCREMENT -offset $i -v log"
144
    `dirname $0`/rebuild_zebra.pl -b -v -v -x -nosanitize -d $TEMPDIRECTORY -k -length $INCREMENT -offset $i > `dirname $0`/logs/rebuild$INCREMENT.$i.log 2> `dirname $0`/logs/rebuild$INCREMENT.$i.err
145
    if (($INCREMENT >1 ));
146
    then
147
        if { grep -q "previous transaction" `dirname $0`/logs/rebuild$INCREMENT.$i.err } ;
148
        then
149
            echo "I must split $i (increment $INCREMENT) because previous transaction didn't reach commit"
150
            ((subincrement=$INCREMENT/10))
151
            if (($i+$INCREMENT >= $BIBLIOEND ));
152
            then
153
                ((newBIBLIOEND=$BIBLIOEND))
154
            else
155
                ((newBIBLIOEND=$i+$INCREMENT))
156
            fi
157
            echo "$0 $subincrement $i $newBIBLIOEND $TEMPDIRECTORY no"
158
            $0 $subincrement $i $newBIBLIOEND $TEMPDIRECTORY no
159
        elif { ! grep -q "Records: $INCREMENT" `dirname $0`/logs/rebuild$INCREMENT.$i.err } ;
160
        then
161
            echo "I must split $i (increment $INCREMENT) because index was uncomplete, less than $INCREMENT records indexed"
162
            ((subincrement=$INCREMENT/10))
163
            if (($i+$INCREMENT >= $BIBLIOEND ));
164
            then
165
                ((newBIBLIOEND=$BIBLIOEND))
166
            else
167
                ((newBIBLIOEND=$i+$INCREMENT))
168
            fi
169
            echo "$0 $subincrement $i $newBIBLIOEND $TEMPDIRECTORY no"
170
            $0 $subincrement $i $newBIBLIOEND $TEMPDIRECTORY no
171
        fi
172
    fi
173
done
174
exit 0

Return to bug 7286