Line 0
Link Here
|
0 |
- |
1 |
#!/usr/bin/zsh |
|
|
2 |
|
3 |
# Copyright 2011 BibLibre SARL |
4 |
# This file is part of Koha. |
5 |
# |
6 |
# Koha is free software; you can redistribute it and/or modify it under the |
7 |
# terms of the GNU General Public License as published by the Free Software |
8 |
# Foundation; either version 2 of the License, or (at your option) any later |
9 |
# version. |
10 |
# |
11 |
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY |
12 |
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR |
13 |
# A PARTICULAR PURPOSE. See the GNU General Public License for more details. |
14 |
# |
15 |
# You should have received a copy of the GNU General Public License along |
16 |
# with Koha; if not, write to the Free Software Foundation, Inc., |
17 |
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
18 |
|
19 |
|
20 |
INCREMENT=$1 |
21 |
BIBLIOSTART=$2 |
22 |
BIBLIOEND=$3 |
23 |
TEMPDIRECTORY=$4 |
24 |
RMLOGS=$5 |
25 |
|
26 |
#echo " | $INCREMENT , $BIBLIOSTART , $BIBLIOEND , $TEMPDIRECTORY , $RMLOGS | "; |
27 |
# this script rebuild the zebra index recursively |
28 |
# INCREMENT specify how many records we must try at once |
29 |
# BIBLIOSTART is the record number to BIBLIOSTART on |
30 |
# BIBLIOEND is the record number to BIBLIOEND on, if it's greater than the last biblionumber it will be redefine |
31 |
# TEMPDIRECTORY define a temporary directory to rebuild_zebra |
32 |
# RMLOGS is used to remove files into logs/, it's use and define by script itself so you have not to define it |
33 |
# At the BIBLIOEND of each "chunk", this script checks if the indexing process has been OK |
34 |
# if it hasn't, the slice is splitted in 10, and the reindexing is called again on each smaller chunk |
35 |
# if the increment goes to 1, it means we tried to reindex 1 by 1, and the failing indexing concern wrong records |
36 |
|
37 |
# the logs are stored in a directory called logs/ that must be a subdirectory of reindex.zsh |
38 |
|
39 |
# at the BIBLIOEND of the script, just type : |
40 |
#grep -l "previous transaction" `ls rebuild1.*.err` |
41 |
# the result contains all the biblios that have not been indexed |
42 |
# WARNING : the numbers are not the biblionumber but the record number, they can be reached by : |
43 |
# SELECT biblionumber FROM biblio LIMIT YourNumberHere,1; |
44 |
|
45 |
# EXAMPLE to run the script on a 800 000 biblios database : |
46 |
# ./reindex.zsh 50000 0 800000 |
47 |
# will reindex the DB, BIBLIOSTARTing with chunks of 50k biblios |
48 |
#if { grep -E "previous transaction" `dirname $0`/logs/rebuild$INCREMENT.$i.err } ; then |
49 |
|
50 |
lastbiblionumber=`perl -e '#!/usr/bin/perl |
51 |
use C4::Context; |
52 |
my $dbh = C4::Context->dbh; |
53 |
my $querylastbiblionumber = "SELECT max(biblionumber) FROM biblio;"; |
54 |
my $sthlastbiblionumber = $dbh->prepare($querylastbiblionumber); |
55 |
$sthlastbiblionumber->execute(); |
56 |
my ( $lastbiblionumber ) = $sthlastbiblionumber->fetchrow_array; print $lastbiblionumber;'` |
57 |
#echo $lastbiblionumber; |
58 |
let "maxbiblionumber = $lastbiblionumber + 1" |
59 |
if [ $# = 4 ] |
60 |
then |
61 |
RMLOGS=yes |
62 |
elif [ $# = 3 ] |
63 |
then |
64 |
RMLOGS=yes |
65 |
TEMPDIRECTORY=/tmp/rebuild |
66 |
elif [ $# = 2 ] |
67 |
then |
68 |
RMLOGS=yes |
69 |
TEMPDIRECTORY=/tmp/rebuild |
70 |
BIBLIOEND=$lastbiblionumber |
71 |
elif [ $# = 1 ] |
72 |
then |
73 |
RMLOGS=yes |
74 |
TEMPDIRECTORY=/tmp/rebuild |
75 |
BIBLIOSTART=0 |
76 |
BIBLIOEND=$lastbiblionumber |
77 |
elif [ $# = 0 ] |
78 |
then |
79 |
RMLOGS=yes |
80 |
TEMPDIRECTORY=/tmp/rebuild |
81 |
INCREMENT=10000 |
82 |
BIBLIOSTART=0 |
83 |
BIBLIOEND=$lastbiblionumber |
84 |
fi |
85 |
if [[ $INCREMENT =~ ^10*$ ]] |
86 |
then |
87 |
else |
88 |
echo "The first argument (INCREMENT) must be 1 or a multiple of 10" |
89 |
exit 2 |
90 |
fi |
91 |
if [[ $BIBLIOSTART =~ ^[0-9]*$ ]] |
92 |
then |
93 |
else |
94 |
echo "The second argument (BIBLIOSTART) must be an integer" |
95 |
exit 2 |
96 |
fi |
97 |
if [[ $BIBLIOEND =~ ^[0-9]*$ ]] |
98 |
then |
99 |
else |
100 |
echo "The third argument (BIBLIOEND) must be an integer" |
101 |
exit 2 |
102 |
fi |
103 |
if [ $BIBLIOSTART -ge $BIBLIOEND ] |
104 |
then |
105 |
echo "The second argument (BIBLIOSTART) must be lower than the third argument (BIBLIOEND)" |
106 |
exit 2 |
107 |
fi |
108 |
if [[ $maxbiblionumber =~ ^[0-9]*$ ]] |
109 |
then |
110 |
else |
111 |
echo "error : last biblionumber is not an integer" |
112 |
exit 2 |
113 |
fi |
114 |
if [ $BIBLIOEND -gt $lastbiblionumber ]; |
115 |
then |
116 |
BIBLIOEND=$lastbiblionumber |
117 |
fi |
118 |
ls `dirname $0`/logs/ >/dev/null 2>&1 |
119 |
if [ $? != 0 ] |
120 |
then |
121 |
mkdir `dirname $0`/logs |
122 |
else |
123 |
if [ $RMLOGS = "yes" ] |
124 |
then |
125 |
rm `dirname $0`/logs/* |
126 |
fi |
127 |
fi |
128 |
ls $TEMPDIRECTORY >/dev/null 2>&1 |
129 |
if [ $? != 0 ] |
130 |
then |
131 |
mkdir $TEMPDIRECTORY |
132 |
if [ $? != 0 ] |
133 |
then |
134 |
echo "can't create dir $TEMPDIRECTORY" |
135 |
exit 2 |
136 |
fi |
137 |
fi |
138 |
#/home/koha/src/misc/migration_tools/rebuild_zebra.pl -r -b -v -x -nosanitize -length 1 -offset 1 |
139 |
for ((i=$BIBLIOSTART ; i<=$BIBLIOEND ; i=i+$INCREMENT)) |
140 |
do |
141 |
echo "I = " $i "with increment " $INCREMENT |
142 |
#echo "BIBLIOSTART $BIBLIOSTART INCREMENT $INCREMENT BIBLIOEND $BIBLIOEND i $i" |
143 |
#echo "`dirname $0`/rebuild_zebra.pl -b -v -x -nosanitize -d $TEMPDIRECTORY -k -length $INCREMENT -offset $i -v log" |
144 |
`dirname $0`/rebuild_zebra.pl -b -v -v -x -nosanitize -d $TEMPDIRECTORY -k -length $INCREMENT -offset $i > `dirname $0`/logs/rebuild$INCREMENT.$i.log 2> `dirname $0`/logs/rebuild$INCREMENT.$i.err |
145 |
if (($INCREMENT >1 )); |
146 |
then |
147 |
if { grep -q "previous transaction" `dirname $0`/logs/rebuild$INCREMENT.$i.err } ; |
148 |
then |
149 |
echo "I must split $i (increment $INCREMENT) because previous transaction didn't reach commit" |
150 |
((subincrement=$INCREMENT/10)) |
151 |
if (($i+$INCREMENT >= $BIBLIOEND )); |
152 |
then |
153 |
((newBIBLIOEND=$BIBLIOEND)) |
154 |
else |
155 |
((newBIBLIOEND=$i+$INCREMENT)) |
156 |
fi |
157 |
echo "$0 $subincrement $i $newBIBLIOEND $TEMPDIRECTORY no" |
158 |
$0 $subincrement $i $newBIBLIOEND $TEMPDIRECTORY no |
159 |
elif { ! grep -q "Records: $INCREMENT" `dirname $0`/logs/rebuild$INCREMENT.$i.err } ; |
160 |
then |
161 |
echo "I must split $i (increment $INCREMENT) because index was uncomplete, less than $INCREMENT records indexed" |
162 |
((subincrement=$INCREMENT/10)) |
163 |
if (($i+$INCREMENT >= $BIBLIOEND )); |
164 |
then |
165 |
((newBIBLIOEND=$BIBLIOEND)) |
166 |
else |
167 |
((newBIBLIOEND=$i+$INCREMENT)) |
168 |
fi |
169 |
echo "$0 $subincrement $i $newBIBLIOEND $TEMPDIRECTORY no" |
170 |
$0 $subincrement $i $newBIBLIOEND $TEMPDIRECTORY no |
171 |
fi |
172 |
fi |
173 |
done |
174 |
exit 0 |