|
Line 0
Link Here
|
| 0 |
- |
1 |
#!/usr/bin/zsh |
|
|
2 |
|
| 3 |
# Copyright 2011 BibLibre SARL |
| 4 |
# This file is part of Koha. |
| 5 |
# |
| 6 |
# Koha is free software; you can redistribute it and/or modify it under the |
| 7 |
# terms of the GNU General Public License as published by the Free Software |
| 8 |
# Foundation; either version 2 of the License, or (at your option) any later |
| 9 |
# version. |
| 10 |
# |
| 11 |
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY |
| 12 |
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR |
| 13 |
# A PARTICULAR PURPOSE. See the GNU General Public License for more details. |
| 14 |
# |
| 15 |
# You should have received a copy of the GNU General Public License along |
| 16 |
# with Koha; if not, write to the Free Software Foundation, Inc., |
| 17 |
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
| 18 |
|
| 19 |
|
| 20 |
INCREMENT=$1 |
| 21 |
BIBLIOSTART=$2 |
| 22 |
BIBLIOEND=$3 |
| 23 |
TEMPDIRECTORY=$4 |
| 24 |
RMLOGS=$5 |
| 25 |
|
| 26 |
#echo " | $INCREMENT , $BIBLIOSTART , $BIBLIOEND , $TEMPDIRECTORY , $RMLOGS | "; |
| 27 |
# this script rebuild the zebra index recursively |
| 28 |
# INCREMENT specify how many records we must try at once |
| 29 |
# BIBLIOSTART is the record number to BIBLIOSTART on |
| 30 |
# BIBLIOEND is the record number to BIBLIOEND on, if it's greater than the last biblionumber it will be redefine |
| 31 |
# TEMPDIRECTORY define a temporary directory to rebuild_zebra |
| 32 |
# RMLOGS is used to remove files into logs/, it's use and define by script itself so you have not to define it |
| 33 |
# At the BIBLIOEND of each "chunk", this script checks if the indexing process has been OK |
| 34 |
# if it hasn't, the slice is splitted in 10, and the reindexing is called again on each smaller chunk |
| 35 |
# if the increment goes to 1, it means we tried to reindex 1 by 1, and the failing indexing concern wrong records |
| 36 |
|
| 37 |
# the logs are stored in a directory called logs/ that must be a subdirectory of reindex.zsh |
| 38 |
|
| 39 |
# at the BIBLIOEND of the script, just type : |
| 40 |
#grep -l "previous transaction" `ls rebuild1.*.err` |
| 41 |
# the result contains all the biblios that have not been indexed |
| 42 |
# WARNING : the numbers are not the biblionumber but the record number, they can be reached by : |
| 43 |
# SELECT biblionumber FROM biblio LIMIT YourNumberHere,1; |
| 44 |
|
| 45 |
# EXAMPLE to run the script on a 800 000 biblios database : |
| 46 |
# ./reindex.zsh 50000 0 800000 |
| 47 |
# will reindex the DB, BIBLIOSTARTing with chunks of 50k biblios |
| 48 |
#if { grep -E "previous transaction" `dirname $0`/logs/rebuild$INCREMENT.$i.err } ; then |
| 49 |
|
| 50 |
lastbiblionumber=`perl -e '#!/usr/bin/perl |
| 51 |
use C4::Context; |
| 52 |
my $dbh = C4::Context->dbh; |
| 53 |
my $querylastbiblionumber = "SELECT max(biblionumber) FROM biblio;"; |
| 54 |
my $sthlastbiblionumber = $dbh->prepare($querylastbiblionumber); |
| 55 |
$sthlastbiblionumber->execute(); |
| 56 |
my ( $lastbiblionumber ) = $sthlastbiblionumber->fetchrow_array; print $lastbiblionumber;'` |
| 57 |
#echo $lastbiblionumber; |
| 58 |
let "maxbiblionumber = $lastbiblionumber + 1" |
| 59 |
if [ $# = 4 ] |
| 60 |
then |
| 61 |
RMLOGS=yes |
| 62 |
elif [ $# = 3 ] |
| 63 |
then |
| 64 |
RMLOGS=yes |
| 65 |
TEMPDIRECTORY=/tmp/rebuild |
| 66 |
elif [ $# = 2 ] |
| 67 |
then |
| 68 |
RMLOGS=yes |
| 69 |
TEMPDIRECTORY=/tmp/rebuild |
| 70 |
BIBLIOEND=$lastbiblionumber |
| 71 |
elif [ $# = 1 ] |
| 72 |
then |
| 73 |
RMLOGS=yes |
| 74 |
TEMPDIRECTORY=/tmp/rebuild |
| 75 |
BIBLIOSTART=0 |
| 76 |
BIBLIOEND=$lastbiblionumber |
| 77 |
elif [ $# = 0 ] |
| 78 |
then |
| 79 |
RMLOGS=yes |
| 80 |
TEMPDIRECTORY=/tmp/rebuild |
| 81 |
INCREMENT=10000 |
| 82 |
BIBLIOSTART=0 |
| 83 |
BIBLIOEND=$lastbiblionumber |
| 84 |
fi |
| 85 |
if [[ $INCREMENT =~ ^10*$ ]] |
| 86 |
then |
| 87 |
else |
| 88 |
echo "The first argument (INCREMENT) must be 1 or a multiple of 10" |
| 89 |
exit 2 |
| 90 |
fi |
| 91 |
if [[ $BIBLIOSTART =~ ^[0-9]*$ ]] |
| 92 |
then |
| 93 |
else |
| 94 |
echo "The second argument (BIBLIOSTART) must be an integer" |
| 95 |
exit 2 |
| 96 |
fi |
| 97 |
if [[ $BIBLIOEND =~ ^[0-9]*$ ]] |
| 98 |
then |
| 99 |
else |
| 100 |
echo "The third argument (BIBLIOEND) must be an integer" |
| 101 |
exit 2 |
| 102 |
fi |
| 103 |
if [ $BIBLIOSTART -ge $BIBLIOEND ] |
| 104 |
then |
| 105 |
echo "The second argument (BIBLIOSTART) must be lower than the third argument (BIBLIOEND)" |
| 106 |
exit 2 |
| 107 |
fi |
| 108 |
if [[ $maxbiblionumber =~ ^[0-9]*$ ]] |
| 109 |
then |
| 110 |
else |
| 111 |
echo "error : last biblionumber is not an integer" |
| 112 |
exit 2 |
| 113 |
fi |
| 114 |
if [ $BIBLIOEND -gt $lastbiblionumber ]; |
| 115 |
then |
| 116 |
BIBLIOEND=$lastbiblionumber |
| 117 |
fi |
| 118 |
ls `dirname $0`/logs/ >/dev/null 2>&1 |
| 119 |
if [ $? != 0 ] |
| 120 |
then |
| 121 |
mkdir `dirname $0`/logs |
| 122 |
else |
| 123 |
if [ $RMLOGS = "yes" ] |
| 124 |
then |
| 125 |
rm `dirname $0`/logs/* |
| 126 |
fi |
| 127 |
fi |
| 128 |
ls $TEMPDIRECTORY >/dev/null 2>&1 |
| 129 |
if [ $? != 0 ] |
| 130 |
then |
| 131 |
mkdir $TEMPDIRECTORY |
| 132 |
if [ $? != 0 ] |
| 133 |
then |
| 134 |
echo "can't create dir $TEMPDIRECTORY" |
| 135 |
exit 2 |
| 136 |
fi |
| 137 |
fi |
| 138 |
#/home/koha/src/misc/migration_tools/rebuild_zebra.pl -r -b -v -x -nosanitize -length 1 -offset 1 |
| 139 |
for ((i=$BIBLIOSTART ; i<=$BIBLIOEND ; i=i+$INCREMENT)) |
| 140 |
do |
| 141 |
echo "I = " $i "with increment " $INCREMENT |
| 142 |
#echo "BIBLIOSTART $BIBLIOSTART INCREMENT $INCREMENT BIBLIOEND $BIBLIOEND i $i" |
| 143 |
#echo "`dirname $0`/rebuild_zebra.pl -b -v -x -nosanitize -d $TEMPDIRECTORY -k -length $INCREMENT -offset $i -v log" |
| 144 |
`dirname $0`/rebuild_zebra.pl -b -v -v -x -nosanitize -d $TEMPDIRECTORY -k -length $INCREMENT -offset $i > `dirname $0`/logs/rebuild$INCREMENT.$i.log 2> `dirname $0`/logs/rebuild$INCREMENT.$i.err |
| 145 |
if (($INCREMENT >1 )); |
| 146 |
then |
| 147 |
if { grep -q "previous transaction" `dirname $0`/logs/rebuild$INCREMENT.$i.err } ; |
| 148 |
then |
| 149 |
echo "I must split $i (increment $INCREMENT) because previous transaction didn't reach commit" |
| 150 |
((subincrement=$INCREMENT/10)) |
| 151 |
if (($i+$INCREMENT >= $BIBLIOEND )); |
| 152 |
then |
| 153 |
((newBIBLIOEND=$BIBLIOEND)) |
| 154 |
else |
| 155 |
((newBIBLIOEND=$i+$INCREMENT)) |
| 156 |
fi |
| 157 |
echo "$0 $subincrement $i $newBIBLIOEND $TEMPDIRECTORY no" |
| 158 |
$0 $subincrement $i $newBIBLIOEND $TEMPDIRECTORY no |
| 159 |
elif { ! grep -q "Records: $INCREMENT" `dirname $0`/logs/rebuild$INCREMENT.$i.err } ; |
| 160 |
then |
| 161 |
echo "I must split $i (increment $INCREMENT) because index was uncomplete, less than $INCREMENT records indexed" |
| 162 |
((subincrement=$INCREMENT/10)) |
| 163 |
if (($i+$INCREMENT >= $BIBLIOEND )); |
| 164 |
then |
| 165 |
((newBIBLIOEND=$BIBLIOEND)) |
| 166 |
else |
| 167 |
((newBIBLIOEND=$i+$INCREMENT)) |
| 168 |
fi |
| 169 |
echo "$0 $subincrement $i $newBIBLIOEND $TEMPDIRECTORY no" |
| 170 |
$0 $subincrement $i $newBIBLIOEND $TEMPDIRECTORY no |
| 171 |
fi |
| 172 |
fi |
| 173 |
done |
| 174 |
exit 0 |