From 7659643bc5b8038a1ac7a8755f63fcc0977bf15f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Delaune?= Date: Mon, 13 Feb 2012 17:00:22 +0100 Subject: [PATCH 2/2] Bug 7286: fix rebuild_zebra.pl to add rebuild_zebra_sliced.zsh --- misc/migration_tools/rebuild_zebra_sliced.zsh | 174 +++++++++++++++++++++++++ 1 files changed, 174 insertions(+), 0 deletions(-) create mode 100755 misc/migration_tools/rebuild_zebra_sliced.zsh diff --git a/misc/migration_tools/rebuild_zebra_sliced.zsh b/misc/migration_tools/rebuild_zebra_sliced.zsh new file mode 100755 index 0000000..c015343 --- /dev/null +++ b/misc/migration_tools/rebuild_zebra_sliced.zsh @@ -0,0 +1,174 @@ +#!/usr/bin/zsh + +# Copyright 2011 BibLibre SARL +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +INCREMENT=$1 +BIBLIOSTART=$2 +BIBLIOEND=$3 +TEMPDIRECTORY=$4 +RMLOGS=$5 + +#echo " | $INCREMENT , $BIBLIOSTART , $BIBLIOEND , $TEMPDIRECTORY , $RMLOGS | "; +# this script rebuild the zebra index recursively +# INCREMENT specify how many records we must try at once +# BIBLIOSTART is the record number to BIBLIOSTART on +# BIBLIOEND is the record number to BIBLIOEND on, if it's greater than the last biblionumber it will be redefine +# TEMPDIRECTORY define a temporary directory to rebuild_zebra +# RMLOGS is used to remove files into logs/, it's use and define by script itself so you have not to define it +# At the BIBLIOEND of each "chunk", this script checks if the indexing process has been OK +# if it hasn't, the slice is splitted in 10, and the reindexing is called again on each smaller chunk +# if the increment goes to 1, it means we tried to reindex 1 by 1, and the failing indexing concern wrong records + +# the logs are stored in a directory called logs/ that must be a subdirectory of reindex.zsh + +# at the BIBLIOEND of the script, just type : +#grep -l "previous transaction" `ls rebuild1.*.err` +# the result contains all the biblios that have not been indexed +# WARNING : the numbers are not the biblionumber but the record number, they can be reached by : +# SELECT biblionumber FROM biblio LIMIT YourNumberHere,1; + +# EXAMPLE to run the script on a 800 000 biblios database : +# ./reindex.zsh 50000 0 800000 +# will reindex the DB, BIBLIOSTARTing with chunks of 50k biblios +#if { grep -E "previous transaction" `dirname $0`/logs/rebuild$INCREMENT.$i.err } ; then + +lastbiblionumber=`perl -e '#!/usr/bin/perl +use C4::Context; +my $dbh = C4::Context->dbh; +my $querylastbiblionumber = "SELECT max(biblionumber) FROM biblio;"; +my $sthlastbiblionumber = $dbh->prepare($querylastbiblionumber); +$sthlastbiblionumber->execute(); +my ( $lastbiblionumber ) = $sthlastbiblionumber->fetchrow_array; print $lastbiblionumber;'` +#echo $lastbiblionumber; +let "maxbiblionumber = $lastbiblionumber + 1" +if [ $# = 4 ] +then + RMLOGS=yes +elif [ $# = 3 ] +then + RMLOGS=yes + TEMPDIRECTORY=/tmp/rebuild +elif [ $# = 2 ] +then + RMLOGS=yes + TEMPDIRECTORY=/tmp/rebuild + BIBLIOEND=$lastbiblionumber +elif [ $# = 1 ] +then + RMLOGS=yes + TEMPDIRECTORY=/tmp/rebuild + BIBLIOSTART=0 + BIBLIOEND=$lastbiblionumber +elif [ $# = 0 ] +then + RMLOGS=yes + TEMPDIRECTORY=/tmp/rebuild + INCREMENT=10000 + BIBLIOSTART=0 + BIBLIOEND=$lastbiblionumber +fi +if [[ $INCREMENT =~ ^10*$ ]] +then +else + echo "The first argument (INCREMENT) must be 1 or a multiple of 10" + exit 2 +fi +if [[ $BIBLIOSTART =~ ^[0-9]*$ ]] +then +else + echo "The second argument (BIBLIOSTART) must be an integer" + exit 2 +fi +if [[ $BIBLIOEND =~ ^[0-9]*$ ]] +then +else + echo "The third argument (BIBLIOEND) must be an integer" + exit 2 +fi +if [ $BIBLIOSTART -ge $BIBLIOEND ] +then + echo "The second argument (BIBLIOSTART) must be lower than the third argument (BIBLIOEND)" + exit 2 +fi +if [[ $maxbiblionumber =~ ^[0-9]*$ ]] +then +else + echo "error : last biblionumber is not an integer" + exit 2 +fi +if [ $BIBLIOEND -gt $lastbiblionumber ]; +then + BIBLIOEND=$lastbiblionumber +fi +ls `dirname $0`/logs/ >/dev/null 2>&1 +if [ $? != 0 ] +then + mkdir `dirname $0`/logs +else + if [ $RMLOGS = "yes" ] + then + rm `dirname $0`/logs/* + fi +fi +ls $TEMPDIRECTORY >/dev/null 2>&1 +if [ $? != 0 ] +then + mkdir $TEMPDIRECTORY + if [ $? != 0 ] + then + echo "can't create dir $TEMPDIRECTORY" + exit 2 + fi +fi +#/home/koha/src/misc/migration_tools/rebuild_zebra.pl -r -b -v -x -nosanitize -length 1 -offset 1 +for ((i=$BIBLIOSTART ; i<=$BIBLIOEND ; i=i+$INCREMENT)) +do + echo "I = " $i "with increment " $INCREMENT + #echo "BIBLIOSTART $BIBLIOSTART INCREMENT $INCREMENT BIBLIOEND $BIBLIOEND i $i" + #echo "`dirname $0`/rebuild_zebra.pl -b -v -x -nosanitize -d $TEMPDIRECTORY -k -length $INCREMENT -offset $i -v log" + `dirname $0`/rebuild_zebra.pl -b -v -v -x -nosanitize -d $TEMPDIRECTORY -k -length $INCREMENT -offset $i > `dirname $0`/logs/rebuild$INCREMENT.$i.log 2> `dirname $0`/logs/rebuild$INCREMENT.$i.err + if (($INCREMENT >1 )); + then + if { grep -q "previous transaction" `dirname $0`/logs/rebuild$INCREMENT.$i.err } ; + then + echo "I must split $i (increment $INCREMENT) because previous transaction didn't reach commit" + ((subincrement=$INCREMENT/10)) + if (($i+$INCREMENT >= $BIBLIOEND )); + then + ((newBIBLIOEND=$BIBLIOEND)) + else + ((newBIBLIOEND=$i+$INCREMENT)) + fi + echo "$0 $subincrement $i $newBIBLIOEND $TEMPDIRECTORY no" + $0 $subincrement $i $newBIBLIOEND $TEMPDIRECTORY no + elif { ! grep -q "Records: $INCREMENT" `dirname $0`/logs/rebuild$INCREMENT.$i.err } ; + then + echo "I must split $i (increment $INCREMENT) because index was uncomplete, less than $INCREMENT records indexed" + ((subincrement=$INCREMENT/10)) + if (($i+$INCREMENT >= $BIBLIOEND )); + then + ((newBIBLIOEND=$BIBLIOEND)) + else + ((newBIBLIOEND=$i+$INCREMENT)) + fi + echo "$0 $subincrement $i $newBIBLIOEND $TEMPDIRECTORY no" + $0 $subincrement $i $newBIBLIOEND $TEMPDIRECTORY no + fi + fi +done +exit 0 -- 1.7.0.4