From 7126aa9c566e7a77acfa279b90559cc080419f59 Mon Sep 17 00:00:00 2001 From: Julian Maurice <julian.maurice@biblibre.com> Date: Mon, 2 Jul 2012 13:57:31 +0200 Subject: [PATCH] Bug 7286: rebuild_zebra_sliced for biblios and authorities Complete rewrite of rebuild_zebra_sliced.zsh (renamed to .sh). Main improvements are: - both biblio and authority records are handled - records are exported only once It also add an option --skip-index to rebuild_zebra.pl that permit to use rebuild_zebra.pl as an 'export only' script. Description: Index Koha records by chunks. It is useful when some record causes errors and stop the indexation process. With this script, if indexation of one chunk fails, chunk is splitted in 2 (or 3) chunks, and indexation continue on these chunks. rebuild_zebra.pl is called only once to export records. Splitting and indexing is handled by this script (using yaz-marcdump and zebraidx). --- misc/migration_tools/rebuild_zebra.pl | 44 ++++-- misc/migration_tools/rebuild_zebra_sliced.sh | 216 ++++++++++++++++++++++++++ 2 files changed, 243 insertions(+), 17 deletions(-) create mode 100755 misc/migration_tools/rebuild_zebra_sliced.sh diff --git a/misc/migration_tools/rebuild_zebra.pl b/misc/migration_tools/rebuild_zebra.pl index dbbb140..6902d2f 100755 --- a/misc/migration_tools/rebuild_zebra.pl +++ b/misc/migration_tools/rebuild_zebra.pl @@ -24,6 +24,7 @@ my $directory; my $nosanitize; my $skip_export; my $keep_export; +my $skip_index; my $reset; my $biblios; my $authorities; @@ -44,6 +45,7 @@ my $result = GetOptions( 'r|reset' => \$reset, 's' => \$skip_export, 'k' => \$keep_export, + 'I|skip-index' => \$skip_index, 'nosanitize' => \$nosanitize, 'b' => \$biblios, 'noxml' => \$noxml, @@ -134,13 +136,13 @@ if ($do_munge) { } if ($authorities) { - index_records('authority', $directory, $skip_export, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $authorityserverdir); + index_records('authority', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $authorityserverdir); } else { print "skipping authorities\n" if ( $verbose_logging ); } if ($biblios) { - index_records('biblio', $directory, $skip_export, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $biblioserverdir); + index_records('biblio', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $biblioserverdir); } else { print "skipping biblios\n" if ( $verbose_logging ); } @@ -191,7 +193,7 @@ sub check_zebra_dirs { } # ---------- end of subroutine check_zebra_dirs ---------- sub index_records { - my ($record_type, $directory, $skip_export, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_; + my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_; my $num_records_exported = 0; my $records_deleted; @@ -230,24 +232,32 @@ sub index_records { } } } - + # # and reindexing everything # - if ( $verbose_logging ) { - print "====================\n"; - print "REINDEXING zebra\n"; - print "====================\n"; - } - my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ; - if ($process_zebraqueue) { - do_indexing($record_type, 'delete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) - if %$records_deleted; - do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) - if $num_records_exported; + if ($skip_index) { + if ($verbose_logging) { + print "====================\n"; + print "SKIPPING $record_type indexing\n"; + print "====================\n"; + } } else { - do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) - if ($num_records_exported or $skip_export); + if ( $verbose_logging ) { + print "====================\n"; + print "REINDEXING zebra\n"; + print "====================\n"; + } + my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ; + if ($process_zebraqueue) { + do_indexing($record_type, 'delete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) + if %$records_deleted; + do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) + if $num_records_exported; + } else { + do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) + if ($num_records_exported or $skip_export); + } } } diff --git a/misc/migration_tools/rebuild_zebra_sliced.sh b/misc/migration_tools/rebuild_zebra_sliced.sh new file mode 100755 index 0000000..9c90218 --- /dev/null +++ b/misc/migration_tools/rebuild_zebra_sliced.sh @@ -0,0 +1,216 @@ +#!/bin/sh + +usage() { + local scriptname=$(basename $0) + cat <<EOF +$scriptname + +Index Koha records by chunks. It is useful when some record causes errors and +stop the indexation process. With this script, if indexation of one chunk fails, +chunk is splitted in two or more chunks, and indexation continue on these chunks. +rebuild_zebra.pl is called only once to export records. Splitting and indexing +is handled by this script (using yaz-marcdump and zebraidx). + +Usage: +$scriptname -t type -l X [-o X] [-s X] [-d /export/dir] [-L /log/dir] [-r] [-f] +$scriptname -h + + -o | --offset Offset parameter of rebuild_zebra.pl + -l | --length Length parameter of rebuild_zebra.pl + -s | --chunks-size Initial chunk size (number of records indexed at once) + -d | --export-dir Where rebuild_zebra.pl will export data + -L | --log-dir Log directory + -r | --remove-logs Clean log directory before start + -t | --type Record type ('biblios' or 'authorities') + -f | --force Don't ask for confirmation before start + -h | --help Display this help message +EOF +} + +indexfile() { + local file=$1 + local chunkssize=$2 + + if [ $chunkssize -lt 1 ]; then + echo "Fail on file $file" + else + + local prefix="${file}_${chunkssize}_" + echo "Splitting file in chunks of $chunkssize" + YAZMARCDUMP_CMD="$YAZMARCDUMP -n -s $prefix -C $chunkssize $file" + $YAZMARCDUMP_CMD + + dir=$(dirname $prefix) + local files="$(find $dir -regex $prefix[0-9]+ | sort | tr '\n' ' ')" + for chunkfile in $files; do + echo "Indexing $chunkfile" + size=$($YAZMARCDUMP -p $chunkfile | grep '<!-- Record [0-9]\+ offset .* -->' | wc -l) + logfile="$LOGDIR/zebraidx.$(basename $chunkfile).log" + ZEBRAIDX_CMD="$ZEBRAIDX -c $CONFIGFILE -d $TYPE -g iso2709 update $chunkfile" + $ZEBRAIDX_CMD >$logfile 2>&1 + grep "Records: $size" $logfile >/dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "Indexing failed. Split file and continue..." + indexfile $chunkfile $(($chunkssize/2)) + fi + done + fi +} + +OFFSET=0 +LENGTH= +CHUNKSSIZE=10000 +EXPORTDIR=/tmp/rebuild/export +LOGDIR=/tmp/rebuild/logs +RMLOGS=no +NOCONFIRM=no +TYPE=biblios +HELP=no + +# Get parameters +while [ $1 ]; do + case $1 in + -o | --offset ) + shift + OFFSET=$1 + ;; + -l | --length ) + shift + LENGTH=$1 + ;; + -s | --chunks-size ) + shift + CHUNKSSIZE=$1 + ;; + -d | --export-dir ) + shift + EXPORTDIR=$1 + ;; + -L | --log-dir ) + shift + LOGDIR=$1 + ;; + -r | --remove-logs ) + RMLOGS=yes + ;; + -t | --type ) + shift + TYPE=$1 + ;; + -f | --force ) + NOCONFIRM=yes + ;; + -h | --help) + HELP=yes + ;; + * ) + usage + exit 1 + esac + shift +done + +if [ $HELP = "yes" ]; then + usage + exit 0 +fi + +if [ -z $LENGTH ]; then + echo "--length parameter is mandatory" + exit 1 +fi + +TYPESWITCH= +case $TYPE in + biblios ) + TYPESWITCH=-b + ;; + authorities ) + TYPESWITCH=-a + ;; + * ) + echo "'$TYPE' is an unknown type. Defaulting to 'biblios'" + TYPESWITCH=-b + TYPE=biblios +esac + +ZEBRAIDX=`which zebraidx` +if [ -z $ZEBRAIDX ]; then + echo "zebraidx not found" + exit 1 +fi + +YAZMARCDUMP=`which yaz-marcdump` +if [ -z $YAZMARCDUMP ]; then + echo "yaz-marcdump not found" + exit 1 +fi + +REBUILDZEBRA="`dirname $0`/rebuild_zebra.pl" +if [ ! -f $REBUILDZEBRA ]; then + echo "$REBUILDZEBRA: file not found" + exit 1 +fi + +echo "" +echo "Configuration" +echo "=========================================================================" +echo "Start at offset: $OFFSET" +echo "Total number of records to index: $LENGTH" +echo "Initial chunk size: $CHUNKSSIZE" +echo "Export directory: $EXPORTDIR" +echo "Log directory: $LOGDIR" +echo "Remove logs before start? $RMLOGS" +echo "Type of record: $TYPE" +echo "-------------------------------------------------------------------------" +echo "zebraidx path: $ZEBRAIDX" +echo "yaz-marcdump path: $YAZMARCDUMP" +echo "rebuild_zebra path: $REBUILDZEBRA" +echo "=========================================================================" + +if [ $NOCONFIRM != "yes" ]; then + confirm=y + echo -n "Confirm ? [Y/n] " + read response + if [ $response ] && [ $response != "yes" ] && [ $response != "y" ]; then + confirm=n + fi + + if [ $confirm = "n" ]; then + exit 0 + fi +fi + +mkdir -p $EXPORTDIR +if [ $? -ne 0 ]; then + echo "Failed to create directory $EXPORTDIR. Aborting." + exit 1 +fi + +mkdir -p $LOGDIR +if [ $? -ne 0 ]; then + echo "Failed to create directory $LOGDIR. Aborting." + exit 1 +fi + +REBUILDZEBRA_CMD="$REBUILDZEBRA $TYPESWITCH -v -k -d $EXPORTDIR --offset $OFFSET --length $LENGTH --skip-index" +echo "\n$REBUILDZEBRA_CMD" +$REBUILDZEBRA_CMD + +EXPORTFILE= +case $TYPE in + biblios ) + EXPORTFILE="$EXPORTDIR/biblio/exported_records" + ;; + authorities ) + EXPORTFILE="$EXPORTDIR/authority/exported_records" + ;; + * ) + echo "Error: TYPE '$TYPE' is not supported" + exit 1 +esac + +CONFIGFILE="$(dirname $KOHA_CONF)/zebradb/zebra-$TYPE.cfg" + + +indexfile $EXPORTFILE $CHUNKSSIZE -- 1.7.10