From 7126aa9c566e7a77acfa279b90559cc080419f59 Mon Sep 17 00:00:00 2001
From: Julian Maurice <julian.maurice@biblibre.com>
Date: Mon, 2 Jul 2012 13:57:31 +0200
Subject: [PATCH] Bug 7286: rebuild_zebra_sliced for biblios and authorities

Complete rewrite of rebuild_zebra_sliced.zsh (renamed to .sh). Main
improvements are:
  - both biblio and authority records are handled
  - records are exported only once

It also add an option --skip-index to rebuild_zebra.pl that permit to
use rebuild_zebra.pl as an 'export only' script.

Description:
Index Koha records by chunks. It is useful when some record causes
errors and stop the indexation process. With this script, if indexation
of one chunk fails, chunk is splitted in 2 (or 3) chunks, and
indexation continue on these chunks.
rebuild_zebra.pl is called only once to export records.
Splitting and indexing is handled by this script (using yaz-marcdump and
zebraidx).
---
 misc/migration_tools/rebuild_zebra.pl        |   44 ++++--
 misc/migration_tools/rebuild_zebra_sliced.sh |  216 ++++++++++++++++++++++++++
 2 files changed, 243 insertions(+), 17 deletions(-)
 create mode 100755 misc/migration_tools/rebuild_zebra_sliced.sh

diff --git a/misc/migration_tools/rebuild_zebra.pl b/misc/migration_tools/rebuild_zebra.pl
index dbbb140..6902d2f 100755
--- a/misc/migration_tools/rebuild_zebra.pl
+++ b/misc/migration_tools/rebuild_zebra.pl
@@ -24,6 +24,7 @@ my $directory;
 my $nosanitize;
 my $skip_export;
 my $keep_export;
+my $skip_index;
 my $reset;
 my $biblios;
 my $authorities;
@@ -44,6 +45,7 @@ my $result = GetOptions(
     'r|reset'       => \$reset,
     's'             => \$skip_export,
     'k'             => \$keep_export,
+    'I|skip-index'    => \$skip_index,
     'nosanitize'    => \$nosanitize,
     'b'             => \$biblios,
     'noxml'         => \$noxml,
@@ -134,13 +136,13 @@ if ($do_munge) {
 }
 
 if ($authorities) {
-    index_records('authority', $directory, $skip_export, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $authorityserverdir);
+    index_records('authority', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $authorityserverdir);
 } else {
     print "skipping authorities\n" if ( $verbose_logging );
 }
 
 if ($biblios) {
-    index_records('biblio', $directory, $skip_export, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $biblioserverdir);
+    index_records('biblio', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $biblioserverdir);
 } else {
     print "skipping biblios\n" if ( $verbose_logging );
 }
@@ -191,7 +193,7 @@ sub check_zebra_dirs {
 }	# ----------  end of subroutine check_zebra_dirs  ----------
 
 sub index_records {
-    my ($record_type, $directory, $skip_export, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
+    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
 
     my $num_records_exported = 0;
     my $records_deleted;
@@ -230,24 +232,32 @@ sub index_records {
             }
         }
     }
-    
+
     #
     # and reindexing everything
     #
-    if ( $verbose_logging ) {
-        print "====================\n";
-        print "REINDEXING zebra\n";
-        print "====================\n";
-    }
-	my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
-    if ($process_zebraqueue) {
-        do_indexing($record_type, 'delete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) 
-            if %$records_deleted;
-        do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
-            if $num_records_exported;
+    if ($skip_index) {
+        if ($verbose_logging) {
+            print "====================\n";
+            print "SKIPPING $record_type indexing\n";
+            print "====================\n";
+        }
     } else {
-        do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
-            if ($num_records_exported or $skip_export);
+        if ( $verbose_logging ) {
+            print "====================\n";
+            print "REINDEXING zebra\n";
+            print "====================\n";
+        }
+        my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
+        if ($process_zebraqueue) {
+            do_indexing($record_type, 'delete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) 
+                if %$records_deleted;
+            do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
+                if $num_records_exported;
+        } else {
+            do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
+                if ($num_records_exported or $skip_export);
+        }
     }
 }
 
diff --git a/misc/migration_tools/rebuild_zebra_sliced.sh b/misc/migration_tools/rebuild_zebra_sliced.sh
new file mode 100755
index 0000000..9c90218
--- /dev/null
+++ b/misc/migration_tools/rebuild_zebra_sliced.sh
@@ -0,0 +1,216 @@
+#!/bin/sh
+
+usage() {
+    local scriptname=$(basename $0)
+    cat <<EOF
+$scriptname
+
+Index Koha records by chunks. It is useful when some record causes errors and
+stop the indexation process. With this script, if indexation of one chunk fails,
+chunk is splitted in two or more chunks, and indexation continue on these chunks.
+rebuild_zebra.pl is called only once to export records. Splitting and indexing
+is handled by this script (using yaz-marcdump and zebraidx).
+
+Usage:
+$scriptname -t type -l X [-o X] [-s X] [-d /export/dir] [-L /log/dir] [-r] [-f]
+$scriptname -h
+
+    -o | --offset         Offset parameter of rebuild_zebra.pl
+    -l | --length         Length parameter of rebuild_zebra.pl
+    -s | --chunks-size    Initial chunk size (number of records indexed at once)
+    -d | --export-dir     Where rebuild_zebra.pl will export data
+    -L | --log-dir        Log directory
+    -r | --remove-logs    Clean log directory before start
+    -t | --type           Record type ('biblios' or 'authorities')
+    -f | --force          Don't ask for confirmation before start
+    -h | --help           Display this help message
+EOF
+}
+
+indexfile() {
+    local file=$1
+    local chunkssize=$2
+
+    if [ $chunkssize -lt 1 ]; then
+        echo "Fail on file $file"
+    else
+
+        local prefix="${file}_${chunkssize}_"
+        echo "Splitting file in chunks of $chunkssize"
+        YAZMARCDUMP_CMD="$YAZMARCDUMP -n -s $prefix -C $chunkssize $file"
+        $YAZMARCDUMP_CMD
+
+        dir=$(dirname $prefix)
+        local files="$(find $dir -regex $prefix[0-9]+ | sort | tr '\n' ' ')"
+        for chunkfile in $files; do
+            echo "Indexing $chunkfile"
+            size=$($YAZMARCDUMP -p $chunkfile | grep '<!-- Record [0-9]\+ offset .* -->' | wc -l)
+            logfile="$LOGDIR/zebraidx.$(basename $chunkfile).log"
+            ZEBRAIDX_CMD="$ZEBRAIDX -c $CONFIGFILE -d $TYPE -g iso2709 update $chunkfile"
+            $ZEBRAIDX_CMD >$logfile 2>&1
+            grep "Records: $size" $logfile >/dev/null 2>&1
+            if [ $? -ne 0 ]; then
+                echo "Indexing failed. Split file and continue..."
+                indexfile $chunkfile $(($chunkssize/2))
+            fi
+        done
+    fi
+}
+
+OFFSET=0
+LENGTH=
+CHUNKSSIZE=10000
+EXPORTDIR=/tmp/rebuild/export
+LOGDIR=/tmp/rebuild/logs
+RMLOGS=no
+NOCONFIRM=no
+TYPE=biblios
+HELP=no
+
+# Get parameters
+while [ $1 ]; do
+    case $1 in
+        -o | --offset )
+            shift
+            OFFSET=$1
+            ;;
+        -l | --length )
+            shift
+            LENGTH=$1
+            ;;
+        -s | --chunks-size )
+            shift
+            CHUNKSSIZE=$1
+            ;;
+        -d | --export-dir )
+            shift
+            EXPORTDIR=$1
+            ;;
+        -L | --log-dir )
+            shift
+            LOGDIR=$1
+            ;;
+        -r | --remove-logs )
+            RMLOGS=yes
+            ;;
+        -t | --type )
+            shift
+            TYPE=$1
+            ;;
+        -f | --force )
+            NOCONFIRM=yes
+            ;;
+        -h | --help)
+            HELP=yes
+            ;;
+        * )
+            usage
+            exit 1
+    esac
+    shift
+done
+
+if [ $HELP = "yes" ]; then
+    usage
+    exit 0
+fi
+
+if [ -z $LENGTH ]; then
+    echo "--length parameter is mandatory"
+    exit 1
+fi
+
+TYPESWITCH=
+case $TYPE in
+    biblios )
+        TYPESWITCH=-b
+        ;;
+    authorities )
+        TYPESWITCH=-a
+        ;;
+    * )
+        echo "'$TYPE' is an unknown type. Defaulting to 'biblios'"
+        TYPESWITCH=-b
+        TYPE=biblios
+esac
+
+ZEBRAIDX=`which zebraidx`
+if [ -z $ZEBRAIDX ]; then
+    echo "zebraidx not found"
+    exit 1
+fi
+
+YAZMARCDUMP=`which yaz-marcdump`
+if [ -z $YAZMARCDUMP ]; then
+    echo "yaz-marcdump not found"
+    exit 1
+fi
+
+REBUILDZEBRA="`dirname $0`/rebuild_zebra.pl"
+if [ ! -f $REBUILDZEBRA ]; then
+    echo "$REBUILDZEBRA: file not found"
+    exit 1
+fi
+
+echo ""
+echo "Configuration"
+echo "========================================================================="
+echo "Start at offset: $OFFSET"
+echo "Total number of records to index: $LENGTH"
+echo "Initial chunk size: $CHUNKSSIZE"
+echo "Export directory: $EXPORTDIR"
+echo "Log directory: $LOGDIR"
+echo "Remove logs before start? $RMLOGS"
+echo "Type of record: $TYPE"
+echo "-------------------------------------------------------------------------"
+echo "zebraidx path: $ZEBRAIDX"
+echo "yaz-marcdump path: $YAZMARCDUMP"
+echo "rebuild_zebra path: $REBUILDZEBRA"
+echo "========================================================================="
+
+if [ $NOCONFIRM != "yes" ]; then
+    confirm=y
+    echo -n "Confirm ? [Y/n] "
+    read response
+    if [ $response ] && [ $response != "yes" ] && [ $response != "y" ]; then
+        confirm=n
+    fi
+
+    if [ $confirm = "n" ]; then
+        exit 0
+    fi
+fi
+
+mkdir -p $EXPORTDIR
+if [ $? -ne 0 ]; then
+    echo "Failed to create directory $EXPORTDIR. Aborting."
+    exit 1
+fi
+
+mkdir -p $LOGDIR
+if [ $? -ne 0 ]; then
+    echo "Failed to create directory $LOGDIR. Aborting."
+    exit 1
+fi
+
+REBUILDZEBRA_CMD="$REBUILDZEBRA $TYPESWITCH -v -k -d $EXPORTDIR --offset $OFFSET --length $LENGTH --skip-index"
+echo "\n$REBUILDZEBRA_CMD"
+$REBUILDZEBRA_CMD
+
+EXPORTFILE=
+case $TYPE in
+    biblios )
+        EXPORTFILE="$EXPORTDIR/biblio/exported_records"
+        ;;
+    authorities )
+        EXPORTFILE="$EXPORTDIR/authority/exported_records"
+        ;;
+    * )
+        echo "Error: TYPE '$TYPE' is not supported"
+        exit 1
+esac
+
+CONFIGFILE="$(dirname $KOHA_CONF)/zebradb/zebra-$TYPE.cfg"
+
+
+indexfile $EXPORTFILE $CHUNKSSIZE
-- 
1.7.10