Attachment #10621 for bug #7286

View | Details | Raw Unified | Return to bug 7286
Collapse All | Expand All




my $nosanitize;
my $skip_export;
my $keep_export;
my $skip_index;
my $reset;
my $biblios;
my $authorities;

    'r|reset'       => \$reset,
    's'             => \$skip_export,
    'k'             => \$keep_export,
    'I|skip-index'    => \$skip_index,
    'nosanitize'    => \$nosanitize,
    'b'             => \$biblios,
    'noxml'         => \$noxml,

}

if ($authorities) {
    index_records('authority', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $authorityserverdir);
} else {
    print "skipping authorities\n" if ( $verbose_logging );
}

if ($biblios) {
    index_records('biblio', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $biblioserverdir);
} else {
    print "skipping biblios\n" if ( $verbose_logging );
}

}	# ----------  end of subroutine check_zebra_dirs  ----------

sub index_records {
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;

    my $num_records_exported = 0;
    my $records_deleted;

            }
        }
    }

    #
    # and reindexing everything
    #
    if ($skip_index) {
        if ($verbose_logging) {
            print "====================\n";
            print "SKIPPING $record_type indexing\n";
            print "====================\n";
        }
    if ($process_zebraqueue) {
        do_indexing($record_type, 'delete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) 
            if %$records_deleted;
        do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
            if $num_records_exported;
    } else {
        if ( $verbose_logging ) {
            print "====================\n";
            print "REINDEXING zebra\n";
            print "====================\n";
        }
        my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
        if ($process_zebraqueue) {
            do_indexing($record_type, 'delete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) 
                if %$records_deleted;
            do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
                if $num_records_exported;
        } else {
            do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
                if ($num_records_exported or $skip_export);
        }
    }
}





#!/bin/sh

usage() {
    local scriptname=$(basename $0)
    cat <<EOF
$scriptname

Index Koha records by chunks. It is useful when some record causes errors and
stop the indexation process. With this script, if indexation of one chunk fails,
chunk is splitted in two or more chunks, and indexation continue on these chunks.
rebuild_zebra.pl is called only once to export records. Splitting and indexing
is handled by this script (using yaz-marcdump and zebraidx).

Usage:
$scriptname -t type -l X [-o X] [-s X] [-d /export/dir] [-L /log/dir] [-r] [-f]
$scriptname -h

    -o | --offset         Offset parameter of rebuild_zebra.pl
    -l | --length         Length parameter of rebuild_zebra.pl
    -s | --chunks-size    Initial chunk size (number of records indexed at once)
    -d | --export-dir     Where rebuild_zebra.pl will export data
    -L | --log-dir        Log directory
    -r | --remove-logs    Clean log directory before start
    -t | --type           Record type ('biblios' or 'authorities')
    -f | --force          Don't ask for confirmation before start
    -h | --help           Display this help message
EOF
}

indexfile() {
    local file=$1
    local chunkssize=$2

    if [ $chunkssize -lt 1 ]; then
        echo "Fail on file $file"
    else

        local prefix="${file}_${chunkssize}_"
        echo "Splitting file in chunks of $chunkssize records"
        YAZMARCDUMP_CMD="$YAZMARCDUMP -n -s $prefix -C $chunkssize $file"
        $YAZMARCDUMP_CMD

        dir=$(dirname $prefix)
        local files="$(find $dir -regex $prefix[0-9]+ | sort | tr '\n' ' ')"
        for chunkfile in $files; do
            echo "Indexing $chunkfile"
            size=$($YAZMARCDUMP -p $chunkfile | grep '<!-- Record [0-9]\+ offset .* -->' | wc -l)
            logfile="$LOGDIR/zebraidx.$(basename $chunkfile).log"
            ZEBRAIDX_CMD="$ZEBRAIDX -c $CONFIGFILE -d $TYPE -g iso2709 update $chunkfile"
            $ZEBRAIDX_CMD >$logfile 2>&1
            grep "Records: $size" $logfile >/dev/null 2>&1
            if [ $? -ne 0 ]; then
                echo "Indexing failed. Split file and continue..."
                indexfile $chunkfile $(($chunkssize/2))
            else
                ZEBRAIDX_CMD="$ZEBRAIDX -c $CONFIGFILE -d $TYPE -g iso2709 commit"
                $ZEBRAIDX_CMD >> $logfile 2>&1
            fi
        done
    fi
}

OFFSET=0
LENGTH=
CHUNKSSIZE=10000
EXPORTDIR=/tmp/rebuild/export
LOGDIR=/tmp/rebuild/logs
RMLOGS=no
NOCONFIRM=no
TYPE=biblios
HELP=no

# Get parameters
while [ $1 ]; do
    case $1 in
        -o | --offset )
            shift
            OFFSET=$1
            ;;
        -l | --length )
            shift
            LENGTH=$1
            ;;
        -s | --chunks-size )
            shift
            CHUNKSSIZE=$1
            ;;
        -d | --export-dir )
            shift
            EXPORTDIR=$1
            ;;
        -L | --log-dir )
            shift
            LOGDIR=$1
            ;;
        -r | --remove-logs )
            RMLOGS=yes
            ;;
        -t | --type )
            shift
            TYPE=$1
            ;;
        -f | --force )
            NOCONFIRM=yes
            ;;
        -h | --help)
            HELP=yes
            ;;
        * )
            usage
            exit 1
    esac
    shift
done

if [ $HELP = "yes" ]; then
    usage
    exit 0
fi

if [ -z $LENGTH ]; then
    echo "--length parameter is mandatory"
    exit 1
fi

TYPESWITCH=
case $TYPE in
    biblios )
        TYPESWITCH=-b
        ;;
    authorities )
        TYPESWITCH=-a
        ;;
    * )
        echo "'$TYPE' is an unknown type. Defaulting to 'biblios'"
        TYPESWITCH=-b
        TYPE=biblios
esac

ZEBRAIDX=`which zebraidx`
if [ -z $ZEBRAIDX ]; then
    echo "zebraidx not found"
    exit 1
fi

YAZMARCDUMP=`which yaz-marcdump`
if [ -z $YAZMARCDUMP ]; then
    echo "yaz-marcdump not found"
    exit 1
fi

REBUILDZEBRA="`dirname $0`/rebuild_zebra.pl"
if [ ! -f $REBUILDZEBRA ]; then
    echo "$REBUILDZEBRA: file not found"
    exit 1
fi

echo ""
echo "Configuration"
echo "========================================================================="
echo "Start at offset: $OFFSET"
echo "Total number of records to index: $LENGTH"
echo "Initial chunk size: $CHUNKSSIZE"
echo "Export directory: $EXPORTDIR"
echo "Log directory: $LOGDIR"
echo "Remove logs before start? $RMLOGS"
echo "Type of record: $TYPE"
echo "-------------------------------------------------------------------------"
echo "zebraidx path: $ZEBRAIDX"
echo "yaz-marcdump path: $YAZMARCDUMP"
echo "rebuild_zebra path: $REBUILDZEBRA"
echo "========================================================================="

if [ $NOCONFIRM != "yes" ]; then
    confirm=y
    echo -n "Confirm ? [Y/n] "
    read response
    if [ $response ] && [ $response != "yes" ] && [ $response != "y" ]; then
        confirm=n
    fi

    if [ $confirm = "n" ]; then
        exit 0
    fi
fi

mkdir -p $EXPORTDIR
if [ $? -ne 0 ]; then
    echo "Failed to create directory $EXPORTDIR. Aborting."
    exit 1
fi

mkdir -p $LOGDIR
if [ $? -ne 0 ]; then
    echo "Failed to create directory $LOGDIR. Aborting."
    exit 1
fi

if [ $RMLOGS = "yes" ]; then
    rm -f $LOGDIR/*.log
fi

REBUILDZEBRA_CMD="$REBUILDZEBRA $TYPESWITCH -v -k -d $EXPORTDIR --offset $OFFSET --length $LENGTH --skip-index"
echo "\n$REBUILDZEBRA_CMD"
$REBUILDZEBRA_CMD

EXPORTFILE=
case $TYPE in
    biblios )
        EXPORTFILE="$EXPORTDIR/biblio/exported_records"
        ;;
    authorities )
        EXPORTFILE="$EXPORTDIR/authority/exported_records"
        ;;
    * )
        echo "Error: TYPE '$TYPE' is not supported"
        exit 1
esac

CONFIGFILE="$(dirname $KOHA_CONF)/zebradb/zebra-$TYPE.cfg"


indexfile $EXPORTFILE $CHUNKSSIZE

Return to bug 7286

Lines 24-29 my $directory; Link Here

(-)a/misc/migration_tools/rebuild_zebra.pl (-17 / +27 lines)
24	my $nosanitize;	24	my $nosanitize;
25	my $skip_export;	25	my $skip_export;
26	my $keep_export;	26	my $keep_export;
		27	my $skip_index;
27	my $reset;	28	my $reset;
28	my $biblios;	29	my $biblios;
29	my $authorities;	30	my $authorities;
Lines 44-49 my $result = GetOptions( Link Here
44	'r\|reset' => \$reset,	45	'r\|reset' => \$reset,
45	's' => \$skip_export,	46	's' => \$skip_export,
46	'k' => \$keep_export,	47	'k' => \$keep_export,
		48	'I\|skip-index' => \$skip_index,
47	'nosanitize' => \$nosanitize,	49	'nosanitize' => \$nosanitize,
48	'b' => \$biblios,	50	'b' => \$biblios,
49	'noxml' => \$noxml,	51	'noxml' => \$noxml,
Lines 134-146 if ($do_munge) { Link Here
134	}	136	}
135		137
136	if ($authorities) {	138	if ($authorities) {
137	index_records('authority', $directory, $skip_export, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $authorityserverdir);	139	index_records('authority', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $authorityserverdir);
138	} else {	140	} else {
139	print "skipping authorities\n" if ( $verbose_logging );	141	print "skipping authorities\n" if ( $verbose_logging );
140	}	142	}
141		143
142	if ($biblios) {	144	if ($biblios) {
143	index_records('biblio', $directory, $skip_export, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $biblioserverdir);	145	index_records('biblio', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $biblioserverdir);
144	} else {	146	} else {
145	print "skipping biblios\n" if ( $verbose_logging );	147	print "skipping biblios\n" if ( $verbose_logging );
146	}	148	}
Lines 191-197 sub check_zebra_dirs { Link Here
191	} # ---------- end of subroutine check_zebra_dirs ----------	193	} # ---------- end of subroutine check_zebra_dirs ----------
192		194
193	sub index_records {	195	sub index_records {
194	my ($record_type, $directory, $skip_export, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;	196	my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
195		197
196	my $num_records_exported = 0;	198	my $num_records_exported = 0;
197	my $records_deleted;	199	my $records_deleted;
Lines 230-253 sub index_records { Link Here
230	}	232	}
231	}	233	}
232	}	234	}
233		235
234	#	236	#
235	# and reindexing everything	237	# and reindexing everything
236	#	238	#
237	if ( $verbose_logging ) {	239	if ($skip_index) {
238	print "====================\n";	240	if ($verbose_logging) {
239	print "REINDEXING zebra\n";	241	print "====================\n";
240	print "====================\n";	242	print "SKIPPING $record_type indexing\n";
241	}	243	print "====================\n";
242	my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;	244	}
243	if ($process_zebraqueue) {
244	do_indexing($record_type, 'delete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
245	if %$records_deleted;
246	do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
247	if $num_records_exported;
248	} else {	245	} else {
249	do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)	246	if ( $verbose_logging ) {
250	if ($num_records_exported or $skip_export);	247	print "====================\n";
		248	print "REINDEXING zebra\n";
		249	print "====================\n";
		250	}
		251	my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
		252	if ($process_zebraqueue) {
		253	do_indexing($record_type, 'delete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
		254	if %$records_deleted;
		255	do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
		256	if $num_records_exported;
		257	} else {
		258	do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
		259	if ($num_records_exported or $skip_export);
		260	}
251	}	261	}
252	}	262	}
253		263

Line 0 Link Here

(-)a/misc/migration_tools/rebuild_zebra_sliced.sh (-1 / +223 lines)
0	-	1	#!/bin/sh
		2
		3	usage() {
		4	local scriptname=$(basename $0)
		5	cat <<EOF
		6	$scriptname
		7
		8	Index Koha records by chunks. It is useful when some record causes errors and
		9	stop the indexation process. With this script, if indexation of one chunk fails,
		10	chunk is splitted in two or more chunks, and indexation continue on these chunks.
		11	rebuild_zebra.pl is called only once to export records. Splitting and indexing
		12	is handled by this script (using yaz-marcdump and zebraidx).
		13
		14	Usage:
		15	$scriptname -t type -l X [-o X] [-s X] [-d /export/dir] [-L /log/dir] [-r] [-f]
		16	$scriptname -h
		17
		18	-o \| --offset Offset parameter of rebuild_zebra.pl
		19	-l \| --length Length parameter of rebuild_zebra.pl
		20	-s \| --chunks-size Initial chunk size (number of records indexed at once)
		21	-d \| --export-dir Where rebuild_zebra.pl will export data
		22	-L \| --log-dir Log directory
		23	-r \| --remove-logs Clean log directory before start
		24	-t \| --type Record type ('biblios' or 'authorities')
		25	-f \| --force Don't ask for confirmation before start
		26	-h \| --help Display this help message
		27	EOF
		28	}
		29
		30	indexfile() {
		31	local file=$1
		32	local chunkssize=$2
		33
		34	if [ $chunkssize -lt 1 ]; then
		35	echo "Fail on file $file"
		36	else
		37
		38	local prefix="${file}_${chunkssize}_"
		39	echo "Splitting file in chunks of $chunkssize records"
		40	YAZMARCDUMP_CMD="$YAZMARCDUMP -n -s $prefix -C $chunkssize $file"
		41	$YAZMARCDUMP_CMD
		42
		43	dir=$(dirname $prefix)
		44	local files="$(find $dir -regex $prefix[0-9]+ \| sort \| tr '\n' ' ')"
		45	for chunkfile in $files; do
		46	echo "Indexing $chunkfile"
		47	size=$($YAZMARCDUMP -p $chunkfile \| grep '<!-- Record [0-9]\+ offset .* -->' \| wc -l)
		48	logfile="$LOGDIR/zebraidx.$(basename $chunkfile).log"
		49	ZEBRAIDX_CMD="$ZEBRAIDX -c $CONFIGFILE -d $TYPE -g iso2709 update $chunkfile"
		50	$ZEBRAIDX_CMD >$logfile 2>&1
		51	grep "Records: $size" $logfile >/dev/null 2>&1
		52	if [ $? -ne 0 ]; then
		53	echo "Indexing failed. Split file and continue..."
		54	indexfile $chunkfile $(($chunkssize/2))
		55	else
		56	ZEBRAIDX_CMD="$ZEBRAIDX -c $CONFIGFILE -d $TYPE -g iso2709 commit"
		57	$ZEBRAIDX_CMD >> $logfile 2>&1
		58	fi
		59	done
		60	fi
		61	}
		62
		63	OFFSET=0
		64	LENGTH=
		65	CHUNKSSIZE=10000
66	EXPORTDIR=/tmp/rebuild/export
67	LOGDIR=/tmp/rebuild/logs
68	RMLOGS=no
69	NOCONFIRM=no
70	TYPE=biblios
71	HELP=no
72
73	# Get parameters
74	while [ $1 ]; do
75	case $1 in
76	-o \| --offset )
77	shift
78	OFFSET=$1
79	;;
80	-l \| --length )
81	shift
82	LENGTH=$1
83	;;
84	-s \| --chunks-size )
85	shift
86	CHUNKSSIZE=$1
87	;;
88	-d \| --export-dir )
89	shift
90	EXPORTDIR=$1
91	;;
92	-L \| --log-dir )
93	shift
94	LOGDIR=$1
95	;;
96	-r \| --remove-logs )
97	RMLOGS=yes
98	;;
99	-t \| --type )
100	shift
101	TYPE=$1
102	;;
103	-f \| --force )
104	NOCONFIRM=yes
105	;;
106	-h \| --help)
107	HELP=yes
108	;;
109	* )
110	usage
111	exit 1
112	esac
113	shift
114	done
115
116	if [ $HELP = "yes" ]; then
117	usage
118	exit 0
119	fi
120
121	if [ -z $LENGTH ]; then
122	echo "--length parameter is mandatory"
123	exit 1
124	fi
125
126	TYPESWITCH=
127	case $TYPE in
128	biblios )
129	TYPESWITCH=-b
130	;;
131	authorities )
132	TYPESWITCH=-a
133	;;
134	* )
135	echo "'$TYPE' is an unknown type. Defaulting to 'biblios'"
136	TYPESWITCH=-b
137	TYPE=biblios
138	esac
139
140	ZEBRAIDX=`which zebraidx`
141	if [ -z $ZEBRAIDX ]; then
142	echo "zebraidx not found"
143	exit 1
144	fi
145
146	YAZMARCDUMP=`which yaz-marcdump`
147	if [ -z $YAZMARCDUMP ]; then
148	echo "yaz-marcdump not found"
149	exit 1
150	fi
151
152	REBUILDZEBRA="`dirname $0`/rebuild_zebra.pl"
153	if [ ! -f $REBUILDZEBRA ]; then
154	echo "$REBUILDZEBRA: file not found"
155	exit 1
156	fi
157
158	echo ""
159	echo "Configuration"
160	echo "========================================================================="
161	echo "Start at offset: $OFFSET"
162	echo "Total number of records to index: $LENGTH"
163	echo "Initial chunk size: $CHUNKSSIZE"
164	echo "Export directory: $EXPORTDIR"
165	echo "Log directory: $LOGDIR"
166	echo "Remove logs before start? $RMLOGS"
167	echo "Type of record: $TYPE"
168	echo "-------------------------------------------------------------------------"
169	echo "zebraidx path: $ZEBRAIDX"
170	echo "yaz-marcdump path: $YAZMARCDUMP"
171	echo "rebuild_zebra path: $REBUILDZEBRA"
172	echo "========================================================================="
173
174	if [ $NOCONFIRM != "yes" ]; then
175	confirm=y
176	echo -n "Confirm ? [Y/n] "
177	read response
178	if [ $response ] && [ $response != "yes" ] && [ $response != "y" ]; then
179	confirm=n
180	fi
181
182	if [ $confirm = "n" ]; then
183	exit 0
184	fi
185	fi
186
187	mkdir -p $EXPORTDIR
188	if [ $? -ne 0 ]; then
189	echo "Failed to create directory $EXPORTDIR. Aborting."
190	exit 1
191	fi
192
193	mkdir -p $LOGDIR
194	if [ $? -ne 0 ]; then
195	echo "Failed to create directory $LOGDIR. Aborting."
196	exit 1
197	fi
198
199	if [ $RMLOGS = "yes" ]; then
200	rm -f $LOGDIR/*.log
201	fi
202
203	REBUILDZEBRA_CMD="$REBUILDZEBRA $TYPESWITCH -v -k -d $EXPORTDIR --offset $OFFSET --length $LENGTH --skip-index"
204	echo "\n$REBUILDZEBRA_CMD"
205	$REBUILDZEBRA_CMD
206
207	EXPORTFILE=
208	case $TYPE in
209	biblios )
210	EXPORTFILE="$EXPORTDIR/biblio/exported_records"
211	;;
212	authorities )
213	EXPORTFILE="$EXPORTDIR/authority/exported_records"
214	;;
215	* )
216	echo "Error: TYPE '$TYPE' is not supported"
217	exit 1
218	esac
219
220	CONFIGFILE="$(dirname $KOHA_CONF)/zebradb/zebra-$TYPE.cfg"
221
222
223	indexfile $EXPORTFILE $CHUNKSSIZE