View | Details | Raw Unified | Return to bug 7286
Collapse All | Expand All

(-)a/misc/migration_tools/rebuild_zebra.pl (-17 / +27 lines)
Lines 24-29 my $directory; Link Here
24
my $nosanitize;
24
my $nosanitize;
25
my $skip_export;
25
my $skip_export;
26
my $keep_export;
26
my $keep_export;
27
my $skip_index;
27
my $reset;
28
my $reset;
28
my $biblios;
29
my $biblios;
29
my $authorities;
30
my $authorities;
Lines 44-49 my $result = GetOptions( Link Here
44
    'r|reset'       => \$reset,
45
    'r|reset'       => \$reset,
45
    's'             => \$skip_export,
46
    's'             => \$skip_export,
46
    'k'             => \$keep_export,
47
    'k'             => \$keep_export,
48
    'I|skip-index'    => \$skip_index,
47
    'nosanitize'    => \$nosanitize,
49
    'nosanitize'    => \$nosanitize,
48
    'b'             => \$biblios,
50
    'b'             => \$biblios,
49
    'noxml'         => \$noxml,
51
    'noxml'         => \$noxml,
Lines 134-146 if ($do_munge) { Link Here
134
}
136
}
135
137
136
if ($authorities) {
138
if ($authorities) {
137
    index_records('authority', $directory, $skip_export, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $authorityserverdir);
139
    index_records('authority', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $authorityserverdir);
138
} else {
140
} else {
139
    print "skipping authorities\n" if ( $verbose_logging );
141
    print "skipping authorities\n" if ( $verbose_logging );
140
}
142
}
141
143
142
if ($biblios) {
144
if ($biblios) {
143
    index_records('biblio', $directory, $skip_export, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $biblioserverdir);
145
    index_records('biblio', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $biblioserverdir);
144
} else {
146
} else {
145
    print "skipping biblios\n" if ( $verbose_logging );
147
    print "skipping biblios\n" if ( $verbose_logging );
146
}
148
}
Lines 191-197 sub check_zebra_dirs { Link Here
191
}	# ----------  end of subroutine check_zebra_dirs  ----------
193
}	# ----------  end of subroutine check_zebra_dirs  ----------
192
194
193
sub index_records {
195
sub index_records {
194
    my ($record_type, $directory, $skip_export, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
196
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
195
197
196
    my $num_records_exported = 0;
198
    my $num_records_exported = 0;
197
    my $records_deleted;
199
    my $records_deleted;
Lines 230-253 sub index_records { Link Here
230
            }
232
            }
231
        }
233
        }
232
    }
234
    }
233
    
235
234
    #
236
    #
235
    # and reindexing everything
237
    # and reindexing everything
236
    #
238
    #
237
    if ( $verbose_logging ) {
239
    if ($skip_index) {
238
        print "====================\n";
240
        if ($verbose_logging) {
239
        print "REINDEXING zebra\n";
241
            print "====================\n";
240
        print "====================\n";
242
            print "SKIPPING $record_type indexing\n";
241
    }
243
            print "====================\n";
242
	my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
244
        }
243
    if ($process_zebraqueue) {
244
        do_indexing($record_type, 'delete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) 
245
            if %$records_deleted;
246
        do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
247
            if $num_records_exported;
248
    } else {
245
    } else {
249
        do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
246
        if ( $verbose_logging ) {
250
            if ($num_records_exported or $skip_export);
247
            print "====================\n";
248
            print "REINDEXING zebra\n";
249
            print "====================\n";
250
        }
251
        my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
252
        if ($process_zebraqueue) {
253
            do_indexing($record_type, 'delete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) 
254
                if %$records_deleted;
255
            do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
256
                if $num_records_exported;
257
        } else {
258
            do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
259
                if ($num_records_exported or $skip_export);
260
        }
251
    }
261
    }
252
}
262
}
253
263
(-)a/misc/migration_tools/rebuild_zebra_sliced.sh (-1 / +223 lines)
Line 0 Link Here
0
- 
1
#!/bin/sh
2
3
usage() {
4
    local scriptname=$(basename $0)
5
    cat <<EOF
6
$scriptname
7
8
Index Koha records by chunks. It is useful when some record causes errors and
9
stop the indexation process. With this script, if indexation of one chunk fails,
10
chunk is splitted in two or more chunks, and indexation continue on these chunks.
11
rebuild_zebra.pl is called only once to export records. Splitting and indexing
12
is handled by this script (using yaz-marcdump and zebraidx).
13
14
Usage:
15
$scriptname -t type -l X [-o X] [-s X] [-d /export/dir] [-L /log/dir] [-r] [-f]
16
$scriptname -h
17
18
    -o | --offset         Offset parameter of rebuild_zebra.pl
19
    -l | --length         Length parameter of rebuild_zebra.pl
20
    -s | --chunks-size    Initial chunk size (number of records indexed at once)
21
    -d | --export-dir     Where rebuild_zebra.pl will export data
22
    -L | --log-dir        Log directory
23
    -r | --remove-logs    Clean log directory before start
24
    -t | --type           Record type ('biblios' or 'authorities')
25
    -f | --force          Don't ask for confirmation before start
26
    -h | --help           Display this help message
27
EOF
28
}
29
30
indexfile() {
31
    local file=$1
32
    local chunkssize=$2
33
34
    if [ $chunkssize -lt 1 ]; then
35
        echo "Fail on file $file"
36
    else
37
38
        local prefix="${file}_${chunkssize}_"
39
        echo "Splitting file in chunks of $chunkssize records"
40
        YAZMARCDUMP_CMD="$YAZMARCDUMP -n -s $prefix -C $chunkssize $file"
41
        $YAZMARCDUMP_CMD
42
43
        dir=$(dirname $prefix)
44
        local files="$(find $dir -regex $prefix[0-9]+ | sort | tr '\n' ' ')"
45
        for chunkfile in $files; do
46
            echo "Indexing $chunkfile"
47
            size=$($YAZMARCDUMP -p $chunkfile | grep '<!-- Record [0-9]\+ offset .* -->' | wc -l)
48
            logfile="$LOGDIR/zebraidx.$(basename $chunkfile).log"
49
            ZEBRAIDX_CMD="$ZEBRAIDX -c $CONFIGFILE -d $TYPE -g iso2709 update $chunkfile"
50
            $ZEBRAIDX_CMD >$logfile 2>&1
51
            grep "Records: $size" $logfile >/dev/null 2>&1
52
            if [ $? -ne 0 ]; then
53
                echo "Indexing failed. Split file and continue..."
54
                indexfile $chunkfile $(($chunkssize/2))
55
            else
56
                ZEBRAIDX_CMD="$ZEBRAIDX -c $CONFIGFILE -d $TYPE -g iso2709 commit"
57
                $ZEBRAIDX_CMD >> $logfile 2>&1
58
            fi
59
        done
60
    fi
61
}
62
63
OFFSET=0
64
LENGTH=
65
CHUNKSSIZE=10000
66
EXPORTDIR=/tmp/rebuild/export
67
LOGDIR=/tmp/rebuild/logs
68
RMLOGS=no
69
NOCONFIRM=no
70
TYPE=biblios
71
HELP=no
72
73
# Get parameters
74
while [ $1 ]; do
75
    case $1 in
76
        -o | --offset )
77
            shift
78
            OFFSET=$1
79
            ;;
80
        -l | --length )
81
            shift
82
            LENGTH=$1
83
            ;;
84
        -s | --chunks-size )
85
            shift
86
            CHUNKSSIZE=$1
87
            ;;
88
        -d | --export-dir )
89
            shift
90
            EXPORTDIR=$1
91
            ;;
92
        -L | --log-dir )
93
            shift
94
            LOGDIR=$1
95
            ;;
96
        -r | --remove-logs )
97
            RMLOGS=yes
98
            ;;
99
        -t | --type )
100
            shift
101
            TYPE=$1
102
            ;;
103
        -f | --force )
104
            NOCONFIRM=yes
105
            ;;
106
        -h | --help)
107
            HELP=yes
108
            ;;
109
        * )
110
            usage
111
            exit 1
112
    esac
113
    shift
114
done
115
116
if [ $HELP = "yes" ]; then
117
    usage
118
    exit 0
119
fi
120
121
if [ -z $LENGTH ]; then
122
    echo "--length parameter is mandatory"
123
    exit 1
124
fi
125
126
TYPESWITCH=
127
case $TYPE in
128
    biblios )
129
        TYPESWITCH=-b
130
        ;;
131
    authorities )
132
        TYPESWITCH=-a
133
        ;;
134
    * )
135
        echo "'$TYPE' is an unknown type. Defaulting to 'biblios'"
136
        TYPESWITCH=-b
137
        TYPE=biblios
138
esac
139
140
ZEBRAIDX=`which zebraidx`
141
if [ -z $ZEBRAIDX ]; then
142
    echo "zebraidx not found"
143
    exit 1
144
fi
145
146
YAZMARCDUMP=`which yaz-marcdump`
147
if [ -z $YAZMARCDUMP ]; then
148
    echo "yaz-marcdump not found"
149
    exit 1
150
fi
151
152
REBUILDZEBRA="`dirname $0`/rebuild_zebra.pl"
153
if [ ! -f $REBUILDZEBRA ]; then
154
    echo "$REBUILDZEBRA: file not found"
155
    exit 1
156
fi
157
158
echo ""
159
echo "Configuration"
160
echo "========================================================================="
161
echo "Start at offset: $OFFSET"
162
echo "Total number of records to index: $LENGTH"
163
echo "Initial chunk size: $CHUNKSSIZE"
164
echo "Export directory: $EXPORTDIR"
165
echo "Log directory: $LOGDIR"
166
echo "Remove logs before start? $RMLOGS"
167
echo "Type of record: $TYPE"
168
echo "-------------------------------------------------------------------------"
169
echo "zebraidx path: $ZEBRAIDX"
170
echo "yaz-marcdump path: $YAZMARCDUMP"
171
echo "rebuild_zebra path: $REBUILDZEBRA"
172
echo "========================================================================="
173
174
if [ $NOCONFIRM != "yes" ]; then
175
    confirm=y
176
    echo -n "Confirm ? [Y/n] "
177
    read response
178
    if [ $response ] && [ $response != "yes" ] && [ $response != "y" ]; then
179
        confirm=n
180
    fi
181
182
    if [ $confirm = "n" ]; then
183
        exit 0
184
    fi
185
fi
186
187
mkdir -p $EXPORTDIR
188
if [ $? -ne 0 ]; then
189
    echo "Failed to create directory $EXPORTDIR. Aborting."
190
    exit 1
191
fi
192
193
mkdir -p $LOGDIR
194
if [ $? -ne 0 ]; then
195
    echo "Failed to create directory $LOGDIR. Aborting."
196
    exit 1
197
fi
198
199
if [ $RMLOGS = "yes" ]; then
200
    rm -f $LOGDIR/*.log
201
fi
202
203
REBUILDZEBRA_CMD="$REBUILDZEBRA $TYPESWITCH -v -k -d $EXPORTDIR --offset $OFFSET --length $LENGTH --skip-index"
204
echo "\n$REBUILDZEBRA_CMD"
205
$REBUILDZEBRA_CMD
206
207
EXPORTFILE=
208
case $TYPE in
209
    biblios )
210
        EXPORTFILE="$EXPORTDIR/biblio/exported_records"
211
        ;;
212
    authorities )
213
        EXPORTFILE="$EXPORTDIR/authority/exported_records"
214
        ;;
215
    * )
216
        echo "Error: TYPE '$TYPE' is not supported"
217
        exit 1
218
esac
219
220
CONFIGFILE="$(dirname $KOHA_CONF)/zebradb/zebra-$TYPE.cfg"
221
222
223
indexfile $EXPORTFILE $CHUNKSSIZE

Return to bug 7286