View | Details | Raw Unified | Return to bug 8746
Collapse All | Expand All

(-)a/misc/migration_tools/rebuild_zebra_sliced.sh (-20 / +91 lines)
Lines 12-32 rebuild_zebra.pl is called only once to export records. Splitting and indexing Link Here
12
is handled by this script (using yaz-marcdump and zebraidx).
12
is handled by this script (using yaz-marcdump and zebraidx).
13
13
14
Usage:
14
Usage:
15
$scriptname -t type -l X [-o X] [-s X] [-d /export/dir] [-L /log/dir] [-r] [-f]
15
$scriptname [-t type] [-l X] [-o X] [-s X] [-d /export/dir] [-L /log/dir] [-r] [-f] [--reset-index]
16
$scriptname -h
16
$scriptname -h
17
17
18
    -o | --offset         Offset parameter of rebuild_zebra.pl
18
    -o | --offset         Offset parameter of rebuild_zebra.pl.
19
    -l | --length         Length parameter of rebuild_zebra.pl
19
                          Default: $OFFSET
20
    -l | --length         Length parameter of rebuild_zebra.pl. If omitted, the
21
                          length is automatically calculated to index all
22
                          records
20
    -s | --chunks-size    Initial chunk size (number of records indexed at once)
23
    -s | --chunks-size    Initial chunk size (number of records indexed at once)
24
                          Default: $CHUNKSSIZE
21
    -d | --export-dir     Where rebuild_zebra.pl will export data
25
    -d | --export-dir     Where rebuild_zebra.pl will export data
26
                          Default: $EXPORTDIR
22
    -L | --log-dir        Log directory
27
    -L | --log-dir        Log directory
28
                          Default: $LOGDIR
23
    -r | --remove-logs    Clean log directory before start
29
    -r | --remove-logs    Clean log directory before start
30
                          Default: $RMLOGS
24
    -t | --type           Record type ('biblios' or 'authorities')
31
    -t | --type           Record type ('biblios' or 'authorities')
32
                          Default: $TYPE
25
    -f | --force          Don't ask for confirmation before start
33
    -f | --force          Don't ask for confirmation before start
26
    -h | --help           Display this help message
34
    -h | --help           Display this help message
35
    --reset-index         Reset Zebra index for 'type'
27
EOF
36
EOF
28
}
37
}
29
38
39
splitfile() {
40
    local file=$1
41
    local prefix=$2
42
    local size=$3
43
    local script='
44
        my $prefix = '"\"$prefix\""';
45
        my $size = '"$size"';
46
        my ($i,$count) = (0,0);
47
        open(my $fh, "<", '"\"$file\""');
48
        open(my $out, ">", sprintf("$prefix%02d", $i));
49
        my $closed = 0;
50
        while (<$fh>) {
51
            my $line = $_;
52
            if ($closed) {
53
                open($out, ">", sprintf("$prefix%02d", $i));
54
                $closed = 0;
55
            }
56
            print $out $line;
57
            $count++ if ($line =~ m|^</record>|);
58
            if ($count == $size) {
59
                $count = 0;
60
                $i++;
61
                close($out);
62
                $closed = 1;
63
            }
64
        }
65
    '
66
    $PERL -e "$script"
67
}
68
30
indexfile() {
69
indexfile() {
31
    local file=$1
70
    local file=$1
32
    local chunkssize=$2
71
    local chunkssize=$2
Lines 37-59 indexfile() { Link Here
37
76
38
        local prefix="${file}_${chunkssize}_"
77
        local prefix="${file}_${chunkssize}_"
39
        echo "Splitting file in chunks of $chunkssize records"
78
        echo "Splitting file in chunks of $chunkssize records"
40
        YAZMARCDUMP_CMD="$YAZMARCDUMP -n -s $prefix -C $chunkssize $file"
79
        splitfile $file $prefix $chunkssize
41
        $YAZMARCDUMP_CMD
42
80
43
        dir=$(dirname $prefix)
81
        dir=$(dirname $prefix)
44
        local files="$(find $dir -regex $prefix[0-9]+ | sort | tr '\n' ' ')"
82
        local files="$(find $dir -regex $prefix[0-9]+ | sort | tr '\n' ' ')"
45
        for chunkfile in $files; do
83
        for chunkfile in $files; do
46
            echo "Indexing $chunkfile"
84
            echo "Indexing $chunkfile"
47
            size=$($YAZMARCDUMP -p $chunkfile | grep '<!-- Record [0-9]\+ offset .* -->' | wc -l)
85
            size=$(grep '^</record>' $chunkfile | wc -l)
48
            logfile="$LOGDIR/zebraidx.$(basename $chunkfile).log"
86
            logfile="$LOGDIR/zebraidx.$(basename $chunkfile).log"
49
            ZEBRAIDX_CMD="$ZEBRAIDX -c $CONFIGFILE -d $TYPE -g iso2709 update $chunkfile"
87
            ZEBRAIDX_CMD="$ZEBRAIDX -c $CONFIGFILE -d $TYPE -g marcxml update $chunkfile"
50
            $ZEBRAIDX_CMD >$logfile 2>&1
88
            $ZEBRAIDX_CMD >$logfile 2>&1
51
            grep "Records: $size" $logfile >/dev/null 2>&1
89
            grep "Records: $size" $logfile >/dev/null 2>&1
52
            if [ $? -ne 0 ]; then
90
            if [ $? -ne 0 ]; then
53
                echo "Indexing failed. Split file and continue..."
91
                echo "Indexing failed. Split file and continue..."
54
                indexfile $chunkfile $(($chunkssize/2))
92
                indexfile $chunkfile $(($chunkssize/2))
55
            else
93
            else
56
                ZEBRAIDX_CMD="$ZEBRAIDX -c $CONFIGFILE -d $TYPE -g iso2709 commit"
94
                ZEBRAIDX_CMD="$ZEBRAIDX -c $CONFIGFILE -d $TYPE -g marcxml commit"
57
                $ZEBRAIDX_CMD >> $logfile 2>&1
95
                $ZEBRAIDX_CMD >> $logfile 2>&1
58
            fi
96
            fi
59
        done
97
        done
Lines 69-74 RMLOGS=no Link Here
69
NOCONFIRM=no
107
NOCONFIRM=no
70
TYPE=biblios
108
TYPE=biblios
71
HELP=no
109
HELP=no
110
RESETINDEX=no
72
111
73
# Get parameters
112
# Get parameters
74
while [ $1 ]; do
113
while [ $1 ]; do
Lines 103-111 while [ $1 ]; do Link Here
103
        -f | --force )
142
        -f | --force )
104
            NOCONFIRM=yes
143
            NOCONFIRM=yes
105
            ;;
144
            ;;
106
        -h | --help)
145
        -h | --help )
107
            HELP=yes
146
            HELP=yes
108
            ;;
147
            ;;
148
        --reset-index )
149
            RESETINDEX=yes
150
            ;;
109
        * )
151
        * )
110
            usage
152
            usage
111
            exit 1
153
            exit 1
Lines 118-151 if [ $HELP = "yes" ]; then Link Here
118
    exit 0
160
    exit 0
119
fi
161
fi
120
162
121
if [ -z $LENGTH ]; then
163
if [ -z $KOHA_CONF ]; then
122
    echo "--length parameter is mandatory"
164
    echo "KOHA_CONF is not set"
123
    exit 1
165
    exit 1
124
fi
166
fi
125
167
168
if [ -z $PERL5LIB ]; then
169
    echo "PERL5LIB is not set"
170
    exit 1
171
fi
172
173
126
TYPESWITCH=
174
TYPESWITCH=
175
SQLTABLE=
127
case $TYPE in
176
case $TYPE in
128
    biblios )
177
    biblios )
129
        TYPESWITCH=-b
178
        TYPESWITCH=-b
179
        SQLTABLE="biblio"
130
        ;;
180
        ;;
131
    authorities )
181
    authorities )
132
        TYPESWITCH=-a
182
        TYPESWITCH=-a
183
        SQLTABLE="auth_header"
133
        ;;
184
        ;;
134
    * )
185
    * )
135
        echo "'$TYPE' is an unknown type. Defaulting to 'biblios'"
186
        echo "'$TYPE' is an unknown type. Defaulting to 'biblios'"
136
        TYPESWITCH=-b
187
        TYPESWITCH=-b
137
        TYPE=biblios
188
        TYPE=biblios
189
        SQLTABLE="biblio"
138
esac
190
esac
139
191
140
ZEBRAIDX=`which zebraidx`
192
PERL=`which perl`
141
if [ -z $ZEBRAIDX ]; then
193
if [ -z $PERL ]; then
142
    echo "zebraidx not found"
194
    echo "perl not found"
143
    exit 1
195
    exit 1
144
fi
196
fi
145
197
146
YAZMARCDUMP=`which yaz-marcdump`
198
if [ -z $LENGTH ]; then
147
if [ -z $YAZMARCDUMP ]; then
199
    LENGTH=$($PERL -e '
148
    echo "yaz-marcdump not found"
200
        use C4::Context;
201
        my ($count) = C4::Context->dbh->selectrow_array(qq{
202
            SELECT COUNT(*) FROM '"$SQLTABLE"'
203
        });
204
        print $count;
205
    ')
206
fi
207
208
ZEBRAIDX=`which zebraidx`
209
if [ -z $ZEBRAIDX ]; then
210
    echo "zebraidx not found"
149
    exit 1
211
    exit 1
150
fi
212
fi
151
213
Lines 158-163 fi Link Here
158
echo ""
220
echo ""
159
echo "Configuration"
221
echo "Configuration"
160
echo "========================================================================="
222
echo "========================================================================="
223
echo "KOHA_CONF: $KOHA_CONF"
224
echo "PERL5LIB: $PERL5LIB"
225
echo "-------------------------------------------------------------------------"
161
echo "Start at offset: $OFFSET"
226
echo "Start at offset: $OFFSET"
162
echo "Total number of records to index: $LENGTH"
227
echo "Total number of records to index: $LENGTH"
163
echo "Initial chunk size: $CHUNKSSIZE"
228
echo "Initial chunk size: $CHUNKSSIZE"
Lines 165-174 echo "Export directory: $EXPORTDIR" Link Here
165
echo "Log directory: $LOGDIR"
230
echo "Log directory: $LOGDIR"
166
echo "Remove logs before start? $RMLOGS"
231
echo "Remove logs before start? $RMLOGS"
167
echo "Type of record: $TYPE"
232
echo "Type of record: $TYPE"
233
echo "Reset index before start? $RESETINDEX"
168
echo "-------------------------------------------------------------------------"
234
echo "-------------------------------------------------------------------------"
169
echo "zebraidx path: $ZEBRAIDX"
235
echo "zebraidx path: $ZEBRAIDX"
170
echo "yaz-marcdump path: $YAZMARCDUMP"
171
echo "rebuild_zebra path: $REBUILDZEBRA"
236
echo "rebuild_zebra path: $REBUILDZEBRA"
237
echo "perl path: $PERL"
172
echo "========================================================================="
238
echo "========================================================================="
173
239
174
if [ $NOCONFIRM != "yes" ]; then
240
if [ $NOCONFIRM != "yes" ]; then
Lines 200-206 if [ $RMLOGS = "yes" ]; then Link Here
200
    rm -f $LOGDIR/*.log
266
    rm -f $LOGDIR/*.log
201
fi
267
fi
202
268
203
REBUILDZEBRA_CMD="$REBUILDZEBRA $TYPESWITCH -v -k -d $EXPORTDIR --offset $OFFSET --length $LENGTH --skip-index"
269
REBUILDZEBRA_CMD="$REBUILDZEBRA $TYPESWITCH -v -x -k -d $EXPORTDIR --offset $OFFSET --length $LENGTH --skip-index"
204
echo "\n$REBUILDZEBRA_CMD"
270
echo "\n$REBUILDZEBRA_CMD"
205
$REBUILDZEBRA_CMD
271
$REBUILDZEBRA_CMD
206
272
Lines 219-223 esac Link Here
219
285
220
CONFIGFILE="$(dirname $KOHA_CONF)/zebradb/zebra-$TYPE.cfg"
286
CONFIGFILE="$(dirname $KOHA_CONF)/zebradb/zebra-$TYPE.cfg"
221
287
288
if [ $RESETINDEX = "yes" ]; then
289
    RESETINDEX_CMD="$ZEBRAIDX -c $CONFIGFILE init"
290
    echo "\n$RESETINDEX_CMD"
291
    $RESETINDEX_CMD
292
    echo ""
293
fi
222
294
223
indexfile $EXPORTFILE $CHUNKSSIZE
295
indexfile $EXPORTFILE $CHUNKSSIZE
224
- 

Return to bug 8746