Lines 12-32
rebuild_zebra.pl is called only once to export records. Splitting and indexing
Link Here
|
12 |
is handled by this script (using yaz-marcdump and zebraidx). |
12 |
is handled by this script (using yaz-marcdump and zebraidx). |
13 |
|
13 |
|
14 |
Usage: |
14 |
Usage: |
15 |
$scriptname -t type -l X [-o X] [-s X] [-d /export/dir] [-L /log/dir] [-r] [-f] |
15 |
$scriptname [-t type] [-l X] [-o X] [-s X] [-d /export/dir] [-L /log/dir] [-r] [-f] [--reset-index] |
16 |
$scriptname -h |
16 |
$scriptname -h |
17 |
|
17 |
|
18 |
-o | --offset Offset parameter of rebuild_zebra.pl |
18 |
-o | --offset Offset parameter of rebuild_zebra.pl. |
19 |
-l | --length Length parameter of rebuild_zebra.pl |
19 |
Default: $OFFSET |
|
|
20 |
-l | --length Length parameter of rebuild_zebra.pl. If omitted, the |
21 |
length is automatically calculated to index all |
22 |
records |
20 |
-s | --chunks-size Initial chunk size (number of records indexed at once) |
23 |
-s | --chunks-size Initial chunk size (number of records indexed at once) |
|
|
24 |
Default: $CHUNKSSIZE |
21 |
-d | --export-dir Where rebuild_zebra.pl will export data |
25 |
-d | --export-dir Where rebuild_zebra.pl will export data |
|
|
26 |
Default: $EXPORTDIR |
22 |
-L | --log-dir Log directory |
27 |
-L | --log-dir Log directory |
|
|
28 |
Default: $LOGDIR |
23 |
-r | --remove-logs Clean log directory before start |
29 |
-r | --remove-logs Clean log directory before start |
|
|
30 |
Default: $RMLOGS |
24 |
-t | --type Record type ('biblios' or 'authorities') |
31 |
-t | --type Record type ('biblios' or 'authorities') |
|
|
32 |
Default: $TYPE |
25 |
-f | --force Don't ask for confirmation before start |
33 |
-f | --force Don't ask for confirmation before start |
26 |
-h | --help Display this help message |
34 |
-h | --help Display this help message |
|
|
35 |
--reset-index Reset Zebra index for 'type' |
27 |
EOF |
36 |
EOF |
28 |
} |
37 |
} |
29 |
|
38 |
|
|
|
39 |
splitfile() { |
40 |
local file=$1 |
41 |
local prefix=$2 |
42 |
local size=$3 |
43 |
local script=' |
44 |
my $prefix = '"\"$prefix\""'; |
45 |
my $size = '"$size"'; |
46 |
my ($i,$count) = (0,0); |
47 |
open(my $fh, "<", '"\"$file\""'); |
48 |
open(my $out, ">", sprintf("$prefix%02d", $i)); |
49 |
my $closed = 0; |
50 |
while (<$fh>) { |
51 |
my $line = $_; |
52 |
if ($closed) { |
53 |
open($out, ">", sprintf("$prefix%02d", $i)); |
54 |
$closed = 0; |
55 |
} |
56 |
print $out $line; |
57 |
$count++ if ($line =~ m|^</record>|); |
58 |
if ($count == $size) { |
59 |
$count = 0; |
60 |
$i++; |
61 |
close($out); |
62 |
$closed = 1; |
63 |
} |
64 |
} |
65 |
' |
66 |
$PERL -e "$script" |
67 |
} |
68 |
|
30 |
indexfile() { |
69 |
indexfile() { |
31 |
local file=$1 |
70 |
local file=$1 |
32 |
local chunkssize=$2 |
71 |
local chunkssize=$2 |
Lines 37-59
indexfile() {
Link Here
|
37 |
|
76 |
|
38 |
local prefix="${file}_${chunkssize}_" |
77 |
local prefix="${file}_${chunkssize}_" |
39 |
echo "Splitting file in chunks of $chunkssize records" |
78 |
echo "Splitting file in chunks of $chunkssize records" |
40 |
YAZMARCDUMP_CMD="$YAZMARCDUMP -n -s $prefix -C $chunkssize $file" |
79 |
splitfile $file $prefix $chunkssize |
41 |
$YAZMARCDUMP_CMD |
|
|
42 |
|
80 |
|
43 |
dir=$(dirname $prefix) |
81 |
dir=$(dirname $prefix) |
44 |
local files="$(find $dir -regex $prefix[0-9]+ | sort | tr '\n' ' ')" |
82 |
local files="$(find $dir -regex $prefix[0-9]+ | sort | tr '\n' ' ')" |
45 |
for chunkfile in $files; do |
83 |
for chunkfile in $files; do |
46 |
echo "Indexing $chunkfile" |
84 |
echo "Indexing $chunkfile" |
47 |
size=$($YAZMARCDUMP -p $chunkfile | grep '<!-- Record [0-9]\+ offset .* -->' | wc -l) |
85 |
size=$(grep '^</record>' $chunkfile | wc -l) |
48 |
logfile="$LOGDIR/zebraidx.$(basename $chunkfile).log" |
86 |
logfile="$LOGDIR/zebraidx.$(basename $chunkfile).log" |
49 |
ZEBRAIDX_CMD="$ZEBRAIDX -c $CONFIGFILE -d $TYPE -g iso2709 update $chunkfile" |
87 |
ZEBRAIDX_CMD="$ZEBRAIDX -c $CONFIGFILE -d $TYPE -g marcxml update $chunkfile" |
50 |
$ZEBRAIDX_CMD >$logfile 2>&1 |
88 |
$ZEBRAIDX_CMD >$logfile 2>&1 |
51 |
grep "Records: $size" $logfile >/dev/null 2>&1 |
89 |
grep "Records: $size" $logfile >/dev/null 2>&1 |
52 |
if [ $? -ne 0 ]; then |
90 |
if [ $? -ne 0 ]; then |
53 |
echo "Indexing failed. Split file and continue..." |
91 |
echo "Indexing failed. Split file and continue..." |
54 |
indexfile $chunkfile $(($chunkssize/2)) |
92 |
indexfile $chunkfile $(($chunkssize/2)) |
55 |
else |
93 |
else |
56 |
ZEBRAIDX_CMD="$ZEBRAIDX -c $CONFIGFILE -d $TYPE -g iso2709 commit" |
94 |
ZEBRAIDX_CMD="$ZEBRAIDX -c $CONFIGFILE -d $TYPE -g marcxml commit" |
57 |
$ZEBRAIDX_CMD >> $logfile 2>&1 |
95 |
$ZEBRAIDX_CMD >> $logfile 2>&1 |
58 |
fi |
96 |
fi |
59 |
done |
97 |
done |
Lines 69-74
RMLOGS=no
Link Here
|
69 |
NOCONFIRM=no |
107 |
NOCONFIRM=no |
70 |
TYPE=biblios |
108 |
TYPE=biblios |
71 |
HELP=no |
109 |
HELP=no |
|
|
110 |
RESETINDEX=no |
72 |
|
111 |
|
73 |
# Get parameters |
112 |
# Get parameters |
74 |
while [ $1 ]; do |
113 |
while [ $1 ]; do |
Lines 103-111
while [ $1 ]; do
Link Here
|
103 |
-f | --force ) |
142 |
-f | --force ) |
104 |
NOCONFIRM=yes |
143 |
NOCONFIRM=yes |
105 |
;; |
144 |
;; |
106 |
-h | --help) |
145 |
-h | --help ) |
107 |
HELP=yes |
146 |
HELP=yes |
108 |
;; |
147 |
;; |
|
|
148 |
--reset-index ) |
149 |
RESETINDEX=yes |
150 |
;; |
109 |
* ) |
151 |
* ) |
110 |
usage |
152 |
usage |
111 |
exit 1 |
153 |
exit 1 |
Lines 118-151
if [ $HELP = "yes" ]; then
Link Here
|
118 |
exit 0 |
160 |
exit 0 |
119 |
fi |
161 |
fi |
120 |
|
162 |
|
121 |
if [ -z $LENGTH ]; then |
163 |
if [ -z $KOHA_CONF ]; then |
122 |
echo "--length parameter is mandatory" |
164 |
echo "KOHA_CONF is not set" |
123 |
exit 1 |
165 |
exit 1 |
124 |
fi |
166 |
fi |
125 |
|
167 |
|
|
|
168 |
if [ -z $PERL5LIB ]; then |
169 |
echo "PERL5LIB is not set" |
170 |
exit 1 |
171 |
fi |
172 |
|
173 |
|
126 |
TYPESWITCH= |
174 |
TYPESWITCH= |
|
|
175 |
SQLTABLE= |
127 |
case $TYPE in |
176 |
case $TYPE in |
128 |
biblios ) |
177 |
biblios ) |
129 |
TYPESWITCH=-b |
178 |
TYPESWITCH=-b |
|
|
179 |
SQLTABLE="biblio" |
130 |
;; |
180 |
;; |
131 |
authorities ) |
181 |
authorities ) |
132 |
TYPESWITCH=-a |
182 |
TYPESWITCH=-a |
|
|
183 |
SQLTABLE="auth_header" |
133 |
;; |
184 |
;; |
134 |
* ) |
185 |
* ) |
135 |
echo "'$TYPE' is an unknown type. Defaulting to 'biblios'" |
186 |
echo "'$TYPE' is an unknown type. Defaulting to 'biblios'" |
136 |
TYPESWITCH=-b |
187 |
TYPESWITCH=-b |
137 |
TYPE=biblios |
188 |
TYPE=biblios |
|
|
189 |
SQLTABLE="biblio" |
138 |
esac |
190 |
esac |
139 |
|
191 |
|
140 |
ZEBRAIDX=`which zebraidx` |
192 |
PERL=`which perl` |
141 |
if [ -z $ZEBRAIDX ]; then |
193 |
if [ -z $PERL ]; then |
142 |
echo "zebraidx not found" |
194 |
echo "perl not found" |
143 |
exit 1 |
195 |
exit 1 |
144 |
fi |
196 |
fi |
145 |
|
197 |
|
146 |
YAZMARCDUMP=`which yaz-marcdump` |
198 |
if [ -z $LENGTH ]; then |
147 |
if [ -z $YAZMARCDUMP ]; then |
199 |
LENGTH=$($PERL -e ' |
148 |
echo "yaz-marcdump not found" |
200 |
use C4::Context; |
|
|
201 |
my ($count) = C4::Context->dbh->selectrow_array(qq{ |
202 |
SELECT COUNT(*) FROM '"$SQLTABLE"' |
203 |
}); |
204 |
print $count; |
205 |
') |
206 |
fi |
207 |
|
208 |
ZEBRAIDX=`which zebraidx` |
209 |
if [ -z $ZEBRAIDX ]; then |
210 |
echo "zebraidx not found" |
149 |
exit 1 |
211 |
exit 1 |
150 |
fi |
212 |
fi |
151 |
|
213 |
|
Lines 158-163
fi
Link Here
|
158 |
echo "" |
220 |
echo "" |
159 |
echo "Configuration" |
221 |
echo "Configuration" |
160 |
echo "=========================================================================" |
222 |
echo "=========================================================================" |
|
|
223 |
echo "KOHA_CONF: $KOHA_CONF" |
224 |
echo "PERL5LIB: $PERL5LIB" |
225 |
echo "-------------------------------------------------------------------------" |
161 |
echo "Start at offset: $OFFSET" |
226 |
echo "Start at offset: $OFFSET" |
162 |
echo "Total number of records to index: $LENGTH" |
227 |
echo "Total number of records to index: $LENGTH" |
163 |
echo "Initial chunk size: $CHUNKSSIZE" |
228 |
echo "Initial chunk size: $CHUNKSSIZE" |
Lines 165-174
echo "Export directory: $EXPORTDIR"
Link Here
|
165 |
echo "Log directory: $LOGDIR" |
230 |
echo "Log directory: $LOGDIR" |
166 |
echo "Remove logs before start? $RMLOGS" |
231 |
echo "Remove logs before start? $RMLOGS" |
167 |
echo "Type of record: $TYPE" |
232 |
echo "Type of record: $TYPE" |
|
|
233 |
echo "Reset index before start? $RESETINDEX" |
168 |
echo "-------------------------------------------------------------------------" |
234 |
echo "-------------------------------------------------------------------------" |
169 |
echo "zebraidx path: $ZEBRAIDX" |
235 |
echo "zebraidx path: $ZEBRAIDX" |
170 |
echo "yaz-marcdump path: $YAZMARCDUMP" |
|
|
171 |
echo "rebuild_zebra path: $REBUILDZEBRA" |
236 |
echo "rebuild_zebra path: $REBUILDZEBRA" |
|
|
237 |
echo "perl path: $PERL" |
172 |
echo "=========================================================================" |
238 |
echo "=========================================================================" |
173 |
|
239 |
|
174 |
if [ $NOCONFIRM != "yes" ]; then |
240 |
if [ $NOCONFIRM != "yes" ]; then |
Lines 200-206
if [ $RMLOGS = "yes" ]; then
Link Here
|
200 |
rm -f $LOGDIR/*.log |
266 |
rm -f $LOGDIR/*.log |
201 |
fi |
267 |
fi |
202 |
|
268 |
|
203 |
REBUILDZEBRA_CMD="$REBUILDZEBRA $TYPESWITCH -v -k -d $EXPORTDIR --offset $OFFSET --length $LENGTH --skip-index" |
269 |
REBUILDZEBRA_CMD="$REBUILDZEBRA $TYPESWITCH -v -x -k -d $EXPORTDIR --offset $OFFSET --length $LENGTH --skip-index" |
204 |
echo "\n$REBUILDZEBRA_CMD" |
270 |
echo "\n$REBUILDZEBRA_CMD" |
205 |
$REBUILDZEBRA_CMD |
271 |
$REBUILDZEBRA_CMD |
206 |
|
272 |
|
Lines 219-223
esac
Link Here
|
219 |
|
285 |
|
220 |
CONFIGFILE="$(dirname $KOHA_CONF)/zebradb/zebra-$TYPE.cfg" |
286 |
CONFIGFILE="$(dirname $KOHA_CONF)/zebradb/zebra-$TYPE.cfg" |
221 |
|
287 |
|
|
|
288 |
if [ $RESETINDEX = "yes" ]; then |
289 |
RESETINDEX_CMD="$ZEBRAIDX -c $CONFIGFILE init" |
290 |
echo "\n$RESETINDEX_CMD" |
291 |
$RESETINDEX_CMD |
292 |
echo "" |
293 |
fi |
222 |
|
294 |
|
223 |
indexfile $EXPORTFILE $CHUNKSSIZE |
295 |
indexfile $EXPORTFILE $CHUNKSSIZE |
224 |
- |
|
|