View | Details | Raw Unified | Return to bug 7286
Collapse All | Expand All

(-)a/misc/migration_tools/rebuild_zebra.pl (-17 / +27 lines)
Lines 24-29 my $directory; Link Here
24
my $nosanitize;
24
my $nosanitize;
25
my $skip_export;
25
my $skip_export;
26
my $keep_export;
26
my $keep_export;
27
my $skip_index;
27
my $reset;
28
my $reset;
28
my $biblios;
29
my $biblios;
29
my $authorities;
30
my $authorities;
Lines 44-49 my $result = GetOptions( Link Here
44
    'r|reset'       => \$reset,
45
    'r|reset'       => \$reset,
45
    's'             => \$skip_export,
46
    's'             => \$skip_export,
46
    'k'             => \$keep_export,
47
    'k'             => \$keep_export,
48
    'I|skip-index'    => \$skip_index,
47
    'nosanitize'    => \$nosanitize,
49
    'nosanitize'    => \$nosanitize,
48
    'b'             => \$biblios,
50
    'b'             => \$biblios,
49
    'noxml'         => \$noxml,
51
    'noxml'         => \$noxml,
Lines 134-146 if ($do_munge) { Link Here
134
}
136
}
135
137
136
if ($authorities) {
138
if ($authorities) {
137
    index_records('authority', $directory, $skip_export, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $authorityserverdir);
139
    index_records('authority', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $authorityserverdir);
138
} else {
140
} else {
139
    print "skipping authorities\n" if ( $verbose_logging );
141
    print "skipping authorities\n" if ( $verbose_logging );
140
}
142
}
141
143
142
if ($biblios) {
144
if ($biblios) {
143
    index_records('biblio', $directory, $skip_export, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $biblioserverdir);
145
    index_records('biblio', $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $biblioserverdir);
144
} else {
146
} else {
145
    print "skipping biblios\n" if ( $verbose_logging );
147
    print "skipping biblios\n" if ( $verbose_logging );
146
}
148
}
Lines 191-197 sub check_zebra_dirs { Link Here
191
}	# ----------  end of subroutine check_zebra_dirs  ----------
193
}	# ----------  end of subroutine check_zebra_dirs  ----------
192
194
193
sub index_records {
195
sub index_records {
194
    my ($record_type, $directory, $skip_export, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
196
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
195
197
196
    my $num_records_exported = 0;
198
    my $num_records_exported = 0;
197
    my $records_deleted;
199
    my $records_deleted;
Lines 230-253 sub index_records { Link Here
230
            }
232
            }
231
        }
233
        }
232
    }
234
    }
233
    
235
234
    #
236
    #
235
    # and reindexing everything
237
    # and reindexing everything
236
    #
238
    #
237
    if ( $verbose_logging ) {
239
    if ($skip_index) {
238
        print "====================\n";
240
        if ($verbose_logging) {
239
        print "REINDEXING zebra\n";
241
            print "====================\n";
240
        print "====================\n";
242
            print "SKIPPING $record_type indexing\n";
241
    }
243
            print "====================\n";
242
	my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
244
        }
243
    if ($process_zebraqueue) {
244
        do_indexing($record_type, 'delete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) 
245
            if %$records_deleted;
246
        do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
247
            if $num_records_exported;
248
    } else {
245
    } else {
249
        do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
246
        if ( $verbose_logging ) {
250
            if ($num_records_exported or $skip_export);
247
            print "====================\n";
248
            print "REINDEXING zebra\n";
249
            print "====================\n";
250
        }
251
        my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
252
        if ($process_zebraqueue) {
253
            do_indexing($record_type, 'delete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) 
254
                if %$records_deleted;
255
            do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
256
                if $num_records_exported;
257
        } else {
258
            do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
259
                if ($num_records_exported or $skip_export);
260
        }
251
    }
261
    }
252
}
262
}
253
263
(-)a/misc/migration_tools/rebuild_zebra_sliced.sh (-1 / +216 lines)
Line 0 Link Here
0
- 
1
#!/bin/sh
2
3
usage() {
4
    local scriptname=$(basename $0)
5
    cat <<EOF
6
$scriptname
7
8
Index Koha records by chunks. It is useful when some record causes errors and
9
stop the indexation process. With this script, if indexation of one chunk fails,
10
chunk is splitted in two or more chunks, and indexation continue on these chunks.
11
rebuild_zebra.pl is called only once to export records. Splitting and indexing
12
is handled by this script (using yaz-marcdump and zebraidx).
13
14
Usage:
15
$scriptname -t type -l X [-o X] [-s X] [-d /export/dir] [-L /log/dir] [-r] [-f]
16
$scriptname -h
17
18
    -o | --offset         Offset parameter of rebuild_zebra.pl
19
    -l | --length         Length parameter of rebuild_zebra.pl
20
    -s | --chunks-size    Initial chunk size (number of records indexed at once)
21
    -d | --export-dir     Where rebuild_zebra.pl will export data
22
    -L | --log-dir        Log directory
23
    -r | --remove-logs    Clean log directory before start
24
    -t | --type           Record type ('biblios' or 'authorities')
25
    -f | --force          Don't ask for confirmation before start
26
    -h | --help           Display this help message
27
EOF
28
}
29
30
indexfile() {
31
    local file=$1
32
    local chunkssize=$2
33
34
    if [ $chunkssize -lt 1 ]; then
35
        echo "Fail on file $file"
36
    else
37
38
        local prefix="${file}_${chunkssize}_"
39
        echo "Splitting file in chunks of $chunkssize"
40
        YAZMARCDUMP_CMD="$YAZMARCDUMP -n -s $prefix -C $chunkssize $file"
41
        $YAZMARCDUMP_CMD
42
43
        dir=$(dirname $prefix)
44
        local files="$(find $dir -regex $prefix[0-9]+ | sort | tr '\n' ' ')"
45
        for chunkfile in $files; do
46
            echo "Indexing $chunkfile"
47
            size=$($YAZMARCDUMP -p $chunkfile | grep '<!-- Record [0-9]\+ offset .* -->' | wc -l)
48
            logfile="$LOGDIR/zebraidx.$(basename $chunkfile).log"
49
            ZEBRAIDX_CMD="$ZEBRAIDX -c $CONFIGFILE -d $TYPE -g iso2709 update $chunkfile"
50
            $ZEBRAIDX_CMD >$logfile 2>&1
51
            grep "Records: $size" $logfile >/dev/null 2>&1
52
            if [ $? -ne 0 ]; then
53
                echo "Indexing failed. Split file and continue..."
54
                indexfile $chunkfile $(($chunkssize/2))
55
            fi
56
        done
57
    fi
58
}
59
60
OFFSET=0
61
LENGTH=
62
CHUNKSSIZE=10000
63
EXPORTDIR=/tmp/rebuild/export
64
LOGDIR=/tmp/rebuild/logs
65
RMLOGS=no
66
NOCONFIRM=no
67
TYPE=biblios
68
HELP=no
69
70
# Get parameters
71
while [ $1 ]; do
72
    case $1 in
73
        -o | --offset )
74
            shift
75
            OFFSET=$1
76
            ;;
77
        -l | --length )
78
            shift
79
            LENGTH=$1
80
            ;;
81
        -s | --chunks-size )
82
            shift
83
            CHUNKSSIZE=$1
84
            ;;
85
        -d | --export-dir )
86
            shift
87
            EXPORTDIR=$1
88
            ;;
89
        -L | --log-dir )
90
            shift
91
            LOGDIR=$1
92
            ;;
93
        -r | --remove-logs )
94
            RMLOGS=yes
95
            ;;
96
        -t | --type )
97
            shift
98
            TYPE=$1
99
            ;;
100
        -f | --force )
101
            NOCONFIRM=yes
102
            ;;
103
        -h | --help)
104
            HELP=yes
105
            ;;
106
        * )
107
            usage
108
            exit 1
109
    esac
110
    shift
111
done
112
113
if [ $HELP = "yes" ]; then
114
    usage
115
    exit 0
116
fi
117
118
if [ -z $LENGTH ]; then
119
    echo "--length parameter is mandatory"
120
    exit 1
121
fi
122
123
TYPESWITCH=
124
case $TYPE in
125
    biblios )
126
        TYPESWITCH=-b
127
        ;;
128
    authorities )
129
        TYPESWITCH=-a
130
        ;;
131
    * )
132
        echo "'$TYPE' is an unknown type. Defaulting to 'biblios'"
133
        TYPESWITCH=-b
134
        TYPE=biblios
135
esac
136
137
ZEBRAIDX=`which zebraidx`
138
if [ -z $ZEBRAIDX ]; then
139
    echo "zebraidx not found"
140
    exit 1
141
fi
142
143
YAZMARCDUMP=`which yaz-marcdump`
144
if [ -z $YAZMARCDUMP ]; then
145
    echo "yaz-marcdump not found"
146
    exit 1
147
fi
148
149
REBUILDZEBRA="`dirname $0`/rebuild_zebra.pl"
150
if [ ! -f $REBUILDZEBRA ]; then
151
    echo "$REBUILDZEBRA: file not found"
152
    exit 1
153
fi
154
155
echo ""
156
echo "Configuration"
157
echo "========================================================================="
158
echo "Start at offset: $OFFSET"
159
echo "Total number of records to index: $LENGTH"
160
echo "Initial chunk size: $CHUNKSSIZE"
161
echo "Export directory: $EXPORTDIR"
162
echo "Log directory: $LOGDIR"
163
echo "Remove logs before start? $RMLOGS"
164
echo "Type of record: $TYPE"
165
echo "-------------------------------------------------------------------------"
166
echo "zebraidx path: $ZEBRAIDX"
167
echo "yaz-marcdump path: $YAZMARCDUMP"
168
echo "rebuild_zebra path: $REBUILDZEBRA"
169
echo "========================================================================="
170
171
if [ $NOCONFIRM != "yes" ]; then
172
    confirm=y
173
    echo -n "Confirm ? [Y/n] "
174
    read response
175
    if [ $response ] && [ $response != "yes" ] && [ $response != "y" ]; then
176
        confirm=n
177
    fi
178
179
    if [ $confirm = "n" ]; then
180
        exit 0
181
    fi
182
fi
183
184
mkdir -p $EXPORTDIR
185
if [ $? -ne 0 ]; then
186
    echo "Failed to create directory $EXPORTDIR. Aborting."
187
    exit 1
188
fi
189
190
mkdir -p $LOGDIR
191
if [ $? -ne 0 ]; then
192
    echo "Failed to create directory $LOGDIR. Aborting."
193
    exit 1
194
fi
195
196
REBUILDZEBRA_CMD="$REBUILDZEBRA $TYPESWITCH -v -k -d $EXPORTDIR --offset $OFFSET --length $LENGTH --skip-index"
197
echo "\n$REBUILDZEBRA_CMD"
198
$REBUILDZEBRA_CMD
199
200
EXPORTFILE=
201
case $TYPE in
202
    biblios )
203
        EXPORTFILE="$EXPORTDIR/biblio/exported_records"
204
        ;;
205
    authorities )
206
        EXPORTFILE="$EXPORTDIR/authority/exported_records"
207
        ;;
208
    * )
209
        echo "Error: TYPE '$TYPE' is not supported"
210
        exit 1
211
esac
212
213
CONFIGFILE="$(dirname $KOHA_CONF)/zebradb/zebra-$TYPE.cfg"
214
215
216
indexfile $EXPORTFILE $CHUNKSSIZE

Return to bug 7286