|
Lines 34-39
B<rebuild_elasticsearch.pl>
Link Here
|
| 34 |
[B<--desc>] |
34 |
[B<--desc>] |
| 35 |
[B<-bn|--bnumber>] |
35 |
[B<-bn|--bnumber>] |
| 36 |
[B<-ai|--authid>] |
36 |
[B<-ai|--authid>] |
|
|
37 |
[B<-w|--where SQL>] |
| 37 |
[B<-p|--processes>] |
38 |
[B<-p|--processes>] |
| 38 |
[B<-v|--verbose>] |
39 |
[B<-v|--verbose>] |
| 39 |
[B<-h|--help>] |
40 |
[B<-h|--help>] |
|
Lines 87-92
repeated.
Link Here
|
| 87 |
Only index the supplied authority id, mostly for testing purposes. May be |
88 |
Only index the supplied authority id, mostly for testing purposes. May be |
| 88 |
repeated. |
89 |
repeated. |
| 89 |
|
90 |
|
|
|
91 |
=item B<-w|--where> |
| 92 |
|
| 93 |
Pass some additional SQL to limit the records to be indexed. |
| 94 |
|
| 90 |
=item B<-p|--processes> |
95 |
=item B<-p|--processes> |
| 91 |
|
96 |
|
| 92 |
Number of processes to use for indexing. This can be used to do more indexing |
97 |
Number of processes to use for indexing. This can be used to do more indexing |
|
Lines 125-137
use Pod::Usage qw( pod2usage );
Link Here
|
| 125 |
use Try::Tiny qw( catch try ); |
130 |
use Try::Tiny qw( catch try ); |
| 126 |
|
131 |
|
| 127 |
my $verbose = 0; |
132 |
my $verbose = 0; |
| 128 |
my $commit = 5000; |
133 |
my $commit = 5000; |
| 129 |
my ($delete, $reset, $help, $man, $processes); |
134 |
my ( $delete, $reset, $help, $man, $processes ); |
| 130 |
my ($index_biblios, $index_authorities); |
135 |
my ( $index_biblios, $index_authorities ); |
| 131 |
my (@biblionumbers,@authids); |
136 |
my ( @biblionumbers, @authids, $where ); |
| 132 |
my $desc; |
137 |
my $desc; |
| 133 |
|
138 |
|
| 134 |
$|=1; # flushes output |
139 |
$| = 1; # flushes output |
| 135 |
|
140 |
|
| 136 |
GetOptions( |
141 |
GetOptions( |
| 137 |
'c|commit=i' => \$commit, |
142 |
'c|commit=i' => \$commit, |
|
Lines 142-147
GetOptions(
Link Here
|
| 142 |
'desc' => \$desc, |
147 |
'desc' => \$desc, |
| 143 |
'bn|bnumber=i' => \@biblionumbers, |
148 |
'bn|bnumber=i' => \@biblionumbers, |
| 144 |
'ai|authid=i' => \@authids, |
149 |
'ai|authid=i' => \@authids, |
|
|
150 |
'w|where=s' => \$where, |
| 145 |
'p|processes=i' => \$processes, |
151 |
'p|processes=i' => \$processes, |
| 146 |
'v|verbose+' => \$verbose, |
152 |
'v|verbose+' => \$verbose, |
| 147 |
'h|help' => \$help, |
153 |
'h|help' => \$help, |
|
Lines 149-210
GetOptions(
Link Here
|
| 149 |
); |
155 |
); |
| 150 |
|
156 |
|
| 151 |
# Default is to do both |
157 |
# Default is to do both |
| 152 |
unless ($index_authorities || $index_biblios) { |
158 |
unless ( $index_authorities || $index_biblios ) { |
| 153 |
$index_authorities = $index_biblios = 1; |
159 |
$index_authorities = $index_biblios = 1; |
| 154 |
} |
160 |
} |
| 155 |
|
161 |
|
| 156 |
if ($processes && ( @biblionumbers || @authids) ) { |
162 |
if ( $processes && ( @biblionumbers || @authids ) ) { |
| 157 |
die "Argument p|processes cannot be combined with bn|bnumber or ai|authid"; |
163 |
die "Argument p|processes cannot be combined with bn|bnumber or ai|authid"; |
| 158 |
} |
164 |
} |
| 159 |
|
165 |
|
| 160 |
pod2usage(1) if $help; |
166 |
pod2usage(1) if $help; |
| 161 |
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; |
167 |
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; |
| 162 |
|
168 |
|
| 163 |
_sanity_check(); |
169 |
_sanity_check(); |
| 164 |
|
170 |
|
| 165 |
if ($reset){ |
171 |
if ($reset) { |
| 166 |
Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings; |
172 |
Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings; |
| 167 |
$delete = 1; |
173 |
$delete = 1; |
| 168 |
} |
174 |
} |
| 169 |
|
175 |
|
| 170 |
_verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete) if ($index_biblios); |
176 |
_verify_index_state( $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete ) if ($index_biblios); |
| 171 |
_verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete) if ($index_authorities); |
177 |
_verify_index_state( $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete ) if ($index_authorities); |
| 172 |
|
178 |
|
| 173 |
my $slice_index = 0; |
179 |
my $slice_index = 0; |
| 174 |
my $slice_count = ( $processes //= 1 ); |
180 |
my $slice_count = ( $processes //= 1 ); |
| 175 |
my %iterator_options; |
181 |
my %iterator_options; |
| 176 |
|
182 |
|
| 177 |
if ($slice_count > 1) { |
183 |
if ( $slice_count > 1 ) { |
|
|
184 |
|
| 178 |
# Fire up child processes for processing slices from 2 on. This main process will handle slice 1. |
185 |
# Fire up child processes for processing slices from 2 on. This main process will handle slice 1. |
| 179 |
$slice_index = 0; |
186 |
$slice_index = 0; |
| 180 |
for (my $proc = 1; $proc < $slice_count; $proc++) { |
187 |
for ( my $proc = 1 ; $proc < $slice_count ; $proc++ ) { |
| 181 |
my $pid = fork(); |
188 |
my $pid = fork(); |
| 182 |
die "Failed to fork a child process\n" unless defined $pid; |
189 |
die "Failed to fork a child process\n" unless defined $pid; |
| 183 |
if ($pid == 0) { |
190 |
if ( $pid == 0 ) { |
|
|
191 |
|
| 184 |
# Child process, give it a slice to process |
192 |
# Child process, give it a slice to process |
| 185 |
$slice_index = $proc; |
193 |
$slice_index = $proc; |
| 186 |
last; |
194 |
last; |
| 187 |
} |
195 |
} |
| 188 |
} |
196 |
} |
|
|
197 |
|
| 189 |
# Fudge the commit count a bit to spread out the Elasticsearch commits |
198 |
# Fudge the commit count a bit to spread out the Elasticsearch commits |
| 190 |
$commit *= 1 + 0.10 * $slice_index; |
199 |
$commit *= 1 + 0.10 * $slice_index; |
| 191 |
$commit = int( $commit ); |
200 |
$commit = int($commit); |
| 192 |
_log(1, "Processing slice @{[$slice_index + 1]} of $slice_count\n"); |
201 |
_log( 1, "Processing slice @{[$slice_index + 1]} of $slice_count\n" ); |
| 193 |
$iterator_options{slice} = { index => $slice_index, count => $slice_count }; |
202 |
$iterator_options{slice} = { index => $slice_index, count => $slice_count }; |
| 194 |
} |
203 |
} |
| 195 |
|
204 |
|
| 196 |
if( $desc ){ |
205 |
if ($desc) { |
| 197 |
$iterator_options{desc} = 1; |
206 |
$iterator_options{desc} = 1; |
| 198 |
} |
207 |
} |
| 199 |
|
208 |
|
|
|
209 |
if ($where) { |
| 210 |
$iterator_options{where} = $where; |
| 211 |
} |
| 212 |
|
| 200 |
my $next; |
213 |
my $next; |
| 201 |
if ($index_biblios) { |
214 |
if ($index_biblios) { |
| 202 |
_log(1, "Indexing biblios\n"); |
215 |
_log( 1, "Indexing biblios\n" ); |
| 203 |
if (@biblionumbers) { |
216 |
if (@biblionumbers) { |
| 204 |
$next = sub { |
217 |
$next = sub { |
| 205 |
my $r = shift @biblionumbers; |
218 |
my $r = shift @biblionumbers; |
| 206 |
return () unless defined $r; |
219 |
return () unless defined $r; |
| 207 |
return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); |
220 |
return ( $r, Koha::BiblioUtils->get_from_biblionumber( $r, item_data => 1 ) ); |
| 208 |
}; |
221 |
}; |
| 209 |
} else { |
222 |
} else { |
| 210 |
my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options); |
223 |
my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options); |
|
Lines 212-227
if ($index_biblios) {
Link Here
|
| 212 |
$records->next(); |
225 |
$records->next(); |
| 213 |
} |
226 |
} |
| 214 |
} |
227 |
} |
| 215 |
_do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); |
228 |
_do_reindex( $next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX ); |
| 216 |
} |
229 |
} |
| 217 |
if ($index_authorities) { |
230 |
if ($index_authorities) { |
| 218 |
_log(1, "Indexing authorities\n"); |
231 |
_log( 1, "Indexing authorities\n" ); |
| 219 |
if (@authids) { |
232 |
if (@authids) { |
| 220 |
$next = sub { |
233 |
$next = sub { |
| 221 |
my $r = shift @authids; |
234 |
my $r = shift @authids; |
| 222 |
return () unless defined $r; |
235 |
return () unless defined $r; |
| 223 |
my $a = Koha::MetadataRecord::Authority->get_from_authid($r); |
236 |
my $a = Koha::MetadataRecord::Authority->get_from_authid($r); |
| 224 |
return ($r, $a); |
237 |
return ( $r, $a ); |
| 225 |
}; |
238 |
}; |
| 226 |
} else { |
239 |
} else { |
| 227 |
my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options); |
240 |
my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options); |
|
Lines 229-240
if ($index_authorities) {
Link Here
|
| 229 |
$records->next(); |
242 |
$records->next(); |
| 230 |
} |
243 |
} |
| 231 |
} |
244 |
} |
| 232 |
_do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX); |
245 |
_do_reindex( $next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX ); |
| 233 |
} |
246 |
} |
| 234 |
|
247 |
|
| 235 |
if ($slice_index == 0) { |
248 |
if ( $slice_index == 0 ) { |
|
|
249 |
|
| 236 |
# Main process, wait for children |
250 |
# Main process, wait for children |
| 237 |
for (my $proc = 1; $proc < $processes; $proc++) { |
251 |
for ( my $proc = 1 ; $proc < $processes ; $proc++ ) { |
| 238 |
wait(); |
252 |
wait(); |
| 239 |
} |
253 |
} |
| 240 |
} |
254 |
} |
|
Lines 252-272
Checks the index state and recreates it if requested.
Link Here
|
| 252 |
sub _verify_index_state { |
266 |
sub _verify_index_state { |
| 253 |
my ( $index_name, $recreate ) = @_; |
267 |
my ( $index_name, $recreate ) = @_; |
| 254 |
|
268 |
|
| 255 |
_log(1, "Checking state of $index_name index\n"); |
269 |
_log( 1, "Checking state of $index_name index\n" ); |
| 256 |
my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } ); |
270 |
my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } ); |
| 257 |
|
271 |
|
| 258 |
if ($recreate) { |
272 |
if ($recreate) { |
| 259 |
_log(1, "Dropping and recreating $index_name index\n"); |
273 |
_log( 1, "Dropping and recreating $index_name index\n" ); |
| 260 |
$indexer->drop_index() if $indexer->index_exists(); |
274 |
$indexer->drop_index() if $indexer->index_exists(); |
| 261 |
$indexer->create_index(); |
275 |
$indexer->create_index(); |
| 262 |
} |
276 |
} elsif ( !$indexer->index_exists ) { |
| 263 |
elsif (!$indexer->index_exists) { |
277 |
|
| 264 |
# Create index if does not exist |
278 |
# Create index if does not exist |
| 265 |
$indexer->create_index(); |
279 |
$indexer->create_index(); |
| 266 |
} elsif ($indexer->is_index_status_ok) { |
280 |
} elsif ( $indexer->is_index_status_ok ) { |
|
|
281 |
|
| 267 |
# Update mapping unless index is some kind of problematic state |
282 |
# Update mapping unless index is some kind of problematic state |
| 268 |
$indexer->update_mappings(); |
283 |
$indexer->update_mappings(); |
| 269 |
} elsif ($indexer->is_index_status_recreate_required) { |
284 |
} elsif ( $indexer->is_index_status_recreate_required ) { |
| 270 |
warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/; |
285 |
warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/; |
| 271 |
} |
286 |
} |
| 272 |
} |
287 |
} |
|
Lines 293-299
sub _do_reindex {
Link Here
|
| 293 |
my $record = $record->record; |
308 |
my $record = $record->record; |
| 294 |
$count++; |
309 |
$count++; |
| 295 |
if ( $verbose == 1 ) { |
310 |
if ( $verbose == 1 ) { |
| 296 |
_log( 1, "$count records processed\n" ) if ( $count % 1000 == 0); |
311 |
_log( 1, "$count records processed\n" ) if ( $count % 1000 == 0 ); |
| 297 |
} else { |
312 |
} else { |
| 298 |
_log( 2, "$id\n" ); |
313 |
_log( 2, "$id\n" ); |
| 299 |
} |
314 |
} |
|
Lines 303-315
sub _do_reindex {
Link Here
|
| 303 |
if ( !( --$commit_count ) ) { |
318 |
if ( !( --$commit_count ) ) { |
| 304 |
_log( 1, "Committing $commit records...\n" ); |
319 |
_log( 1, "Committing $commit records...\n" ); |
| 305 |
my $response; |
320 |
my $response; |
| 306 |
try{ |
321 |
try { |
| 307 |
$response = $indexer->update_index( \@id_buffer, \@commit_buffer ); |
322 |
$response = $indexer->update_index( \@id_buffer, \@commit_buffer ); |
| 308 |
_handle_response($response); |
323 |
_handle_response($response); |
| 309 |
_log( 1, "Commit complete\n" ); |
324 |
_log( 1, "Commit complete\n" ); |
| 310 |
} catch { |
325 |
} catch { |
| 311 |
_log(1,"Elasticsearch exception thrown: ".$_->type."\n"); |
326 |
_log( 1, "Elasticsearch exception thrown: " . $_->type . "\n" ); |
| 312 |
_log(2,"Details: ".$_->details."\n"); |
327 |
_log( 2, "Details: " . $_->details . "\n" ); |
| 313 |
}; |
328 |
}; |
| 314 |
$commit_count = $commit; |
329 |
$commit_count = $commit; |
| 315 |
@id_buffer = (); |
330 |
@id_buffer = (); |
|
Lines 333-338
Checks some basic stuff to ensure that it's sane before we start.
Link Here
|
| 333 |
=cut |
348 |
=cut |
| 334 |
|
349 |
|
| 335 |
sub _sanity_check { |
350 |
sub _sanity_check { |
|
|
351 |
|
| 336 |
# Do we have an elasticsearch block defined? |
352 |
# Do we have an elasticsearch block defined? |
| 337 |
my $conf = C4::Context->config('elasticsearch'); |
353 |
my $conf = C4::Context->config('elasticsearch'); |
| 338 |
die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf ); |
354 |
die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf ); |
|
Lines 346-359
Parse the return from update_index and display errors depending on verbosity of
Link Here
|
| 346 |
|
362 |
|
| 347 |
sub _handle_response { |
363 |
sub _handle_response { |
| 348 |
my ($response) = @_; |
364 |
my ($response) = @_; |
| 349 |
if( $response->{errors} eq 'true' ){ |
365 |
if ( $response->{errors} eq 'true' ) { |
| 350 |
_log( 1, "There were errors during indexing\n" ); |
366 |
_log( 1, "There were errors during indexing\n" ); |
| 351 |
if ( $verbose > 1 ){ |
367 |
if ( $verbose > 1 ) { |
| 352 |
foreach my $item (@{$response->{items}}){ |
368 |
foreach my $item ( @{ $response->{items} } ) { |
| 353 |
next unless defined $item->{index}->{error}; |
369 |
next unless defined $item->{index}->{error}; |
| 354 |
print "Record #" . $item->{index}->{_id} . " " . |
370 |
print "Record #" |
| 355 |
$item->{index}->{error}->{reason} . " (" . $item->{index}->{error}->{type} . ") : " . |
371 |
. $item->{index}->{_id} . " " |
| 356 |
$item->{index}->{error}->{caused_by}->{type} . " (" . $item->{index}->{error}->{caused_by}->{reason} . ")\n"; |
372 |
. $item->{index}->{error}->{reason} . " (" |
|
|
373 |
. $item->{index}->{error}->{type} . ") : " |
| 374 |
. $item->{index}->{error}->{caused_by}->{type} . " (" |
| 375 |
. $item->{index}->{error}->{caused_by}->{reason} . ")\n"; |
| 357 |
} |
376 |
} |
| 358 |
} |
377 |
} |
| 359 |
} |
378 |
} |
|
Lines 371-377
include a trailing newline automatically.
Link Here
|
| 371 |
=cut |
390 |
=cut |
| 372 |
|
391 |
|
| 373 |
sub _log { |
392 |
sub _log { |
| 374 |
my ($level, $msg) = @_; |
393 |
my ( $level, $msg ) = @_; |
| 375 |
|
394 |
|
| 376 |
print "[$$] $msg" if ($verbose >= $level); |
395 |
print "[$$] $msg" if ( $verbose >= $level ); |
| 377 |
} |
396 |
} |
| 378 |
- |
|
|