|
Lines 64-69
Only index the supplied biblionumber, mostly for testing purposes. May be
Link Here
|
| 64 |
repeated. This also applies to authorities via authid, so if you're using it, |
64 |
repeated. This also applies to authorities via authid, so if you're using it, |
| 65 |
you probably only want to do one or the other at a time. |
65 |
you probably only want to do one or the other at a time. |
| 66 |
|
66 |
|
|
|
67 |
=item B<-p|--processes> |
| 68 |
|
| 69 |
Number of processes to use for indexing. This can be used to do more indexing |
| 70 |
work in parallel on multicore systems. By default, a single process is used. |
| 71 |
|
| 67 |
=item B<-v|--verbose> |
72 |
=item B<-v|--verbose> |
| 68 |
|
73 |
|
| 69 |
By default, this program only emits warnings and errors. This makes it talk |
74 |
By default, this program only emits warnings and errors. This makes it talk |
|
Lines 79-84
Full documentation.
Link Here
|
| 79 |
|
84 |
|
| 80 |
=back |
85 |
=back |
| 81 |
|
86 |
|
|
|
87 |
=head1 IMPLEMENTATION |
| 88 |
|
| 82 |
=cut |
89 |
=cut |
| 83 |
|
90 |
|
| 84 |
use autodie; |
91 |
use autodie; |
|
Lines 95-115
use Pod::Usage;
Link Here
|
| 95 |
|
102 |
|
| 96 |
my $verbose = 0; |
103 |
my $verbose = 0; |
| 97 |
my $commit = 5000; |
104 |
my $commit = 5000; |
| 98 |
my ($delete, $help, $man); |
105 |
my ($delete, $help, $man, $processes); |
| 99 |
my ($index_biblios, $index_authorities); |
106 |
my ($index_biblios, $index_authorities); |
| 100 |
my (@biblionumbers); |
107 |
my (@record_numbers); |
| 101 |
|
108 |
|
| 102 |
$|=1; # flushes output |
109 |
$|=1; # flushes output |
| 103 |
|
110 |
|
| 104 |
GetOptions( |
111 |
GetOptions( |
| 105 |
'c|commit=i' => \$commit, |
112 |
'c|commit=i' => \$commit, |
| 106 |
'd|delete' => \$delete, |
113 |
'd|delete' => \$delete, |
| 107 |
'a|authorities' => \$index_authorities, |
114 |
'a|authorities' => \$index_authorities, |
| 108 |
'b|biblios' => \$index_biblios, |
115 |
'b|biblios' => \$index_biblios, |
| 109 |
'bn|bnumber=i' => \@biblionumbers, |
116 |
'bn|bnumber=i' => \@record_numbers, |
| 110 |
'v|verbose+' => \$verbose, |
117 |
'p|processes=i' => \$processes, |
| 111 |
'h|help' => \$help, |
118 |
'v|verbose+' => \$verbose, |
| 112 |
'man' => \$man, |
119 |
'h|help' => \$help, |
|
|
120 |
'man' => \$man, |
| 113 |
); |
121 |
); |
| 114 |
|
122 |
|
| 115 |
# Default is to do both |
123 |
# Default is to do both |
|
Lines 120-168
unless ($index_authorities || $index_biblios) {
Link Here
|
| 120 |
pod2usage(1) if $help; |
128 |
pod2usage(1) if $help; |
| 121 |
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; |
129 |
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; |
| 122 |
|
130 |
|
| 123 |
sanity_check(); |
131 |
_sanity_check(); |
|
|
132 |
|
| 133 |
_verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete) if ($index_biblios); |
| 134 |
_verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete) if ($index_authorities); |
| 135 |
|
| 136 |
my $slice_index = 0; |
| 137 |
my $slice_count = $processes // 1; |
| 138 |
|
| 139 |
if ($slice_count > 1) { |
| 140 |
# Fire up child processes for processing slices from 2 on. This main process will handle slice 1. |
| 141 |
$slice_index = 1; |
| 142 |
for (my $proc = 2; $proc <= $processes; $proc++) { |
| 143 |
my $pid = fork(); |
| 144 |
die "Failed to fork a child process\n" unless defined $pid; |
| 145 |
if ($pid == 0) { |
| 146 |
# Child process, give it a slice to process |
| 147 |
$slice_index = $proc; |
| 148 |
last; |
| 149 |
} |
| 150 |
} |
| 151 |
# Fudge the commit count a bit to spread out the Elasticsearch commits |
| 152 |
$commit *= 1 + 0.10 * ($slice_index - 1); |
| 153 |
} |
| 154 |
|
| 155 |
my %iterator_options; |
| 156 |
if ($slice_index) { |
| 157 |
_log(1, "Processing slice $slice_index of $slice_count\n"); |
| 158 |
$iterator_options{slice} = { index => $slice_index, count => $slice_count }; |
| 159 |
} |
| 124 |
|
160 |
|
| 125 |
my $next; |
161 |
my $next; |
| 126 |
if ($index_biblios) { |
162 |
if ($index_biblios) { |
| 127 |
_log(1, "Indexing biblios\n"); |
163 |
_log(1, "Indexing biblios\n"); |
| 128 |
if (@biblionumbers) { |
164 |
if (@record_numbers) { |
| 129 |
$next = sub { |
165 |
$next = sub { |
| 130 |
my $r = shift @biblionumbers; |
166 |
my $r = shift @record_numbers; |
| 131 |
return () unless defined $r; |
167 |
return () unless defined $r; |
| 132 |
return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); |
168 |
return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); |
| 133 |
}; |
169 |
}; |
| 134 |
} else { |
170 |
} else { |
| 135 |
my $records = Koha::BiblioUtils->get_all_biblios_iterator(); |
171 |
my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options); |
| 136 |
$next = sub { |
172 |
$next = sub { |
| 137 |
$records->next(); |
173 |
$records->next(); |
| 138 |
} |
174 |
} |
| 139 |
} |
175 |
} |
| 140 |
do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); |
176 |
_do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); |
| 141 |
} |
177 |
} |
| 142 |
if ($index_authorities) { |
178 |
if ($index_authorities) { |
| 143 |
_log(1, "Indexing authorities\n"); |
179 |
_log(1, "Indexing authorities\n"); |
| 144 |
if (@biblionumbers) { |
180 |
if (@record_numbers) { |
| 145 |
$next = sub { |
181 |
$next = sub { |
| 146 |
my $r = shift @biblionumbers; |
182 |
my $r = shift @record_numbers; |
| 147 |
return () unless defined $r; |
183 |
return () unless defined $r; |
| 148 |
my $a = Koha::MetadataRecord::Authority->get_from_authid($r); |
184 |
my $a = Koha::MetadataRecord::Authority->get_from_authid($r); |
| 149 |
return ($r, $a->record); |
185 |
return ($r, $a->record); |
| 150 |
}; |
186 |
}; |
| 151 |
} else { |
187 |
} else { |
| 152 |
my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(); |
188 |
my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options); |
| 153 |
$next = sub { |
189 |
$next = sub { |
| 154 |
$records->next(); |
190 |
$records->next(); |
| 155 |
} |
191 |
} |
| 156 |
} |
192 |
} |
| 157 |
do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX); |
193 |
_do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX); |
| 158 |
} |
194 |
} |
| 159 |
|
195 |
|
| 160 |
sub do_reindex { |
196 |
if ($slice_index == 1) { |
| 161 |
my ( $next, $index_name ) = @_; |
197 |
# Main process, wait for children |
|
|
198 |
for (my $proc = 2; $proc <= $processes; $proc++) { |
| 199 |
wait(); |
| 200 |
} |
| 201 |
} |
| 202 |
|
| 203 |
=head2 _verify_index_state |
| 204 |
|
| 205 |
_ verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, 1); |
| 206 |
|
| 207 |
Checks the index state and recreates it if requested. |
| 162 |
|
208 |
|
|
|
209 |
=cut |
| 210 |
|
| 211 |
sub _verify_index_state { |
| 212 |
my ( $index_name, $recreate ) = @_; |
| 213 |
|
| 214 |
_log(1, "Checking state of $index_name index\n"); |
| 163 |
my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } ); |
215 |
my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } ); |
| 164 |
|
216 |
|
| 165 |
if ($delete) { |
217 |
if ($recreate) { |
|
|
218 |
_log(1, "Dropping and recreating $index_name index\n"); |
| 166 |
$indexer->drop_index() if $indexer->index_exists(); |
219 |
$indexer->drop_index() if $indexer->index_exists(); |
| 167 |
$indexer->create_index(); |
220 |
$indexer->create_index(); |
| 168 |
} |
221 |
} |
|
Lines 175-180
sub do_reindex {
Link Here
|
| 175 |
} elsif ($indexer->is_index_status_recreate_required) { |
228 |
} elsif ($indexer->is_index_status_recreate_required) { |
| 176 |
warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/; |
229 |
warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/; |
| 177 |
} |
230 |
} |
|
|
231 |
} |
| 232 |
|
| 233 |
=head2 _do_reindex |
| 234 |
|
| 235 |
_do_reindex($callback, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); |
| 236 |
|
| 237 |
Does the actual reindexing. $callback is a function that always returns the next record. |
| 238 |
|
| 239 |
=cut |
| 240 |
|
| 241 |
sub _do_reindex { |
| 242 |
my ( $next, $index_name ) = @_; |
| 243 |
|
| 244 |
my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } ); |
| 178 |
|
245 |
|
| 179 |
my $count = 0; |
246 |
my $count = 0; |
| 180 |
my $commit_count = $commit; |
247 |
my $commit_count = $commit; |
|
Lines 192-203
sub do_reindex {
Link Here
|
| 192 |
push @id_buffer, $id; |
259 |
push @id_buffer, $id; |
| 193 |
push @commit_buffer, $record; |
260 |
push @commit_buffer, $record; |
| 194 |
if ( !( --$commit_count ) ) { |
261 |
if ( !( --$commit_count ) ) { |
| 195 |
_log( 1, "Committing $commit records..." ); |
262 |
_log( 1, "Committing $commit records...\n" ); |
| 196 |
$indexer->update_index( \@id_buffer, \@commit_buffer ); |
263 |
$indexer->update_index( \@id_buffer, \@commit_buffer ); |
| 197 |
$commit_count = $commit; |
264 |
$commit_count = $commit; |
| 198 |
@id_buffer = (); |
265 |
@id_buffer = (); |
| 199 |
@commit_buffer = (); |
266 |
@commit_buffer = (); |
| 200 |
_log( 1, " done\n" ); |
267 |
_log( 1, "Commit complete\n" ); |
| 201 |
} |
268 |
} |
| 202 |
} |
269 |
} |
| 203 |
|
270 |
|
|
Lines 207-227
sub do_reindex {
Link Here
|
| 207 |
_log( 1, "Total $count records indexed\n" ); |
274 |
_log( 1, "Total $count records indexed\n" ); |
| 208 |
} |
275 |
} |
| 209 |
|
276 |
|
| 210 |
# Checks some basic stuff to ensure that it's sane before we start. |
277 |
=head2 _sanity_check |
| 211 |
sub sanity_check { |
278 |
|
|
|
279 |
_sanity_check(); |
| 280 |
|
| 281 |
Checks some basic stuff to ensure that it's sane before we start. |
| 282 |
|
| 283 |
=cut |
| 284 |
|
| 285 |
sub _sanity_check { |
| 212 |
# Do we have an elasticsearch block defined? |
286 |
# Do we have an elasticsearch block defined? |
| 213 |
my $conf = C4::Context->config('elasticsearch'); |
287 |
my $conf = C4::Context->config('elasticsearch'); |
| 214 |
die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf ); |
288 |
die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf ); |
| 215 |
} |
289 |
} |
| 216 |
|
290 |
|
| 217 |
# Output progress information. |
291 |
=head2 _log |
| 218 |
# |
292 |
|
| 219 |
# _log($level, $msg); |
293 |
_log($level, "Message\n"); |
| 220 |
# |
294 |
|
| 221 |
# Will output $msg if the verbosity setting is set to $level or more. Will |
295 |
Output progress information. |
| 222 |
# not include a trailing newline. |
296 |
|
|
|
297 |
Will output the message if verbosity level is set to $level or more. Will not |
| 298 |
include a trailing newline automatically. |
| 299 |
|
| 300 |
=cut |
| 301 |
|
| 223 |
sub _log { |
302 |
sub _log { |
| 224 |
my ($level, $msg) = @_; |
303 |
my ($level, $msg) = @_; |
| 225 |
|
304 |
|
| 226 |
print $msg if ($verbose >= $level); |
305 |
print "[$$] $msg" if ($verbose >= $level); |
| 227 |
} |
306 |
} |
| 228 |
- |
|
|