|
Lines 64-69
Only index the supplied biblionumber, mostly for testing purposes. May be
Link Here
|
| 64 |
repeated. This also applies to authorities via authid, so if you're using it, |
64 |
repeated. This also applies to authorities via authid, so if you're using it, |
| 65 |
you probably only want to do one or the other at a time. |
65 |
you probably only want to do one or the other at a time. |
| 66 |
|
66 |
|
|
|
67 |
=item B<-p|--processes> |
| 68 |
|
| 69 |
Number of processes to use for indexing. This can be used to do more indexing |
| 70 |
work in parallel on multicore systems. By default, a single process is used. |
| 71 |
|
| 67 |
=item B<-v|--verbose> |
72 |
=item B<-v|--verbose> |
| 68 |
|
73 |
|
| 69 |
By default, this program only emits warnings and errors. This makes it talk |
74 |
By default, this program only emits warnings and errors. This makes it talk |
|
Lines 94-114
use Pod::Usage;
Link Here
|
| 94 |
|
99 |
|
| 95 |
my $verbose = 0; |
100 |
my $verbose = 0; |
| 96 |
my $commit = 5000; |
101 |
my $commit = 5000; |
| 97 |
my ($delete, $help, $man); |
102 |
my ($delete, $help, $man, $processes); |
| 98 |
my ($index_biblios, $index_authorities); |
103 |
my ($index_biblios, $index_authorities); |
| 99 |
my (@biblionumbers); |
104 |
my (@record_numbers); |
| 100 |
|
105 |
|
| 101 |
$|=1; # flushes output |
106 |
$|=1; # flushes output |
| 102 |
|
107 |
|
| 103 |
GetOptions( |
108 |
GetOptions( |
| 104 |
'c|commit=i' => \$commit, |
109 |
'c|commit=i' => \$commit, |
| 105 |
'd|delete' => \$delete, |
110 |
'd|delete' => \$delete, |
| 106 |
'a|authorities' => \$index_authorities, |
111 |
'a|authorities' => \$index_authorities, |
| 107 |
'b|biblios' => \$index_biblios, |
112 |
'b|biblios' => \$index_biblios, |
| 108 |
'bn|bnumber=i' => \@biblionumbers, |
113 |
'bn|bnumber=i' => \@record_numbers, |
| 109 |
'v|verbose+' => \$verbose, |
114 |
'p|processes=i' => \$processes, |
| 110 |
'h|help' => \$help, |
115 |
'v|verbose+' => \$verbose, |
| 111 |
'man' => \$man, |
116 |
'h|help' => \$help, |
|
|
117 |
'man' => \$man, |
| 112 |
); |
118 |
); |
| 113 |
|
119 |
|
| 114 |
# Default is to do both |
120 |
# Default is to do both |
|
Lines 119-167
unless ($index_authorities || $index_biblios) {
Link Here
|
| 119 |
pod2usage(1) if $help; |
125 |
pod2usage(1) if $help; |
| 120 |
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; |
126 |
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; |
| 121 |
|
127 |
|
| 122 |
sanity_check(); |
128 |
_sanity_check(); |
|
|
129 |
|
| 130 |
_verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete) if ($index_biblios); |
| 131 |
_verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete) if ($index_authorities); |
| 132 |
|
| 133 |
my $slice_index = 0; |
| 134 |
my $slice_count = $processes // 1; |
| 135 |
|
| 136 |
if ($slice_count > 1) { |
| 137 |
# Fire up child processes for processing slices from 2 on. This main process will handle slice 1. |
| 138 |
$slice_index = 1; |
| 139 |
for (my $proc = 2; $proc <= $processes; $proc++) { |
| 140 |
my $pid = fork(); |
| 141 |
die "Failed to fork a child process\n" unless defined $pid; |
| 142 |
if ($pid == 0) { |
| 143 |
# Child process, give it a slice to process |
| 144 |
$slice_index = $proc; |
| 145 |
last; |
| 146 |
} |
| 147 |
} |
| 148 |
# Fudge the commit count a bit to spread out the Elasticsearch commits |
| 149 |
$commit *= 1 + 0.10 * ($slice_index - 1); |
| 150 |
} |
| 151 |
|
| 152 |
my %iterator_options; |
| 153 |
if ($slice_index) { |
| 154 |
_log(1, "Processing slice $slice_index of $slice_count\n"); |
| 155 |
$iterator_options{slice} = { index => $slice_index, count => $slice_count }; |
| 156 |
} |
| 123 |
|
157 |
|
| 124 |
my $next; |
158 |
my $next; |
| 125 |
if ($index_biblios) { |
159 |
if ($index_biblios) { |
| 126 |
_log(1, "Indexing biblios\n"); |
160 |
_log(1, "Indexing biblios\n"); |
| 127 |
if (@biblionumbers) { |
161 |
if (@record_numbers) { |
| 128 |
$next = sub { |
162 |
$next = sub { |
| 129 |
my $r = shift @biblionumbers; |
163 |
my $r = shift @record_numbers; |
| 130 |
return () unless defined $r; |
164 |
return () unless defined $r; |
| 131 |
return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); |
165 |
return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); |
| 132 |
}; |
166 |
}; |
| 133 |
} else { |
167 |
} else { |
| 134 |
my $records = Koha::BiblioUtils->get_all_biblios_iterator(); |
168 |
my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options); |
| 135 |
$next = sub { |
169 |
$next = sub { |
| 136 |
$records->next(); |
170 |
$records->next(); |
| 137 |
} |
171 |
} |
| 138 |
} |
172 |
} |
| 139 |
do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); |
173 |
_do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); |
| 140 |
} |
174 |
} |
| 141 |
if ($index_authorities) { |
175 |
if ($index_authorities) { |
| 142 |
_log(1, "Indexing authorities\n"); |
176 |
_log(1, "Indexing authorities\n"); |
| 143 |
if (@biblionumbers) { |
177 |
if (@record_numbers) { |
| 144 |
$next = sub { |
178 |
$next = sub { |
| 145 |
my $r = shift @biblionumbers; |
179 |
my $r = shift @record_numbers; |
| 146 |
return () unless defined $r; |
180 |
return () unless defined $r; |
| 147 |
my $a = Koha::MetadataRecord::Authority->get_from_authid($r); |
181 |
my $a = Koha::MetadataRecord::Authority->get_from_authid($r); |
| 148 |
return ($r, $a->record); |
182 |
return ($r, $a->record); |
| 149 |
}; |
183 |
}; |
| 150 |
} else { |
184 |
} else { |
| 151 |
my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(); |
185 |
my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options); |
| 152 |
$next = sub { |
186 |
$next = sub { |
| 153 |
$records->next(); |
187 |
$records->next(); |
| 154 |
} |
188 |
} |
| 155 |
} |
189 |
} |
| 156 |
do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX); |
190 |
_do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX); |
| 157 |
} |
191 |
} |
| 158 |
|
192 |
|
| 159 |
sub do_reindex { |
193 |
if ($slice_index == 1) { |
| 160 |
my ( $next, $index_name ) = @_; |
194 |
# Main process, wait for children |
|
|
195 |
for (my $proc = 2; $proc <= $processes; $proc++) { |
| 196 |
wait(); |
| 197 |
} |
| 198 |
} |
| 199 |
|
| 200 |
=head2 _verify_index_state |
| 201 |
|
| 202 |
_ verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, 1); |
| 203 |
|
| 204 |
Checks the index state and recreates it if requested. |
| 161 |
|
205 |
|
|
|
206 |
=cut |
| 207 |
|
| 208 |
sub _verify_index_state { |
| 209 |
my ( $index_name, $recreate ) = @_; |
| 210 |
|
| 211 |
_log(1, "Checking state of $index_name index\n"); |
| 162 |
my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } ); |
212 |
my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } ); |
| 163 |
|
213 |
|
| 164 |
if ($delete) { |
214 |
if ($recreate) { |
|
|
215 |
_log(1, "Dropping and recreating $index_name index\n"); |
| 165 |
$indexer->drop_index() if $indexer->index_exists(); |
216 |
$indexer->drop_index() if $indexer->index_exists(); |
| 166 |
$indexer->create_index(); |
217 |
$indexer->create_index(); |
| 167 |
} |
218 |
} |
|
Lines 174-179
sub do_reindex {
Link Here
|
| 174 |
} elsif ($indexer->is_index_status_recreate_required) { |
225 |
} elsif ($indexer->is_index_status_recreate_required) { |
| 175 |
warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/; |
226 |
warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/; |
| 176 |
} |
227 |
} |
|
|
228 |
} |
| 229 |
|
| 230 |
=head2 _do_reindex |
| 231 |
|
| 232 |
_do_reindex($callback, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); |
| 233 |
|
| 234 |
Does the actual reindexing. $callback is a function that always returns the next record. |
| 235 |
|
| 236 |
=cut |
| 237 |
|
| 238 |
sub _do_reindex { |
| 239 |
my ( $next, $index_name ) = @_; |
| 240 |
|
| 241 |
my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } ); |
| 177 |
|
242 |
|
| 178 |
my $count = 0; |
243 |
my $count = 0; |
| 179 |
my $commit_count = $commit; |
244 |
my $commit_count = $commit; |
|
Lines 191-202
sub do_reindex {
Link Here
|
| 191 |
push @id_buffer, $id; |
256 |
push @id_buffer, $id; |
| 192 |
push @commit_buffer, $record; |
257 |
push @commit_buffer, $record; |
| 193 |
if ( !( --$commit_count ) ) { |
258 |
if ( !( --$commit_count ) ) { |
| 194 |
_log( 1, "Committing $commit records..." ); |
259 |
_log( 1, "Committing $commit records...\n" ); |
| 195 |
$indexer->update_index( \@id_buffer, \@commit_buffer ); |
260 |
$indexer->update_index( \@id_buffer, \@commit_buffer ); |
| 196 |
$commit_count = $commit; |
261 |
$commit_count = $commit; |
| 197 |
@id_buffer = (); |
262 |
@id_buffer = (); |
| 198 |
@commit_buffer = (); |
263 |
@commit_buffer = (); |
| 199 |
_log( 1, " done\n" ); |
264 |
_log( 1, "Commit complete\n" ); |
| 200 |
} |
265 |
} |
| 201 |
} |
266 |
} |
| 202 |
|
267 |
|
|
Lines 206-226
sub do_reindex {
Link Here
|
| 206 |
_log( 1, "Total $count records indexed\n" ); |
271 |
_log( 1, "Total $count records indexed\n" ); |
| 207 |
} |
272 |
} |
| 208 |
|
273 |
|
| 209 |
# Checks some basic stuff to ensure that it's sane before we start. |
274 |
=head2 _sanity_check |
| 210 |
sub sanity_check { |
275 |
|
|
|
276 |
_sanity_check(); |
| 277 |
|
| 278 |
Checks some basic stuff to ensure that it's sane before we start. |
| 279 |
|
| 280 |
=cut |
| 281 |
|
| 282 |
sub _sanity_check { |
| 211 |
# Do we have an elasticsearch block defined? |
283 |
# Do we have an elasticsearch block defined? |
| 212 |
my $conf = C4::Context->config('elasticsearch'); |
284 |
my $conf = C4::Context->config('elasticsearch'); |
| 213 |
die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf ); |
285 |
die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf ); |
| 214 |
} |
286 |
} |
| 215 |
|
287 |
|
| 216 |
# Output progress information. |
288 |
=head2 _log |
| 217 |
# |
289 |
|
| 218 |
# _log($level, $msg); |
290 |
_log($level, "Message\n"); |
| 219 |
# |
291 |
|
| 220 |
# Will output $msg if the verbosity setting is set to $level or more. Will |
292 |
Output progress information. |
| 221 |
# not include a trailing newline. |
293 |
|
|
|
294 |
Will output the message if verbosity level is set to $level or more. Will not |
| 295 |
include a trailing newline automatically. |
| 296 |
|
| 297 |
=cut |
| 298 |
|
| 222 |
sub _log { |
299 |
sub _log { |
| 223 |
my ($level, $msg) = @_; |
300 |
my ($level, $msg) = @_; |
| 224 |
|
301 |
|
| 225 |
print $msg if ($verbose >= $level); |
302 |
print "[$$] $msg" if ($verbose >= $level); |
| 226 |
} |
303 |
} |
| 227 |
- |
|
|