Lines 64-69
Only index the supplied biblionumber, mostly for testing purposes. May be
Link Here
|
64 |
repeated. This also applies to authorities via authid, so if you're using it, |
64 |
repeated. This also applies to authorities via authid, so if you're using it, |
65 |
you probably only want to do one or the other at a time. |
65 |
you probably only want to do one or the other at a time. |
66 |
|
66 |
|
|
|
67 |
=item B<-p|--processes> |
68 |
|
69 |
Number of processes to use for indexing. This can be used to do more indexing |
70 |
work in parallel on multicore systems. By default, a single process is used. |
71 |
|
67 |
=item B<-v|--verbose> |
72 |
=item B<-v|--verbose> |
68 |
|
73 |
|
69 |
By default, this program only emits warnings and errors. This makes it talk |
74 |
By default, this program only emits warnings and errors. This makes it talk |
Lines 94-114
use Pod::Usage;
Link Here
|
94 |
|
99 |
|
95 |
my $verbose = 0; |
100 |
my $verbose = 0; |
96 |
my $commit = 5000; |
101 |
my $commit = 5000; |
97 |
my ($delete, $help, $man); |
102 |
my ($delete, $help, $man, $processes); |
98 |
my ($index_biblios, $index_authorities); |
103 |
my ($index_biblios, $index_authorities); |
99 |
my (@biblionumbers); |
104 |
my (@record_numbers); |
100 |
|
105 |
|
101 |
$|=1; # flushes output |
106 |
$|=1; # flushes output |
102 |
|
107 |
|
103 |
GetOptions( |
108 |
GetOptions( |
104 |
'c|commit=i' => \$commit, |
109 |
'c|commit=i' => \$commit, |
105 |
'd|delete' => \$delete, |
110 |
'd|delete' => \$delete, |
106 |
'a|authorities' => \$index_authorities, |
111 |
'a|authorities' => \$index_authorities, |
107 |
'b|biblios' => \$index_biblios, |
112 |
'b|biblios' => \$index_biblios, |
108 |
'bn|bnumber=i' => \@biblionumbers, |
113 |
'bn|bnumber=i' => \@record_numbers, |
109 |
'v|verbose+' => \$verbose, |
114 |
'p|processes=i' => \$processes, |
110 |
'h|help' => \$help, |
115 |
'v|verbose+' => \$verbose, |
111 |
'man' => \$man, |
116 |
'h|help' => \$help, |
|
|
117 |
'man' => \$man, |
112 |
); |
118 |
); |
113 |
|
119 |
|
114 |
# Default is to do both |
120 |
# Default is to do both |
Lines 119-167
unless ($index_authorities || $index_biblios) {
Link Here
|
119 |
pod2usage(1) if $help; |
125 |
pod2usage(1) if $help; |
120 |
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; |
126 |
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; |
121 |
|
127 |
|
122 |
sanity_check(); |
128 |
_sanity_check(); |
|
|
129 |
|
130 |
_verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete) if ($index_biblios); |
131 |
_verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete) if ($index_authorities); |
132 |
|
133 |
my $slice_index = 0; |
134 |
my $slice_count = $processes // 1; |
135 |
|
136 |
if ($slice_count > 1) { |
137 |
# Fire up child processes for processing slices from 2 on. This main process will handle slice 1. |
138 |
$slice_index = 1; |
139 |
for (my $proc = 2; $proc <= $processes; $proc++) { |
140 |
my $pid = fork(); |
141 |
die "Failed to fork a child process\n" unless defined $pid; |
142 |
if ($pid == 0) { |
143 |
# Child process, give it a slice to process |
144 |
$slice_index = $proc; |
145 |
last; |
146 |
} |
147 |
} |
148 |
# Fudge the commit count a bit to spread out the Elasticsearch commits |
149 |
$commit *= 1 + 0.10 * ($slice_index - 1); |
150 |
} |
151 |
|
152 |
my %iterator_options; |
153 |
if ($slice_index) { |
154 |
_log(1, "Processing slice $slice_index of $slice_count\n"); |
155 |
$iterator_options{slice} = { index => $slice_index, count => $slice_count }; |
156 |
} |
123 |
|
157 |
|
124 |
my $next; |
158 |
my $next; |
125 |
if ($index_biblios) { |
159 |
if ($index_biblios) { |
126 |
_log(1, "Indexing biblios\n"); |
160 |
_log(1, "Indexing biblios\n"); |
127 |
if (@biblionumbers) { |
161 |
if (@record_numbers) { |
128 |
$next = sub { |
162 |
$next = sub { |
129 |
my $r = shift @biblionumbers; |
163 |
my $r = shift @record_numbers; |
130 |
return () unless defined $r; |
164 |
return () unless defined $r; |
131 |
return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); |
165 |
return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); |
132 |
}; |
166 |
}; |
133 |
} else { |
167 |
} else { |
134 |
my $records = Koha::BiblioUtils->get_all_biblios_iterator(); |
168 |
my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options); |
135 |
$next = sub { |
169 |
$next = sub { |
136 |
$records->next(); |
170 |
$records->next(); |
137 |
} |
171 |
} |
138 |
} |
172 |
} |
139 |
do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); |
173 |
_do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); |
140 |
} |
174 |
} |
141 |
if ($index_authorities) { |
175 |
if ($index_authorities) { |
142 |
_log(1, "Indexing authorities\n"); |
176 |
_log(1, "Indexing authorities\n"); |
143 |
if (@biblionumbers) { |
177 |
if (@record_numbers) { |
144 |
$next = sub { |
178 |
$next = sub { |
145 |
my $r = shift @biblionumbers; |
179 |
my $r = shift @record_numbers; |
146 |
return () unless defined $r; |
180 |
return () unless defined $r; |
147 |
my $a = Koha::MetadataRecord::Authority->get_from_authid($r); |
181 |
my $a = Koha::MetadataRecord::Authority->get_from_authid($r); |
148 |
return ($r, $a->record); |
182 |
return ($r, $a->record); |
149 |
}; |
183 |
}; |
150 |
} else { |
184 |
} else { |
151 |
my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(); |
185 |
my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options); |
152 |
$next = sub { |
186 |
$next = sub { |
153 |
$records->next(); |
187 |
$records->next(); |
154 |
} |
188 |
} |
155 |
} |
189 |
} |
156 |
do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX); |
190 |
_do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX); |
157 |
} |
191 |
} |
158 |
|
192 |
|
159 |
sub do_reindex { |
193 |
if ($slice_index == 1) { |
160 |
my ( $next, $index_name ) = @_; |
194 |
# Main process, wait for children |
|
|
195 |
for (my $proc = 2; $proc <= $processes; $proc++) { |
196 |
wait(); |
197 |
} |
198 |
} |
199 |
|
200 |
=head2 _verify_index_state |
201 |
|
202 |
_ verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, 1); |
203 |
|
204 |
Checks the index state and recreates it if requested. |
161 |
|
205 |
|
|
|
206 |
=cut |
207 |
|
208 |
sub _verify_index_state { |
209 |
my ( $index_name, $recreate ) = @_; |
210 |
|
211 |
_log(1, "Checking state of $index_name index\n"); |
162 |
my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } ); |
212 |
my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } ); |
163 |
|
213 |
|
164 |
if ($delete) { |
214 |
if ($recreate) { |
|
|
215 |
_log(1, "Dropping and recreating $index_name index\n"); |
165 |
$indexer->drop_index() if $indexer->index_exists(); |
216 |
$indexer->drop_index() if $indexer->index_exists(); |
166 |
$indexer->create_index(); |
217 |
$indexer->create_index(); |
167 |
} |
218 |
} |
Lines 174-179
sub do_reindex {
Link Here
|
174 |
} elsif ($indexer->is_index_status_recreate_required) { |
225 |
} elsif ($indexer->is_index_status_recreate_required) { |
175 |
warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/; |
226 |
warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/; |
176 |
} |
227 |
} |
|
|
228 |
} |
229 |
|
230 |
=head2 _do_reindex |
231 |
|
232 |
_do_reindex($callback, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); |
233 |
|
234 |
Does the actual reindexing. $callback is a function that always returns the next record. |
235 |
|
236 |
=cut |
237 |
|
238 |
sub _do_reindex { |
239 |
my ( $next, $index_name ) = @_; |
240 |
|
241 |
my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } ); |
177 |
|
242 |
|
178 |
my $count = 0; |
243 |
my $count = 0; |
179 |
my $commit_count = $commit; |
244 |
my $commit_count = $commit; |
Lines 191-202
sub do_reindex {
Link Here
|
191 |
push @id_buffer, $id; |
256 |
push @id_buffer, $id; |
192 |
push @commit_buffer, $record; |
257 |
push @commit_buffer, $record; |
193 |
if ( !( --$commit_count ) ) { |
258 |
if ( !( --$commit_count ) ) { |
194 |
_log( 1, "Committing $commit records..." ); |
259 |
_log( 1, "Committing $commit records...\n" ); |
195 |
$indexer->update_index( \@id_buffer, \@commit_buffer ); |
260 |
$indexer->update_index( \@id_buffer, \@commit_buffer ); |
196 |
$commit_count = $commit; |
261 |
$commit_count = $commit; |
197 |
@id_buffer = (); |
262 |
@id_buffer = (); |
198 |
@commit_buffer = (); |
263 |
@commit_buffer = (); |
199 |
_log( 1, " done\n" ); |
264 |
_log( 1, "Commit complete\n" ); |
200 |
} |
265 |
} |
201 |
} |
266 |
} |
202 |
|
267 |
|
Lines 206-226
sub do_reindex {
Link Here
|
206 |
_log( 1, "Total $count records indexed\n" ); |
271 |
_log( 1, "Total $count records indexed\n" ); |
207 |
} |
272 |
} |
208 |
|
273 |
|
209 |
# Checks some basic stuff to ensure that it's sane before we start. |
274 |
=head2 _sanity_check |
210 |
sub sanity_check { |
275 |
|
|
|
276 |
_sanity_check(); |
277 |
|
278 |
Checks some basic stuff to ensure that it's sane before we start. |
279 |
|
280 |
=cut |
281 |
|
282 |
sub _sanity_check { |
211 |
# Do we have an elasticsearch block defined? |
283 |
# Do we have an elasticsearch block defined? |
212 |
my $conf = C4::Context->config('elasticsearch'); |
284 |
my $conf = C4::Context->config('elasticsearch'); |
213 |
die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf ); |
285 |
die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf ); |
214 |
} |
286 |
} |
215 |
|
287 |
|
216 |
# Output progress information. |
288 |
=head2 _log |
217 |
# |
289 |
|
218 |
# _log($level, $msg); |
290 |
_log($level, "Message\n"); |
219 |
# |
291 |
|
220 |
# Will output $msg if the verbosity setting is set to $level or more. Will |
292 |
Output progress information. |
221 |
# not include a trailing newline. |
293 |
|
|
|
294 |
Will output the message if verbosity level is set to $level or more. Will not |
295 |
include a trailing newline automatically. |
296 |
|
297 |
=cut |
298 |
|
222 |
sub _log { |
299 |
sub _log { |
223 |
my ($level, $msg) = @_; |
300 |
my ($level, $msg) = @_; |
224 |
|
301 |
|
225 |
print $msg if ($verbose >= $level); |
302 |
print "[$$] $msg" if ($verbose >= $level); |
226 |
} |
303 |
} |
227 |
- |
|
|