|
Lines 64-69
Only index the supplied biblionumber, mostly for testing purposes. May be
Link Here
|
| 64 |
repeated. This also applies to authorities via authid, so if you're using it, |
64 |
repeated. This also applies to authorities via authid, so if you're using it, |
| 65 |
you probably only want to do one or the other at a time. |
65 |
you probably only want to do one or the other at a time. |
| 66 |
|
66 |
|
|
|
67 |
=item B<-p|--processes> |
| 68 |
|
| 69 |
Number of processes to use for indexing. This can be used to do more indexing |
| 70 |
work in parallel on multicore systems. By default, a single process is used. |
| 71 |
|
| 67 |
=item B<-v|--verbose> |
72 |
=item B<-v|--verbose> |
| 68 |
|
73 |
|
| 69 |
By default, this program only emits warnings and errors. This makes it talk |
74 |
By default, this program only emits warnings and errors. This makes it talk |
|
Lines 79-84
Full documentation.
Link Here
|
| 79 |
|
84 |
|
| 80 |
=back |
85 |
=back |
| 81 |
|
86 |
|
|
|
87 |
=head1 IMPLEMENTATION |
| 88 |
|
| 82 |
=cut |
89 |
=cut |
| 83 |
|
90 |
|
| 84 |
use autodie; |
91 |
use autodie; |
|
Lines 94-114
use Pod::Usage;
Link Here
|
| 94 |
|
101 |
|
| 95 |
my $verbose = 0; |
102 |
my $verbose = 0; |
| 96 |
my $commit = 5000; |
103 |
my $commit = 5000; |
| 97 |
my ($delete, $help, $man); |
104 |
my ($delete, $help, $man, $processes); |
| 98 |
my ($index_biblios, $index_authorities); |
105 |
my ($index_biblios, $index_authorities); |
| 99 |
my (@biblionumbers); |
106 |
my (@record_numbers); |
| 100 |
|
107 |
|
| 101 |
$|=1; # flushes output |
108 |
$|=1; # flushes output |
| 102 |
|
109 |
|
| 103 |
GetOptions( |
110 |
GetOptions( |
| 104 |
'c|commit=i' => \$commit, |
111 |
'c|commit=i' => \$commit, |
| 105 |
'd|delete' => \$delete, |
112 |
'd|delete' => \$delete, |
| 106 |
'a|authorities' => \$index_authorities, |
113 |
'a|authorities' => \$index_authorities, |
| 107 |
'b|biblios' => \$index_biblios, |
114 |
'b|biblios' => \$index_biblios, |
| 108 |
'bn|bnumber=i' => \@biblionumbers, |
115 |
'bn|bnumber=i' => \@record_numbers, |
| 109 |
'v|verbose+' => \$verbose, |
116 |
'p|processes=i' => \$processes, |
| 110 |
'h|help' => \$help, |
117 |
'v|verbose+' => \$verbose, |
| 111 |
'man' => \$man, |
118 |
'h|help' => \$help, |
|
|
119 |
'man' => \$man, |
| 112 |
); |
120 |
); |
| 113 |
|
121 |
|
| 114 |
# Default is to do both |
122 |
# Default is to do both |
|
Lines 119-167
unless ($index_authorities || $index_biblios) {
Link Here
|
| 119 |
pod2usage(1) if $help; |
127 |
pod2usage(1) if $help; |
| 120 |
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; |
128 |
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; |
| 121 |
|
129 |
|
| 122 |
sanity_check(); |
130 |
_sanity_check(); |
|
|
131 |
|
| 132 |
_verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete) if ($index_biblios); |
| 133 |
_verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete) if ($index_authorities); |
| 134 |
|
| 135 |
my $slice_index = 0; |
| 136 |
my $slice_count = $processes // 1; |
| 137 |
|
| 138 |
if ($slice_count > 1) { |
| 139 |
# Fire up child processes for processing slices from 2 on. This main process will handle slice 1. |
| 140 |
$slice_index = 1; |
| 141 |
for (my $proc = 2; $proc <= $processes; $proc++) { |
| 142 |
my $pid = fork(); |
| 143 |
die "Failed to fork a child process\n" unless defined $pid; |
| 144 |
if ($pid == 0) { |
| 145 |
# Child process, give it a slice to process |
| 146 |
$slice_index = $proc; |
| 147 |
last; |
| 148 |
} |
| 149 |
} |
| 150 |
# Fudge the commit count a bit to spread out the Elasticsearch commits |
| 151 |
$commit *= 1 + 0.10 * ($slice_index - 1); |
| 152 |
} |
| 153 |
|
| 154 |
my %iterator_options; |
| 155 |
if ($slice_index) { |
| 156 |
_log(1, "Processing slice $slice_index of $slice_count\n"); |
| 157 |
$iterator_options{slice} = { index => $slice_index, count => $slice_count }; |
| 158 |
} |
| 123 |
|
159 |
|
| 124 |
my $next; |
160 |
my $next; |
| 125 |
if ($index_biblios) { |
161 |
if ($index_biblios) { |
| 126 |
_log(1, "Indexing biblios\n"); |
162 |
_log(1, "Indexing biblios\n"); |
| 127 |
if (@biblionumbers) { |
163 |
if (@record_numbers) { |
| 128 |
$next = sub { |
164 |
$next = sub { |
| 129 |
my $r = shift @biblionumbers; |
165 |
my $r = shift @record_numbers; |
| 130 |
return () unless defined $r; |
166 |
return () unless defined $r; |
| 131 |
return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); |
167 |
return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 )); |
| 132 |
}; |
168 |
}; |
| 133 |
} else { |
169 |
} else { |
| 134 |
my $records = Koha::BiblioUtils->get_all_biblios_iterator(); |
170 |
my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options); |
| 135 |
$next = sub { |
171 |
$next = sub { |
| 136 |
$records->next(); |
172 |
$records->next(); |
| 137 |
} |
173 |
} |
| 138 |
} |
174 |
} |
| 139 |
do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); |
175 |
_do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); |
| 140 |
} |
176 |
} |
| 141 |
if ($index_authorities) { |
177 |
if ($index_authorities) { |
| 142 |
_log(1, "Indexing authorities\n"); |
178 |
_log(1, "Indexing authorities\n"); |
| 143 |
if (@biblionumbers) { |
179 |
if (@record_numbers) { |
| 144 |
$next = sub { |
180 |
$next = sub { |
| 145 |
my $r = shift @biblionumbers; |
181 |
my $r = shift @record_numbers; |
| 146 |
return () unless defined $r; |
182 |
return () unless defined $r; |
| 147 |
my $a = Koha::MetadataRecord::Authority->get_from_authid($r); |
183 |
my $a = Koha::MetadataRecord::Authority->get_from_authid($r); |
| 148 |
return ($r, $a->record); |
184 |
return ($r, $a->record); |
| 149 |
}; |
185 |
}; |
| 150 |
} else { |
186 |
} else { |
| 151 |
my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(); |
187 |
my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options); |
| 152 |
$next = sub { |
188 |
$next = sub { |
| 153 |
$records->next(); |
189 |
$records->next(); |
| 154 |
} |
190 |
} |
| 155 |
} |
191 |
} |
| 156 |
do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX); |
192 |
_do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX); |
| 157 |
} |
193 |
} |
| 158 |
|
194 |
|
| 159 |
sub do_reindex { |
195 |
if ($slice_index == 1) { |
| 160 |
my ( $next, $index_name ) = @_; |
196 |
# Main process, wait for children |
|
|
197 |
for (my $proc = 2; $proc <= $processes; $proc++) { |
| 198 |
wait(); |
| 199 |
} |
| 200 |
} |
| 201 |
|
| 202 |
=head2 _verify_index_state |
| 203 |
|
| 204 |
_ verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, 1); |
| 205 |
|
| 206 |
Checks the index state and recreates it if requested. |
| 161 |
|
207 |
|
|
|
208 |
=cut |
| 209 |
|
| 210 |
sub _verify_index_state { |
| 211 |
my ( $index_name, $recreate ) = @_; |
| 212 |
|
| 213 |
_log(1, "Checking state of $index_name index\n"); |
| 162 |
my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } ); |
214 |
my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } ); |
| 163 |
|
215 |
|
| 164 |
if ($delete) { |
216 |
if ($recreate) { |
|
|
217 |
_log(1, "Dropping and recreating $index_name index\n"); |
| 165 |
$indexer->drop_index() if $indexer->index_exists(); |
218 |
$indexer->drop_index() if $indexer->index_exists(); |
| 166 |
$indexer->create_index(); |
219 |
$indexer->create_index(); |
| 167 |
} |
220 |
} |
|
Lines 174-179
sub do_reindex {
Link Here
|
| 174 |
} elsif ($indexer->is_index_status_recreate_required) { |
227 |
} elsif ($indexer->is_index_status_recreate_required) { |
| 175 |
warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/; |
228 |
warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/; |
| 176 |
} |
229 |
} |
|
|
230 |
} |
| 231 |
|
| 232 |
=head2 _do_reindex |
| 233 |
|
| 234 |
_do_reindex($callback, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); |
| 235 |
|
| 236 |
Does the actual reindexing. $callback is a function that always returns the next record. |
| 237 |
|
| 238 |
=cut |
| 239 |
|
| 240 |
sub _do_reindex { |
| 241 |
my ( $next, $index_name ) = @_; |
| 242 |
|
| 243 |
my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } ); |
| 177 |
|
244 |
|
| 178 |
my $count = 0; |
245 |
my $count = 0; |
| 179 |
my $commit_count = $commit; |
246 |
my $commit_count = $commit; |
|
Lines 191-202
sub do_reindex {
Link Here
|
| 191 |
push @id_buffer, $id; |
258 |
push @id_buffer, $id; |
| 192 |
push @commit_buffer, $record; |
259 |
push @commit_buffer, $record; |
| 193 |
if ( !( --$commit_count ) ) { |
260 |
if ( !( --$commit_count ) ) { |
| 194 |
_log( 1, "Committing $commit records..." ); |
261 |
_log( 1, "Committing $commit records...\n" ); |
| 195 |
$indexer->update_index( \@id_buffer, \@commit_buffer ); |
262 |
$indexer->update_index( \@id_buffer, \@commit_buffer ); |
| 196 |
$commit_count = $commit; |
263 |
$commit_count = $commit; |
| 197 |
@id_buffer = (); |
264 |
@id_buffer = (); |
| 198 |
@commit_buffer = (); |
265 |
@commit_buffer = (); |
| 199 |
_log( 1, " done\n" ); |
266 |
_log( 1, "Commit complete\n" ); |
| 200 |
} |
267 |
} |
| 201 |
} |
268 |
} |
| 202 |
|
269 |
|
|
Lines 206-226
sub do_reindex {
Link Here
|
| 206 |
_log( 1, "Total $count records indexed\n" ); |
273 |
_log( 1, "Total $count records indexed\n" ); |
| 207 |
} |
274 |
} |
| 208 |
|
275 |
|
| 209 |
# Checks some basic stuff to ensure that it's sane before we start. |
276 |
=head2 _sanity_check |
| 210 |
sub sanity_check { |
277 |
|
|
|
278 |
_sanity_check(); |
| 279 |
|
| 280 |
Checks some basic stuff to ensure that it's sane before we start. |
| 281 |
|
| 282 |
=cut |
| 283 |
|
| 284 |
sub _sanity_check { |
| 211 |
# Do we have an elasticsearch block defined? |
285 |
# Do we have an elasticsearch block defined? |
| 212 |
my $conf = C4::Context->config('elasticsearch'); |
286 |
my $conf = C4::Context->config('elasticsearch'); |
| 213 |
die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf ); |
287 |
die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf ); |
| 214 |
} |
288 |
} |
| 215 |
|
289 |
|
| 216 |
# Output progress information. |
290 |
=head2 _log |
| 217 |
# |
291 |
|
| 218 |
# _log($level, $msg); |
292 |
_log($level, "Message\n"); |
| 219 |
# |
293 |
|
| 220 |
# Will output $msg if the verbosity setting is set to $level or more. Will |
294 |
Output progress information. |
| 221 |
# not include a trailing newline. |
295 |
|
|
|
296 |
Will output the message if verbosity level is set to $level or more. Will not |
| 297 |
include a trailing newline automatically. |
| 298 |
|
| 299 |
=cut |
| 300 |
|
| 222 |
sub _log { |
301 |
sub _log { |
| 223 |
my ($level, $msg) = @_; |
302 |
my ($level, $msg) = @_; |
| 224 |
|
303 |
|
| 225 |
print $msg if ($verbose >= $level); |
304 |
print "[$$] $msg" if ($verbose >= $level); |
| 226 |
} |
305 |
} |
| 227 |
- |
|
|