Lines 20-26
package Koha::SearchEngine::Elasticsearch::Indexer;
Link Here
|
20 |
use Carp; |
20 |
use Carp; |
21 |
use Modern::Perl; |
21 |
use Modern::Perl; |
22 |
use base qw(Koha::SearchEngine::Elasticsearch); |
22 |
use base qw(Koha::SearchEngine::Elasticsearch); |
23 |
use Data::Dumper; |
23 |
|
|
|
24 |
use Koha::BiblioUtils; |
25 |
use Koha::MetadataRecord::Authority; |
24 |
|
26 |
|
25 |
# For now just marc, but we can do anything here really |
27 |
# For now just marc, but we can do anything here really |
26 |
use Catmandu::Importer::MARC; |
28 |
use Catmandu::Importer::MARC; |
Lines 89-95
sub update_index {
Link Here
|
89 |
|
91 |
|
90 |
my $from = $self->_convert_marc_to_json($records); |
92 |
my $from = $self->_convert_marc_to_json($records); |
91 |
|
93 |
|
92 |
print Data::Dumper::Dumper( $from->to_array ); |
94 |
#print Data::Dumper::Dumper( $from->to_array ); |
93 |
$self->store->bag->add_many($from); |
95 |
$self->store->bag->add_many($from); |
94 |
$self->store->bag->commit; |
96 |
$self->store->bag->commit; |
95 |
return 1; |
97 |
return 1; |
Lines 110-116
it to be updated by a regular index cron job in the future.
Link Here
|
110 |
|
112 |
|
111 |
sub update_index_background { |
113 |
sub update_index_background { |
112 |
my $self = shift; |
114 |
my $self = shift; |
113 |
$self->update_index(@_); |
115 |
warn Data::Dumper::Dumper( @_ ); |
|
|
116 |
$self->do_reindex(undef, undef, 0, @_); |
114 |
} |
117 |
} |
115 |
|
118 |
|
116 |
=head2 $indexer->delete_index($biblionums) |
119 |
=head2 $indexer->delete_index($biblionums) |
Lines 142-149
sub delete_index_background {
Link Here
|
142 |
|
145 |
|
143 |
=head2 $indexer->drop_index(); |
146 |
=head2 $indexer->drop_index(); |
144 |
|
147 |
|
145 |
Drops the index from the elasticsearch server. Calling C<update_index> |
148 |
Drops the index from the elasticsearch server. |
146 |
after this will recreate it again. |
149 |
We need to create a new object once this has been done. |
147 |
|
150 |
|
148 |
=cut |
151 |
=cut |
149 |
|
152 |
|
Lines 151-177
sub drop_index {
Link Here
|
151 |
my ($self) = @_; |
154 |
my ($self) = @_; |
152 |
|
155 |
|
153 |
$self->store->drop(); |
156 |
$self->store->drop(); |
154 |
my $params = $self->get_elasticsearch_params(); |
|
|
155 |
$self->store( |
156 |
Catmandu::Store::ElasticSearch->new( |
157 |
%$params, |
158 |
index_settings => $self->get_elasticsearch_settings(), |
159 |
index_mappings => $self->get_elasticsearch_mappings(), |
160 |
) |
161 |
); |
162 |
} |
157 |
} |
163 |
|
158 |
|
164 |
|
159 |
|
165 |
=head2 $indexer->do_reindex(); |
160 |
=head2 $indexer->do_reindex($delete, $commit, $verbose, $biblionumbers); |
|
|
161 |
|
162 |
Performs a reindex, full or patial is based on whether biblionumbers are passed in |
163 |
We do an additional check if an alias exists - if so and only indexing a list of bibs we use it, |
164 |
if not, or if doing a full reindex, we index into a new index and adjust the existing alias then drop the old. |
166 |
|
165 |
|
167 |
Performs a reindex based on records passed in |
166 |
C<$delete> indicates if we should delete the existing index - only affects partial indexes as otherwise we use a new index in either case. |
|
|
167 |
|
168 |
C<$commit> specifies at how many records we commit the update - higher is faster but uses more memory - default is 5000. |
169 |
|
170 |
C<$verbose> specifies warning level - only for command line use currently. |
171 |
|
172 |
C<$biblionums> is an arrayref containing the biblionumbers for the records. |
168 |
|
173 |
|
169 |
=cut |
174 |
=cut |
170 |
|
175 |
|
171 |
sub do_reindex { |
176 |
sub do_reindex { |
172 |
my ($self, $next, $delete, $commit, $verbose, $biblionumbers) = @_; |
177 |
my ($self, $delete, $commit, $verbose, $biblionumbers) = @_; |
173 |
|
178 |
|
174 |
_log(1, $verbose, "Indexing ".$self->index."\n"); |
179 |
_log(1, $verbose, "Indexing ".$self->index."\n"); |
|
|
180 |
my $next; |
175 |
|
181 |
|
176 |
if ( scalar @$biblionumbers ) { |
182 |
if ( scalar @$biblionumbers ) { |
177 |
if ( $self->index eq 'biblios' ) { |
183 |
if ( $self->index eq 'biblios' ) { |
Lines 197-204
sub do_reindex {
Link Here
|
197 |
$next = sub { $records->next(); }; |
203 |
$next = sub { $records->next(); }; |
198 |
} |
204 |
} |
199 |
|
205 |
|
200 |
warn |
206 |
my $current_index = $self->get_alias_index(); |
201 |
$self->drop_index() if ( $delete ); |
207 |
|
|
|
208 |
if ( scalar @$biblionumbers && !$delete && $current_index) { |
209 |
|
210 |
#In this one case we index onto existing alias |
211 |
$self->_index_records( $next, $commit, $verbose ); |
212 |
|
213 |
} else { |
214 |
|
215 |
#In all other cases we are going to use a new index then switch our alias to this index |
216 |
#'time' is used for providing a unique name |
217 |
my $alias_name = $self->store->{index_name}; |
218 |
|
219 |
my $new_indexer = |
220 |
Koha::SearchEngine::Elasticsearch::Indexer->new( { |
221 |
index => $self->index.time |
222 |
}); |
223 |
|
224 |
$new_indexer->_index_records( $next, $commit, $verbose ); |
225 |
|
226 |
if ( $current_index ){ #If the alias already exists |
227 |
$self->update_alias({ #let's point the alias to the new index |
228 |
old_index => $current_index, |
229 |
new_index => $new_indexer->store->{index_name} |
230 |
}); |
231 |
$current_index =~ s/koha_kohadev_//; #Then we need short index name to build object |
232 |
|
233 |
my $old_index = |
234 |
Koha::SearchEngine::Elasticsearch::Indexer->new({ |
235 |
index => $current_index |
236 |
}); |
237 |
|
238 |
$old_index->drop_index(); #And we drop the specific index that the alias pointed to originally |
239 |
} else { #The alias was not created |
240 |
$self->drop_index(); #Drop the index we created initially |
241 |
#It will have the same name as our alias so must be dropped first |
242 |
#It should be empty (we indexed into a new one) |
243 |
$new_indexer->store->es->indices->put_alias( |
244 |
name => $alias_name, |
245 |
index => $new_indexer->store->{index_name} |
246 |
); |
247 |
} |
248 |
|
249 |
} |
250 |
} |
251 |
|
252 |
=head2 $indexer->index_records( $next, $commit, $verbose ); |
253 |
|
254 |
Performs a reindex, full or patial is based on whether biblionumbers are passed in |
255 |
We do an additional check if an alias exists - if so and only indexing a list of bibs we use it, |
256 |
if not, or if doing a full reindex, we index into a new index and adjust the existing alias then drop the old. |
257 |
|
258 |
C<$next> is a function reference, used for iterating over records. |
259 |
|
260 |
C<$commit> specifies at how many records we commit the update - higher is faster but uses more memory - default is 5000. |
261 |
|
262 |
C<$verbose> specifies warning level - only for command line use currently. |
263 |
|
264 |
=cut |
265 |
|
266 |
sub _index_records { |
267 |
my ( $self, $next, $commit, $verbose ) = @_; |
268 |
$commit ||= 5000; |
202 |
|
269 |
|
203 |
my $count = 0; |
270 |
my $count = 0; |
204 |
my $commit_count = $commit; |
271 |
my $commit_count = $commit; |
Lines 219-229
warn
Link Here
|
219 |
@commit_buffer = (); |
286 |
@commit_buffer = (); |
220 |
} |
287 |
} |
221 |
} |
288 |
} |
222 |
# There are probably uncommitted records (beacuse we didn't hit exact commit count) |
289 |
# There are probably uncommitted records (because we didn't hit exact commit count) |
223 |
$self->update_index( \@id_buffer, \@commit_buffer ); |
290 |
$self->update_index( \@id_buffer, \@commit_buffer ); |
224 |
_log( 1, $verbose, "$count records indexed.\n" ); |
291 |
_log( 1, $verbose, "$count records indexed.\n" ); |
225 |
} |
292 |
} |
226 |
|
293 |
|
|
|
294 |
=head2 $indexer->_log( $level, $verbose, $msg ) |
295 |
|
296 |
Internal function, print msg to screen if verbosity > level |
297 |
|
298 |
=cut |
227 |
|
299 |
|
228 |
sub _log { |
300 |
sub _log { |
229 |
my ($level, $verbose, $msg) = @_; |
301 |
my ($level, $verbose, $msg) = @_; |
Lines 273-276
__END__
Link Here
|
273 |
|
345 |
|
274 |
=item Robin Sheat C<< <robin@catalyst.net.nz> >> |
346 |
=item Robin Sheat C<< <robin@catalyst.net.nz> >> |
275 |
|
347 |
|
|
|
348 |
=item Nick Clemens C<< <nick@bywatersolutions.com> >> |
349 |
|
276 |
=back |
350 |
=back |