Lines 20-26
package Koha::SearchEngine::Elasticsearch::Indexer;
Link Here
|
20 |
use Carp; |
20 |
use Carp; |
21 |
use Modern::Perl; |
21 |
use Modern::Perl; |
22 |
use base qw(Koha::SearchEngine::Elasticsearch); |
22 |
use base qw(Koha::SearchEngine::Elasticsearch); |
23 |
use Data::Dumper; |
23 |
use Koha::BiblioUtils; |
|
|
24 |
use Koha::MetadataRecord::Authority; |
24 |
|
25 |
|
25 |
# For now just marc, but we can do anything here really |
26 |
# For now just marc, but we can do anything here really |
26 |
use Catmandu::Importer::MARC; |
27 |
use Catmandu::Importer::MARC; |
Lines 89-95
sub update_index {
Link Here
|
89 |
|
90 |
|
90 |
my $from = $self->_convert_marc_to_json($records); |
91 |
my $from = $self->_convert_marc_to_json($records); |
91 |
|
92 |
|
92 |
print Data::Dumper::Dumper( $from->to_array ); |
93 |
#print Data::Dumper::Dumper( $from->to_array );$self->store->bag->add_many($from); |
93 |
$self->store->bag->add_many($from); |
94 |
$self->store->bag->add_many($from); |
94 |
$self->store->bag->commit; |
95 |
$self->store->bag->commit; |
95 |
return 1; |
96 |
return 1; |
Lines 110-116
it to be updated by a regular index cron job in the future.
Link Here
|
110 |
|
111 |
|
111 |
sub update_index_background { |
112 |
sub update_index_background { |
112 |
my $self = shift; |
113 |
my $self = shift; |
113 |
$self->update_index(@_); |
114 |
$self->do_reindex(undef, undef, 0, @_); |
114 |
} |
115 |
} |
115 |
|
116 |
|
116 |
=head2 $indexer->delete_index($biblionums) |
117 |
=head2 $indexer->delete_index($biblionums) |
Lines 142-149
sub delete_index_background {
Link Here
|
142 |
|
143 |
|
143 |
=head2 $indexer->drop_index(); |
144 |
=head2 $indexer->drop_index(); |
144 |
|
145 |
|
145 |
Drops the index from the elasticsearch server. Calling C<update_index> |
146 |
Drops the index from the elasticsearch server. |
146 |
after this will recreate it again. |
147 |
We need to create a new object once this has been done. |
147 |
|
148 |
|
148 |
=cut |
149 |
=cut |
149 |
|
150 |
|
Lines 151-177
sub drop_index {
Link Here
|
151 |
my ($self) = @_; |
152 |
my ($self) = @_; |
152 |
|
153 |
|
153 |
$self->store->drop(); |
154 |
$self->store->drop(); |
154 |
my $params = $self->get_elasticsearch_params(); |
|
|
155 |
$self->store( |
156 |
Catmandu::Store::ElasticSearch->new( |
157 |
%$params, |
158 |
index_settings => $self->get_elasticsearch_settings(), |
159 |
index_mappings => $self->get_elasticsearch_mappings(), |
160 |
) |
161 |
); |
162 |
} |
155 |
} |
163 |
|
156 |
|
164 |
|
157 |
|
165 |
=head2 $indexer->do_reindex(); |
158 |
=head2 $indexer->do_reindex($delete, $commit, $verbose, $biblionumbers); |
|
|
159 |
|
160 |
Performs a reindex, full or patial is based on whether biblionumbers are passed in |
161 |
We do an additional check if an alias exists - if so and only indexing a list of bibs we use it, |
162 |
if not, or if doing a full reindex, we index into a new index and adjust the existing alias then drop the old. |
163 |
|
164 |
C<$delete> indicates if we should delete the existing index - only affects partial indexes as otherwise we use a new index in either case. |
165 |
|
166 |
C<$commit> specifies at how many records we commit the update - higher is faster but uses more memory - default is 5000. |
167 |
|
168 |
C<$verbose> specifies warning level - only for command line use currently. |
166 |
|
169 |
|
167 |
Performs a reindex based on records passed in |
170 |
C<$biblionums> is an arrayref containing the biblionumbers for the records. |
168 |
|
171 |
|
169 |
=cut |
172 |
=cut |
170 |
|
173 |
|
171 |
sub do_reindex { |
174 |
sub do_reindex { |
172 |
my ($self, $next, $delete, $commit, $verbose, $biblionumbers) = @_; |
175 |
my ($self, $delete, $commit, $verbose, $biblionumbers) = @_; |
173 |
|
176 |
|
174 |
_log(1, $verbose, "Indexing ".$self->index."\n"); |
177 |
_log(1, $verbose, "Indexing ".$self->index."\n"); |
|
|
178 |
my $next; |
175 |
|
179 |
|
176 |
if ( scalar @$biblionumbers ) { |
180 |
if ( scalar @$biblionumbers ) { |
177 |
if ( $self->index eq 'biblios' ) { |
181 |
if ( $self->index eq 'biblios' ) { |
Lines 197-205
sub do_reindex {
Link Here
|
197 |
$next = sub { $records->next(); }; |
201 |
$next = sub { $records->next(); }; |
198 |
} |
202 |
} |
199 |
|
203 |
|
200 |
warn |
204 |
my $current_index = $self->get_alias_index(); |
201 |
$self->drop_index() if ( $delete ); |
205 |
if ( scalar @$biblionumbers && !$delete && $current_index) { |
|
|
206 |
|
207 |
#In this one case we index onto existing alias |
208 |
$self->_index_records( $next, $commit, $verbose ); |
209 |
|
210 |
} else { |
211 |
|
212 |
#In all other cases we are going to use a new index then switch our alias to this index |
213 |
#'time' is used for providing a unique name |
214 |
my $alias_name = $self->store->{index_name}; |
215 |
|
216 |
my $new_indexer = |
217 |
Koha::SearchEngine::Elasticsearch::Indexer->new( { |
218 |
index => $self->index.time |
219 |
}); |
220 |
|
221 |
$new_indexer->_index_records( $next, $commit, $verbose ); |
222 |
|
223 |
if ( $current_index ){ #If the alias already exists |
224 |
$self->update_alias({ #let's point the alias to the new index |
225 |
old_index => $current_index, |
226 |
new_index => $new_indexer->store->{index_name} |
227 |
}); |
228 |
$current_index =~ s/koha_kohadev_//; #Then we need short index name to build object |
229 |
|
230 |
my $old_index = |
231 |
Koha::SearchEngine::Elasticsearch::Indexer->new({ |
232 |
index => $current_index |
233 |
}); |
234 |
|
235 |
$old_index->drop_index(); #And we drop the specific index that the alias pointed to originally |
236 |
} else { #The alias was not created |
237 |
$self->drop_index(); #Drop the index we created initially |
238 |
#It will have the same name as our alias so must be dropped first |
239 |
#It should be empty (we indexed into a new one) |
240 |
$new_indexer->store->es->indices->put_alias( |
241 |
name => $alias_name, |
242 |
index => $new_indexer->store->{index_name} |
243 |
); |
244 |
} |
245 |
|
246 |
|
247 |
=head2 $indexer->index_records( $next, $commit, $verbose ); |
248 |
|
249 |
Performs a reindex, full or patial is based on whether biblionumbers are passed in |
250 |
We do an additional check if an alias exists - if so and only indexing a list of bibs we use it, |
251 |
if not, or if doing a full reindex, we index into a new index and adjust the existing alias then drop the old. |
252 |
|
253 |
C<$next> is a function reference, used for iterating over records. |
202 |
|
254 |
|
|
|
255 |
C<$commit> specifies at how many records we commit the update - higher is faster but uses more memory - default is 5000. |
256 |
|
257 |
C<$verbose> specifies warning level - only for command line use currently. |
258 |
|
259 |
=cut |
260 |
|
261 |
sub _index_records { |
262 |
my ( $self, $next, $commit, $verbose ) = @_; |
263 |
$commit ||= 5000; |
203 |
my $count = 0; |
264 |
my $count = 0; |
204 |
my $commit_count = $commit; |
265 |
my $commit_count = $commit; |
205 |
my ( @id_buffer, @commit_buffer ); |
266 |
my ( @id_buffer, @commit_buffer ); |
Lines 269-276
__END__
Link Here
|
269 |
|
330 |
|
270 |
=over 4 |
331 |
=over 4 |
271 |
|
332 |
|
272 |
=item Chris Cormack C<< <chrisc@catalyst.net.nz> >> |
333 |
=item Chris Cormack C<< <chrisc@catalyst.net.nz> >>=item Robin Sheat C<< <robin@catalyst.net.nz> >> |
273 |
|
334 |
back |
274 |
=item Robin Sheat C<< <robin@catalyst.net.nz> >> |
|
|
275 |
|
276 |
=back |
277 |
- |
|
|