|
Lines 20-26
package Koha::SearchEngine::Elasticsearch::Indexer;
Link Here
|
| 20 |
use Carp; |
20 |
use Carp; |
| 21 |
use Modern::Perl; |
21 |
use Modern::Perl; |
| 22 |
use base qw(Koha::SearchEngine::Elasticsearch); |
22 |
use base qw(Koha::SearchEngine::Elasticsearch); |
| 23 |
use Data::Dumper; |
23 |
|
|
|
24 |
use Koha::BiblioUtils; |
| 25 |
use Koha::MetadataRecord::Authority; |
| 24 |
|
26 |
|
| 25 |
# For now just marc, but we can do anything here really |
27 |
# For now just marc, but we can do anything here really |
| 26 |
use Catmandu::Importer::MARC; |
28 |
use Catmandu::Importer::MARC; |
|
Lines 89-95
sub update_index {
Link Here
|
| 89 |
|
91 |
|
| 90 |
my $from = $self->_convert_marc_to_json($records); |
92 |
my $from = $self->_convert_marc_to_json($records); |
| 91 |
|
93 |
|
| 92 |
print Data::Dumper::Dumper( $from->to_array ); |
94 |
#print Data::Dumper::Dumper( $from->to_array ); |
| 93 |
$self->store->bag->add_many($from); |
95 |
$self->store->bag->add_many($from); |
| 94 |
$self->store->bag->commit; |
96 |
$self->store->bag->commit; |
| 95 |
return 1; |
97 |
return 1; |
|
Lines 110-116
it to be updated by a regular index cron job in the future.
Link Here
|
| 110 |
|
112 |
|
| 111 |
sub update_index_background { |
113 |
sub update_index_background { |
| 112 |
my $self = shift; |
114 |
my $self = shift; |
| 113 |
$self->update_index(@_); |
115 |
warn Data::Dumper::Dumper( @_ ); |
|
|
116 |
$self->do_reindex(undef, undef, 0, @_); |
| 114 |
} |
117 |
} |
| 115 |
|
118 |
|
| 116 |
=head2 $indexer->delete_index($biblionums) |
119 |
=head2 $indexer->delete_index($biblionums) |
|
Lines 142-149
sub delete_index_background {
Link Here
|
| 142 |
|
145 |
|
| 143 |
=head2 $indexer->drop_index(); |
146 |
=head2 $indexer->drop_index(); |
| 144 |
|
147 |
|
| 145 |
Drops the index from the elasticsearch server. Calling C<update_index> |
148 |
Drops the index from the elasticsearch server. |
| 146 |
after this will recreate it again. |
149 |
We need to create a new object once this has been done. |
| 147 |
|
150 |
|
| 148 |
=cut |
151 |
=cut |
| 149 |
|
152 |
|
|
Lines 151-177
sub drop_index {
Link Here
|
| 151 |
my ($self) = @_; |
154 |
my ($self) = @_; |
| 152 |
|
155 |
|
| 153 |
$self->store->drop(); |
156 |
$self->store->drop(); |
| 154 |
my $params = $self->get_elasticsearch_params(); |
|
|
| 155 |
$self->store( |
| 156 |
Catmandu::Store::ElasticSearch->new( |
| 157 |
%$params, |
| 158 |
index_settings => $self->get_elasticsearch_settings(), |
| 159 |
index_mappings => $self->get_elasticsearch_mappings(), |
| 160 |
) |
| 161 |
); |
| 162 |
} |
157 |
} |
| 163 |
|
158 |
|
| 164 |
|
159 |
|
| 165 |
=head2 $indexer->do_reindex(); |
160 |
=head2 $indexer->do_reindex($delete, $commit, $verbose, $biblionumbers); |
|
|
161 |
|
| 162 |
Performs a reindex, full or patial is based on whether biblionumbers are passed in |
| 163 |
We do an additional check if an alias exists - if so and only indexing a list of bibs we use it, |
| 164 |
if not, or if doing a full reindex, we index into a new index and adjust the existing alias then drop the old. |
| 166 |
|
165 |
|
| 167 |
Performs a reindex based on records passed in |
166 |
C<$delete> indicates if we should delete the existing index - only affects partial indexes as otherwise we use a new index in either case. |
|
|
167 |
|
| 168 |
C<$commit> specifies at how many records we commit the update - higher is faster but uses more memory - default is 5000. |
| 169 |
|
| 170 |
C<$verbose> specifies warning level - only for command line use currently. |
| 171 |
|
| 172 |
C<$biblionums> is an arrayref containing the biblionumbers for the records. |
| 168 |
|
173 |
|
| 169 |
=cut |
174 |
=cut |
| 170 |
|
175 |
|
| 171 |
sub do_reindex { |
176 |
sub do_reindex { |
| 172 |
my ($self, $next, $delete, $commit, $verbose, $biblionumbers) = @_; |
177 |
my ($self, $delete, $commit, $verbose, $biblionumbers) = @_; |
| 173 |
|
178 |
|
| 174 |
_log(1, $verbose, "Indexing ".$self->index."\n"); |
179 |
_log(1, $verbose, "Indexing ".$self->index."\n"); |
|
|
180 |
my $next; |
| 175 |
|
181 |
|
| 176 |
if ( scalar @$biblionumbers ) { |
182 |
if ( scalar @$biblionumbers ) { |
| 177 |
if ( $self->index eq 'biblios' ) { |
183 |
if ( $self->index eq 'biblios' ) { |
|
Lines 197-204
sub do_reindex {
Link Here
|
| 197 |
$next = sub { $records->next(); }; |
203 |
$next = sub { $records->next(); }; |
| 198 |
} |
204 |
} |
| 199 |
|
205 |
|
| 200 |
warn |
206 |
my $current_index = $self->get_alias_index(); |
| 201 |
$self->drop_index() if ( $delete ); |
207 |
|
|
|
208 |
if ( scalar @$biblionumbers && !$delete && $current_index) { |
| 209 |
|
| 210 |
#In this one case we index onto existing alias |
| 211 |
$self->_index_records( $next, $commit, $verbose ); |
| 212 |
|
| 213 |
} else { |
| 214 |
|
| 215 |
#In all other cases we are going to use a new index then switch our alias to this index |
| 216 |
#'time' is used for providing a unique name |
| 217 |
my $alias_name = $self->store->{index_name}; |
| 218 |
|
| 219 |
my $new_indexer = |
| 220 |
Koha::SearchEngine::Elasticsearch::Indexer->new( { |
| 221 |
index => $self->index.time |
| 222 |
}); |
| 223 |
|
| 224 |
$new_indexer->_index_records( $next, $commit, $verbose ); |
| 225 |
|
| 226 |
if ( $current_index ){ #If the alias already exists |
| 227 |
$self->update_alias({ #let's point the alias to the new index |
| 228 |
old_index => $current_index, |
| 229 |
new_index => $new_indexer->store->{index_name} |
| 230 |
}); |
| 231 |
$current_index =~ s/koha_kohadev_//; #Then we need short index name to build object |
| 232 |
|
| 233 |
my $old_index = |
| 234 |
Koha::SearchEngine::Elasticsearch::Indexer->new({ |
| 235 |
index => $current_index |
| 236 |
}); |
| 237 |
|
| 238 |
$old_index->drop_index(); #And we drop the specific index that the alias pointed to originally |
| 239 |
} else { #The alias was not created |
| 240 |
$self->drop_index(); #Drop the index we created initially |
| 241 |
#It will have the same name as our alias so must be dropped first |
| 242 |
#It should be empty (we indexed into a new one) |
| 243 |
$new_indexer->store->es->indices->put_alias( |
| 244 |
name => $alias_name, |
| 245 |
index => $new_indexer->store->{index_name} |
| 246 |
); |
| 247 |
} |
| 248 |
|
| 249 |
} |
| 250 |
} |
| 251 |
|
| 252 |
=head2 $indexer->index_records( $next, $commit, $verbose ); |
| 253 |
|
| 254 |
Performs a reindex, full or patial is based on whether biblionumbers are passed in |
| 255 |
We do an additional check if an alias exists - if so and only indexing a list of bibs we use it, |
| 256 |
if not, or if doing a full reindex, we index into a new index and adjust the existing alias then drop the old. |
| 257 |
|
| 258 |
C<$next> is a function reference, used for iterating over records. |
| 259 |
|
| 260 |
C<$commit> specifies at how many records we commit the update - higher is faster but uses more memory - default is 5000. |
| 261 |
|
| 262 |
C<$verbose> specifies warning level - only for command line use currently. |
| 263 |
|
| 264 |
=cut |
| 265 |
|
| 266 |
sub _index_records { |
| 267 |
my ( $self, $next, $commit, $verbose ) = @_; |
| 268 |
$commit ||= 5000; |
| 202 |
|
269 |
|
| 203 |
my $count = 0; |
270 |
my $count = 0; |
| 204 |
my $commit_count = $commit; |
271 |
my $commit_count = $commit; |
|
Lines 219-229
warn
Link Here
|
| 219 |
@commit_buffer = (); |
286 |
@commit_buffer = (); |
| 220 |
} |
287 |
} |
| 221 |
} |
288 |
} |
| 222 |
# There are probably uncommitted records (beacuse we didn't hit exact commit count) |
289 |
# There are probably uncommitted records (because we didn't hit exact commit count) |
| 223 |
$self->update_index( \@id_buffer, \@commit_buffer ); |
290 |
$self->update_index( \@id_buffer, \@commit_buffer ); |
| 224 |
_log( 1, $verbose, "$count records indexed.\n" ); |
291 |
_log( 1, $verbose, "$count records indexed.\n" ); |
| 225 |
} |
292 |
} |
| 226 |
|
293 |
|
|
|
294 |
=head2 $indexer->_log( $level, $verbose, $msg ) |
| 295 |
|
| 296 |
Internal function, print msg to screen if verbosity > level |
| 297 |
|
| 298 |
=cut |
| 227 |
|
299 |
|
| 228 |
sub _log { |
300 |
sub _log { |
| 229 |
my ($level, $verbose, $msg) = @_; |
301 |
my ($level, $verbose, $msg) = @_; |
|
Lines 273-276
__END__
Link Here
|
| 273 |
|
345 |
|
| 274 |
=item Robin Sheat C<< <robin@catalyst.net.nz> >> |
346 |
=item Robin Sheat C<< <robin@catalyst.net.nz> >> |
| 275 |
|
347 |
|
|
|
348 |
=item Nick Clemens C<< <nick@bywatersolutions.com> >> |
| 349 |
|
| 276 |
=back |
350 |
=back |