|
Lines 20-26
package Koha::SearchEngine::Elasticsearch::Indexer;
Link Here
|
| 20 |
use Carp; |
20 |
use Carp; |
| 21 |
use Modern::Perl; |
21 |
use Modern::Perl; |
| 22 |
use base qw(Koha::SearchEngine::Elasticsearch); |
22 |
use base qw(Koha::SearchEngine::Elasticsearch); |
| 23 |
use Data::Dumper; |
23 |
use Koha::BiblioUtils; |
|
|
24 |
use Koha::MetadataRecord::Authority; |
| 24 |
|
25 |
|
| 25 |
# For now just marc, but we can do anything here really |
26 |
# For now just marc, but we can do anything here really |
| 26 |
use Catmandu::Importer::MARC; |
27 |
use Catmandu::Importer::MARC; |
|
Lines 89-95
sub update_index {
Link Here
|
| 89 |
|
90 |
|
| 90 |
my $from = $self->_convert_marc_to_json($records); |
91 |
my $from = $self->_convert_marc_to_json($records); |
| 91 |
|
92 |
|
| 92 |
print Data::Dumper::Dumper( $from->to_array ); |
93 |
#print Data::Dumper::Dumper( $from->to_array );$self->store->bag->add_many($from); |
| 93 |
$self->store->bag->add_many($from); |
94 |
$self->store->bag->add_many($from); |
| 94 |
$self->store->bag->commit; |
95 |
$self->store->bag->commit; |
| 95 |
return 1; |
96 |
return 1; |
|
Lines 110-116
it to be updated by a regular index cron job in the future.
Link Here
|
| 110 |
|
111 |
|
| 111 |
sub update_index_background { |
112 |
sub update_index_background { |
| 112 |
my $self = shift; |
113 |
my $self = shift; |
| 113 |
$self->update_index(@_); |
114 |
$self->do_reindex(undef, undef, 0, @_); |
| 114 |
} |
115 |
} |
| 115 |
|
116 |
|
| 116 |
=head2 $indexer->delete_index($biblionums) |
117 |
=head2 $indexer->delete_index($biblionums) |
|
Lines 142-149
sub delete_index_background {
Link Here
|
| 142 |
|
143 |
|
| 143 |
=head2 $indexer->drop_index(); |
144 |
=head2 $indexer->drop_index(); |
| 144 |
|
145 |
|
| 145 |
Drops the index from the elasticsearch server. Calling C<update_index> |
146 |
Drops the index from the elasticsearch server. |
| 146 |
after this will recreate it again. |
147 |
We need to create a new object once this has been done. |
| 147 |
|
148 |
|
| 148 |
=cut |
149 |
=cut |
| 149 |
|
150 |
|
|
Lines 151-177
sub drop_index {
Link Here
|
| 151 |
my ($self) = @_; |
152 |
my ($self) = @_; |
| 152 |
|
153 |
|
| 153 |
$self->store->drop(); |
154 |
$self->store->drop(); |
| 154 |
my $params = $self->get_elasticsearch_params(); |
|
|
| 155 |
$self->store( |
| 156 |
Catmandu::Store::ElasticSearch->new( |
| 157 |
%$params, |
| 158 |
index_settings => $self->get_elasticsearch_settings(), |
| 159 |
index_mappings => $self->get_elasticsearch_mappings(), |
| 160 |
) |
| 161 |
); |
| 162 |
} |
155 |
} |
| 163 |
|
156 |
|
| 164 |
|
157 |
|
| 165 |
=head2 $indexer->do_reindex(); |
158 |
=head2 $indexer->do_reindex($delete, $commit, $verbose, $biblionumbers); |
|
|
159 |
|
| 160 |
Performs a reindex, full or patial is based on whether biblionumbers are passed in |
| 161 |
We do an additional check if an alias exists - if so and only indexing a list of bibs we use it, |
| 162 |
if not, or if doing a full reindex, we index into a new index and adjust the existing alias then drop the old. |
| 163 |
|
| 164 |
C<$delete> indicates if we should delete the existing index - only affects partial indexes as otherwise we use a new index in either case. |
| 165 |
|
| 166 |
C<$commit> specifies at how many records we commit the update - higher is faster but uses more memory - default is 5000. |
| 167 |
|
| 168 |
C<$verbose> specifies warning level - only for command line use currently. |
| 166 |
|
169 |
|
| 167 |
Performs a reindex based on records passed in |
170 |
C<$biblionums> is an arrayref containing the biblionumbers for the records. |
| 168 |
|
171 |
|
| 169 |
=cut |
172 |
=cut |
| 170 |
|
173 |
|
| 171 |
sub do_reindex { |
174 |
sub do_reindex { |
| 172 |
my ($self, $next, $delete, $commit, $verbose, $biblionumbers) = @_; |
175 |
my ($self, $delete, $commit, $verbose, $biblionumbers) = @_; |
| 173 |
|
176 |
|
| 174 |
_log(1, $verbose, "Indexing ".$self->index."\n"); |
177 |
_log(1, $verbose, "Indexing ".$self->index."\n"); |
|
|
178 |
my $next; |
| 175 |
|
179 |
|
| 176 |
if ( scalar @$biblionumbers ) { |
180 |
if ( scalar @$biblionumbers ) { |
| 177 |
if ( $self->index eq 'biblios' ) { |
181 |
if ( $self->index eq 'biblios' ) { |
|
Lines 197-205
sub do_reindex {
Link Here
|
| 197 |
$next = sub { $records->next(); }; |
201 |
$next = sub { $records->next(); }; |
| 198 |
} |
202 |
} |
| 199 |
|
203 |
|
| 200 |
warn |
204 |
my $current_index = $self->get_alias_index(); |
| 201 |
$self->drop_index() if ( $delete ); |
205 |
if ( scalar @$biblionumbers && !$delete && $current_index) { |
|
|
206 |
|
| 207 |
#In this one case we index onto existing alias |
| 208 |
$self->_index_records( $next, $commit, $verbose ); |
| 209 |
|
| 210 |
} else { |
| 211 |
|
| 212 |
#In all other cases we are going to use a new index then switch our alias to this index |
| 213 |
#'time' is used for providing a unique name |
| 214 |
my $alias_name = $self->store->{index_name}; |
| 215 |
|
| 216 |
my $new_indexer = |
| 217 |
Koha::SearchEngine::Elasticsearch::Indexer->new( { |
| 218 |
index => $self->index.time |
| 219 |
}); |
| 220 |
|
| 221 |
$new_indexer->_index_records( $next, $commit, $verbose ); |
| 222 |
|
| 223 |
if ( $current_index ){ #If the alias already exists |
| 224 |
$self->update_alias({ #let's point the alias to the new index |
| 225 |
old_index => $current_index, |
| 226 |
new_index => $new_indexer->store->{index_name} |
| 227 |
}); |
| 228 |
$current_index =~ s/koha_kohadev_//; #Then we need short index name to build object |
| 229 |
|
| 230 |
my $old_index = |
| 231 |
Koha::SearchEngine::Elasticsearch::Indexer->new({ |
| 232 |
index => $current_index |
| 233 |
}); |
| 234 |
|
| 235 |
$old_index->drop_index(); #And we drop the specific index that the alias pointed to originally |
| 236 |
} else { #The alias was not created |
| 237 |
$self->drop_index(); #Drop the index we created initially |
| 238 |
#It will have the same name as our alias so must be dropped first |
| 239 |
#It should be empty (we indexed into a new one) |
| 240 |
$new_indexer->store->es->indices->put_alias( |
| 241 |
name => $alias_name, |
| 242 |
index => $new_indexer->store->{index_name} |
| 243 |
); |
| 244 |
} |
| 245 |
|
| 246 |
|
| 247 |
=head2 $indexer->index_records( $next, $commit, $verbose ); |
| 248 |
|
| 249 |
Performs a reindex, full or patial is based on whether biblionumbers are passed in |
| 250 |
We do an additional check if an alias exists - if so and only indexing a list of bibs we use it, |
| 251 |
if not, or if doing a full reindex, we index into a new index and adjust the existing alias then drop the old. |
| 252 |
|
| 253 |
C<$next> is a function reference, used for iterating over records. |
| 202 |
|
254 |
|
|
|
255 |
C<$commit> specifies at how many records we commit the update - higher is faster but uses more memory - default is 5000. |
| 256 |
|
| 257 |
C<$verbose> specifies warning level - only for command line use currently. |
| 258 |
|
| 259 |
=cut |
| 260 |
|
| 261 |
sub _index_records { |
| 262 |
my ( $self, $next, $commit, $verbose ) = @_; |
| 263 |
$commit ||= 5000; |
| 203 |
my $count = 0; |
264 |
my $count = 0; |
| 204 |
my $commit_count = $commit; |
265 |
my $commit_count = $commit; |
| 205 |
my ( @id_buffer, @commit_buffer ); |
266 |
my ( @id_buffer, @commit_buffer ); |
|
Lines 269-276
__END__
Link Here
|
| 269 |
|
330 |
|
| 270 |
=over 4 |
331 |
=over 4 |
| 271 |
|
332 |
|
| 272 |
=item Chris Cormack C<< <chrisc@catalyst.net.nz> >> |
333 |
=item Chris Cormack C<< <chrisc@catalyst.net.nz> >>=item Robin Sheat C<< <robin@catalyst.net.nz> >> |
| 273 |
|
334 |
back |
| 274 |
=item Robin Sheat C<< <robin@catalyst.net.nz> >> |
|
|
| 275 |
|
| 276 |
=back |
| 277 |
- |
|
|