|
Lines 44-88
Koha::SearchEngine::Elasticsearch::Indexer - handles adding new records to the i
Link Here
|
| 44 |
$indexer->drop_index(); |
44 |
$indexer->drop_index(); |
| 45 |
$indexer->update_index(\@biblionumbers, \@records); |
45 |
$indexer->update_index(\@biblionumbers, \@records); |
| 46 |
|
46 |
|
| 47 |
=head1 FUNCTIONS |
|
|
| 48 |
|
47 |
|
| 49 |
=head2 $indexer->update_index($biblionums, $records); |
48 |
=head1 CONSTANTS |
| 50 |
|
49 |
|
| 51 |
C<$biblionums> is an arrayref containing the biblionumbers for the records. |
50 |
=over 4 |
| 52 |
|
51 |
|
| 53 |
C<$records> is an arrayref containing the L<MARC::Record>s themselves. |
52 |
=item C<Koha::SearchEngine::Elasticsearch::Indexer::INDEX_STATUS_OK> |
| 54 |
|
53 |
|
| 55 |
The values in the arrays must match up, and the 999$c value in the MARC record |
54 |
Represents an index state where index is created and in a working state. |
| 56 |
will be rewritten using the values in C<$biblionums> to ensure they are correct. |
|
|
| 57 |
If C<$biblionums> is C<undef>, this won't happen, but you should be sure that |
| 58 |
999$c is correct on your own then. |
| 59 |
|
55 |
|
| 60 |
Note that this will modify the original record if C<$biblionums> is supplied. |
56 |
=item C<Koha::SearchEngine::Elasticsearch::Indexer::INDEX_STATUS_REINDEX_REQUIRED> |
| 61 |
If that's a problem, clone them first. |
57 |
|
|
|
58 |
Not currently used, but could be useful later, for example if can detect when new field or mapping added. |
| 59 |
|
| 60 |
=item C<Koha::SearchEngine::Elasticsearch::Indexer::INDEX_STATUS_RECREATE_REQUIRED> |
| 61 |
|
| 62 |
Representings an index state where index needs to be recreated and is not in a working state. |
| 63 |
|
| 64 |
=back |
| 62 |
|
65 |
|
| 63 |
=cut |
66 |
=cut |
| 64 |
|
67 |
|
| 65 |
use constant { |
68 |
use constant { |
| 66 |
INDEX_STATUS_OK => 0, |
69 |
INDEX_STATUS_OK => 0, |
| 67 |
INDEX_STATUS_REINDEX_REQUIRED => 1, # Not currently used, but could be useful later, for example if can detect when new field or mapping added |
70 |
INDEX_STATUS_REINDEX_REQUIRED => 1, |
| 68 |
INDEX_STATUS_RECREATE_REQUIRED => 2, |
71 |
INDEX_STATUS_RECREATE_REQUIRED => 2, |
| 69 |
}; |
72 |
}; |
| 70 |
|
73 |
|
|
|
74 |
=head1 FUNCTIONS |
| 75 |
|
| 76 |
=head2 update_index($biblionums, $records) |
| 77 |
|
| 78 |
try { |
| 79 |
$self->update_index($biblionums, $records); |
| 80 |
} catch { |
| 81 |
die("Something whent wrong trying to update index:" . $_[0]); |
| 82 |
} |
| 83 |
|
| 84 |
Converts C<MARC::Records> C<$records> to Elasticsearch documents and perform |
| 85 |
an update request for these records on the Elasticsearch index. |
| 86 |
|
| 87 |
The values in the arrays must match up, and the 999$c value in the MARC record |
| 88 |
will be rewritten using the values in C<$biblionums> to ensure they are correct. |
| 89 |
If C<$biblionums> is C<undef>, this won't happen, so in that case you should make |
| 90 |
sure that 999$c is correct. |
| 91 |
|
| 92 |
Note that this will modify the original record if C<$biblionums> is supplied. |
| 93 |
If that's a problem, clone them first. |
| 94 |
|
| 95 |
=over 4 |
| 96 |
|
| 97 |
=item C<$biblionums> |
| 98 |
|
| 99 |
Arrayref of biblio numbers for the C<$records>, the order must be the same |
| 100 |
as C<$records>. |
| 101 |
|
| 102 |
=item C<$records> |
| 103 |
|
| 104 |
Arrayref of C<MARC::Record>s. |
| 105 |
|
| 106 |
=back |
| 107 |
|
| 108 |
=cut |
| 109 |
|
| 71 |
sub update_index { |
110 |
sub update_index { |
| 72 |
my ($self, $biblionums, $records) = @_; |
111 |
my ($self, $biblionums, $records) = @_; |
| 73 |
|
112 |
|
| 74 |
# TODO should have a separate path for dealing with a large number |
|
|
| 75 |
# of records at once where we use the bulk update functions in ES. |
| 76 |
if ($biblionums) { |
113 |
if ($biblionums) { |
| 77 |
$self->_sanitise_records($biblionums, $records); |
114 |
$self->_sanitise_records($biblionums, $records); |
| 78 |
} |
115 |
} |
| 79 |
|
116 |
|
| 80 |
$self->bulk_index($records); |
|
|
| 81 |
return 1; |
| 82 |
} |
| 83 |
|
| 84 |
sub bulk_index { |
| 85 |
my ($self, $records) = @_; |
| 86 |
my $conf = $self->get_elasticsearch_params(); |
117 |
my $conf = $self->get_elasticsearch_params(); |
| 87 |
my $elasticsearch = $self->get_elasticsearch(); |
118 |
my $elasticsearch = $self->get_elasticsearch(); |
| 88 |
my $documents = $self->marc_records_to_documents($records); |
119 |
my $documents = $self->marc_records_to_documents($records); |
|
Lines 108-134
sub bulk_index {
Link Here
|
| 108 |
return 1; |
139 |
return 1; |
| 109 |
} |
140 |
} |
| 110 |
|
141 |
|
| 111 |
sub index_status_ok { |
142 |
=head2 set_index_status_ok |
|
|
143 |
|
| 144 |
Convenience method for setting index status to C<INDEX_STATUS_OK>. |
| 145 |
|
| 146 |
=cut |
| 147 |
|
| 148 |
sub set_index_status_ok { |
| 112 |
my ($self, $set) = @_; |
149 |
my ($self, $set) = @_; |
| 113 |
return defined $set ? |
150 |
$self->index_status(INDEX_STATUS_OK); |
| 114 |
$self->index_status(INDEX_STATUS_OK) : |
|
|
| 115 |
$self->index_status == INDEX_STATUS_OK; |
| 116 |
} |
151 |
} |
| 117 |
|
152 |
|
| 118 |
sub index_status_reindex_required { |
153 |
=head2 is_index_status_ok |
| 119 |
my ($self, $set) = @_; |
154 |
|
| 120 |
return defined $set ? |
155 |
Convenience method for checking if index status is C<INDEX_STATUS_OK>. |
| 121 |
$self->index_status(INDEX_STATUS_REINDEX_REQUIRED) : |
156 |
|
| 122 |
$self->index_status == INDEX_STATUS_REINDEX_REQUIRED; |
157 |
=cut |
|
|
158 |
|
| 159 |
sub is_index_status_ok { |
| 160 |
return $self->index_status == INDEX_STATUS_OK; |
| 123 |
} |
161 |
} |
| 124 |
|
162 |
|
| 125 |
sub index_status_recreate_required { |
163 |
=head2 set_index_status_reindex_required |
| 126 |
my ($self, $set) = @_; |
164 |
|
| 127 |
return defined $set ? |
165 |
Convenience method for setting index status to C<INDEX_REINDEX_REQUIRED>. |
| 128 |
$self->index_status(INDEX_STATUS_RECREATE_REQUIRED) : |
166 |
|
| 129 |
$self->index_status == INDEX_STATUS_RECREATE_REQUIRED; |
167 |
=cut |
|
|
168 |
|
| 169 |
sub set_index_status_reindex_required { |
| 170 |
$self->index_status(INDEX_STATUS_REINDEX_REQUIRED); |
| 171 |
} |
| 172 |
|
| 173 |
=head2 is_index_status_reindex_required |
| 174 |
|
| 175 |
Convenience method for checking if index status is C<INDEX_STATUS_REINDEX_REQUIRED>. |
| 176 |
|
| 177 |
=cut |
| 178 |
|
| 179 |
sub is_index_status_reindex_required { |
| 180 |
return $self->index_status == INDEX_STATUS_REINDEX_REQUIRED; |
| 181 |
} |
| 182 |
|
| 183 |
=head2 set_index_status_recreate_required |
| 184 |
|
| 185 |
Convenience method for setting index status to C<INDEX_STATUS_RECREATE_REQUIRED>. |
| 186 |
|
| 187 |
=cut |
| 188 |
|
| 189 |
sub set_index_status_recreate_required { |
| 190 |
$self->index_status(INDEX_STATUS_RECREATE_REQUIRED); |
| 191 |
} |
| 192 |
|
| 193 |
=head2 is_index_status_recreate_required |
| 194 |
|
| 195 |
Convenience method for checking if index status is C<INDEX_STATUS_RECREATE_REQUIRED>. |
| 196 |
|
| 197 |
=cut |
| 198 |
|
| 199 |
sub is_index_status_recreate_required { |
| 200 |
return $self->index_status == INDEX_STATUS_RECREATE_REQUIRED; |
| 130 |
} |
201 |
} |
| 131 |
|
202 |
|
|
|
203 |
=head2 index_status($status) |
| 204 |
|
| 205 |
Will either set the current index status to C<$status> and return C<$status>, |
| 206 |
or return the current index status if called with no arguments. |
| 207 |
|
| 208 |
=over 4 |
| 209 |
|
| 210 |
=item C<$status> |
| 211 |
|
| 212 |
Optional argument. If passed will set current index status to C<$status> if C<$status> is |
| 213 |
a valid status. See L</CONSTANTS>. |
| 214 |
|
| 215 |
=back |
| 216 |
|
| 217 |
=cut |
| 218 |
|
| 132 |
sub index_status { |
219 |
sub index_status { |
| 133 |
my ($self, $status) = @_; |
220 |
my ($self, $status) = @_; |
| 134 |
my $key = 'ElasticsearchIndexStatus_' . $self->index; |
221 |
my $key = 'ElasticsearchIndexStatus_' . $self->index; |
|
Lines 150-155
sub index_status {
Link Here
|
| 150 |
} |
237 |
} |
| 151 |
} |
238 |
} |
| 152 |
|
239 |
|
|
|
240 |
=head2 update_mappings |
| 241 |
|
| 242 |
Generate Elasticsearch mappings from mappings stored in database and |
| 243 |
perform a request to update Elasticsearch index mappings. Will throw an |
| 244 |
error and set index status to C<INDEX_STATUS_RECREATE_REQUIRED> if update |
| 245 |
failes. |
| 246 |
|
| 247 |
=cut |
| 248 |
|
| 153 |
sub update_mappings { |
249 |
sub update_mappings { |
| 154 |
my ($self) = @_; |
250 |
my ($self) = @_; |
| 155 |
my $conf = $self->get_elasticsearch_params(); |
251 |
my $conf = $self->get_elasticsearch_params(); |
|
Lines 166-200
sub update_mappings {
Link Here
|
| 166 |
} |
262 |
} |
| 167 |
); |
263 |
); |
| 168 |
} catch { |
264 |
} catch { |
| 169 |
$self->index_status_recreate_required(1); |
265 |
$self->set_index_status_recreate_required(); |
| 170 |
my $reason = $_[0]->{vars}->{body}->{error}->{reason}; |
266 |
my $reason = $_[0]->{vars}->{body}->{error}->{reason}; |
| 171 |
Koha::Exceptions::Exception->throw( |
267 |
Koha::Exceptions::Exception->throw( |
| 172 |
error => "Unable to update mappings for index \"$conf->{index_name}\". Reason was: \"$reason\". Index needs to be recreated and reindexed", |
268 |
error => "Unable to update mappings for index \"$conf->{index_name}\". Reason was: \"$reason\". Index needs to be recreated and reindexed", |
| 173 |
); |
269 |
); |
| 174 |
}; |
270 |
}; |
| 175 |
} |
271 |
} |
| 176 |
$self->index_status_ok(1); |
272 |
$self->set_index_status_ok(); |
| 177 |
} |
273 |
} |
| 178 |
|
274 |
|
| 179 |
=head2 $indexer->update_index_background($biblionums, $records) |
275 |
=head2 update_index_background($biblionums, $records) |
| 180 |
|
276 |
|
| 181 |
This has exactly the same API as C<update_index_background> however it'll |
277 |
This has exactly the same API as C<update_index> however it'll |
| 182 |
return immediately. It'll start a background process that does the adding. |
278 |
return immediately. It'll start a background process that does the adding. |
| 183 |
|
279 |
|
| 184 |
If it fails to add to Elasticsearch then it'll add to a queue that will cause |
280 |
If it fails to add to Elasticsearch then it'll add to a queue that will cause |
| 185 |
it to be updated by a regular index cron job in the future. |
281 |
it to be updated by a regular index cron job in the future. |
| 186 |
|
282 |
|
|
|
283 |
=cut |
| 284 |
|
| 187 |
# TODO implement in the future - I don't know the best way of doing this yet. |
285 |
# TODO implement in the future - I don't know the best way of doing this yet. |
| 188 |
# If fork: make sure process group is changed so apache doesn't wait for us. |
286 |
# If fork: make sure process group is changed so apache doesn't wait for us. |
| 189 |
|
287 |
|
| 190 |
=cut |
|
|
| 191 |
|
| 192 |
sub update_index_background { |
288 |
sub update_index_background { |
| 193 |
my $self = shift; |
289 |
my $self = shift; |
| 194 |
$self->update_index(@_); |
290 |
$self->update_index(@_); |
| 195 |
} |
291 |
} |
| 196 |
|
292 |
|
| 197 |
=head2 $indexer->delete_index($biblionums) |
293 |
=head2 delete_index($biblionums) |
| 198 |
|
294 |
|
| 199 |
C<$biblionums> is an arrayref of biblionumbers to delete from the index. |
295 |
C<$biblionums> is an arrayref of biblionumbers to delete from the index. |
| 200 |
|
296 |
|
|
Lines 206-212
sub delete_index {
Link Here
|
| 206 |
if ( !$self->store ) { |
302 |
if ( !$self->store ) { |
| 207 |
my $params = $self->get_elasticsearch_params(); |
303 |
my $params = $self->get_elasticsearch_params(); |
| 208 |
$self->store( |
304 |
$self->store( |
| 209 |
Catmandu::Store::ElasticSearch->new( |
305 |
Catmandu::Store::Elasticsearch->new( |
| 210 |
%$params, |
306 |
%$params, |
| 211 |
index_settings => $self->get_elasticsearch_settings(), |
307 |
index_settings => $self->get_elasticsearch_settings(), |
| 212 |
index_mappings => $self->get_elasticsearch_mappings(), |
308 |
index_mappings => $self->get_elasticsearch_mappings(), |
|
Lines 217-239
sub delete_index {
Link Here
|
| 217 |
$self->store->bag->commit; |
313 |
$self->store->bag->commit; |
| 218 |
} |
314 |
} |
| 219 |
|
315 |
|
| 220 |
=head2 $indexer->delete_index_background($biblionums) |
316 |
=head2 delete_index_background($biblionums) |
| 221 |
|
317 |
|
| 222 |
Identical to L<delete_index>, this will return immediately and start a |
318 |
Identical to L</delete_index($biblionums)> |
| 223 |
background process to do the actual deleting. |
|
|
| 224 |
|
319 |
|
| 225 |
=cut |
320 |
=cut |
| 226 |
|
321 |
|
| 227 |
# TODO implement in the future |
322 |
# TODO: Should be made async |
| 228 |
|
|
|
| 229 |
sub delete_index_background { |
323 |
sub delete_index_background { |
| 230 |
my $self = shift; |
324 |
my $self = shift; |
| 231 |
$self->delete_index(@_); |
325 |
$self->delete_index(@_); |
| 232 |
} |
326 |
} |
| 233 |
|
327 |
|
| 234 |
=head2 $indexer->drop_index(); |
328 |
=head2 drop_index |
| 235 |
|
329 |
|
| 236 |
Drops the index from the elasticsearch server. |
330 |
Drops the index from the Elasticsearch server. |
| 237 |
|
331 |
|
| 238 |
=cut |
332 |
=cut |
| 239 |
|
333 |
|
|
Lines 243-252
sub drop_index {
Link Here
|
| 243 |
my $conf = $self->get_elasticsearch_params(); |
337 |
my $conf = $self->get_elasticsearch_params(); |
| 244 |
my $elasticsearch = $self->get_elasticsearch(); |
338 |
my $elasticsearch = $self->get_elasticsearch(); |
| 245 |
$elasticsearch->indices->delete(index => $conf->{index_name}); |
339 |
$elasticsearch->indices->delete(index => $conf->{index_name}); |
| 246 |
$self->index_status_recreate_required(1); |
340 |
$self->set_index_status_recreate_required(); |
| 247 |
} |
341 |
} |
| 248 |
} |
342 |
} |
| 249 |
|
343 |
|
|
|
344 |
=head2 create_index |
| 345 |
|
| 346 |
Creates the index (including mappings) on the Elasticsearch server. |
| 347 |
|
| 348 |
=cut |
| 349 |
|
| 250 |
sub create_index { |
350 |
sub create_index { |
| 251 |
my ($self) = @_; |
351 |
my ($self) = @_; |
| 252 |
my $conf = $self->get_elasticsearch_params(); |
352 |
my $conf = $self->get_elasticsearch_params(); |
|
Lines 261-266
sub create_index {
Link Here
|
| 261 |
$self->update_mappings(); |
361 |
$self->update_mappings(); |
| 262 |
} |
362 |
} |
| 263 |
|
363 |
|
|
|
364 |
=head2 index_exists |
| 365 |
|
| 366 |
Checks if index has been created on the ElasticSearch server. Returns C<1> or the |
| 367 |
empty string to indicate whether index exists or not. |
| 368 |
|
| 369 |
=cut |
| 370 |
|
| 264 |
sub index_exists { |
371 |
sub index_exists { |
| 265 |
my ($self) = @_; |
372 |
my ($self) = @_; |
| 266 |
my $conf = $self->get_elasticsearch_params(); |
373 |
my $conf = $self->get_elasticsearch_params(); |