Lines 44-88
Koha::SearchEngine::Elasticsearch::Indexer - handles adding new records to the i
Link Here
|
44 |
$indexer->drop_index(); |
44 |
$indexer->drop_index(); |
45 |
$indexer->update_index(\@biblionumbers, \@records); |
45 |
$indexer->update_index(\@biblionumbers, \@records); |
46 |
|
46 |
|
47 |
=head1 FUNCTIONS |
|
|
48 |
|
47 |
|
49 |
=head2 $indexer->update_index($biblionums, $records); |
48 |
=head1 CONSTANTS |
50 |
|
49 |
|
51 |
C<$biblionums> is an arrayref containing the biblionumbers for the records. |
50 |
=over 4 |
52 |
|
51 |
|
53 |
C<$records> is an arrayref containing the L<MARC::Record>s themselves. |
52 |
=item C<Koha::SearchEngine::Elasticsearch::Indexer::INDEX_STATUS_OK> |
54 |
|
53 |
|
55 |
The values in the arrays must match up, and the 999$c value in the MARC record |
54 |
Represents an index state where index is created and in a working state. |
56 |
will be rewritten using the values in C<$biblionums> to ensure they are correct. |
|
|
57 |
If C<$biblionums> is C<undef>, this won't happen, but you should be sure that |
58 |
999$c is correct on your own then. |
59 |
|
55 |
|
60 |
Note that this will modify the original record if C<$biblionums> is supplied. |
56 |
=item C<Koha::SearchEngine::Elasticsearch::Indexer::INDEX_STATUS_REINDEX_REQUIRED> |
61 |
If that's a problem, clone them first. |
57 |
|
|
|
58 |
Not currently used, but could be useful later, for example if can detect when new field or mapping added. |
59 |
|
60 |
=item C<Koha::SearchEngine::Elasticsearch::Indexer::INDEX_STATUS_RECREATE_REQUIRED> |
61 |
|
62 |
Representings an index state where index needs to be recreated and is not in a working state. |
63 |
|
64 |
=back |
62 |
|
65 |
|
63 |
=cut |
66 |
=cut |
64 |
|
67 |
|
65 |
use constant { |
68 |
use constant { |
66 |
INDEX_STATUS_OK => 0, |
69 |
INDEX_STATUS_OK => 0, |
67 |
INDEX_STATUS_REINDEX_REQUIRED => 1, # Not currently used, but could be useful later, for example if can detect when new field or mapping added |
70 |
INDEX_STATUS_REINDEX_REQUIRED => 1, |
68 |
INDEX_STATUS_RECREATE_REQUIRED => 2, |
71 |
INDEX_STATUS_RECREATE_REQUIRED => 2, |
69 |
}; |
72 |
}; |
70 |
|
73 |
|
|
|
74 |
=head1 FUNCTIONS |
75 |
|
76 |
=head2 update_index($biblionums, $records) |
77 |
|
78 |
try { |
79 |
$self->update_index($biblionums, $records); |
80 |
} catch { |
81 |
die("Something whent wrong trying to update index:" . $_[0]); |
82 |
} |
83 |
|
84 |
Converts C<MARC::Records> C<$records> to Elasticsearch documents and perform |
85 |
an update request for these records on the Elasticsearch index. |
86 |
|
87 |
The values in the arrays must match up, and the 999$c value in the MARC record |
88 |
will be rewritten using the values in C<$biblionums> to ensure they are correct. |
89 |
If C<$biblionums> is C<undef>, this won't happen, so in that case you should make |
90 |
sure that 999$c is correct. |
91 |
|
92 |
Note that this will modify the original record if C<$biblionums> is supplied. |
93 |
If that's a problem, clone them first. |
94 |
|
95 |
=over 4 |
96 |
|
97 |
=item C<$biblionums> |
98 |
|
99 |
Arrayref of biblio numbers for the C<$records>, the order must be the same |
100 |
as C<$records>. |
101 |
|
102 |
=item C<$records> |
103 |
|
104 |
Arrayref of C<MARC::Record>s. |
105 |
|
106 |
=back |
107 |
|
108 |
=cut |
109 |
|
71 |
sub update_index { |
110 |
sub update_index { |
72 |
my ($self, $biblionums, $records) = @_; |
111 |
my ($self, $biblionums, $records) = @_; |
73 |
|
112 |
|
74 |
# TODO should have a separate path for dealing with a large number |
|
|
75 |
# of records at once where we use the bulk update functions in ES. |
76 |
if ($biblionums) { |
113 |
if ($biblionums) { |
77 |
$self->_sanitise_records($biblionums, $records); |
114 |
$self->_sanitise_records($biblionums, $records); |
78 |
} |
115 |
} |
79 |
|
116 |
|
80 |
$self->bulk_index($records); |
|
|
81 |
return 1; |
82 |
} |
83 |
|
84 |
sub bulk_index { |
85 |
my ($self, $records) = @_; |
86 |
my $conf = $self->get_elasticsearch_params(); |
117 |
my $conf = $self->get_elasticsearch_params(); |
87 |
my $elasticsearch = $self->get_elasticsearch(); |
118 |
my $elasticsearch = $self->get_elasticsearch(); |
88 |
my $documents = $self->marc_records_to_documents($records); |
119 |
my $documents = $self->marc_records_to_documents($records); |
Lines 108-134
sub bulk_index {
Link Here
|
108 |
return 1; |
139 |
return 1; |
109 |
} |
140 |
} |
110 |
|
141 |
|
111 |
sub index_status_ok { |
142 |
=head2 set_index_status_ok |
|
|
143 |
|
144 |
Convenience method for setting index status to C<INDEX_STATUS_OK>. |
145 |
|
146 |
=cut |
147 |
|
148 |
sub set_index_status_ok { |
112 |
my ($self, $set) = @_; |
149 |
my ($self, $set) = @_; |
113 |
return defined $set ? |
150 |
$self->index_status(INDEX_STATUS_OK); |
114 |
$self->index_status(INDEX_STATUS_OK) : |
|
|
115 |
$self->index_status == INDEX_STATUS_OK; |
116 |
} |
151 |
} |
117 |
|
152 |
|
118 |
sub index_status_reindex_required { |
153 |
=head2 is_index_status_ok |
119 |
my ($self, $set) = @_; |
154 |
|
120 |
return defined $set ? |
155 |
Convenience method for checking if index status is C<INDEX_STATUS_OK>. |
121 |
$self->index_status(INDEX_STATUS_REINDEX_REQUIRED) : |
156 |
|
122 |
$self->index_status == INDEX_STATUS_REINDEX_REQUIRED; |
157 |
=cut |
|
|
158 |
|
159 |
sub is_index_status_ok { |
160 |
return $self->index_status == INDEX_STATUS_OK; |
123 |
} |
161 |
} |
124 |
|
162 |
|
125 |
sub index_status_recreate_required { |
163 |
=head2 set_index_status_reindex_required |
126 |
my ($self, $set) = @_; |
164 |
|
127 |
return defined $set ? |
165 |
Convenience method for setting index status to C<INDEX_REINDEX_REQUIRED>. |
128 |
$self->index_status(INDEX_STATUS_RECREATE_REQUIRED) : |
166 |
|
129 |
$self->index_status == INDEX_STATUS_RECREATE_REQUIRED; |
167 |
=cut |
|
|
168 |
|
169 |
sub set_index_status_reindex_required { |
170 |
$self->index_status(INDEX_STATUS_REINDEX_REQUIRED); |
171 |
} |
172 |
|
173 |
=head2 is_index_status_reindex_required |
174 |
|
175 |
Convenience method for checking if index status is C<INDEX_STATUS_REINDEX_REQUIRED>. |
176 |
|
177 |
=cut |
178 |
|
179 |
sub is_index_status_reindex_required { |
180 |
return $self->index_status == INDEX_STATUS_REINDEX_REQUIRED; |
181 |
} |
182 |
|
183 |
=head2 set_index_status_recreate_required |
184 |
|
185 |
Convenience method for setting index status to C<INDEX_STATUS_RECREATE_REQUIRED>. |
186 |
|
187 |
=cut |
188 |
|
189 |
sub set_index_status_recreate_required { |
190 |
$self->index_status(INDEX_STATUS_RECREATE_REQUIRED); |
191 |
} |
192 |
|
193 |
=head2 is_index_status_recreate_required |
194 |
|
195 |
Convenience method for checking if index status is C<INDEX_STATUS_RECREATE_REQUIRED>. |
196 |
|
197 |
=cut |
198 |
|
199 |
sub is_index_status_recreate_required { |
200 |
return $self->index_status == INDEX_STATUS_RECREATE_REQUIRED; |
130 |
} |
201 |
} |
131 |
|
202 |
|
|
|
203 |
=head2 index_status($status) |
204 |
|
205 |
Will either set the current index status to C<$status> and return C<$status>, |
206 |
or return the current index status if called with no arguments. |
207 |
|
208 |
=over 4 |
209 |
|
210 |
=item C<$status> |
211 |
|
212 |
Optional argument. If passed will set current index status to C<$status> if C<$status> is |
213 |
a valid status. See L</CONSTANTS>. |
214 |
|
215 |
=back |
216 |
|
217 |
=cut |
218 |
|
132 |
sub index_status { |
219 |
sub index_status { |
133 |
my ($self, $status) = @_; |
220 |
my ($self, $status) = @_; |
134 |
my $key = 'ElasticsearchIndexStatus_' . $self->index; |
221 |
my $key = 'ElasticsearchIndexStatus_' . $self->index; |
Lines 150-155
sub index_status {
Link Here
|
150 |
} |
237 |
} |
151 |
} |
238 |
} |
152 |
|
239 |
|
|
|
240 |
=head2 update_mappings |
241 |
|
242 |
Generate Elasticsearch mappings from mappings stored in database and |
243 |
perform a request to update Elasticsearch index mappings. Will throw an |
244 |
error and set index status to C<INDEX_STATUS_RECREATE_REQUIRED> if update |
245 |
failes. |
246 |
|
247 |
=cut |
248 |
|
153 |
sub update_mappings { |
249 |
sub update_mappings { |
154 |
my ($self) = @_; |
250 |
my ($self) = @_; |
155 |
my $conf = $self->get_elasticsearch_params(); |
251 |
my $conf = $self->get_elasticsearch_params(); |
Lines 166-200
sub update_mappings {
Link Here
|
166 |
} |
262 |
} |
167 |
); |
263 |
); |
168 |
} catch { |
264 |
} catch { |
169 |
$self->index_status_recreate_required(1); |
265 |
$self->set_index_status_recreate_required(); |
170 |
my $reason = $_[0]->{vars}->{body}->{error}->{reason}; |
266 |
my $reason = $_[0]->{vars}->{body}->{error}->{reason}; |
171 |
Koha::Exceptions::Exception->throw( |
267 |
Koha::Exceptions::Exception->throw( |
172 |
error => "Unable to update mappings for index \"$conf->{index_name}\". Reason was: \"$reason\". Index needs to be recreated and reindexed", |
268 |
error => "Unable to update mappings for index \"$conf->{index_name}\". Reason was: \"$reason\". Index needs to be recreated and reindexed", |
173 |
); |
269 |
); |
174 |
}; |
270 |
}; |
175 |
} |
271 |
} |
176 |
$self->index_status_ok(1); |
272 |
$self->set_index_status_ok(); |
177 |
} |
273 |
} |
178 |
|
274 |
|
179 |
=head2 $indexer->update_index_background($biblionums, $records) |
275 |
=head2 update_index_background($biblionums, $records) |
180 |
|
276 |
|
181 |
This has exactly the same API as C<update_index_background> however it'll |
277 |
This has exactly the same API as C<update_index> however it'll |
182 |
return immediately. It'll start a background process that does the adding. |
278 |
return immediately. It'll start a background process that does the adding. |
183 |
|
279 |
|
184 |
If it fails to add to Elasticsearch then it'll add to a queue that will cause |
280 |
If it fails to add to Elasticsearch then it'll add to a queue that will cause |
185 |
it to be updated by a regular index cron job in the future. |
281 |
it to be updated by a regular index cron job in the future. |
186 |
|
282 |
|
|
|
283 |
=cut |
284 |
|
187 |
# TODO implement in the future - I don't know the best way of doing this yet. |
285 |
# TODO implement in the future - I don't know the best way of doing this yet. |
188 |
# If fork: make sure process group is changed so apache doesn't wait for us. |
286 |
# If fork: make sure process group is changed so apache doesn't wait for us. |
189 |
|
287 |
|
190 |
=cut |
|
|
191 |
|
192 |
sub update_index_background { |
288 |
sub update_index_background { |
193 |
my $self = shift; |
289 |
my $self = shift; |
194 |
$self->update_index(@_); |
290 |
$self->update_index(@_); |
195 |
} |
291 |
} |
196 |
|
292 |
|
197 |
=head2 $indexer->delete_index($biblionums) |
293 |
=head2 delete_index($biblionums) |
198 |
|
294 |
|
199 |
C<$biblionums> is an arrayref of biblionumbers to delete from the index. |
295 |
C<$biblionums> is an arrayref of biblionumbers to delete from the index. |
200 |
|
296 |
|
Lines 206-212
sub delete_index {
Link Here
|
206 |
if ( !$self->store ) { |
302 |
if ( !$self->store ) { |
207 |
my $params = $self->get_elasticsearch_params(); |
303 |
my $params = $self->get_elasticsearch_params(); |
208 |
$self->store( |
304 |
$self->store( |
209 |
Catmandu::Store::ElasticSearch->new( |
305 |
Catmandu::Store::Elasticsearch->new( |
210 |
%$params, |
306 |
%$params, |
211 |
index_settings => $self->get_elasticsearch_settings(), |
307 |
index_settings => $self->get_elasticsearch_settings(), |
212 |
index_mappings => $self->get_elasticsearch_mappings(), |
308 |
index_mappings => $self->get_elasticsearch_mappings(), |
Lines 217-239
sub delete_index {
Link Here
|
217 |
$self->store->bag->commit; |
313 |
$self->store->bag->commit; |
218 |
} |
314 |
} |
219 |
|
315 |
|
220 |
=head2 $indexer->delete_index_background($biblionums) |
316 |
=head2 delete_index_background($biblionums) |
221 |
|
317 |
|
222 |
Identical to L<delete_index>, this will return immediately and start a |
318 |
Identical to L</delete_index($biblionums)> |
223 |
background process to do the actual deleting. |
|
|
224 |
|
319 |
|
225 |
=cut |
320 |
=cut |
226 |
|
321 |
|
227 |
# TODO implement in the future |
322 |
# TODO: Should be made async |
228 |
|
|
|
229 |
sub delete_index_background { |
323 |
sub delete_index_background { |
230 |
my $self = shift; |
324 |
my $self = shift; |
231 |
$self->delete_index(@_); |
325 |
$self->delete_index(@_); |
232 |
} |
326 |
} |
233 |
|
327 |
|
234 |
=head2 $indexer->drop_index(); |
328 |
=head2 drop_index |
235 |
|
329 |
|
236 |
Drops the index from the elasticsearch server. |
330 |
Drops the index from the Elasticsearch server. |
237 |
|
331 |
|
238 |
=cut |
332 |
=cut |
239 |
|
333 |
|
Lines 243-252
sub drop_index {
Link Here
|
243 |
my $conf = $self->get_elasticsearch_params(); |
337 |
my $conf = $self->get_elasticsearch_params(); |
244 |
my $elasticsearch = $self->get_elasticsearch(); |
338 |
my $elasticsearch = $self->get_elasticsearch(); |
245 |
$elasticsearch->indices->delete(index => $conf->{index_name}); |
339 |
$elasticsearch->indices->delete(index => $conf->{index_name}); |
246 |
$self->index_status_recreate_required(1); |
340 |
$self->set_index_status_recreate_required(); |
247 |
} |
341 |
} |
248 |
} |
342 |
} |
249 |
|
343 |
|
|
|
344 |
=head2 create_index |
345 |
|
346 |
Creates the index (including mappings) on the Elasticsearch server. |
347 |
|
348 |
=cut |
349 |
|
250 |
sub create_index { |
350 |
sub create_index { |
251 |
my ($self) = @_; |
351 |
my ($self) = @_; |
252 |
my $conf = $self->get_elasticsearch_params(); |
352 |
my $conf = $self->get_elasticsearch_params(); |
Lines 261-266
sub create_index {
Link Here
|
261 |
$self->update_mappings(); |
361 |
$self->update_mappings(); |
262 |
} |
362 |
} |
263 |
|
363 |
|
|
|
364 |
=head2 index_exists |
365 |
|
366 |
Checks if index has been created on the ElasticSearch server. Returns C<1> or the |
367 |
empty string to indicate whether index exists or not. |
368 |
|
369 |
=cut |
370 |
|
264 |
sub index_exists { |
371 |
sub index_exists { |
265 |
my ($self) = @_; |
372 |
my ($self) = @_; |
266 |
my $conf = $self->get_elasticsearch_params(); |
373 |
my $conf = $self->get_elasticsearch_params(); |