View | Details | Raw Unified | Return to bug 19893
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-8 / +55 lines)
Lines 73-78 sub new { Link Here
73
    return $self;
73
    return $self;
74
}
74
}
75
75
76
=head2 get_elasticsearch
77
78
    my $elasticsearch_client = $self->get_elasticsearch();
79
80
Returns a C<Search::Elasticsearch> client. The client is cached on a C<Koha::SearchEngine::ElasticSearch>
81
instance level and will be reused if method is called multiple times.
82
83
=cut
84
76
sub get_elasticsearch {
85
sub get_elasticsearch {
77
    my $self = shift @_;
86
    my $self = shift @_;
78
    unless (defined $self->{elasticsearch}) {
87
    unless (defined $self->{elasticsearch}) {
Lines 145-152 sub get_elasticsearch_params { Link Here
145
154
146
    my $settings = $self->get_elasticsearch_settings();
155
    my $settings = $self->get_elasticsearch_settings();
147
156
148
This provides the settings provided to elasticsearch when an index is created.
157
This provides the settings provided to Elasticsearch when an index is created.
149
These can do things like define tokenisation methods.
158
These can do things like define tokenization methods.
150
159
151
A hashref containing the settings is returned.
160
A hashref containing the settings is returned.
152
161
Lines 170-176 sub get_elasticsearch_settings { Link Here
170
179
171
    my $mappings = $self->get_elasticsearch_mappings();
180
    my $mappings = $self->get_elasticsearch_mappings();
172
181
173
This provides the mappings that get passed to elasticsearch when an index is
182
This provides the mappings that get passed to Elasticsearch when an index is
174
created.
183
created.
175
184
176
=cut
185
=cut
Lines 232-238 sub get_elasticsearch_mappings { Link Here
232
241
233
=head2 _get_elasticsearch_mapping
242
=head2 _get_elasticsearch_mapping
234
243
235
Get the ES mappings for the given purpose and data type
244
Get the Elasticsearch mappings for the given purpose and data type.
236
245
237
$mapping = _get_elasticsearch_mapping('search', 'text');
246
$mapping = _get_elasticsearch_mapping('search', 'text');
238
247
Lines 301-313 sub sort_fields { Link Here
301
    return $self->_sort_fields_accessor();
310
    return $self->_sort_fields_accessor();
302
}
311
}
303
312
313
=head2 marc_records_to_documents($marc_records)
314
315
    my @record_documents = $self->marc_records_to_documents($marc_records);
316
317
Using mappings stored in database convert C<$marc_records> to ElasticSearch documents.
318
319
Returns array of hash references, representing ElasticSearch documents,
320
acceptable as body payload in C<Search::Elasticsearch> requests.
321
322
=over 4
323
324
=item C<$marc_documents>
325
326
Reference to array of C<MARC::Record> objects to be converted to ElasticSearch documents.
327
328
=back
329
330
=cut
331
304
sub marc_records_to_documents {
332
sub marc_records_to_documents {
305
    my ($self, $records) = @_;
333
    my ($self, $records) = @_;
306
    my $rules = $self->get_marc_mapping_rules();
334
    my $rules = $self->_get_marc_mapping_rules();
307
    my $control_fields_rules = $rules->{control_fields};
335
    my $control_fields_rules = $rules->{control_fields};
308
    my $data_fields_rules = $rules->{data_fields};
336
    my $data_fields_rules = $rules->{data_fields};
309
    my $marcflavour = lc C4::Context->preference('marcflavour');
337
    my $marcflavour = lc C4::Context->preference('marcflavour');
310
    my $serialization_format = C4::Context->preference('ElasticsearchMARCSerializationFormat');
311
338
312
    my @record_documents;
339
    my @record_documents;
313
340
Lines 426-433 sub marc_records_to_documents { Link Here
426
    return \@record_documents;
453
    return \@record_documents;
427
}
454
}
428
455
429
# Provides the rules for marc to Elasticsearch JSON document conversion.
456
=head2 _get_marc_mapping_rules
430
sub get_marc_mapping_rules {
457
458
    my $mapping_rules = $self->_get_marc_mapping_rules()
459
460
Generates rules from mappings stored in database for MARC records to Elasticsearch JSON document conversion.
461
462
Since field retrieval is slow in C<MARC::Records> (all fields are itereted through for
463
each call to C<MARC::Record>->field) we create an optimized structure of mapping
464
rules keyed by MARC field tags holding all the mapping rules for that particular tag.
465
466
We can then iterate through all MARC fields for each record and apply all relevant
467
rules once per fields instead of retreiving fields multiple times for each mapping rule
468
wich is terribly slow.
469
470
=cut
471
472
# TODO: This structure can be used for processing multiple MARC::Records so is currently
473
# rebuilt for each batch. Since it is cacheable it could also be stored in an in
474
# memory cache which it is currently not. The performance gain of caching
475
# would probably be marginal, but to do this could be a further improvement.
476
477
sub _get_marc_mapping_rules {
431
    my ($self) = @_;
478
    my ($self) = @_;
432
479
433
    my $marcflavour = lc C4::Context->preference('marcflavour');
480
    my $marcflavour = lc C4::Context->preference('marcflavour');
(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-49 / +161 lines)
Lines 44-88 Koha::SearchEngine::Elasticsearch::Indexer - handles adding new records to the i Link Here
44
    $indexer->drop_index();
44
    $indexer->drop_index();
45
    $indexer->update_index(\@biblionumbers, \@records);
45
    $indexer->update_index(\@biblionumbers, \@records);
46
46
47
=head1 FUNCTIONS
48
47
49
=head2 $indexer->update_index($biblionums, $records);
48
=head1 CONSTANTS
50
49
51
C<$biblionums> is an arrayref containing the biblionumbers for the records.
50
=over 4
52
51
53
C<$records> is an arrayref containing the L<MARC::Record>s themselves.
52
=item C<Koha::SearchEngine::Elasticsearch::Indexer::INDEX_STATUS_OK>
54
53
55
The values in the arrays must match up, and the 999$c value in the MARC record
54
Represents an index state where index is created and in a working state.
56
will be rewritten using the values in C<$biblionums> to ensure they are correct.
57
If C<$biblionums> is C<undef>, this won't happen, but you should be sure that
58
999$c is correct on your own then.
59
55
60
Note that this will modify the original record if C<$biblionums> is supplied.
56
=item C<Koha::SearchEngine::Elasticsearch::Indexer::INDEX_STATUS_REINDEX_REQUIRED>
61
If that's a problem, clone them first.
57
58
Not currently used, but could be useful later, for example if can detect when new field or mapping added.
59
60
=item C<Koha::SearchEngine::Elasticsearch::Indexer::INDEX_STATUS_RECREATE_REQUIRED>
61
62
Representings an index state where index needs to be recreated and is not in a working state.
63
64
=back
62
65
63
=cut
66
=cut
64
67
65
use constant {
68
use constant {
66
    INDEX_STATUS_OK => 0,
69
    INDEX_STATUS_OK => 0,
67
    INDEX_STATUS_REINDEX_REQUIRED => 1, # Not currently used, but could be useful later, for example if can detect when new field or mapping added
70
    INDEX_STATUS_REINDEX_REQUIRED => 1,
68
    INDEX_STATUS_RECREATE_REQUIRED => 2,
71
    INDEX_STATUS_RECREATE_REQUIRED => 2,
69
};
72
};
70
73
74
=head1 FUNCTIONS
75
76
=head2 update_index($biblionums, $records)
77
78
    try {
79
        $self->update_index($biblionums, $records);
80
    } catch {
81
        die("Something whent wrong trying to update index:" .  $_[0]);
82
    }
83
84
Converts C<MARC::Records> C<$records> to Elasticsearch documents and performs
85
an update request for these records on the Elasticsearch index.
86
87
The values in the arrays must match up, and the 999$c value in the MARC record
88
will be rewritten using the values in C<$biblionums> to ensure they are correct.
89
If C<$biblionums> is C<undef>, this won't happen, so in that case you should make
90
sure that 999$c is correct.
91
92
Note that this will modify the original record if C<$biblionums> is supplied.
93
If that's a problem, clone them first.
94
95
=over 4
96
97
=item C<$biblionums>
98
99
Arrayref of biblio numbers for the C<$records>, the order must be the same as
100
and match up with C<$records>.
101
102
=item C<$records>
103
104
Arrayref of C<MARC::Record>s.
105
106
=back
107
108
=cut
109
71
sub update_index {
110
sub update_index {
72
    my ($self, $biblionums, $records) = @_;
111
    my ($self, $biblionums, $records) = @_;
73
112
74
    # TODO should have a separate path for dealing with a large number
75
    # of records at once where we use the bulk update functions in ES.
76
    if ($biblionums) {
113
    if ($biblionums) {
77
        $self->_sanitise_records($biblionums, $records);
114
        $self->_sanitise_records($biblionums, $records);
78
    }
115
    }
79
116
80
    $self->bulk_index($records);
81
    return 1;
82
}
83
84
sub bulk_index {
85
    my ($self, $records) = @_;
86
    my $conf = $self->get_elasticsearch_params();
117
    my $conf = $self->get_elasticsearch_params();
87
    my $elasticsearch = $self->get_elasticsearch();
118
    my $elasticsearch = $self->get_elasticsearch();
88
    my $documents = $self->marc_records_to_documents($records);
119
    my $documents = $self->marc_records_to_documents($records);
Lines 108-134 sub bulk_index { Link Here
108
    return 1;
139
    return 1;
109
}
140
}
110
141
111
sub index_status_ok {
142
=head2 set_index_status_ok
112
    my ($self, $set) = @_;
143
113
    return defined $set ?
144
Convenience method for setting index status to C<INDEX_STATUS_OK>.
114
        $self->index_status(INDEX_STATUS_OK) :
145
115
        $self->index_status == INDEX_STATUS_OK;
146
=cut
147
148
sub set_index_status_ok {
149
    my ($self) = @_;
150
    $self->index_status(INDEX_STATUS_OK);
116
}
151
}
117
152
118
sub index_status_reindex_required {
153
=head2 is_index_status_ok
119
    my ($self, $set) = @_;
154
120
    return defined $set ?
155
Convenience method for checking if index status is C<INDEX_STATUS_OK>.
121
        $self->index_status(INDEX_STATUS_REINDEX_REQUIRED) :
156
122
        $self->index_status == INDEX_STATUS_REINDEX_REQUIRED;
157
=cut
158
159
sub is_index_status_ok {
160
    my ($self) = @_;
161
    return $self->index_status == INDEX_STATUS_OK;
123
}
162
}
124
163
125
sub index_status_recreate_required {
164
=head2 set_index_status_reindex_required
126
    my ($self, $set) = @_;
165
127
    return defined $set ?
166
Convenience method for setting index status to C<INDEX_REINDEX_REQUIRED>.
128
        $self->index_status(INDEX_STATUS_RECREATE_REQUIRED) :
167
129
        $self->index_status == INDEX_STATUS_RECREATE_REQUIRED;
168
=cut
169
170
sub set_index_status_reindex_required {
171
    my ($self) = @_;
172
    $self->index_status(INDEX_STATUS_REINDEX_REQUIRED);
130
}
173
}
131
174
175
=head2 is_index_status_reindex_required
176
177
Convenience method for checking if index status is C<INDEX_STATUS_REINDEX_REQUIRED>.
178
179
=cut
180
181
sub is_index_status_reindex_required {
182
    my ($self) = @_;
183
    return $self->index_status == INDEX_STATUS_REINDEX_REQUIRED;
184
}
185
186
=head2 set_index_status_recreate_required
187
188
Convenience method for setting index status to C<INDEX_STATUS_RECREATE_REQUIRED>.
189
190
=cut
191
192
sub set_index_status_recreate_required {
193
    my ($self) = @_;
194
    $self->index_status(INDEX_STATUS_RECREATE_REQUIRED);
195
}
196
197
=head2 is_index_status_recreate_required
198
199
Convenience method for checking if index status is C<INDEX_STATUS_RECREATE_REQUIRED>.
200
201
=cut
202
203
sub is_index_status_recreate_required {
204
    my ($self) = @_;
205
    return $self->index_status == INDEX_STATUS_RECREATE_REQUIRED;
206
}
207
208
=head2 index_status($status)
209
210
Will either set the current index status to C<$status> and return C<$status>,
211
or return the current index status if called with no arguments.
212
213
=over 4
214
215
=item C<$status>
216
217
Optional argument. If passed will set current index status to C<$status> if C<$status> is
218
a valid status. See L</CONSTANTS>.
219
220
=back
221
222
=cut
223
132
sub index_status {
224
sub index_status {
133
    my ($self, $status) = @_;
225
    my ($self, $status) = @_;
134
    my $key = 'ElasticsearchIndexStatus_' . $self->index;
226
    my $key = 'ElasticsearchIndexStatus_' . $self->index;
Lines 150-155 sub index_status { Link Here
150
    }
242
    }
151
}
243
}
152
244
245
=head2 update_mappings
246
247
Generate Elasticsearch mappings from mappings stored in database and
248
perform a request to update Elasticsearch index mappings. Will throw an
249
error and set index status to C<INDEX_STATUS_RECREATE_REQUIRED> if update
250
failes.
251
252
=cut
253
153
sub update_mappings {
254
sub update_mappings {
154
    my ($self) = @_;
255
    my ($self) = @_;
155
    my $conf = $self->get_elasticsearch_params();
256
    my $conf = $self->get_elasticsearch_params();
Lines 166-200 sub update_mappings { Link Here
166
                }
267
                }
167
            );
268
            );
168
        } catch {
269
        } catch {
169
            $self->index_status_recreate_required(1);
270
            $self->set_index_status_recreate_required();
170
            my $reason = $_[0]->{vars}->{body}->{error}->{reason};
271
            my $reason = $_[0]->{vars}->{body}->{error}->{reason};
171
            Koha::Exceptions::Exception->throw(
272
            Koha::Exceptions::Exception->throw(
172
                error => "Unable to update mappings for index \"$conf->{index_name}\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
273
                error => "Unable to update mappings for index \"$conf->{index_name}\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
173
            );
274
            );
174
        };
275
        };
175
    }
276
    }
176
    $self->index_status_ok(1);
277
    $self->set_index_status_ok();
177
}
278
}
178
279
179
=head2 $indexer->update_index_background($biblionums, $records)
280
=head2 update_index_background($biblionums, $records)
180
281
181
This has exactly the same API as C<update_index_background> however it'll
282
This has exactly the same API as C<update_index> however it'll
182
return immediately. It'll start a background process that does the adding.
283
return immediately. It'll start a background process that does the adding.
183
284
184
If it fails to add to Elasticsearch then it'll add to a queue that will cause
285
If it fails to add to Elasticsearch then it'll add to a queue that will cause
185
it to be updated by a regular index cron job in the future.
286
it to be updated by a regular index cron job in the future.
186
287
288
=cut
289
187
# TODO implement in the future - I don't know the best way of doing this yet.
290
# TODO implement in the future - I don't know the best way of doing this yet.
188
# If fork: make sure process group is changed so apache doesn't wait for us.
291
# If fork: make sure process group is changed so apache doesn't wait for us.
189
292
190
=cut
191
192
sub update_index_background {
293
sub update_index_background {
193
    my $self = shift;
294
    my $self = shift;
194
    $self->update_index(@_);
295
    $self->update_index(@_);
195
}
296
}
196
297
197
=head2 $indexer->delete_index($biblionums)
298
=head2 delete_index($biblionums)
198
299
199
C<$biblionums> is an arrayref of biblionumbers to delete from the index.
300
C<$biblionums> is an arrayref of biblionumbers to delete from the index.
200
301
Lines 217-239 sub delete_index { Link Here
217
    $self->store->bag->commit;
318
    $self->store->bag->commit;
218
}
319
}
219
320
220
=head2 $indexer->delete_index_background($biblionums)
321
=head2 delete_index_background($biblionums)
221
322
222
Identical to L<delete_index>, this will return immediately and start a
323
Identical to L</delete_index($biblionums)>
223
background process to do the actual deleting.
224
324
225
=cut
325
=cut
226
326
227
# TODO implement in the future
327
# TODO: Should be made async
228
229
sub delete_index_background {
328
sub delete_index_background {
230
    my $self = shift;
329
    my $self = shift;
231
    $self->delete_index(@_);
330
    $self->delete_index(@_);
232
}
331
}
233
332
234
=head2 $indexer->drop_index();
333
=head2 drop_index
235
334
236
Drops the index from the elasticsearch server.
335
Drops the index from the Elasticsearch server.
237
336
238
=cut
337
=cut
239
338
Lines 243-252 sub drop_index { Link Here
243
        my $conf = $self->get_elasticsearch_params();
342
        my $conf = $self->get_elasticsearch_params();
244
        my $elasticsearch = $self->get_elasticsearch();
343
        my $elasticsearch = $self->get_elasticsearch();
245
        $elasticsearch->indices->delete(index => $conf->{index_name});
344
        $elasticsearch->indices->delete(index => $conf->{index_name});
246
        $self->index_status_recreate_required(1);
345
        $self->set_index_status_recreate_required();
247
    }
346
    }
248
}
347
}
249
348
349
=head2 create_index
350
351
Creates the index (including mappings) on the Elasticsearch server.
352
353
=cut
354
250
sub create_index {
355
sub create_index {
251
    my ($self) = @_;
356
    my ($self) = @_;
252
    my $conf = $self->get_elasticsearch_params();
357
    my $conf = $self->get_elasticsearch_params();
Lines 261-266 sub create_index { Link Here
261
    $self->update_mappings();
366
    $self->update_mappings();
262
}
367
}
263
368
369
=head2 index_exists
370
371
Checks if index has been created on the ElasticSearch server. Returns C<1> or the
372
empty string to indicate whether index exists or not.
373
374
=cut
375
264
sub index_exists {
376
sub index_exists {
265
    my ($self) = @_;
377
    my ($self) = @_;
266
    my $conf = $self->get_elasticsearch_params();
378
    my $conf = $self->get_elasticsearch_params();
(-)a/admin/searchengine/elasticsearch/mappings.pl (-3 / +3 lines)
Lines 135-150 my @indexes; Link Here
135
135
136
for my $index_name (@index_names) {
136
for my $index_name (@index_names) {
137
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
137
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
138
    if (!$indexer->index_status_ok) {
138
    if (!$indexer->is_index_status_ok) {
139
        my $conf = $indexer->get_elasticsearch_params();
139
        my $conf = $indexer->get_elasticsearch_params();
140
        if ($indexer->index_status_reindex_required) {
140
        if ($indexer->is_index_status_reindex_required) {
141
            push @messages, {
141
            push @messages, {
142
                type => 'error',
142
                type => 'error',
143
                code => 'reindex_required',
143
                code => 'reindex_required',
144
                index => $conf->{index_name},
144
                index => $conf->{index_name},
145
            };
145
            };
146
        }
146
        }
147
        elsif($indexer->index_status_recreate_required) {
147
        elsif($indexer->is_index_status_recreate_required) {
148
            push @messages, {
148
            push @messages, {
149
                type => 'error',
149
                type => 'error',
150
                code => 'recreate_required',
150
                code => 'recreate_required',
(-)a/misc/search_tools/rebuild_elastic_search.pl (-3 / +3 lines)
Lines 165-177 sub do_reindex { Link Here
165
        $indexer->drop_index() if $indexer->index_exists();
165
        $indexer->drop_index() if $indexer->index_exists();
166
        $indexer->create_index();
166
        $indexer->create_index();
167
    }
167
    }
168
    elsif (!$indexer->index_exists()) {
168
    elsif (!$indexer->index_exists) {
169
        # Create index if does not exist
169
        # Create index if does not exist
170
        $indexer->create_index();
170
        $indexer->create_index();
171
    } elsif ($indexer->index_status_ok) {
171
    } elsif ($indexer->is_index_status_ok) {
172
        # Update mapping unless index is some kind of problematic state
172
        # Update mapping unless index is some kind of problematic state
173
        $indexer->update_mappings();
173
        $indexer->update_mappings();
174
    } elsif ($indexer->index_status_recreate_required) {
174
    } elsif ($indexer->is_index_status_recreate_required) {
175
        warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;
175
        warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;
176
    }
176
    }
177
177
(-)a/t/Koha/SearchEngine/Elasticsearch.t (-2 / +26 lines)
Lines 115-121 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
115
115
116
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
116
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
117
117
118
    plan tests => 30;
118
    plan tests => 32;
119
119
120
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
120
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
121
121
Lines 315-320 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
315
315
316
    ok(defined $docs->[0][1]->{marc_format}, 'First document marc_format field should be set');
316
    ok(defined $docs->[0][1]->{marc_format}, 'First document marc_format field should be set');
317
317
318
    is($docs->[0][1]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');
319
318
    is(scalar @{$docs->[0][1]->{type_of_record}}, 1, 'First document type_of_record field should have one value');
320
    is(scalar @{$docs->[0][1]->{type_of_record}}, 1, 'First document type_of_record field should have one value');
319
    is_deeply(
321
    is_deeply(
320
        $docs->[0][1]->{type_of_record},
322
        $docs->[0][1]->{type_of_record},
Lines 351-354 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
351
353
352
    ok(!(defined $docs->[0][1]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
354
    ok(!(defined $docs->[0][1]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
353
355
356
    # Marc serialization format fallback for records exceeding ISO2709 max record size
357
358
    my $large_marc_record = MARC::Record->new();
359
    $large_marc_record->leader('     cam  22      a 4500');
360
361
    $large_marc_record->append_fields(
362
        MARC::Field->new('100', '', '', a => 'Author 1'),
363
        MARC::Field->new('110', '', '', a => 'Corp Author'),
364
        MARC::Field->new('210', '', '', a => 'Title 1'),
365
        MARC::Field->new('245', '', '', a => 'Title:', b => 'large record'),
366
        MARC::Field->new('999', '', '', c => '1234567'),
367
    );
368
369
    my $item_field = MARC::Field->new('952', '', '', o => '123456789123456789123456789', p => '123456789', z => 'test');
370
    my $items_count = 1638;
371
    while(--$items_count) {
372
        $large_marc_record->append_fields($item_field);
373
    }
374
375
    $docs = $see->marc_records_to_documents([$large_marc_record]);
376
377
    is($docs->[0][1]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
378
354
};
379
};
355
- 

Return to bug 19893