View | Details | Raw Unified | Return to bug 19893
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-8 / +55 lines)
Lines 73-78 sub new { Link Here
73
    return $self;
73
    return $self;
74
}
74
}
75
75
76
=head2 get_elasticsearch
77
78
    my $elasticsearch_client = $self->get_elasticsearch();
79
80
Returns a C<Search::Elasticsearch> client. The client is cached on a C<Koha::SearchEngine::ElasticSearch>
81
instance level and will be reused if method is called multiple times.
82
83
=cut
84
76
sub get_elasticsearch {
85
sub get_elasticsearch {
77
    my $self = shift @_;
86
    my $self = shift @_;
78
    unless (defined $self->{elasticsearch}) {
87
    unless (defined $self->{elasticsearch}) {
Lines 145-152 sub get_elasticsearch_params { Link Here
145
154
146
    my $settings = $self->get_elasticsearch_settings();
155
    my $settings = $self->get_elasticsearch_settings();
147
156
148
This provides the settings provided to elasticsearch when an index is created.
157
This provides the settings provided to Elasticsearch when an index is created.
149
These can do things like define tokenisation methods.
158
These can do things like define tokenization methods.
150
159
151
A hashref containing the settings is returned.
160
A hashref containing the settings is returned.
152
161
Lines 170-176 sub get_elasticsearch_settings { Link Here
170
179
171
    my $mappings = $self->get_elasticsearch_mappings();
180
    my $mappings = $self->get_elasticsearch_mappings();
172
181
173
This provides the mappings that get passed to elasticsearch when an index is
182
This provides the mappings that get passed to Elasticsearch when an index is
174
created.
183
created.
175
184
176
=cut
185
=cut
Lines 232-238 sub get_elasticsearch_mappings { Link Here
232
241
233
=head2 _get_elasticsearch_mapping
242
=head2 _get_elasticsearch_mapping
234
243
235
Get the ES mappings for the given purpose and data type
244
Get the Elasticsearch mappings for the given purpose and data type.
236
245
237
$mapping = _get_elasticsearch_mapping('search', 'text');
246
$mapping = _get_elasticsearch_mapping('search', 'text');
238
247
Lines 301-313 sub sort_fields { Link Here
301
    return $self->_sort_fields_accessor();
310
    return $self->_sort_fields_accessor();
302
}
311
}
303
312
313
=head2 marc_records_to_documents($marc_records)
314
315
    my @record_documents = $self->marc_records_to_documents($marc_records);
316
317
Using mappings stored in databse convert C<$marc_records> to ElasticSearch documents.
318
319
Returns array of hash references, representing ElasticSearch documents,
320
acceptable as body payload in C<Search::Elasticsearch> requests.
321
322
=over 4
323
324
=item C<$marc_documents>
325
326
Reference to array of C<MARC::Record> objects to be converted to ElasticSearch documents.
327
328
=back
329
330
=cut
331
304
sub marc_records_to_documents {
332
sub marc_records_to_documents {
305
    my ($self, $records) = @_;
333
    my ($self, $records) = @_;
306
    my $rules = $self->get_marc_mapping_rules();
334
    my $rules = $self->_get_marc_mapping_rules();
307
    my $control_fields_rules = $rules->{control_fields};
335
    my $control_fields_rules = $rules->{control_fields};
308
    my $data_fields_rules = $rules->{data_fields};
336
    my $data_fields_rules = $rules->{data_fields};
309
    my $marcflavour = lc C4::Context->preference('marcflavour');
337
    my $marcflavour = lc C4::Context->preference('marcflavour');
310
    my $serialization_format = C4::Context->preference('ElasticsearchMARCSerializationFormat');
311
338
312
    my @record_documents;
339
    my @record_documents;
313
340
Lines 426-433 sub marc_records_to_documents { Link Here
426
    return \@record_documents;
453
    return \@record_documents;
427
}
454
}
428
455
429
# Provides the rules for marc to Elasticsearch JSON document conversion.
456
=head2 _get_marc_mapping_rules
430
sub get_marc_mapping_rules {
457
458
    my $mapping_rules = $self->_get_marc_mapping_rules()
459
460
Generates rules from mappings stored in databse for MARC records to Elasticsearch JSON document conversion.
461
462
Since field retrieval is slow in C<MARC::Records> (all fields are itereted through for
463
each call to C<MARC::Record>->field) we create an optimized structure of mapping
464
rules keyed by MARC field tags holding all the mapping rules for that particular tag.
465
466
We can then iterate through all MARC fields for each record and apply all relevant
467
rules once per fields instead of retreiving field multiple times for each mapping rule
468
wich is terribly slow.
469
470
=cut
471
472
# TODO: This structure can be used for processing multiple MARC::Records so is currently
473
# rebuilt for each batch. Since it is cacheable it could also be stored in an in
474
# memory cache which it is currently not. The performance gain of caching
475
# would probably be marginal, but to do this could be a further improvement.
476
477
sub _get_marc_mapping_rules {
431
    my ($self) = @_;
478
    my ($self) = @_;
432
479
433
    my $marcflavour = lc C4::Context->preference('marcflavour');
480
    my $marcflavour = lc C4::Context->preference('marcflavour');
(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-49 / +156 lines)
Lines 44-88 Koha::SearchEngine::Elasticsearch::Indexer - handles adding new records to the i Link Here
44
    $indexer->drop_index();
44
    $indexer->drop_index();
45
    $indexer->update_index(\@biblionumbers, \@records);
45
    $indexer->update_index(\@biblionumbers, \@records);
46
46
47
=head1 FUNCTIONS
48
47
49
=head2 $indexer->update_index($biblionums, $records);
48
=head1 CONSTANTS
50
49
51
C<$biblionums> is an arrayref containing the biblionumbers for the records.
50
=over 4
52
51
53
C<$records> is an arrayref containing the L<MARC::Record>s themselves.
52
=item C<Koha::SearchEngine::Elasticsearch::Indexer::INDEX_STATUS_OK>
54
53
55
The values in the arrays must match up, and the 999$c value in the MARC record
54
Represents an index state where index is created and in a working state.
56
will be rewritten using the values in C<$biblionums> to ensure they are correct.
57
If C<$biblionums> is C<undef>, this won't happen, but you should be sure that
58
999$c is correct on your own then.
59
55
60
Note that this will modify the original record if C<$biblionums> is supplied.
56
=item C<Koha::SearchEngine::Elasticsearch::Indexer::INDEX_STATUS_REINDEX_REQUIRED>
61
If that's a problem, clone them first.
57
58
Not currently used, but could be useful later, for example if can detect when new field or mapping added.
59
60
=item C<Koha::SearchEngine::Elasticsearch::Indexer::INDEX_STATUS_RECREATE_REQUIRED>
61
62
Representings an index state where index needs to be recreated and is not in a working state.
63
64
=back
62
65
63
=cut
66
=cut
64
67
65
use constant {
68
use constant {
66
    INDEX_STATUS_OK => 0,
69
    INDEX_STATUS_OK => 0,
67
    INDEX_STATUS_REINDEX_REQUIRED => 1, # Not currently used, but could be useful later, for example if can detect when new field or mapping added
70
    INDEX_STATUS_REINDEX_REQUIRED => 1,
68
    INDEX_STATUS_RECREATE_REQUIRED => 2,
71
    INDEX_STATUS_RECREATE_REQUIRED => 2,
69
};
72
};
70
73
74
=head1 FUNCTIONS
75
76
=head2 update_index($biblionums, $records)
77
78
    try {
79
        $self->update_index($biblionums, $records);
80
    } catch {
81
        die("Something whent wrong trying to update index:" .  $_[0]);
82
    }
83
84
Converts C<MARC::Records> C<$records> to Elasticsearch documents and perform
85
an update request for these records on the Elasticsearch index.
86
87
The values in the arrays must match up, and the 999$c value in the MARC record
88
will be rewritten using the values in C<$biblionums> to ensure they are correct.
89
If C<$biblionums> is C<undef>, this won't happen, so in that case you should make
90
sure that 999$c is correct.
91
92
Note that this will modify the original record if C<$biblionums> is supplied.
93
If that's a problem, clone them first.
94
95
=over 4
96
97
=item C<$biblionums>
98
99
Arrayref of biblio numbers for the C<$records>, the order must be the same
100
as C<$records>.
101
102
=item C<$records>
103
104
Arrayref of C<MARC::Record>s.
105
106
=back
107
108
=cut
109
71
sub update_index {
110
sub update_index {
72
    my ($self, $biblionums, $records) = @_;
111
    my ($self, $biblionums, $records) = @_;
73
112
74
    # TODO should have a separate path for dealing with a large number
75
    # of records at once where we use the bulk update functions in ES.
76
    if ($biblionums) {
113
    if ($biblionums) {
77
        $self->_sanitise_records($biblionums, $records);
114
        $self->_sanitise_records($biblionums, $records);
78
    }
115
    }
79
116
80
    $self->bulk_index($records);
81
    return 1;
82
}
83
84
sub bulk_index {
85
    my ($self, $records) = @_;
86
    my $conf = $self->get_elasticsearch_params();
117
    my $conf = $self->get_elasticsearch_params();
87
    my $elasticsearch = $self->get_elasticsearch();
118
    my $elasticsearch = $self->get_elasticsearch();
88
    my $documents = $self->marc_records_to_documents($records);
119
    my $documents = $self->marc_records_to_documents($records);
Lines 108-134 sub bulk_index { Link Here
108
    return 1;
139
    return 1;
109
}
140
}
110
141
111
sub index_status_ok {
142
=head2 set_index_status_ok
143
144
Convenience method for setting index status to C<INDEX_STATUS_OK>.
145
146
=cut
147
148
sub set_index_status_ok {
112
    my ($self, $set) = @_;
149
    my ($self, $set) = @_;
113
    return defined $set ?
150
    $self->index_status(INDEX_STATUS_OK);
114
        $self->index_status(INDEX_STATUS_OK) :
115
        $self->index_status == INDEX_STATUS_OK;
116
}
151
}
117
152
118
sub index_status_reindex_required {
153
=head2 is_index_status_ok
119
    my ($self, $set) = @_;
154
120
    return defined $set ?
155
Convenience method for checking if index status is C<INDEX_STATUS_OK>.
121
        $self->index_status(INDEX_STATUS_REINDEX_REQUIRED) :
156
122
        $self->index_status == INDEX_STATUS_REINDEX_REQUIRED;
157
=cut
158
159
sub is_index_status_ok {
160
    return $self->index_status == INDEX_STATUS_OK;
123
}
161
}
124
162
125
sub index_status_recreate_required {
163
=head2 set_index_status_reindex_required
126
    my ($self, $set) = @_;
164
127
    return defined $set ?
165
Convenience method for setting index status to C<INDEX_REINDEX_REQUIRED>.
128
        $self->index_status(INDEX_STATUS_RECREATE_REQUIRED) :
166
129
        $self->index_status == INDEX_STATUS_RECREATE_REQUIRED;
167
=cut
168
169
sub set_index_status_reindex_required {
170
    $self->index_status(INDEX_STATUS_REINDEX_REQUIRED);
171
}
172
173
=head2 is_index_status_reindex_required
174
175
Convenience method for checking if index status is C<INDEX_STATUS_REINDEX_REQUIRED>.
176
177
=cut
178
179
sub is_index_status_reindex_required {
180
    return $self->index_status == INDEX_STATUS_REINDEX_REQUIRED;
181
}
182
183
=head2 set_index_status_recreate_required
184
185
Convenience method for setting index status to C<INDEX_STATUS_RECREATE_REQUIRED>.
186
187
=cut
188
189
sub set_index_status_recreate_required {
190
    $self->index_status(INDEX_STATUS_RECREATE_REQUIRED);
191
}
192
193
=head2 is_index_status_recreate_required
194
195
Convenience method for checking if index status is C<INDEX_STATUS_RECREATE_REQUIRED>.
196
197
=cut
198
199
sub is_index_status_recreate_required {
200
    return $self->index_status == INDEX_STATUS_RECREATE_REQUIRED;
130
}
201
}
131
202
203
=head2 index_status($status)
204
205
Will either set the current index status to C<$status> and return C<$status>,
206
or return the current index status if called with no arguments.
207
208
=over 4
209
210
=item C<$status>
211
212
Optional argument. If passed will set current index status to C<$status> if C<$status> is
213
a valid status. See L</CONSTANTS>.
214
215
=back
216
217
=cut
218
132
sub index_status {
219
sub index_status {
133
    my ($self, $status) = @_;
220
    my ($self, $status) = @_;
134
    my $key = 'ElasticsearchIndexStatus_' . $self->index;
221
    my $key = 'ElasticsearchIndexStatus_' . $self->index;
Lines 150-155 sub index_status { Link Here
150
    }
237
    }
151
}
238
}
152
239
240
=head2 update_mappings
241
242
Generate Elasticsearch mappings from mappings stored in database and
243
perform a request to update Elasticsearch index mappings. Will throw an
244
error and set index status to C<INDEX_STATUS_RECREATE_REQUIRED> if update
245
failes.
246
247
=cut
248
153
sub update_mappings {
249
sub update_mappings {
154
    my ($self) = @_;
250
    my ($self) = @_;
155
    my $conf = $self->get_elasticsearch_params();
251
    my $conf = $self->get_elasticsearch_params();
Lines 166-200 sub update_mappings { Link Here
166
                }
262
                }
167
            );
263
            );
168
        } catch {
264
        } catch {
169
            $self->index_status_recreate_required(1);
265
            $self->set_index_status_recreate_required();
170
            my $reason = $_[0]->{vars}->{body}->{error}->{reason};
266
            my $reason = $_[0]->{vars}->{body}->{error}->{reason};
171
            Koha::Exceptions::Exception->throw(
267
            Koha::Exceptions::Exception->throw(
172
                error => "Unable to update mappings for index \"$conf->{index_name}\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
268
                error => "Unable to update mappings for index \"$conf->{index_name}\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
173
            );
269
            );
174
        };
270
        };
175
    }
271
    }
176
    $self->index_status_ok(1);
272
    $self->set_index_status_ok();
177
}
273
}
178
274
179
=head2 $indexer->update_index_background($biblionums, $records)
275
=head2 update_index_background($biblionums, $records)
180
276
181
This has exactly the same API as C<update_index_background> however it'll
277
This has exactly the same API as C<update_index> however it'll
182
return immediately. It'll start a background process that does the adding.
278
return immediately. It'll start a background process that does the adding.
183
279
184
If it fails to add to Elasticsearch then it'll add to a queue that will cause
280
If it fails to add to Elasticsearch then it'll add to a queue that will cause
185
it to be updated by a regular index cron job in the future.
281
it to be updated by a regular index cron job in the future.
186
282
283
=cut
284
187
# TODO implement in the future - I don't know the best way of doing this yet.
285
# TODO implement in the future - I don't know the best way of doing this yet.
188
# If fork: make sure process group is changed so apache doesn't wait for us.
286
# If fork: make sure process group is changed so apache doesn't wait for us.
189
287
190
=cut
191
192
sub update_index_background {
288
sub update_index_background {
193
    my $self = shift;
289
    my $self = shift;
194
    $self->update_index(@_);
290
    $self->update_index(@_);
195
}
291
}
196
292
197
=head2 $indexer->delete_index($biblionums)
293
=head2 delete_index($biblionums)
198
294
199
C<$biblionums> is an arrayref of biblionumbers to delete from the index.
295
C<$biblionums> is an arrayref of biblionumbers to delete from the index.
200
296
Lines 206-212 sub delete_index { Link Here
206
    if ( !$self->store ) {
302
    if ( !$self->store ) {
207
        my $params  = $self->get_elasticsearch_params();
303
        my $params  = $self->get_elasticsearch_params();
208
        $self->store(
304
        $self->store(
209
            Catmandu::Store::ElasticSearch->new(
305
            Catmandu::Store::Elasticsearch->new(
210
                %$params,
306
                %$params,
211
                index_settings => $self->get_elasticsearch_settings(),
307
                index_settings => $self->get_elasticsearch_settings(),
212
                index_mappings => $self->get_elasticsearch_mappings(),
308
                index_mappings => $self->get_elasticsearch_mappings(),
Lines 217-239 sub delete_index { Link Here
217
    $self->store->bag->commit;
313
    $self->store->bag->commit;
218
}
314
}
219
315
220
=head2 $indexer->delete_index_background($biblionums)
316
=head2 delete_index_background($biblionums)
221
317
222
Identical to L<delete_index>, this will return immediately and start a
318
Identical to L</delete_index($biblionums)>
223
background process to do the actual deleting.
224
319
225
=cut
320
=cut
226
321
227
# TODO implement in the future
322
# TODO: Should be made async
228
229
sub delete_index_background {
323
sub delete_index_background {
230
    my $self = shift;
324
    my $self = shift;
231
    $self->delete_index(@_);
325
    $self->delete_index(@_);
232
}
326
}
233
327
234
=head2 $indexer->drop_index();
328
=head2 drop_index
235
329
236
Drops the index from the elasticsearch server.
330
Drops the index from the Elasticsearch server.
237
331
238
=cut
332
=cut
239
333
Lines 243-252 sub drop_index { Link Here
243
        my $conf = $self->get_elasticsearch_params();
337
        my $conf = $self->get_elasticsearch_params();
244
        my $elasticsearch = $self->get_elasticsearch();
338
        my $elasticsearch = $self->get_elasticsearch();
245
        $elasticsearch->indices->delete(index => $conf->{index_name});
339
        $elasticsearch->indices->delete(index => $conf->{index_name});
246
        $self->index_status_recreate_required(1);
340
        $self->set_index_status_recreate_required();
247
    }
341
    }
248
}
342
}
249
343
344
=head2 create_index
345
346
Creates the index (including mappings) on the Elasticsearch server.
347
348
=cut
349
250
sub create_index {
350
sub create_index {
251
    my ($self) = @_;
351
    my ($self) = @_;
252
    my $conf = $self->get_elasticsearch_params();
352
    my $conf = $self->get_elasticsearch_params();
Lines 261-266 sub create_index { Link Here
261
    $self->update_mappings();
361
    $self->update_mappings();
262
}
362
}
263
363
364
=head2 index_exists
365
366
Checks if index has been created on the ElasticSearch server. Returns C<1> or the
367
empty string to indicate whether index exists or not.
368
369
=cut
370
264
sub index_exists {
371
sub index_exists {
265
    my ($self) = @_;
372
    my ($self) = @_;
266
    my $conf = $self->get_elasticsearch_params();
373
    my $conf = $self->get_elasticsearch_params();
(-)a/misc/search_tools/rebuild_elastic_search.pl (-3 / +3 lines)
Lines 165-177 sub do_reindex { Link Here
165
        $indexer->drop_index() if $indexer->index_exists();
165
        $indexer->drop_index() if $indexer->index_exists();
166
        $indexer->create_index();
166
        $indexer->create_index();
167
    }
167
    }
168
    elsif (!$indexer->index_exists()) {
168
    elsif (!$indexer->index_exists) {
169
        # Create index if does not exist
169
        # Create index if does not exist
170
        $indexer->create_index();
170
        $indexer->create_index();
171
    } elsif ($indexer->index_status_ok) {
171
    } elsif ($indexer->is_index_status_ok) {
172
        # Update mapping unless index is some kind of problematic state
172
        # Update mapping unless index is some kind of problematic state
173
        $indexer->update_mappings();
173
        $indexer->update_mappings();
174
    } elsif ($indexer->index_status_recreate_required) {
174
    } elsif ($indexer->is_index_status_recreate_required) {
175
        warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;
175
        warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;
176
    }
176
    }
177
177
(-)a/t/Koha/SearchEngine/Elasticsearch.t (-2 / +26 lines)
Lines 115-121 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
115
115
116
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
116
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
117
117
118
    plan tests => 30;
118
    plan tests => 32;
119
119
120
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
120
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
121
121
Lines 315-320 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
315
315
316
    ok(defined $docs->[0][1]->{marc_format}, 'First document marc_format field should be set');
316
    ok(defined $docs->[0][1]->{marc_format}, 'First document marc_format field should be set');
317
317
318
    is($docs->[0][1]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');
319
318
    is(scalar @{$docs->[0][1]->{type_of_record}}, 1, 'First document type_of_record field should have one value');
320
    is(scalar @{$docs->[0][1]->{type_of_record}}, 1, 'First document type_of_record field should have one value');
319
    is_deeply(
321
    is_deeply(
320
        $docs->[0][1]->{type_of_record},
322
        $docs->[0][1]->{type_of_record},
Lines 351-354 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
351
353
352
    ok(!(defined $docs->[0][1]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
354
    ok(!(defined $docs->[0][1]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
353
355
356
    # Marc serialization format fallback for records exceeding ISO2709 max record size
357
358
    my $large_marc_record = MARC::Record->new();
359
    $large_marc_record->leader('     cam  22      a 4500');
360
361
    $large_marc_record->append_fields(
362
        MARC::Field->new('100', '', '', a => 'Author 1'),
363
        MARC::Field->new('110', '', '', a => 'Corp Author'),
364
        MARC::Field->new('210', '', '', a => 'Title 1'),
365
        MARC::Field->new('245', '', '', a => 'Title:', b => 'large record'),
366
        MARC::Field->new('999', '', '', c => '1234567'),
367
    );
368
369
    my $item_field = MARC::Field->new('952', '', '', o => '123456789123456789123456789', p => '123456789', z => 'test');
370
    my $items_count = 1638;
371
    while(--$items_count) {
372
        $large_marc_record->append_fields($item_field);
373
    }
374
375
    $docs = $see->marc_records_to_documents([$large_marc_record]);
376
377
    is($docs->[0][1]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
378
354
};
379
};
355
- 

Return to bug 19893