View | Details | Raw Unified | Return to bug 19893
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (+211 lines)
Lines 34-39 use Search::Elasticsearch; Link Here
34
use Try::Tiny;
34
use Try::Tiny;
35
use YAML::Syck;
35
use YAML::Syck;
36
36
37
use List::Util qw( sum0 reduce );
38
use Search::Elasticsearch;
39
use MARC::File::XML;
40
37
__PACKAGE__->mk_ro_accessors(qw( index ));
41
__PACKAGE__->mk_ro_accessors(qw( index ));
38
__PACKAGE__->mk_accessors(qw( sort_fields ));
42
__PACKAGE__->mk_accessors(qw( sort_fields ));
39
43
Lines 67-72 sub new { Link Here
67
    return $self;
71
    return $self;
68
}
72
}
69
73
74
sub get_elasticsearch {
75
    my $self = shift @_;
76
    unless (defined $self->{elasticsearch}) {
77
        my $conf = $self->get_elasticsearch_params();
78
        $self->{elasticsearch} = Search::Elasticsearch->new(
79
            client => "5_0::Direct",
80
            nodes => $conf->{nodes},
81
            cxn_pool => 'Sniff'
82
        );
83
    }
84
    return $self->{elasticsearch};
85
}
86
70
=head2 get_elasticsearch_params
87
=head2 get_elasticsearch_params
71
88
72
    my $params = $self->get_elasticsearch_params();
89
    my $params = $self->get_elasticsearch_params();
Lines 281-286 sub sort_fields { Link Here
281
    return $self->_sort_fields_accessor();
298
    return $self->_sort_fields_accessor();
282
}
299
}
283
300
301
sub marc_records_to_documents {
302
    my ($self, $records) = @_;
303
    my $rules = $self->get_marc_mapping_rules();
304
    my $control_fields_rules = $rules->{control_fields};
305
    my $data_fields_rules = $rules->{data_fields};
306
    my $marcflavour = lc C4::Context->preference('marcflavour');
307
308
    my @record_documents;
309
310
    sub _process_mappings {
311
        my ($mappings, $data, $record_document) = @_;
312
        foreach my $mapping (@{$mappings}) {
313
            my ($target, $options) = @{$mapping};
314
            # Copy (scalar) data since can have multiple targets
315
            # with differing options for (possibly) mutating data
316
            # so need a different copy for each
317
            my $_data = $data;
318
            $record_document->{$target} //= [];
319
            if (defined $options->{substr}) {
320
                my ($offset, $length) = @{$options->{substr}};
321
                $_data = length($data) > $offset ? substr $data, $offset, $length : '';
322
            }
323
            if (defined $options->{value_callbacks}) {
324
                $_data = reduce { $b->($a) } ($_data, @{$options->{value_callbacks}});
325
            }
326
            if (defined $options->{property}) {
327
                $_data = {
328
                    $options->{property} => $_data
329
                }
330
            }
331
            push @{$record_document->{$target}}, $_data;
332
        }
333
    }
334
    foreach my $record (@{$records}) {
335
        my $record_document = {};
336
        my $mappings = $rules->{leader};
337
        if ($mappings) {
338
            _process_mappings($mappings, $record->leader(), $record_document);
339
        }
340
        foreach my $field ($record->fields()) {
341
            if($field->is_control_field()) {
342
                my $mappings = $control_fields_rules->{$field->tag()};
343
                if ($mappings) {
344
                    _process_mappings($mappings, $field->data(), $record_document);
345
                }
346
            }
347
            else {
348
                my $subfields_mappings = $data_fields_rules->{$field->tag()};
349
                if ($subfields_mappings) {
350
                    my $wildcard_mappings = $subfields_mappings->{'*'};
351
                    foreach my $subfield ($field->subfields()) {
352
                        my ($code, $data) = @{$subfield};
353
                        my $mappings = $subfields_mappings->{$code} // [];
354
                        if ($wildcard_mappings) {
355
                            $mappings = [@{$mappings}, @{$wildcard_mappings}];
356
                        }
357
                        if (@{$mappings}) {
358
                            _process_mappings($mappings, $data, $record_document);
359
                        }
360
                    }
361
                }
362
            }
363
        }
364
        foreach my $field (keys %{$rules->{defaults}}) {
365
            unless (defined $record_document->{$field}) {
366
                $record_document->{$field} = $rules->{defaults}->{$field};
367
            }
368
        }
369
        foreach my $field (@{$rules->{sum}}) {
370
            if (defined $record_document->{$field}) {
371
                # TODO: validate numeric? filter?
372
                # TODO: Or should only accept fields without nested values?
373
                # TODO: Quick and dirty, improve if needed
374
                $record_document->{$field} = sum0(grep { ref($_) eq 'SCALAR' && m/\d+([.,]\d+)?/} @{$record_document->{$field}});
375
            }
376
        }
377
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
378
        $record->encoding('UTF-8');
379
        $record_document->{'marc_xml'} = $record->as_xml_record($marcflavour);
380
        my $id = $record->subfield('999', 'c');
381
        push @record_documents, [$id, $record_document];
382
    }
383
    return \@record_documents;
384
}
385
386
# Provides the rules for marc to Elasticsearch JSON document conversion.
387
sub get_marc_mapping_rules {
388
    my ($self) = @_;
389
390
    my $marcflavour = lc C4::Context->preference('marcflavour');
391
    my @rules;
392
393
    sub _field_mappings {
394
        my ($facet, $suggestible, $sort, $target_name, $target_type, $range) = @_;
395
        my %mapping_defaults = ();
396
        my @mappings;
397
398
        my $substr_args = undef;
399
        if ($range) {
400
            my ($offset, $end) = map(int, split /-/, $range, 2);
401
            $substr_args = [$offset];
402
            push @{$substr_args}, (defined $end ? $end - $offset : 1);
403
        }
404
        my $default_options = {};
405
        if ($substr_args) {
406
            $default_options->{substr} = $substr_args;
407
        }
408
409
        # TODO: Should probably have per type value callback/hook
410
        # but hard code for now
411
        if ($target_type eq 'boolean') {
412
            $default_options->{value_callbacks} //= [];
413
            push @{$default_options->{value_callbacks}}, sub {
414
                my ($value) = @_;
415
                # Trim whitespace at both ends
416
                $value =~ s/^\s+|\s+$//g;
417
                return $value ? 'true' : 'false';
418
            };
419
        }
420
421
        my $mapping = [$target_name, $default_options];
422
        push @mappings, $mapping;
423
424
        my @suffixes = ();
425
        push @suffixes, 'facet' if $facet;
426
        push @suffixes, 'suggestion' if $suggestible;
427
        push @suffixes, 'sort' if !defined $sort || $sort;
428
429
        foreach my $suffix (@suffixes) {
430
            my $mapping = ["${target_name}__$suffix"];
431
            # Hack, fix later in less hideous manner
432
            if ($suffix eq 'suggestion') {
433
                push @{$mapping}, {%{$default_options}, property => 'input'};
434
            }
435
            else {
436
                push @{$mapping}, $default_options;
437
            }
438
            push @mappings, $mapping;
439
        }
440
        return @mappings;
441
    };
442
    my $field_spec_regexp = qr/^([0-9]{3})([0-9a-z]+)?(?:_\/(\d+(?:-\d+)?))?$/;
443
    my $leader_regexp = qr/^leader(?:_\/(\d+(?:-\d+)?))?$/;
444
    my $rules = {
445
        'leader' => [],
446
        'control_fields' => {},
447
        'data_fields' => {},
448
        'sum' => [],
449
        'defaults' => {}
450
    };
451
452
    $self->_foreach_mapping(sub {
453
        my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_;
454
        return if $marc_type ne $marcflavour;
455
456
        if ($type eq 'sum') {
457
            push @{$rules->{sum}}, $name;
458
        }
459
        elsif($type eq 'boolean') {
460
            # boolean gets special handling, if value doesn't exist for a field,
461
            # it is set to false
462
            $rules->{defaults}->{$name} = 'false';
463
        }
464
465
        if ($marc_field =~ $field_spec_regexp) {
466
            my $field_tag = $1;
467
            my $subfields = defined $2 ? $2 : '*';
468
            my $range = defined $3 ? $3 : undef;
469
            if ($field_tag < 10) {
470
                $rules->{control_fields}->{$field_tag} //= [];
471
                my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
472
                push @{$rules->{control_fields}->{$field_tag}}, @mappings;
473
            }
474
            else {
475
                $rules->{data_fields}->{$field_tag} //= {};
476
                foreach my $subfield (split //, $subfields) {
477
                    $rules->{data_fields}->{$field_tag}->{$subfield} //= [];
478
                    my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
479
                    push @{$rules->{data_fields}->{$field_tag}->{$subfield}}, @mappings;
480
                }
481
            }
482
        }
483
        elsif ($marc_field =~ $leader_regexp) {
484
            my $range = defined $1 ? $1 : undef;
485
            my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
486
            push @{$rules->{leader}}, @mappings;
487
        }
488
        else {
489
            die("Invalid marc field: $marc_field");
490
        }
491
    });
492
    return $rules;
493
}
494
284
# Provides the rules for data conversion.
495
# Provides the rules for data conversion.
285
sub get_fixer_rules {
496
sub get_fixer_rules {
286
    my ($self) = @_;
497
    my ($self) = @_;
(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-27 / +117 lines)
Lines 66-89 sub update_index { Link Here
66
        $self->_sanitise_records($biblionums, $records);
66
        $self->_sanitise_records($biblionums, $records);
67
    }
67
    }
68
68
69
    my $from    = $self->_convert_marc_to_json($records);
69
    if (C4::Context->preference('ExperimentalElasticsearchIndexing')) {
70
    if ( !$self->store ) {
70
        $self->ensure_mappings_updated();
71
        my $params  = $self->get_elasticsearch_params();
71
        $self->bulk_index($records);
72
        $self->store(
72
        return 1;
73
            Catmandu::Store::ElasticSearch->new(
74
                %$params,
75
                index_settings => $self->get_elasticsearch_settings(),
76
                index_mappings => $self->get_elasticsearch_mappings(),
77
            )
78
        );
79
    }
73
    }
74
    else {
75
        my $from = $self->_convert_marc_to_json($records);
76
        if ( !$self->store ) {
77
            my $params  = $self->get_elasticsearch_params();
78
            $self->store(
79
                Catmandu::Store::ElasticSearch->new(
80
                    %$params,
81
                    index_settings => $self->get_elasticsearch_settings(),
82
                    index_mappings => $self->get_elasticsearch_mappings(),
83
                )
84
            );
85
        }
80
86
81
    #print Data::Dumper::Dumper( $from->to_array );
87
        #print Data::Dumper::Dumper( $from->to_array );
82
    $self->store->bag->add_many($from);
88
        $self->store->bag->add_many($from);
83
    $self->store->bag->commit;
89
        $self->store->bag->commit;
90
        return 1;
91
    }
92
}
93
94
sub bulk_index {
95
    my ($self, $records) = @_;
96
    my $conf = $self->get_elasticsearch_params();
97
    my $elasticsearch = $self->get_elasticsearch();
98
    my $documents = $self->marc_records_to_documents($records);
99
    my @body;
100
101
    foreach my $document_info (@{$documents}) {
102
        my ($id, $document) = @{$document_info};
103
        push @body, {
104
            index => {
105
                _id => $id
106
            }
107
        };
108
        push @body, $document;
109
    }
110
    my $response = $elasticsearch->bulk(
111
        index => $conf->{index_name},
112
        type => 'data', # is just hard coded in Indexer.pm?
113
        body => \@body
114
    );
115
    # TODO: handle response
84
    return 1;
116
    return 1;
85
}
117
}
86
118
119
sub ensure_mappings_updated {
120
    my ($self) = @_;
121
    unless ($self->{_mappings_updated}) {
122
        $self->update_mappings();
123
    }
124
}
125
126
sub update_mappings {
127
    my ($self) = @_;
128
    my $conf = $self->get_elasticsearch_params();
129
    my $elasticsearch = $self->get_elasticsearch();
130
    my $mappings = $self->get_elasticsearch_mappings();
131
132
    foreach my $type (keys %{$mappings}) {
133
        my $response = $elasticsearch->indices->put_mapping(
134
            index => $conf->{index_name},
135
            type => $type,
136
            body => {
137
                $type => $mappings->{$type}
138
            }
139
        );
140
        # TODO: process response, produce errors etc
141
    }
142
    $self->{_mappings_updated} = 1;
143
}
144
87
=head2 $indexer->update_index_background($biblionums, $records)
145
=head2 $indexer->update_index_background($biblionums, $records)
88
146
89
This has exactly the same API as C<update_index_background> however it'll
147
This has exactly the same API as C<update_index_background> however it'll
Lines 148-168 after this will recreate it again. Link Here
148
206
149
sub drop_index {
207
sub drop_index {
150
    my ($self) = @_;
208
    my ($self) = @_;
151
209
    if (C4::Context->preference('ExperimentalElasticsearchIndexing')) {
152
    if (!$self->store) {
210
        if ($self->index_exists) {
153
        # If this index doesn't exist, this will create it. Then it'll be
211
            my $conf = $self->get_elasticsearch_params();
154
        # deleted. That's not the end of the world however.
212
            my $elasticsearch = $self->get_elasticsearch();
155
        my $params  = $self->get_elasticsearch_params();
213
            my $response = $elasticsearch->indices->delete(index => $conf->{index_name});
156
        $self->store(
214
            # TODO: Handle response? Convert errors to exceptions/die
157
            Catmandu::Store::ElasticSearch->new(
215
        }
158
                %$params,
216
    }
159
                index_settings => $self->get_elasticsearch_settings(),
217
    else {
160
                index_mappings => $self->get_elasticsearch_mappings(),
218
        if (!$self->store) {
161
            )
219
            # If this index doesn't exist, this will create it. Then it'll be
162
        );
220
            # deleted. That's not the end of the world however.
221
            my $params  = $self->get_elasticsearch_params();
222
            $self->store(
223
                Catmandu::Store::ElasticSearch->new(
224
                    %$params,
225
                    index_settings => $self->get_elasticsearch_settings(),
226
                    index_mappings => $self->get_elasticsearch_mappings(),
227
                )
228
            );
229
        }
230
        $self->store->drop();
231
        $self->store(undef);
163
    }
232
    }
164
    $self->store->drop();
233
}
165
    $self->store(undef);
234
235
sub create_index {
236
    my ($self) = @_;
237
    my $conf = $self->get_elasticsearch_params();
238
    my $settings = $self->get_elasticsearch_settings();
239
    my $elasticsearch = $self->get_elasticsearch();
240
    my $response = $elasticsearch->indices->create(
241
        index => $conf->{index_name},
242
        body => {
243
            settings => $settings
244
        }
245
    );
246
    # TODO: Handle response? Convert errors to exceptions/die
247
}
248
249
sub index_exists {
250
    my ($self) = @_;
251
    my $conf = $self->get_elasticsearch_params();
252
    my $elasticsearch = $self->get_elasticsearch();
253
    return $elasticsearch->indices->exists(
254
        index => $conf->{index_name},
255
    );
166
}
256
}
167
257
168
sub _sanitise_records {
258
sub _sanitise_records {
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-15 / +31 lines)
Lines 48-54 use Koha::SearchEngine::QueryBuilder; Link Here
48
use Koha::SearchEngine::Search;
48
use Koha::SearchEngine::Search;
49
use MARC::Record;
49
use MARC::Record;
50
use Catmandu::Store::ElasticSearch;
50
use Catmandu::Store::ElasticSearch;
51
51
use MARC::File::XML;
52
use Data::Dumper; #TODO remove
52
use Data::Dumper; #TODO remove
53
use Carp qw(cluck);
53
use Carp qw(cluck);
54
54
Lines 156-170 sub search_compat { Link Here
156
    my $results = $self->search($query, undef, $results_per_page, %options);
156
    my $results = $self->search($query, undef, $results_per_page, %options);
157
157
158
    # Convert each result into a MARC::Record
158
    # Convert each result into a MARC::Record
159
    my (@records, $index);
159
    my @records;
160
    $index = $offset; # opac-search expects results to be put in the
160
    # opac-search expects results to be put in the
161
        # right place in the array, according to $offset
161
    # right place in the array, according to $offset
162
    my $index = $offset;
162
    $results->each(sub {
163
    $results->each(sub {
163
            # The results come in an array for some reason
164
        $records[$index++] = $self->decode_record_from_result(@_);
164
            my $marc_json = $_[0]->{record};
165
    });
165
            my $marc = $self->json2marc($marc_json);
166
            $records[$index++] = $marc;
167
        });
168
    # consumers of this expect a name-spaced result, we provide the default
166
    # consumers of this expect a name-spaced result, we provide the default
169
    # configuration.
167
    # configuration.
170
    my %result;
168
    my %result;
Lines 195-208 sub search_auth_compat { Link Here
195
    $res->each(
193
    $res->each(
196
        sub {
194
        sub {
197
            my %result;
195
            my %result;
198
            my $record    = $_[0];
199
            my $marc_json = $record->{record};
200
196
201
            # I wonder if these should be real values defined in the mapping
197
            # I wonder if these should be real values defined in the mapping
202
            # rather than hard-coded conversions.
198
            # rather than hard-coded conversions.
199
            my $record    = $_[0];
203
            # Handle legacy nested arrays indexed with splitting enabled.
200
            # Handle legacy nested arrays indexed with splitting enabled.
204
            my $authid = $record->{ 'Local-number' }[0];
201
            my $authid = $record->{ 'Local-number' }[0];
205
            $authid = @$authid[0] if (ref $authid eq 'ARRAY');
202
            $authid = @$authid[0] if (ref $authid eq 'ARRAY');
203
206
            $result{authid} = $authid;
204
            $result{authid} = $authid;
207
205
208
            # TODO put all this info into the record at index time so we
206
            # TODO put all this info into the record at index time so we
Lines 218-224 sub search_auth_compat { Link Here
218
            # it's not reproduced here yet.
216
            # it's not reproduced here yet.
219
            my $authtype           = $rs->single;
217
            my $authtype           = $rs->single;
220
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
218
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
221
            my $marc               = $self->json2marc($marc_json);
219
            my $marc               = $self->decode_record_from_result(@_);
222
            my $mainentry          = $marc->field($auth_tag_to_report);
220
            my $mainentry          = $marc->field($auth_tag_to_report);
223
            my $reported_tag;
221
            my $reported_tag;
224
            if ($mainentry) {
222
            if ($mainentry) {
Lines 337-345 sub simple_search_compat { Link Here
337
    my $results = $self->search($query, undef, $max_results, %options);
335
    my $results = $self->search($query, undef, $max_results, %options);
338
    my @records;
336
    my @records;
339
    $results->each(sub {
337
    $results->each(sub {
340
            # The results come in an array for some reason
338
            my $marc = $self->decode_record_from_result(@_);
341
            my $marc_json = $_[0]->{record};
342
            my $marc = $self->json2marc($marc_json);
343
            push @records, $marc;
339
            push @records, $marc;
344
        });
340
        });
345
    return (undef, \@records, $results->total);
341
    return (undef, \@records, $results->total);
Lines 360-365 sub extract_biblionumber { Link Here
360
    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
356
    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
361
}
357
}
362
358
359
=head2 decode_record_from_result
360
    my $marc_record = $self->decode_record_from_result(@result);
361
362
Extracts marc data from Elasticsearch result and decodes to MARC::Record object
363
364
=cut
365
366
sub decode_record_from_result {
367
    # Result is passed in as array, will get flattened
368
    # and first element will be $result
369
    my ( $self, $result ) = @_;
370
    if (C4::Context->preference('ExperimentalElasticsearchIndexing')) {
371
        return MARC::Record->new_from_xml($result->{marc_xml}, 'UTF-8', uc C4::Context->preference('marcflavour'));
372
    }
373
    else {
374
        return $self->json2marc($result->{record});
375
    }
376
}
377
378
363
=head2 json2marc
379
=head2 json2marc
364
380
365
    my $marc = $self->json2marc($marc_json);
381
    my $marc = $self->json2marc($marc_json);
(-)a/etc/searchengine/elasticsearch/field_config.yaml (+5 lines)
Lines 8-13 general: Link Here
8
    record:
8
    record:
9
      store: true
9
      store: true
10
      type: text
10
      type: text
11
    marc_xml:
12
      store: true
13
      type: text
14
      analyzer: keyword
15
      index: false
11
# Search fields
16
# Search fields
12
search:
17
search:
13
  boolean:
18
  boolean:
(-)a/installer/data/mysql/atomicupdate/bug_19893_experimental_indexing_elasticsearch_syspref.sql (+1 lines)
Line 0 Link Here
1
INSERT IGNORE INTO systempreferences (variable, value, explanation, options, type) VALUES ('ExperimentalElasticsearchIndexing', '0', 'Enable optimized experimental Elasticsearch indexing', NULL, 'YesNo');
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref (+6 lines)
Lines 425-427 Administration: Link Here
425
              choices:
425
              choices:
426
                Zebra: Zebra
426
                Zebra: Zebra
427
                Elasticsearch: Elasticsearch
427
                Elasticsearch: Elasticsearch
428
        -
429
            - pref: ExperimentalElasticsearchIndexing
430
              choices:
431
                yes: Enable
432
                no: "Don't enable"
433
            - "experimental faster indexing, only relevant if using Elasticsearch."
(-)a/misc/search_tools/rebuild_elastic_search.pl (-2 / +10 lines)
Lines 160-170 sub do_reindex { Link Here
160
    my ( $next, $index_name ) = @_;
160
    my ( $next, $index_name ) = @_;
161
161
162
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
162
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
163
    if ($delete) {
164
163
164
    if ($delete) {
165
        # We know it's safe to not recreate the indexer because update_index
165
        # We know it's safe to not recreate the indexer because update_index
166
        # hasn't been called yet.
166
        # hasn't been called yet.
167
        $indexer->drop_index();
167
        $indexer->drop_index();
168
        if (C4::Context->preference('ExperimentalElasticsearchIndexing')) {
169
            # Catmandu will create index for us in update_index, so without it we
170
            # to create it ourselves
171
            $indexer->create_index();
172
        }
173
    }
174
    elsif (C4::Context->preference('ExperimentalElasticsearchIndexing') && !$indexer->index_exists) {
175
        # Create index if does not exist
176
        $indexer->create_index();
168
    }
177
    }
169
178
170
    my $count        = 0;
179
    my $count        = 0;
171
- 

Return to bug 19893