View | Details | Raw Unified | Return to bug 19893
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (+192 lines)
Lines 34-39 use Search::Elasticsearch; Link Here
34
use Try::Tiny;
34
use Try::Tiny;
35
use YAML::Syck;
35
use YAML::Syck;
36
36
37
use List::Util qw( sum0 );
38
use Search::Elasticsearch;
39
use MARC::File::XML;
40
37
__PACKAGE__->mk_ro_accessors(qw( index ));
41
__PACKAGE__->mk_ro_accessors(qw( index ));
38
__PACKAGE__->mk_accessors(qw( sort_fields ));
42
__PACKAGE__->mk_accessors(qw( sort_fields ));
39
43
Lines 67-72 sub new { Link Here
67
    return $self;
71
    return $self;
68
}
72
}
69
73
74
sub get_elasticsearch {
75
    my $self = shift @_;
76
    unless (defined $self->{elasticsearch}) {
77
        my $conf = $self->get_elasticsearch_params();
78
        $self->{elasticsearch} = Search::Elasticsearch->new(
79
            client => "5_0::Direct",
80
            nodes => $conf->{nodes},
81
            cxn_pool => 'Sniff'
82
        );
83
    }
84
    return $self->{elasticsearch};
85
}
86
70
=head2 get_elasticsearch_params
87
=head2 get_elasticsearch_params
71
88
72
    my $params = $self->get_elasticsearch_params();
89
    my $params = $self->get_elasticsearch_params();
Lines 281-286 sub sort_fields { Link Here
281
    return $self->_sort_fields_accessor();
298
    return $self->_sort_fields_accessor();
282
}
299
}
283
300
301
sub marc_records_to_documents {
302
    my ($self, $records) = @_;
303
    my $rules = $self->get_marc_mapping_rules();
304
    my $control_fields_rules = $rules->{control_fields};
305
    my $data_fields_rules = $rules->{data_fields};
306
    my $marcflavour = lc C4::Context->preference('marcflavour');
307
308
    my @record_documents;
309
310
    sub _process_mappings {
311
        my ($mappings, $data, $record_document) = @_;
312
        foreach my $mapping (@{$mappings}) {
313
            my ($target, $options) = @{$mapping};
314
            my $_data = $data;
315
            $record_document->{$target} //= [];
316
            if ($options->{substr}) {
317
                my ($offset, $length) = @{$options->{substr}};
318
                $_data = substr $data, $offset, $length;
319
            }
320
            if ($options->{property}) {
321
                $_data = {
322
                    $options->{property} => $_data
323
                }
324
            }
325
            push @{$record_document->{$target}}, $_data;
326
        }
327
    }
328
    foreach my $record (@{$records}) {
329
        my $record_document = {};
330
        my $mappings = $rules->{leader};
331
        if ($mappings) {
332
            _process_mappings($mappings, $record->leader(), $record_document);
333
        }
334
        foreach my $field ($record->fields()) {
335
            if($field->is_control_field()) {
336
                my $mappings = $control_fields_rules->{$field->tag()};
337
                if ($mappings) {
338
                    _process_mappings($mappings, $field->data(), $record_document);
339
                }
340
            }
341
            else {
342
                my $subfields_mappings = $data_fields_rules->{$field->tag()};
343
                if ($subfields_mappings) {
344
                    my $wildcard_mappings = $subfields_mappings->{'*'};
345
                    foreach my $subfield ($field->subfields()) {
346
                        my ($code, $data) = @{$subfield};
347
                        my $mappings = $subfields_mappings->{$code} // [];
348
                        if ($wildcard_mappings) {
349
                            $mappings = [@{$mappings}, @{$wildcard_mappings}];
350
                        }
351
                        if (@{$mappings}) {
352
                            _process_mappings($mappings, $data, $record_document);
353
                        }
354
                    }
355
                }
356
            }
357
        }
358
        foreach my $field (keys %{$rules->{defaults}}) {
359
            unless (defined $record_document->{$field}) {
360
                $record_document->{$field} = $rules->{defaults}->{$field};
361
            }
362
        }
363
        foreach my $field (@{$rules->{sum}}) {
364
            if (defined $record_document->{$field}) {
365
                # TODO: validate numeric? filter?
366
                # TODO: Or should only accept fields without nested values?
367
                # TODO: Quick and dirty, improve if needed
368
                $record_document->{$field} = sum0(grep { ref($_) eq 'SCALAR' && m/\d+([.,]\d+)?/} @{$record_document->{$field}});
369
            }
370
        }
371
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
372
        $record->encoding('UTF-8');
373
        $record_document->{'marc_xml'} = $record->as_xml_record($marcflavour);
374
        my $id = $record->subfield('999', 'c');
375
        push @record_documents, [$id, $record_document];
376
    }
377
    return \@record_documents;
378
}
379
380
# Provides the rules for marc to Elasticsearch JSON document conversion.
381
sub get_marc_mapping_rules {
382
    my ($self) = @_;
383
384
    my $marcflavour = lc C4::Context->preference('marcflavour');
385
    my @rules;
386
387
    sub _field_mappings {
388
        my ($facet, $suggestible, $sort, $target_name, $target_type, $range) = @_;
389
        my %mapping_defaults = ();
390
        my @mappings;
391
392
        my $substr_args = undef;
393
        if ($range) {
394
            my ($offset, $end) = map(int, split /-/, $range, 2);
395
            $substr_args = [$offset];
396
            push @{$substr_args}, (defined $end ? $end - $offset : 1);
397
        }
398
        my $default_options = {};
399
        if ($substr_args) {
400
            $default_options->{substr} = $substr_args;
401
        }
402
403
        my $mapping = [$target_name, $default_options];
404
        push @mappings, $mapping;
405
406
        my @suffixes = ();
407
        push @suffixes, 'facet' if $facet;
408
        push @suffixes, 'suggestion' if $suggestible; # Check condition, also if undef?
409
        push @suffixes, 'sort' if $sort;
410
        foreach my $suffix (@suffixes) {
411
            my $mapping = ["${target_name}__$suffix"];
412
            # Hack, fix later in less hideous manner
413
            if ($suffix eq 'suggestion') {
414
                push @{$mapping}, {%{$default_options}, property => 'input'};
415
            }
416
            else {
417
                push @{$mapping}, $default_options;
418
            }
419
            push @mappings, $mapping;
420
        }
421
        return @mappings;
422
    };
423
    my $field_spec_regexp = qr/^([0-9]{3})([0-9a-z]+)?(?:_\/(\d+(?:-\d+)?))?$/;
424
    my $leader_regexp = qr/^leader(?:_\/(\d+(?:-\d+)?))?$/;
425
    my $rules = {
426
        'leader' => [],
427
        'control_fields' => {},
428
        'data_fields' => {},
429
        'sum' => [],
430
        'defaults' => {}
431
    };
432
433
    $self->_foreach_mapping(sub {
434
        my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_;
435
        return if $marc_type ne $marcflavour;
436
437
        if ($type eq 'sum') {
438
            push @{$rules->{sum}}, $name;
439
        }
440
        elsif($type eq 'boolean') {
441
            # boolean gets special handling, if value doesn't exist for a field,
442
            # it is set to false
443
            $rules->{defaults}->{$name} = 0;
444
        }
445
446
        if ($marc_field =~ $field_spec_regexp) {
447
            my $field_tag = $1;
448
            my $subfields = defined $2 ? $2 : '*';
449
            my $range = defined $3 ? $3 : undef;
450
            if ($field_tag < 10) {
451
                $rules->{control_fields}->{$field_tag} //= [];
452
                my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
453
                push @{$rules->{control_fields}->{$field_tag}}, @mappings;
454
            }
455
            else {
456
                $rules->{data_fields}->{$field_tag} //= {};
457
                foreach my $subfield (split //, $subfields) {
458
                    $rules->{data_fields}->{$field_tag}->{$subfield} //= [];
459
                    my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
460
                    push @{$rules->{data_fields}->{$field_tag}->{$subfield}}, @mappings;
461
                }
462
            }
463
        }
464
        elsif ($marc_field =~ $leader_regexp) {
465
            my $range = defined $1 ? $1 : undef;
466
            my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
467
            push @{$rules->{leader}}, @mappings;
468
        }
469
        else {
470
            die("Invalid marc field: $marc_field");
471
        }
472
    });
473
    return $rules;
474
}
475
284
# Provides the rules for data conversion.
476
# Provides the rules for data conversion.
285
sub get_fixer_rules {
477
sub get_fixer_rules {
286
    my ($self) = @_;
478
    my ($self) = @_;
(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-27 / +106 lines)
Lines 66-89 sub update_index { Link Here
66
        $self->_sanitise_records($biblionums, $records);
66
        $self->_sanitise_records($biblionums, $records);
67
    }
67
    }
68
68
69
    my $from    = $self->_convert_marc_to_json($records);
69
    if (C4::Context->preference('ExperimentalElasticsearchIndexing')) {
70
    if ( !$self->store ) {
70
        $self->ensure_mappings_updated();
71
        my $params  = $self->get_elasticsearch_params();
71
        $self->bulk_index($records);
72
        $self->store(
72
        return 1;
73
            Catmandu::Store::ElasticSearch->new(
73
    }
74
                %$params,
74
    else {
75
                index_settings => $self->get_elasticsearch_settings(),
75
        my $from = $self->_convert_marc_to_json($records);
76
                index_mappings => $self->get_elasticsearch_mappings(),
76
        if ( !$self->store ) {
77
            )
77
            my $params  = $self->get_elasticsearch_params();
78
        );
78
            $self->store(
79
                Catmandu::Store::ElasticSearch->new(
80
                    %$params,
81
                    index_settings => $self->get_elasticsearch_settings(),
82
                    index_mappings => $self->get_elasticsearch_mappings(),
83
                )
84
            );
85
        }
86
87
        #print Data::Dumper::Dumper( $from->to_array );
88
        $self->store->bag->add_many($from);
89
        $self->store->bag->commit;
90
        return 1;
79
    }
91
    }
92
}
80
93
81
    #print Data::Dumper::Dumper( $from->to_array );
94
sub bulk_index {
82
    $self->store->bag->add_many($from);
95
    my ($self, $records) = @_;
83
    $self->store->bag->commit;
96
    my $conf = $self->get_elasticsearch_params();
97
    my $elasticsearch = $self->get_elasticsearch();
98
    my $documents = $self->marc_records_to_documents($records);
99
    my @body;
100
101
    foreach my $document_info (@{$documents}) {
102
        my ($id, $document) = @{$document_info};
103
        push @body, {
104
            index => {
105
                _id => $id
106
            }
107
        };
108
        push @body, $document;
109
    }
110
    my $response = $elasticsearch->bulk(
111
        index => $conf->{index_name},
112
        type => 'data', # is just hard coded in Indexer.pm?
113
        body => \@body
114
    );
115
    # TODO: handle response
84
    return 1;
116
    return 1;
85
}
117
}
86
118
119
sub ensure_mappings_updated {
120
    my ($self) = @_;
121
    unless ($self->{_mappings_updated}) {
122
        $self->update_mappings();
123
    }
124
}
125
126
sub update_mappings {
127
    my ($self) = @_;
128
    my $conf = $self->get_elasticsearch_params();
129
    my $elasticsearch = $self->get_elasticsearch();
130
    my $mappings = $self->get_elasticsearch_mappings();
131
132
    foreach my $type (keys %{$mappings}) {
133
        my $response = $elasticsearch->indices->put_mapping(
134
            index => $conf->{index_name},
135
            type => $type,
136
            body => {
137
                $type => $mappings->{$type}
138
            }
139
        );
140
        # TODO: process response, produce errors etc
141
    }
142
    $self->{_mappings_updated} = 1;
143
}
144
87
=head2 $indexer->update_index_background($biblionums, $records)
145
=head2 $indexer->update_index_background($biblionums, $records)
88
146
89
This has exactly the same API as C<update_index_background> however it'll
147
This has exactly the same API as C<update_index_background> however it'll
Lines 148-168 after this will recreate it again. Link Here
148
206
149
sub drop_index {
207
sub drop_index {
150
    my ($self) = @_;
208
    my ($self) = @_;
151
209
    if (C4::Context->preference('ExperimentalElasticsearchIndexing')) {
152
    if (!$self->store) {
210
        my $conf = $self->get_elasticsearch_params();
153
        # If this index doesn't exist, this will create it. Then it'll be
211
        my $elasticsearch = $self->get_elasticsearch();
154
        # deleted. That's not the end of the world however.
212
        my $response = $elasticsearch->indices->delete(index => $conf->{index_name});
155
        my $params  = $self->get_elasticsearch_params();
213
        # TODO: Handle response? Convert errors to exceptions/die
156
        $self->store(
214
    }
157
            Catmandu::Store::ElasticSearch->new(
215
    else {
158
                %$params,
216
        if (!$self->store) {
159
                index_settings => $self->get_elasticsearch_settings(),
217
            # If this index doesn't exist, this will create it. Then it'll be
160
                index_mappings => $self->get_elasticsearch_mappings(),
218
            # deleted. That's not the end of the world however.
161
            )
219
            my $params  = $self->get_elasticsearch_params();
162
        );
220
            $self->store(
221
                Catmandu::Store::ElasticSearch->new(
222
                    %$params,
223
                    index_settings => $self->get_elasticsearch_settings(),
224
                    index_mappings => $self->get_elasticsearch_mappings(),
225
                )
226
            );
227
        }
228
        $self->store->drop();
229
        $self->store(undef);
163
    }
230
    }
164
    $self->store->drop();
231
}
165
    $self->store(undef);
232
233
sub create_index {
234
    my ($self) = @_;
235
    my $conf = $self->get_elasticsearch_params();
236
    my $settings = $self->get_elasticsearch_settings();
237
    my $elasticsearch = $self->get_elasticsearch();
238
    my $response = $elasticsearch->indices->create(
239
        index => $conf->{index_name},
240
        body => {
241
            settings => $settings
242
        }
243
    );
244
    # TODO: Handle response? Convert errors to exceptions/die
166
}
245
}
167
246
168
sub _sanitise_records {
247
sub _sanitise_records {
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-15 / +31 lines)
Lines 48-54 use Koha::SearchEngine::QueryBuilder; Link Here
48
use Koha::SearchEngine::Search;
48
use Koha::SearchEngine::Search;
49
use MARC::Record;
49
use MARC::Record;
50
use Catmandu::Store::ElasticSearch;
50
use Catmandu::Store::ElasticSearch;
51
51
use MARC::File::XML;
52
use Data::Dumper; #TODO remove
52
use Data::Dumper; #TODO remove
53
use Carp qw(cluck);
53
use Carp qw(cluck);
54
54
Lines 156-170 sub search_compat { Link Here
156
    my $results = $self->search($query, undef, $results_per_page, %options);
156
    my $results = $self->search($query, undef, $results_per_page, %options);
157
157
158
    # Convert each result into a MARC::Record
158
    # Convert each result into a MARC::Record
159
    my (@records, $index);
159
    my @records;
160
    $index = $offset; # opac-search expects results to be put in the
160
    # opac-search expects results to be put in the
161
        # right place in the array, according to $offset
161
    # right place in the array, according to $offset
162
    my $index = $offset;
162
    $results->each(sub {
163
    $results->each(sub {
163
            # The results come in an array for some reason
164
        $records[$index++] = $self->decode_record_from_result(@_);
164
            my $marc_json = $_[0]->{record};
165
    });
165
            my $marc = $self->json2marc($marc_json);
166
            $records[$index++] = $marc;
167
        });
168
    # consumers of this expect a name-spaced result, we provide the default
166
    # consumers of this expect a name-spaced result, we provide the default
169
    # configuration.
167
    # configuration.
170
    my %result;
168
    my %result;
Lines 195-208 sub search_auth_compat { Link Here
195
    $res->each(
193
    $res->each(
196
        sub {
194
        sub {
197
            my %result;
195
            my %result;
198
            my $record    = $_[0];
199
            my $marc_json = $record->{record};
200
196
201
            # I wonder if these should be real values defined in the mapping
197
            # I wonder if these should be real values defined in the mapping
202
            # rather than hard-coded conversions.
198
            # rather than hard-coded conversions.
199
            my $record    = $_[0];
203
            # Handle legacy nested arrays indexed with splitting enabled.
200
            # Handle legacy nested arrays indexed with splitting enabled.
204
            my $authid = $record->{ 'Local-number' }[0];
201
            my $authid = $record->{ 'Local-number' }[0];
205
            $authid = @$authid[0] if (ref $authid eq 'ARRAY');
202
            $authid = @$authid[0] if (ref $authid eq 'ARRAY');
203
206
            $result{authid} = $authid;
204
            $result{authid} = $authid;
207
205
208
            # TODO put all this info into the record at index time so we
206
            # TODO put all this info into the record at index time so we
Lines 218-224 sub search_auth_compat { Link Here
218
            # it's not reproduced here yet.
216
            # it's not reproduced here yet.
219
            my $authtype           = $rs->single;
217
            my $authtype           = $rs->single;
220
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
218
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
221
            my $marc               = $self->json2marc($marc_json);
219
            my $marc               = $self->decode_record_from_result(@_);
222
            my $mainentry          = $marc->field($auth_tag_to_report);
220
            my $mainentry          = $marc->field($auth_tag_to_report);
223
            my $reported_tag;
221
            my $reported_tag;
224
            if ($mainentry) {
222
            if ($mainentry) {
Lines 337-345 sub simple_search_compat { Link Here
337
    my $results = $self->search($query, undef, $max_results, %options);
335
    my $results = $self->search($query, undef, $max_results, %options);
338
    my @records;
336
    my @records;
339
    $results->each(sub {
337
    $results->each(sub {
340
            # The results come in an array for some reason
338
            my $marc = $self->decode_record_from_result(@_);
341
            my $marc_json = $_[0]->{record};
342
            my $marc = $self->json2marc($marc_json);
343
            push @records, $marc;
339
            push @records, $marc;
344
        });
340
        });
345
    return (undef, \@records, $results->total);
341
    return (undef, \@records, $results->total);
Lines 360-365 sub extract_biblionumber { Link Here
360
    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
356
    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
361
}
357
}
362
358
359
=head2 decode_record_from_result
360
    my $marc_record = $self->decode_record_from_result(@result);
361
362
Extracts marc data from Elasticsearch result and decodes to MARC::Record object
363
364
=cut
365
366
sub decode_record_from_result {
367
    # Result is passed in as array, will get flattened
368
    # and first element will be $result
369
    my ( $self, $result ) = @_;
370
    if (C4::Context->preference('ExperimentalElasticsearchIndexing')) {
371
        return MARC::Record->new_from_xml($result->{marc_xml}, 'UTF-8', uc C4::Context->preference('marcflavour'));
372
    }
373
    else {
374
        return $self->json2marc($result->{record});
375
    }
376
}
377
378
363
=head2 json2marc
379
=head2 json2marc
364
380
365
    my $marc = $self->json2marc($marc_json);
381
    my $marc = $self->json2marc($marc_json);
(-)a/etc/searchengine/elasticsearch/field_config.yaml (+5 lines)
Lines 8-13 general: Link Here
8
    record:
8
    record:
9
      store: true
9
      store: true
10
      type: text
10
      type: text
11
    marc_xml:
12
      store: true
13
      type: text
14
      analyzer: keyword
15
      index: false
11
# Search fields
16
# Search fields
12
search:
17
search:
13
  boolean:
18
  boolean:
(-)a/installer/data/mysql/atomicupdate/bug_19893_experimental_indexing_elasticsearch_syspref.sql (+1 lines)
Line 0 Link Here
1
INSERT IGNORE INTO systempreferences (variable, value, explanation, options, type) VALUES ('ExperimentalElasticsearchIndexing', '0', 'Enable optimized experimental Elasticsearch indexing', NULL, 'YesNo');
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref (+6 lines)
Lines 425-427 Administration: Link Here
425
              choices:
425
              choices:
426
                Zebra: Zebra
426
                Zebra: Zebra
427
                Elasticsearch: Elasticsearch
427
                Elasticsearch: Elasticsearch
428
        -
429
            - pref: ExperimentalElasticsearchIndexing
430
              choices:
431
                yes: Enable
432
                no: "Don't enable"
433
            - "experimental faster indexing, only relevant if using Elasticsearch."
(-)a/misc/search_tools/rebuild_elastic_search.pl (-2 / +5 lines)
Lines 161-170 sub do_reindex { Link Here
161
161
162
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
162
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
163
    if ($delete) {
163
    if ($delete) {
164
165
        # We know it's safe to not recreate the indexer because update_index
164
        # We know it's safe to not recreate the indexer because update_index
166
        # hasn't been called yet.
165
        # hasn't been called yet.
167
        $indexer->drop_index();
166
        $indexer->drop_index();
167
        if (C4::Context->preference('ExperimentalElasticsearchIndexing')) {
168
            # Catmandu will create index for us in update_index, so without it we
169
            # to create it ourselves
170
            $indexer->create_index();
171
        }
168
    }
172
    }
169
173
170
    my $count        = 0;
174
    my $count        = 0;
171
- 

Return to bug 19893