View | Details | Raw Unified | Return to bug 19893
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (+199 lines)
Lines 34-39 use Search::Elasticsearch; Link Here
34
use Try::Tiny;
34
use Try::Tiny;
35
use YAML::Syck;
35
use YAML::Syck;
36
36
37
use List::Util qw( sum0 );
38
use Search::Elasticsearch;
39
use MARC::File::XML;
40
37
use Data::Dumper;    # TODO remove
41
use Data::Dumper;    # TODO remove
38
42
39
__PACKAGE__->mk_ro_accessors(qw( index ));
43
__PACKAGE__->mk_ro_accessors(qw( index ));
Lines 69-74 sub new { Link Here
69
    return $self;
73
    return $self;
70
}
74
}
71
75
76
sub get_elasticsearch {
77
    my $self = shift @_;
78
    unless (defined $self->{elasticsearch}) {
79
        my $conf = $self->get_elasticsearch_params();
80
        $self->{elasticsearch} = Search::Elasticsearch->new(
81
            client => "5_0::Direct",
82
            nodes => $conf->{nodes},
83
            cxn_pool => 'Sniff'
84
        );
85
    }
86
    return $self->{elasticsearch};
87
}
88
72
=head2 get_elasticsearch_params
89
=head2 get_elasticsearch_params
73
90
74
    my $params = $self->get_elasticsearch_params();
91
    my $params = $self->get_elasticsearch_params();
Lines 180-185 sub get_elasticsearch_mappings { Link Here
180
                    include_in_all => JSON::false,
197
                    include_in_all => JSON::false,
181
                    type           => "text",
198
                    type           => "text",
182
                },
199
                },
200
                marc_xml => {
201
                    store => JSON::true,
202
                    analyzer => "keyword",
203
                    index => JSON::false,
204
                    include_in_all => JSON::false,
205
                    type => "text",
206
                },
183
            }
207
            }
184
        }
208
        }
185
    };
209
    };
Lines 312-317 sub sort_fields { Link Here
312
    return $self->_sort_fields_accessor();
336
    return $self->_sort_fields_accessor();
313
}
337
}
314
338
339
sub marc_records_to_documents {
340
    my ($self, $records) = @_;
341
    my $rules = $self->get_marc_mapping_rules();
342
    my $control_fields_rules = $rules->{control_fields};
343
    my $data_fields_rules = $rules->{data_fields};
344
    my $marcflavour = lc C4::Context->preference('marcflavour');
345
346
    my @record_documents;
347
348
    sub _process_mappings {
349
        my ($mappings, $data, $record_document) = @_;
350
        foreach my $mapping (@{$mappings}) {
351
            my ($target, $options) = @{$mapping};
352
            my $_data = $data;
353
            $record_document->{$target} //= [];
354
            if ($options->{substr}) {
355
                my ($offset, $length) = @{$options->{substr}};
356
                $_data = substr $data, $offset, $length;
357
            }
358
            if ($options->{property}) {
359
                $_data = {
360
                    $options->{property} => $_data
361
                }
362
            }
363
            push @{$record_document->{$target}}, $_data;
364
        }
365
    }
366
    foreach my $record (@{$records}) {
367
        my $record_document = {};
368
        my $mappings = $rules->{leader};
369
        if ($mappings) {
370
            _process_mappings($mappings, $record->leader(), $record_document);
371
        }
372
        foreach my $field ($record->fields()) {
373
            if($field->is_control_field()) {
374
                my $mappings = $control_fields_rules->{$field->tag()};
375
                if ($mappings) {
376
                    _process_mappings($mappings, $field->data(), $record_document);
377
                }
378
            }
379
            else {
380
                my $subfields_mappings = $data_fields_rules->{$field->tag()};
381
                if ($subfields_mappings) {
382
                    my $wildcard_mappings = $subfields_mappings->{'*'};
383
                    foreach my $subfield ($field->subfields()) {
384
                        my ($code, $data) = @{$subfield};
385
                        my $mappings = $subfields_mappings->{$code} // [];
386
                        if ($wildcard_mappings) {
387
                            $mappings = [@{$mappings}, @{$wildcard_mappings}];
388
                        }
389
                        if (@{$mappings}) {
390
                            _process_mappings($mappings, $data, $record_document);
391
                        }
392
                    }
393
                }
394
            }
395
        }
396
        foreach my $field (keys %{$rules->{defaults}}) {
397
            unless (defined $record_document->{$field}) {
398
                $record_document->{$field} = $rules->{defaults}->{$field};
399
            }
400
        }
401
        foreach my $field (@{$rules->{sum}}) {
402
            if (defined $record_document->{$field}) {
403
                # TODO: validate numeric? filter?
404
                # TODO: Or should only accept fields without nested values?
405
                # TODO: Quick and dirty, improve if needed
406
                $record_document->{$field} = sum0(grep { ref($_) eq 'SCALAR' && m/\d+([.,]\d+)?/} @{$record_document->{$field}});
407
            }
408
        }
409
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
410
        $record->encoding('UTF-8');
411
        $record_document->{'marc_xml'} = $record->as_xml_record($marcflavour);
412
        my $id = $record->subfield('999', 'c');
413
        push @record_documents, [$id, $record_document];
414
    }
415
    return \@record_documents;
416
}
417
418
# Provides the rules for marc to Elasticsearch JSON document conversion.
419
sub get_marc_mapping_rules {
420
    my ($self) = @_;
421
422
    my $marcflavour = lc C4::Context->preference('marcflavour');
423
    my @rules;
424
425
    sub _field_mappings {
426
        my ($facet, $suggestible, $sort, $target_name, $target_type, $range) = @_;
427
        my %mapping_defaults = ();
428
        my @mappings;
429
430
        my $substr_args = undef;
431
        if ($range) {
432
            my ($offset, $end) = map(int, split /-/, $range, 2);
433
            $substr_args = [$offset];
434
            push @{$substr_args}, (defined $end ? $end - $offset : 1);
435
        }
436
        my $default_options = {};
437
        if ($substr_args) {
438
            $default_options->{substr} = $substr_args;
439
        }
440
441
        my $mapping = [$target_name, $default_options];
442
        push @mappings, $mapping;
443
444
        my @suffixes = ();
445
        push @suffixes, 'facet' if $facet;
446
        push @suffixes, 'suggestion' if $suggestible; # Check condition, also if undef?
447
        push @suffixes, 'sort' if $sort;
448
        foreach my $suffix (@suffixes) {
449
            my $mapping = ["${target_name}__$suffix"];
450
            # Hack, fix later in less hideous manner
451
            if ($suffix eq 'suggestion') {
452
                push @{$mapping}, {%{$default_options}, property => 'input'};
453
            }
454
            else {
455
                push @{$mapping}, $default_options;
456
            }
457
            push @mappings, $mapping;
458
        }
459
        return @mappings;
460
    };
461
    my $field_spec_regexp = qr/^([0-9]{3})([0-9a-z]+)?(?:_\/(\d+(?:-\d+)?))?$/;
462
    my $leader_regexp = qr/^leader(?:_\/(\d+(?:-\d+)?))?$/;
463
    my $rules = {
464
        'leader' => [],
465
        'control_fields' => {},
466
        'data_fields' => {},
467
        'sum' => [],
468
        'defaults' => {}
469
    };
470
471
    $self->_foreach_mapping(sub {
472
        my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_;
473
        return if $marc_type ne $marcflavour;
474
475
        if ($type eq 'sum') {
476
            push @{$rules->{sum}}, $name;
477
        }
478
        elsif($type eq 'boolean') {
479
            # boolean gets special handling, if value doesn't exist for a field,
480
            # it is set to false
481
            $rules->{defaults}->{$name} = 0;
482
        }
483
484
        if ($marc_field =~ $field_spec_regexp) {
485
            my $field_tag = $1;
486
            my $subfields = defined $2 ? $2 : '*';
487
            my $range = defined $3 ? $3 : undef;
488
            if ($field_tag < 10) {
489
                $rules->{control_fields}->{$field_tag} //= [];
490
                my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
491
                push @{$rules->{control_fields}->{$field_tag}}, @mappings;
492
            }
493
            else {
494
                $rules->{data_fields}->{$field_tag} //= {};
495
                foreach my $subfield (split //, $subfields) {
496
                    $rules->{data_fields}->{$field_tag}->{$subfield} //= [];
497
                    my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
498
                    push @{$rules->{data_fields}->{$field_tag}->{$subfield}}, @mappings;
499
                }
500
            }
501
        }
502
        elsif ($marc_field =~ $leader_regexp) {
503
            my $range = defined $1 ? $1 : undef;
504
            my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
505
            push @{$rules->{leader}}, @mappings;
506
        }
507
        else {
508
            die("Invalid marc field: $marc_field");
509
        }
510
    });
511
    return $rules;
512
}
513
315
# Provides the rules for data conversion.
514
# Provides the rules for data conversion.
316
sub get_fixer_rules {
515
sub get_fixer_rules {
317
    my ($self) = @_;
516
    my ($self) = @_;
(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-27 / +106 lines)
Lines 66-89 sub update_index { Link Here
66
        $self->_sanitise_records($biblionums, $records);
66
        $self->_sanitise_records($biblionums, $records);
67
    }
67
    }
68
68
69
    my $from    = $self->_convert_marc_to_json($records);
69
    if (C4::Context->preference('ExperimentalElasticsearchIndexing')) {
70
    if ( !$self->store ) {
70
        $self->ensure_mappings_updated();
71
        my $params  = $self->get_elasticsearch_params();
71
        $self->bulk_index($records);
72
        $self->store(
72
        return 1;
73
            Catmandu::Store::ElasticSearch->new(
73
    }
74
                %$params,
74
    else {
75
                index_settings => $self->get_elasticsearch_settings(),
75
        my $from = $self->_convert_marc_to_json($records);
76
                index_mappings => $self->get_elasticsearch_mappings(),
76
        if ( !$self->store ) {
77
            )
77
            my $params  = $self->get_elasticsearch_params();
78
        );
78
            $self->store(
79
                Catmandu::Store::ElasticSearch->new(
80
                    %$params,
81
                    index_settings => $self->get_elasticsearch_settings(),
82
                    index_mappings => $self->get_elasticsearch_mappings(),
83
                )
84
            );
85
        }
86
87
        #print Data::Dumper::Dumper( $from->to_array );
88
        $self->store->bag->add_many($from);
89
        $self->store->bag->commit;
90
        return 1;
79
    }
91
    }
92
}
80
93
81
    #print Data::Dumper::Dumper( $from->to_array );
94
sub bulk_index {
82
    $self->store->bag->add_many($from);
95
    my ($self, $records) = @_;
83
    $self->store->bag->commit;
96
    my $conf = $self->get_elasticsearch_params();
97
    my $elasticsearch = $self->get_elasticsearch();
98
    my $documents = $self->marc_records_to_documents($records);
99
    my @body;
100
101
    foreach my $document_info (@{$documents}) {
102
        my ($id, $document) = @{$document_info};
103
        push @body, {
104
            index => {
105
                _id => $id
106
            }
107
        };
108
        push @body, $document;
109
    }
110
    my $response = $elasticsearch->bulk(
111
        index => $conf->{index_name},
112
        type => 'data', # is just hard coded in Indexer.pm?
113
        body => \@body
114
    );
115
    # TODO: handle response
84
    return 1;
116
    return 1;
85
}
117
}
86
118
119
sub ensure_mappings_updated {
120
    my ($self) = @_;
121
    unless ($self->{_mappings_updated}) {
122
        $self->update_mappings();
123
    }
124
}
125
126
sub update_mappings {
127
    my ($self) = @_;
128
    my $conf = $self->get_elasticsearch_params();
129
    my $elasticsearch = $self->get_elasticsearch();
130
    my $mappings = $self->get_elasticsearch_mappings();
131
132
    foreach my $type (keys %{$mappings}) {
133
        my $response = $elasticsearch->indices->put_mapping(
134
            index => $conf->{index_name},
135
            type => $type,
136
            body => {
137
                $type => $mappings->{$type}
138
            }
139
        );
140
        # TODO: process response, produce errors etc
141
    }
142
    $self->{_mappings_updated} = 1;
143
}
144
87
=head2 $indexer->update_index_background($biblionums, $records)
145
=head2 $indexer->update_index_background($biblionums, $records)
88
146
89
This has exactly the same API as C<update_index_background> however it'll
147
This has exactly the same API as C<update_index_background> however it'll
Lines 148-168 after this will recreate it again. Link Here
148
206
149
sub drop_index {
207
sub drop_index {
150
    my ($self) = @_;
208
    my ($self) = @_;
151
209
    if (C4::Context->preference('ExperimentalElasticsearchIndexing')) {
152
    if (!$self->store) {
210
        my $conf = $self->get_elasticsearch_params();
153
        # If this index doesn't exist, this will create it. Then it'll be
211
        my $elasticsearch = $self->get_elasticsearch();
154
        # deleted. That's not the end of the world however.
212
        my $response = $elasticsearch->indices->delete(index => $conf->{index_name});
155
        my $params  = $self->get_elasticsearch_params();
213
        # TODO: Handle response? Convert errors to exceptions/die
156
        $self->store(
214
    }
157
            Catmandu::Store::ElasticSearch->new(
215
    else {
158
                %$params,
216
        if (!$self->store) {
159
                index_settings => $self->get_elasticsearch_settings(),
217
            # If this index doesn't exist, this will create it. Then it'll be
160
                index_mappings => $self->get_elasticsearch_mappings(),
218
            # deleted. That's not the end of the world however.
161
            )
219
            my $params  = $self->get_elasticsearch_params();
162
        );
220
            $self->store(
221
                Catmandu::Store::ElasticSearch->new(
222
                    %$params,
223
                    index_settings => $self->get_elasticsearch_settings(),
224
                    index_mappings => $self->get_elasticsearch_mappings(),
225
                )
226
            );
227
        }
228
        $self->store->drop();
229
        $self->store(undef);
163
    }
230
    }
164
    $self->store->drop();
231
}
165
    $self->store(undef);
232
233
sub create_index {
234
    my ($self) = @_;
235
    my $conf = $self->get_elasticsearch_params();
236
    my $settings = $self->get_elasticsearch_settings();
237
    my $elasticsearch = $self->get_elasticsearch();
238
    my $response = $elasticsearch->indices->create(
239
        index => $conf->{index_name},
240
        body => {
241
            settings => $settings
242
        }
243
    );
244
    # TODO: Handle response? Convert errors to exceptions/die
166
}
245
}
167
246
168
sub _sanitise_records {
247
sub _sanitise_records {
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-16 / +34 lines)
Lines 48-54 use Koha::SearchEngine::QueryBuilder; Link Here
48
use Koha::SearchEngine::Search;
48
use Koha::SearchEngine::Search;
49
use MARC::Record;
49
use MARC::Record;
50
use Catmandu::Store::ElasticSearch;
50
use Catmandu::Store::ElasticSearch;
51
51
use MARC::File::XML;
52
use Data::Dumper; #TODO remove
52
use Data::Dumper; #TODO remove
53
use Carp qw(cluck);
53
use Carp qw(cluck);
54
54
Lines 156-170 sub search_compat { Link Here
156
    my $results = $self->search($query, undef, $results_per_page, %options);
156
    my $results = $self->search($query, undef, $results_per_page, %options);
157
157
158
    # Convert each result into a MARC::Record
158
    # Convert each result into a MARC::Record
159
    my (@records, $index);
159
    my @records;
160
    $index = $offset; # opac-search expects results to be put in the
160
    # opac-search expects results to be put in the
161
        # right place in the array, according to $offset
161
    # right place in the array, according to $offset
162
    my $index = $offset;
162
    $results->each(sub {
163
    $results->each(sub {
163
            # The results come in an array for some reason
164
        $records[$index++] = $self->decode_record_from_result(@_);
164
            my $marc_json = $_[0]->{record};
165
    });
165
            my $marc = $self->json2marc($marc_json);
166
            $records[$index++] = $marc;
167
        });
168
    # consumers of this expect a name-spaced result, we provide the default
166
    # consumers of this expect a name-spaced result, we provide the default
169
    # configuration.
167
    # configuration.
170
    my %result;
168
    my %result;
Lines 195-208 sub search_auth_compat { Link Here
195
    $res->each(
193
    $res->each(
196
        sub {
194
        sub {
197
            my %result;
195
            my %result;
198
            my $record    = $_[0];
199
            my $marc_json = $record->{record};
200
196
201
            # I wonder if these should be real values defined in the mapping
197
            # I wonder if these should be real values defined in the mapping
202
            # rather than hard-coded conversions.
198
            # rather than hard-coded conversions.
203
            # Our results often come through as nested arrays, to fix this
199
            # Our results often come through as nested arrays, to fix this
204
            # requires changes in catmandu.
200
            # requires changes in catmandu.
205
            my $authid = $record->{ 'Local-number' }[0][0];
201
            my $record    = $_[0];
202
            my $authid = C4::Context->preference('ExperimentalElasticsearchIndexing') ?
203
                $record->{ 'Local-number' }[0] :
204
                $record->{ 'Local-number' }[0][0];
205
206
            $result{authid} = $authid;
206
            $result{authid} = $authid;
207
207
208
            # TODO put all this info into the record at index time so we
208
            # TODO put all this info into the record at index time so we
Lines 218-224 sub search_auth_compat { Link Here
218
            # it's not reproduced here yet.
218
            # it's not reproduced here yet.
219
            my $authtype           = $rs->single;
219
            my $authtype           = $rs->single;
220
            my $auth_tag_to_report = $authtype->auth_tag_to_report;
220
            my $auth_tag_to_report = $authtype->auth_tag_to_report;
221
            my $marc               = $self->json2marc($marc_json);
221
            my $marc               = $self->decode_record_from_result(@_);
222
            my $mainentry          = $marc->field($auth_tag_to_report);
222
            my $mainentry          = $marc->field($auth_tag_to_report);
223
            my $reported_tag;
223
            my $reported_tag;
224
            if ($mainentry) {
224
            if ($mainentry) {
Lines 341-349 sub simple_search_compat { Link Here
341
    my $results = $self->search($query, undef, $max_results, %options);
341
    my $results = $self->search($query, undef, $max_results, %options);
342
    my @records;
342
    my @records;
343
    $results->each(sub {
343
    $results->each(sub {
344
            # The results come in an array for some reason
344
            my $marc = $self->decode_record_from_result(@_);
345
            my $marc_json = $_[0]->{record};
346
            my $marc = $self->json2marc($marc_json);
347
            push @records, $marc;
345
            push @records, $marc;
348
        });
346
        });
349
    return (undef, \@records, $results->total);
347
    return (undef, \@records, $results->total);
Lines 364-369 sub extract_biblionumber { Link Here
364
    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
362
    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
365
}
363
}
366
364
365
=head2 decode_record_from_result
366
    my $marc_record = $self->decode_record_from_result(@result);
367
368
Extracts marc data from Elasticsearch result and decodes to MARC::Record object
369
370
=cut
371
372
sub decode_record_from_result {
373
    # Result is passed in as array, will get flattened
374
    # and first element will be $result
375
    my ( $self, $result ) = @_;
376
    if (C4::Context->preference('ExperimentalElasticsearchIndexing')) {
377
        return MARC::Record->new_from_xml($result->{marc_xml}, 'UTF-8', uc C4::Context->preference('marcflavour'));
378
    }
379
    else {
380
        return $self->json2marc($result->{record});
381
    }
382
}
383
384
367
=head2 json2marc
385
=head2 json2marc
368
386
369
    my $marc = $self->json2marc($marc_json);
387
    my $marc = $self->json2marc($marc_json);
(-)a/installer/data/mysql/atomicupdate/bug_19893_experimental_indexing_elasticsearch_syspref.sql (+1 lines)
Line 0 Link Here
1
INSERT IGNORE INTO systempreferences (variable, value, explanation, options, type) VALUES ('ExperimentalElasticsearchIndexing', '0', 'Enable optimized experimental Elasticsearch indexing', NULL, 'YesNo');
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref (+6 lines)
Lines 425-427 Administration: Link Here
425
              choices:
425
              choices:
426
                Zebra: Zebra
426
                Zebra: Zebra
427
                Elasticsearch: Elasticsearch
427
                Elasticsearch: Elasticsearch
428
        -
429
            - pref: ExperimentalElasticsearchIndexing
430
              choices:
431
                yes: Enable
432
                no: "Don't enable"
433
            - "experimental faster indexing, only relevant if using Elasticsearch."
(-)a/misc/search_tools/rebuild_elastic_search.pl (-2 / +5 lines)
Lines 161-170 sub do_reindex { Link Here
161
161
162
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
162
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
163
    if ($delete) {
163
    if ($delete) {
164
165
        # We know it's safe to not recreate the indexer because update_index
164
        # We know it's safe to not recreate the indexer because update_index
166
        # hasn't been called yet.
165
        # hasn't been called yet.
167
        $indexer->drop_index();
166
        $indexer->drop_index();
167
        if (C4::Context->preference('ExperimentalElasticsearchIndexing')) {
168
            # Catmandu will create index for us in update_index, so without it we
169
            # to create it ourselves
170
            $indexer->create_index();
171
        }
168
    }
172
    }
169
173
170
    my $count        = 0;
174
    my $count        = 0;
171
- 

Return to bug 19893