View | Details | Raw Unified | Return to bug 19893
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-29 / +198 lines)
Lines 34-39 use Search::Elasticsearch; Link Here
34
use Try::Tiny;
34
use Try::Tiny;
35
use YAML::Syck;
35
use YAML::Syck;
36
36
37
use List::Util qw( sum0 reduce );
38
use Search::Elasticsearch;
39
use MARC::File::XML;
40
37
__PACKAGE__->mk_ro_accessors(qw( index ));
41
__PACKAGE__->mk_ro_accessors(qw( index ));
38
__PACKAGE__->mk_accessors(qw( sort_fields ));
42
__PACKAGE__->mk_accessors(qw( sort_fields ));
39
43
Lines 67-72 sub new { Link Here
67
    return $self;
71
    return $self;
68
}
72
}
69
73
74
sub get_elasticsearch {
75
    my $self = shift @_;
76
    unless (defined $self->{elasticsearch}) {
77
        my $conf = $self->get_elasticsearch_params();
78
        $self->{elasticsearch} = Search::Elasticsearch->new(
79
            client => "5_0::Direct",
80
            nodes => $conf->{nodes},
81
            cxn_pool => 'Sniff'
82
        );
83
    }
84
    return $self->{elasticsearch};
85
}
86
70
=head2 get_elasticsearch_params
87
=head2 get_elasticsearch_params
71
88
72
    my $params = $self->get_elasticsearch_params();
89
    my $params = $self->get_elasticsearch_params();
Lines 281-328 sub sort_fields { Link Here
281
    return $self->_sort_fields_accessor();
298
    return $self->_sort_fields_accessor();
282
}
299
}
283
300
284
# Provides the rules for data conversion.
301
sub marc_records_to_documents {
285
sub get_fixer_rules {
302
    my ($self, $records) = @_;
303
    my $rules = $self->get_marc_mapping_rules();
304
    my $control_fields_rules = $rules->{control_fields};
305
    my $data_fields_rules = $rules->{data_fields};
306
    my $marcflavour = lc C4::Context->preference('marcflavour');
307
308
    my @record_documents;
309
310
    sub _process_mappings {
311
        my ($mappings, $data, $record_document) = @_;
312
        foreach my $mapping (@{$mappings}) {
313
            my ($target, $options) = @{$mapping};
314
            # Copy (scalar) data since can have multiple targets
315
            # with differing options for (possibly) mutating data
316
            # so need a different copy for each
317
            my $_data = $data;
318
            $record_document->{$target} //= [];
319
            if (defined $options->{substr}) {
320
                my ($start, $length) = @{$options->{substr}};
321
                $_data = length($data) > $start ? substr $data, $start, $length : '';
322
            }
323
            if (defined $options->{value_callbacks}) {
324
                $_data = reduce { $b->($a) } ($_data, @{$options->{value_callbacks}});
325
            }
326
            if (defined $options->{property}) {
327
                $_data = {
328
                    $options->{property} => $_data
329
                }
330
            }
331
            push @{$record_document->{$target}}, $_data;
332
        }
333
    }
334
    foreach my $record (@{$records}) {
335
        my $record_document = {};
336
        my $mappings = $rules->{leader};
337
        if ($mappings) {
338
            _process_mappings($mappings, $record->leader(), $record_document);
339
        }
340
        foreach my $field ($record->fields()) {
341
            if($field->is_control_field()) {
342
                my $mappings = $control_fields_rules->{$field->tag()};
343
                if ($mappings) {
344
                    _process_mappings($mappings, $field->data(), $record_document);
345
                }
346
            }
347
            else {
348
                my $subfields_mappings = $data_fields_rules->{$field->tag()};
349
                if ($subfields_mappings) {
350
                    my $wildcard_mappings = $subfields_mappings->{'*'};
351
                    foreach my $subfield ($field->subfields()) {
352
                        my ($code, $data) = @{$subfield};
353
                        my $mappings = $subfields_mappings->{$code} // [];
354
                        if ($wildcard_mappings) {
355
                            $mappings = [@{$mappings}, @{$wildcard_mappings}];
356
                        }
357
                        if (@{$mappings}) {
358
                            _process_mappings($mappings, $data, $record_document);
359
                        }
360
                    }
361
                }
362
            }
363
        }
364
        foreach my $field (keys %{$rules->{defaults}}) {
365
            unless (defined $record_document->{$field}) {
366
                $record_document->{$field} = $rules->{defaults}->{$field};
367
            }
368
        }
369
        foreach my $field (@{$rules->{sum}}) {
370
            if (defined $record_document->{$field}) {
371
                use Data::Dumper;
372
                # TODO: validate numeric? filter?
373
                # TODO: Or should only accept fields without nested values?
374
                # TODO: Quick and dirty, improve if needed
375
                $record_document->{$field} = sum0(grep { !ref($_) && m/\d+(\.\d+)?/} @{$record_document->{$field}});
376
            }
377
        }
378
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
379
        $record->encoding('UTF-8');
380
        $record_document->{'marc_xml'} = $record->as_xml_record($marcflavour);
381
        my $id = $record->subfield('999', 'c');
382
        push @record_documents, [$id, $record_document];
383
    }
384
    return \@record_documents;
385
}
386
387
# Provides the rules for marc to Elasticsearch JSON document conversion.
388
sub get_marc_mapping_rules {
286
    my ($self) = @_;
389
    my ($self) = @_;
287
390
288
    my $marcflavour = lc C4::Context->preference('marcflavour');
391
    my $marcflavour = lc C4::Context->preference('marcflavour');
289
    my @rules;
392
    my @rules;
290
393
291
    $self->_foreach_mapping(
394
    sub _field_mappings {
292
        sub {
395
        my ($facet, $suggestible, $sort, $target_name, $target_type, $range) = @_;
293
            my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_;
396
        my %mapping_defaults = ();
294
            return if $marc_type ne $marcflavour;
397
        my @mappings;
295
            my $options ='';
398
399
        my $substr_args = undef;
400
        if ($range) {
401
            # TODO: use value_callback instead?
402
            my ($start, $end) = map(int, split /-/, $range, 2);
403
            $substr_args = [$start];
404
            push @{$substr_args}, (defined $end ? $end - $start + 1 : 1);
405
        }
406
        my $default_options = {};
407
        if ($substr_args) {
408
            $default_options->{substr} = $substr_args;
409
        }
410
411
        # TODO: Should probably have per type value callback/hook
412
        # but hard code for now
413
        if ($target_type eq 'boolean') {
414
            $default_options->{value_callbacks} //= [];
415
            push @{$default_options->{value_callbacks}}, sub {
416
                my ($value) = @_;
417
                # Trim whitespace at both ends
418
                $value =~ s/^\s+|\s+$//g;
419
                return $value ? 'true' : 'false';
420
            };
421
        }
296
422
297
            push @rules, "marc_map('$marc_field','${name}.\$append', $options)";
423
        my $mapping = [$target_name, $default_options];
298
            if ($facet) {
424
        push @mappings, $mapping;
299
                push @rules, "marc_map('$marc_field','${name}__facet.\$append', $options)";
425
426
        my @suffixes = ();
427
        push @suffixes, 'facet' if $facet;
428
        push @suffixes, 'suggestion' if $suggestible;
429
        push @suffixes, 'sort' if !defined $sort || $sort;
430
431
        foreach my $suffix (@suffixes) {
432
            my $mapping = ["${target_name}__$suffix"];
433
            # Hack, fix later in less hideous manner
434
            if ($suffix eq 'suggestion') {
435
                push @{$mapping}, {%{$default_options}, property => 'input'};
300
            }
436
            }
301
            if ($suggestible) {
437
            else {
302
                push @rules,
438
                push @{$mapping}, $default_options;
303
                    #"marc_map('$marc_field','${name}__suggestion.input.\$append', '')"; #must not have nested data structures in .input
304
                    "marc_map('$marc_field','${name}__suggestion.input.\$append')";
305
            }
439
            }
306
            if ( $type eq 'boolean' ) {
440
            push @mappings, $mapping;
441
        }
442
        return @mappings;
443
    };
444
    my $field_spec_regexp = qr/^([0-9]{3})([0-9a-z]+)?(?:_\/(\d+(?:-\d+)?))?$/;
445
    my $leader_regexp = qr/^leader(?:_\/(\d+(?:-\d+)?))?$/;
446
    my $rules = {
447
        'leader' => [],
448
        'control_fields' => {},
449
        'data_fields' => {},
450
        'sum' => [],
451
        'defaults' => {}
452
    };
453
454
    $self->_foreach_mapping(sub {
455
        my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_;
456
        return if $marc_type ne $marcflavour;
457
458
        if ($type eq 'sum') {
459
            push @{$rules->{sum}}, $name;
460
        }
461
        elsif($type eq 'boolean') {
462
            # boolean gets special handling, if value doesn't exist for a field,
463
            # it is set to false
464
            $rules->{defaults}->{$name} = 'false';
465
        }
307
466
308
                # boolean gets special handling, basically if it doesn't exist,
467
        if ($marc_field =~ $field_spec_regexp) {
309
                # it's added and set to false. Otherwise we can't query it.
468
            my $field_tag = $1;
310
                push @rules,
469
            my $subfields = defined $2 ? $2 : '*';
311
                  "unless exists('$name') add_field('$name', 0) end";
470
            my $range = defined $3 ? $3 : undef;
312
            }
471
            if ($field_tag < 10) {
313
            if ($type eq 'sum' ) {
472
                $rules->{control_fields}->{$field_tag} //= [];
314
                push @rules, "sum('$name')";
473
                my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
474
                push @{$rules->{control_fields}->{$field_tag}}, @mappings;
315
            }
475
            }
316
            if ($self->sort_fields()->{$name}) {
476
            else {
317
                if ($sort || !defined $sort) {
477
                $rules->{data_fields}->{$field_tag} //= {};
318
                    push @rules, "marc_map('$marc_field','${name}__sort.\$append', $options)";
478
                foreach my $subfield (split //, $subfields) {
479
                    $rules->{data_fields}->{$field_tag}->{$subfield} //= [];
480
                    my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
481
                    push @{$rules->{data_fields}->{$field_tag}->{$subfield}}, @mappings;
319
                }
482
                }
320
            }
483
            }
321
        }
484
        }
322
    );
485
        elsif ($marc_field =~ $leader_regexp) {
323
486
            my $range = defined $1 ? $1 : undef;
324
    push @rules, "move_field(_id,es_id)"; #Also you must set the Catmandu::Store::ElasticSearch->new(key_prefix: 'es_');
487
            my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
325
    return \@rules;
488
            push @{$rules->{leader}}, @mappings;
489
        }
490
        else {
491
            die("Invalid marc field: $marc_field");
492
        }
493
    });
494
    return $rules;
326
}
495
}
327
496
328
=head2 _foreach_mapping
497
=head2 _foreach_mapping
(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-38 / +80 lines)
Lines 66-89 sub update_index { Link Here
66
        $self->_sanitise_records($biblionums, $records);
66
        $self->_sanitise_records($biblionums, $records);
67
    }
67
    }
68
68
69
    my $from    = $self->_convert_marc_to_json($records);
69
    $self->ensure_mappings_updated();
70
    if ( !$self->store ) {
70
    $self->bulk_index($records);
71
        my $params  = $self->get_elasticsearch_params();
71
    return 1;
72
        $self->store(
72
}
73
            Catmandu::Store::ElasticSearch->new(
74
                %$params,
75
                index_settings => $self->get_elasticsearch_settings(),
76
                index_mappings => $self->get_elasticsearch_mappings(),
77
            )
78
        );
79
    }
80
73
81
    #print Data::Dumper::Dumper( $from->to_array );
74
sub bulk_index {
82
    $self->store->bag->add_many($from);
75
    my ($self, $records) = @_;
83
    $self->store->bag->commit;
76
    my $conf = $self->get_elasticsearch_params();
77
    my $elasticsearch = $self->get_elasticsearch();
78
    my $documents = $self->marc_records_to_documents($records);
79
    my @body;
80
81
    foreach my $document_info (@{$documents}) {
82
        my ($id, $document) = @{$document_info};
83
        push @body, {
84
            index => {
85
                _id => $id
86
            }
87
        };
88
        push @body, $document;
89
    }
90
    my $response = $elasticsearch->bulk(
91
        index => $conf->{index_name},
92
        type => 'data', # is just hard coded in Indexer.pm?
93
        body => \@body
94
    );
95
    # TODO: handle response
84
    return 1;
96
    return 1;
85
}
97
}
86
98
99
sub ensure_mappings_updated {
100
    my ($self) = @_;
101
    unless ($self->{_mappings_updated}) {
102
        $self->update_mappings();
103
    }
104
}
105
106
sub update_mappings {
107
    my ($self) = @_;
108
    my $conf = $self->get_elasticsearch_params();
109
    my $elasticsearch = $self->get_elasticsearch();
110
    my $mappings = $self->get_elasticsearch_mappings();
111
112
    foreach my $type (keys %{$mappings}) {
113
        my $response = $elasticsearch->indices->put_mapping(
114
            index => $conf->{index_name},
115
            type => $type,
116
            body => {
117
                $type => $mappings->{$type}
118
            }
119
        );
120
        # TODO: process response, produce errors etc
121
    }
122
    $self->{_mappings_updated} = 1;
123
}
124
87
=head2 $indexer->update_index_background($biblionums, $records)
125
=head2 $indexer->update_index_background($biblionums, $records)
88
126
89
This has exactly the same API as C<update_index_background> however it'll
127
This has exactly the same API as C<update_index_background> however it'll
Lines 148-168 after this will recreate it again. Link Here
148
186
149
sub drop_index {
187
sub drop_index {
150
    my ($self) = @_;
188
    my ($self) = @_;
151
189
    if ($self->index_exists) {
152
    if (!$self->store) {
190
        my $conf = $self->get_elasticsearch_params();
153
        # If this index doesn't exist, this will create it. Then it'll be
191
        my $elasticsearch = $self->get_elasticsearch();
154
        # deleted. That's not the end of the world however.
192
        my $response = $elasticsearch->indices->delete(index => $conf->{index_name});
155
        my $params  = $self->get_elasticsearch_params();
193
        # TODO: Handle response? Convert errors to exceptions/die
156
        $self->store(
157
            Catmandu::Store::ElasticSearch->new(
158
                %$params,
159
                index_settings => $self->get_elasticsearch_settings(),
160
                index_mappings => $self->get_elasticsearch_mappings(),
161
            )
162
        );
163
    }
194
    }
164
    $self->store->drop();
195
}
165
    $self->store(undef);
196
197
sub create_index {
198
    my ($self) = @_;
199
    my $conf = $self->get_elasticsearch_params();
200
    my $settings = $self->get_elasticsearch_settings();
201
    my $elasticsearch = $self->get_elasticsearch();
202
    my $response = $elasticsearch->indices->create(
203
        index => $conf->{index_name},
204
        body => {
205
            settings => $settings
206
        }
207
    );
208
    # TODO: Handle response? Convert errors to exceptions/die
209
}
210
211
sub index_exists {
212
    my ($self) = @_;
213
    my $conf = $self->get_elasticsearch_params();
214
    my $elasticsearch = $self->get_elasticsearch();
215
    return $elasticsearch->indices->exists(
216
        index => $conf->{index_name},
217
    );
166
}
218
}
167
219
168
sub _sanitise_records {
220
sub _sanitise_records {
Lines 186-201 sub _sanitise_records { Link Here
186
    }
238
    }
187
}
239
}
188
240
189
sub _convert_marc_to_json {
190
    my $self    = shift;
191
    my $records = shift;
192
    my $importer =
193
      Catmandu::Importer::MARC->new( records => $records, id => '999c' );
194
    my $fixer = Catmandu::Fix->new( fixes => $self->get_fixer_rules() );
195
    $importer = $fixer->fix($importer);
196
    return $importer;
197
}
198
199
1;
241
1;
200
242
201
__END__
243
__END__
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-48 / +19 lines)
Lines 48-54 use Koha::SearchEngine::QueryBuilder; Link Here
48
use Koha::SearchEngine::Search;
48
use Koha::SearchEngine::Search;
49
use MARC::Record;
49
use MARC::Record;
50
use Catmandu::Store::ElasticSearch;
50
use Catmandu::Store::ElasticSearch;
51
51
use MARC::File::XML;
52
use Data::Dumper; #TODO remove
52
use Data::Dumper; #TODO remove
53
use Carp qw(cluck);
53
use Carp qw(cluck);
54
54
Lines 156-170 sub search_compat { Link Here
156
    my $results = $self->search($query, undef, $results_per_page, %options);
156
    my $results = $self->search($query, undef, $results_per_page, %options);
157
157
158
    # Convert each result into a MARC::Record
158
    # Convert each result into a MARC::Record
159
    my (@records, $index);
159
    my @records;
160
    $index = $offset; # opac-search expects results to be put in the
160
    # opac-search expects results to be put in the
161
        # right place in the array, according to $offset
161
    # right place in the array, according to $offset
162
    my $index = $offset;
162
    $results->each(sub {
163
    $results->each(sub {
163
            # The results come in an array for some reason
164
        $records[$index++] = $self->decode_record_from_result(@_);
164
            my $marc_json = $_[0]->{record};
165
    });
165
            my $marc = $self->json2marc($marc_json);
166
            $records[$index++] = $marc;
167
        });
168
    # consumers of this expect a name-spaced result, we provide the default
166
    # consumers of this expect a name-spaced result, we provide the default
169
    # configuration.
167
    # configuration.
170
    my %result;
168
    my %result;
Lines 195-208 sub search_auth_compat { Link Here
195
    $res->each(
193
    $res->each(
196
        sub {
194
        sub {
197
            my %result;
195
            my %result;
198
            my $record    = $_[0];
199
            my $marc_json = $record->{record};
200
196
201
            # I wonder if these should be real values defined in the mapping
197
            # I wonder if these should be real values defined in the mapping
202
            # rather than hard-coded conversions.
198
            # rather than hard-coded conversions.
199
            my $record    = $_[0];
203
            # Handle legacy nested arrays indexed with splitting enabled.
200
            # Handle legacy nested arrays indexed with splitting enabled.
204
            my $authid = $record->{ 'Local-number' }[0];
201
            my $authid = $record->{ 'Local-number' }[0];
205
            $authid = @$authid[0] if (ref $authid eq 'ARRAY');
202
            $authid = @$authid[0] if (ref $authid eq 'ARRAY');
203
206
            $result{authid} = $authid;
204
            $result{authid} = $authid;
207
205
208
            # TODO put all this info into the record at index time so we
206
            # TODO put all this info into the record at index time so we
Lines 218-224 sub search_auth_compat { Link Here
218
            # it's not reproduced here yet.
216
            # it's not reproduced here yet.
219
            my $authtype           = $rs->single;
217
            my $authtype           = $rs->single;
220
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
218
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
221
            my $marc               = $self->json2marc($marc_json);
219
            my $marc               = $self->decode_record_from_result(@_);
222
            my $mainentry          = $marc->field($auth_tag_to_report);
220
            my $mainentry          = $marc->field($auth_tag_to_report);
223
            my $reported_tag;
221
            my $reported_tag;
224
            if ($mainentry) {
222
            if ($mainentry) {
Lines 337-345 sub simple_search_compat { Link Here
337
    my $results = $self->search($query, undef, $max_results, %options);
335
    my $results = $self->search($query, undef, $max_results, %options);
338
    my @records;
336
    my @records;
339
    $results->each(sub {
337
    $results->each(sub {
340
            # The results come in an array for some reason
338
            my $marc = $self->decode_record_from_result(@_);
341
            my $marc_json = $_[0]->{record};
342
            my $marc = $self->json2marc($marc_json);
343
            push @records, $marc;
339
            push @records, $marc;
344
        });
340
        });
345
    return (undef, \@records, $results->total);
341
    return (undef, \@records, $results->total);
Lines 360-402 sub extract_biblionumber { Link Here
360
    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
356
    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
361
}
357
}
362
358
363
=head2 json2marc
359
=head2 decode_record_from_result
364
360
    my $marc_record = $self->decode_record_from_result(@result);
365
    my $marc = $self->json2marc($marc_json);
366
361
367
Converts the form of marc (based on its JSON, but as a Perl structure) that
362
Extracts marc data from Elasticsearch result and decodes to MARC::Record object
368
Catmandu stores into a MARC::Record object.
369
363
370
=cut
364
=cut
371
365
372
sub json2marc {
366
sub decode_record_from_result {
373
    my ( $self, $marcjson ) = @_;
367
    # Result is passed in as array, will get flattened
374
368
    # and first element will be $result
375
    my $marc = MARC::Record->new();
369
    my ( $self, $result ) = @_;
376
    $marc->encoding('UTF-8');
370
    return MARC::Record->new_from_xml($result->{marc_xml}, 'UTF-8', uc C4::Context->preference('marcflavour'));
377
378
    # fields are like:
379
    # [ '245', '1', '2', 'a' => 'Title', 'b' => 'Subtitle' ]
380
    # or
381
    # [ '001', undef, undef, '_', 'a value' ]
382
    # conveniently, this is the form that MARC::Field->new() likes
383
    foreach my $field (@$marcjson) {
384
        next if @$field < 5;
385
        if ( $field->[0] eq 'LDR' ) {
386
            $marc->leader( $field->[4] );
387
        }
388
        else {
389
            my $tag = $field->[0];
390
            my $marc_field;
391
            if ( MARC::Field->is_controlfield_tag( $field->[0] ) ) {
392
                $marc_field = MARC::Field->new($field->[0], $field->[4]);
393
            } else {
394
                $marc_field = MARC::Field->new(@$field);
395
            }
396
            $marc->append_fields($marc_field);
397
        }
398
    }
399
    return $marc;
400
}
371
}
401
372
402
=head2 _convert_facets
373
=head2 _convert_facets
(-)a/etc/searchengine/elasticsearch/field_config.yaml (+5 lines)
Lines 8-13 general: Link Here
8
    record:
8
    record:
9
      store: true
9
      store: true
10
      type: text
10
      type: text
11
    marc_xml:
12
      store: true
13
      type: text
14
      analyzer: keyword
15
      index: false
11
# Search fields
16
# Search fields
12
search:
17
search:
13
  boolean:
18
  boolean:
(-)a/misc/search_tools/rebuild_elastic_search.pl (-4 / +7 lines)
Lines 160-170 sub do_reindex { Link Here
160
    my ( $next, $index_name ) = @_;
160
    my ( $next, $index_name ) = @_;
161
161
162
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
162
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
163
    if ($delete) {
164
163
165
        # We know it's safe to not recreate the indexer because update_index
164
    if ($delete) {
166
        # hasn't been called yet.
165
        $indexer->drop_index() if $indexer->index_exists();
167
        $indexer->drop_index();
166
        $indexer->create_index();
167
    }
168
    elsif (!$indexer->index_exists()) {
169
        # Create index if does not exist
170
        $indexer->create_index();
168
    }
171
    }
169
172
170
    my $count        = 0;
173
    my $count        = 0;
(-)a/t/Koha/SearchEngine/Elasticsearch.t (-1 / +239 lines)
Lines 17-27 Link Here
17
17
18
use Modern::Perl;
18
use Modern::Perl;
19
19
20
use Test::More tests => 3;
20
use Test::More tests => 4;
21
use Test::Exception;
21
use Test::Exception;
22
22
23
use t::lib::Mocks;
23
use t::lib::Mocks;
24
24
25
use Test::MockModule;
26
27
use MARC::Record;
28
25
use Koha::SearchEngine::Elasticsearch;
29
use Koha::SearchEngine::Elasticsearch;
26
30
27
subtest '_read_configuration() tests' => sub {
31
subtest '_read_configuration() tests' => sub {
Lines 108-110 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
108
    $mappings = $es->get_elasticsearch_mappings();
112
    $mappings = $es->get_elasticsearch_mappings();
109
    is( $mappings->{data}{_all}{type}, 'string', 'Field mappings parsed correctly' );
113
    is( $mappings->{data}{_all}{type}, 'string', 'Field mappings parsed correctly' );
110
};
114
};
115
116
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
117
118
    plan tests => 29;
119
120
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
121
122
    my @mappings = (
123
        {
124
            name => 'author',
125
            type => 'string',
126
            facet => 1,
127
            suggestible => 1,
128
            sort => undef,
129
            marc_type => 'marc21',
130
            marc_field => '100a',
131
        },
132
        {
133
            name => 'author',
134
            type => 'string',
135
            facet => 1,
136
            suggestible => 1,
137
            sort => 1,
138
            marc_type => 'marc21',
139
            marc_field => '110a',
140
        },
141
        {
142
            name => 'title',
143
            type => 'string',
144
            facet => 0,
145
            suggestible => 1,
146
            sort => 1,
147
            marc_type => 'marc21',
148
            marc_field => '245a',
149
        },
150
        {
151
            name => 'unimarc_title',
152
            type => 'string',
153
            facet => 0,
154
            suggestible => 1,
155
            sort => 1,
156
            marc_type => 'unimarc',
157
            marc_field => '245a',
158
        },
159
        {
160
            name => 'title',
161
            type => 'string',
162
            facet => 0,
163
            suggestible => undef,
164
            sort => 0,
165
            marc_type => 'marc21',
166
            marc_field => '220',
167
        },
168
        {
169
            name => 'sum_item_price',
170
            type => 'sum',
171
            facet => 0,
172
            suggestible => 0,
173
            sort => 0,
174
            marc_type => 'marc21',
175
            marc_field => '952g',
176
        },
177
        {
178
            name => 'items_withdrawn_status',
179
            type => 'boolean',
180
            facet => 0,
181
            suggestible => 0,
182
            sort => 0,
183
            marc_type => 'marc21',
184
            marc_field => '9520',
185
        },
186
        {
187
            name => 'type_of_record',
188
            type => 'string',
189
            facet => 0,
190
            suggestible => 0,
191
            sort => 0,
192
            marc_type => 'marc21',
193
            marc_field => 'leader_/6',
194
        },
195
        {
196
            name => 'type_of_record_and_bib_level',
197
            type => 'string',
198
            facet => 0,
199
            suggestible => 0,
200
            sort => 0,
201
            marc_type => 'marc21',
202
            marc_field => 'leader_/6-7',
203
        },
204
    );
205
206
    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
207
    $se->mock('_foreach_mapping', sub {
208
        my ($self, $sub) = @_;
209
210
        foreach my $map (@mappings) {
211
            $sub->(
212
                $map->{name},
213
                $map->{type},
214
                $map->{facet},
215
                $map->{suggestible},
216
                $map->{sort},
217
                $map->{marc_type},
218
                $map->{marc_field}
219
            );
220
        }
221
    });
222
223
    my $see = Koha::SearchEngine::Elasticsearch->new({ index => 'biblios' });
224
225
    my $marc_record_1 = MARC::Record->new();
226
    $marc_record_1->leader('     cam  22      a 4500');
227
    $marc_record_1->append_fields(
228
        MARC::Field->new('100', '', '', a => 'Author 1'),
229
        MARC::Field->new('110', '', '', a => 'Corp Author'),
230
        MARC::Field->new('210', '', '', a => 'Title 1'),
231
        MARC::Field->new('245', '', '', a => 'Title: first record'),
232
        MARC::Field->new('999', '', '', c => '1234567'),
233
        # '  ' for testing trimming of white space in boolean value callback:
234
        MARC::Field->new('952', '', '', 0 => '  ', g => '123.30'),
235
        MARC::Field->new('952', '', '', 0 => 0, g => '127.20'),
236
    );
237
    my $marc_record_2 = MARC::Record->new();
238
    $marc_record_2->leader('     cam  22      a 4500');
239
    $marc_record_2->append_fields(
240
        MARC::Field->new('100', '', '', a => 'Author 2'),
241
        # MARC::Field->new('210', '', '', a => 'Title 2'),
242
        # MARC::Field->new('245', '', '', a => 'Title: second record'),
243
        MARC::Field->new('999', '', '', c => '1234568'),
244
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),
245
    );
246
    my $records = [$marc_record_1, $marc_record_2];
247
248
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
249
250
    my $docs = $see->marc_records_to_documents($records);
251
252
    # First record:
253
254
    is(scalar @{$docs}, 2, 'Two records converted to documents');
255
256
    is($docs->[0][0], '1234567', 'First document biblionumber should be set as first element in document touple');
257
258
    is(scalar @{$docs->[0][1]->{author}}, 2, 'First document author field should contain two values');
259
    is_deeply($docs->[0][1]->{author}, ['Author 1', 'Corp Author'], 'First document author field should be set correctly');
260
261
    is(scalar @{$docs->[0][1]->{author__sort}}, 2, 'First document author__sort field should have two values');
262
    is_deeply($docs->[0][1]->{author__sort}, ['Author 1', 'Corp Author'], 'First document author__sort field should be set correctly');
263
264
    is(scalar @{$docs->[0][1]->{title__sort}}, 1, 'First document title__sort field should have one value');
265
    is_deeply($docs->[0][1]->{title__sort}, ['Title: first record'], 'First document title__sort field should be set correctly');
266
267
    is(scalar @{$docs->[0][1]->{author__suggestion}}, 2, 'First document author__suggestion field should contain two values');
268
    is_deeply(
269
        $docs->[0][1]->{author__suggestion},
270
        [
271
            {
272
                'input' => 'Author 1'
273
            },
274
            {
275
                'input' => 'Corp Author'
276
            }
277
        ],
278
        'First document author__suggestion field should be set correctly'
279
    );
280
281
    is(scalar @{$docs->[0][1]->{title__suggestion}}, 1, 'First document title__suggestion field should contain one value');
282
    is_deeply(
283
        $docs->[0][1]->{title__suggestion},
284
        [{ 'input' => 'Title: first record' }],
285
        'First document title__suggestion field should be set correctly'
286
    );
287
288
    ok(!(defined $docs->[0][1]->{title__facet}), 'First document should have no title__facet field');
289
290
    is(scalar @{$docs->[0][1]->{author__facet}}, 2, 'First document author__facet field should have two values');
291
    is_deeply(
292
        $docs->[0][1]->{author__facet},
293
        ['Author 1', 'Corp Author'],
294
        'First document author__facet field should be set correctly'
295
    );
296
297
    is(scalar @{$docs->[0][1]->{items_withdrawn_status}}, 2, 'First document items_withdrawn_status field should have two values');
298
    is_deeply(
299
        $docs->[0][1]->{items_withdrawn_status},
300
        ['false', 'false'],
301
        'First document items_withdrawn_status field should be set correctly'
302
    );
303
304
    is(
305
        $docs->[0][1]->{sum_item_price},
306
        '250.5',
307
        'First document sum_item_price field should be set correctly'
308
    );
309
310
    ok(defined $docs->[0][1]->{marc_xml}, 'First document marc_xml field should be set');
311
312
    is(scalar @{$docs->[0][1]->{type_of_record}}, 1, 'First document type_of_record field should have one value');
313
    is_deeply(
314
        $docs->[0][1]->{type_of_record},
315
        ['a'],
316
        'First document type_of_record field should be set correctly'
317
    );
318
319
    is(scalar @{$docs->[0][1]->{type_of_record_and_bib_level}}, 1, 'First document type_of_record_and_bib_level field should have one value');
320
    is_deeply(
321
        $docs->[0][1]->{type_of_record_and_bib_level},
322
        ['am'],
323
        'First document type_of_record_and_bib_level field should be set correctly'
324
    );
325
326
    # Second record:
327
328
    is(scalar @{$docs->[1][1]->{author}}, 1, 'Second document author field should contain one value');
329
    is_deeply($docs->[1][1]->{author}, ['Author 2'], 'Second document author field should be set correctly');
330
331
    is(scalar @{$docs->[1][1]->{items_withdrawn_status}}, 1, 'Second document items_withdrawn_status field should have one value');
332
    is_deeply(
333
        $docs->[1][1]->{items_withdrawn_status},
334
        ['true'],
335
        'Second document items_withdrawn_status field should be set correctly'
336
    );
337
338
    is(
339
        $docs->[1][1]->{sum_item_price},
340
        0,
341
        'Second document sum_item_price field should be set correctly'
342
    );
343
344
    # Mappings marc_type:
345
346
    ok(!(defined $docs->[0][1]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
347
348
};
(-)a/t/db_dependent/Koha_Elasticsearch.t (-164 lines)
Lines 1-164 Link Here
1
# Copyright 2015 Catalyst IT
2
#
3
# This file is part of Koha.
4
#
5
# Koha is free software; you can redistribute it and/or modify it
6
# under the terms of the GNU General Public License as published by
7
# the Free Software Foundation; either version 3 of the License, or
8
# (at your option) any later version.
9
#
10
# Koha is distributed in the hope that it will be useful, but
11
# WITHOUT ANY WARRANTY; without even the implied warranty of
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
# GNU General Public License for more details.
14
#
15
# You should have received a copy of the GNU General Public License
16
# along with Koha; if not, see <http://www.gnu.org/licenses>
17
18
use Modern::Perl;
19
20
use Test::More tests => 2;
21
use Test::MockModule;
22
23
use t::lib::Mocks;
24
use MARC::Record;
25
26
my $schema = Koha::Database->schema;
27
28
use_ok('Koha::SearchEngine::Elasticsearch');
29
30
subtest 'get_fixer_rules() tests' => sub {
31
32
    plan tests => 49;
33
34
    $schema->storage->txn_begin;
35
36
    t::lib::Mocks::mock_preference( 'marcflavour', 'MARC21' );
37
38
    my @mappings;
39
40
    my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
41
    $se->mock( '_foreach_mapping', sub {
42
        my ($self, $sub ) = @_;
43
44
        foreach my $map ( @mappings ) {
45
            $sub->(
46
                $map->{name},
47
                $map->{type},
48
                $map->{facet},
49
                $map->{suggestible},
50
                $map->{sort},
51
                $map->{marc_type},
52
                $map->{marc_field}
53
            );
54
        }
55
    });
56
57
    my $see = Koha::SearchEngine::Elasticsearch->new({ index => 'biblios' });
58
59
    @mappings = (
60
        {
61
            name => 'author',
62
            type => 'string',
63
            facet => 1,
64
            suggestible => 1,
65
            sort => undef,
66
            marc_type => 'marc21',
67
            marc_field => '100a',
68
        },
69
        {
70
            name => 'author',
71
            type => 'string',
72
            facet => 1,
73
            suggestible => 1,
74
            sort => 1,
75
            marc_type => 'marc21',
76
            marc_field => '110a',
77
        },
78
    );
79
80
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
81
    my $result = $see->get_fixer_rules();
82
    is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
83
    is( $result->[1], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__facet.$append', )});
84
    is( $result->[2], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__suggestion.input.$append')});
85
    is( $result->[3], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__sort.$append', )});
86
    is( $result->[4], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
87
    is( $result->[5], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__facet.$append', )});
88
    is( $result->[6], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__suggestion.input.$append')});
89
    is( $result->[7], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
90
    is( $result->[8], q{move_field(_id,es_id)});
91
92
    $mappings[0]->{type}  = 'boolean';
93
    $mappings[1]->{type}  = 'boolean';
94
    $result = $see->get_fixer_rules();
95
    is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
96
    is( $result->[1], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__facet.$append', )});
97
    is( $result->[2], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__suggestion.input.$append')});
98
    is( $result->[3], q{unless exists('} . $mappings[0]->{name} . q{') add_field('} . $mappings[0]->{name} . q{', 0) end});
99
    is( $result->[4], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__sort.$append', )});
100
    is( $result->[5], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
101
    is( $result->[6], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__facet.$append', )});
102
    is( $result->[7], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__suggestion.input.$append')});
103
    is( $result->[8], q{unless exists('} . $mappings[1]->{name} . q{') add_field('} . $mappings[1]->{name} . q{', 0) end});
104
    is( $result->[9], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
105
    is( $result->[10], q{move_field(_id,es_id)});
106
107
    $mappings[0]->{type}  = 'sum';
108
    $mappings[1]->{type}  = 'sum';
109
    $result = $see->get_fixer_rules();
110
    is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
111
    is( $result->[1], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__facet.$append', )});
112
    is( $result->[2], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__suggestion.input.$append')});
113
    is( $result->[3], q{sum('} . $mappings[0]->{name} . q{')});
114
    is( $result->[4], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__sort.$append', )});
115
    is( $result->[5], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
116
    is( $result->[6], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__facet.$append', )});
117
    is( $result->[7], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__suggestion.input.$append')});
118
    is( $result->[8], q{sum('} . $mappings[1]->{name} . q{')});
119
    is( $result->[9], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
120
    is( $result->[10], q{move_field(_id,es_id)});
121
122
    $mappings[0]->{type}  = 'string';
123
    $mappings[0]->{facet} = 0;
124
    $mappings[1]->{type}  = 'string';
125
    $mappings[1]->{facet} = 0;
126
127
    $result = $see->get_fixer_rules();
128
    is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
129
    is( $result->[1], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__suggestion.input.$append')});
130
    is( $result->[2], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__sort.$append', )});
131
    is( $result->[3], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
132
    is( $result->[4], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__suggestion.input.$append')});
133
    is( $result->[5], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
134
    is( $result->[6], q{move_field(_id,es_id)});
135
136
    $mappings[0]->{suggestible}  = 0;
137
    $mappings[1]->{suggestible}  = 0;
138
139
    $result = $see->get_fixer_rules();
140
    is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
141
    is( $result->[1], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__sort.$append', )});
142
    is( $result->[2], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
143
    is( $result->[3], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
144
    is( $result->[4], q{move_field(_id,es_id)});
145
146
    $mappings[0]->{sort}  = 0;
147
    $mappings[1]->{sort}  = undef;
148
149
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
150
    $result = $see->get_fixer_rules();
151
    is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
152
    is( $result->[1], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
153
    is( $result->[2], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
154
    is( $result->[3], q{move_field(_id,es_id)});
155
156
    t::lib::Mocks::mock_preference( 'marcflavour', 'UNIMARC' );
157
158
    $result = $see->get_fixer_rules();
159
    is( $result->[0], q{move_field(_id,es_id)});
160
    is( $result->[1], undef, q{No mapping when marc_type doesn't match marchflavour} );
161
162
    $schema->storage->txn_rollback;
163
164
};
(-)a/t/db_dependent/Koha_Elasticsearch_Indexer.t (-87 / +5 lines)
Lines 17-23 Link Here
17
17
18
use Modern::Perl;
18
use Modern::Perl;
19
19
20
use Test::More tests => 6;
20
use Test::More tests => 3;
21
use Test::MockModule;
21
use Test::MockModule;
22
use t::lib::Mocks;
22
use t::lib::Mocks;
23
23
Lines 37-52 ok( Link Here
37
37
38
my $marc_record = MARC::Record->new();
38
my $marc_record = MARC::Record->new();
39
$marc_record->append_fields(
39
$marc_record->append_fields(
40
    MARC::Field->new( '001', '1234567' ),
40
	MARC::Field->new( '001', '1234567' ),
41
    MARC::Field->new( '020', '', '', 'a' => '1234567890123' ),
41
	MARC::Field->new( '020', '', '', 'a' => '1234567890123' ),
42
    MARC::Field->new( '245', '', '', 'a' => 'Title' )
42
	MARC::Field->new( '245', '', '', 'a' => 'Title' )
43
);
43
);
44
45
my $records = [$marc_record];
44
my $records = [$marc_record];
46
ok( my $converted = $indexer->_convert_marc_to_json($records),
47
    'Convert some records' );
48
49
is( $converted->count, 1, 'One converted record' );
50
45
51
SKIP: {
46
SKIP: {
52
47
Lines 55-135 SKIP: { Link Here
55
    skip 'ElasticSeatch configuration not available', 1
50
    skip 'ElasticSeatch configuration not available', 1
56
        if $@;
51
        if $@;
57
52
58
    ok( $indexer->update_index(undef,$records), 'Update Index' );
53
    ok( $indexer->update_index(undef, $records), 'Update Index' );
59
}
54
}
60
61
subtest '_convert_marc_to_json() tests' => sub {
62
63
    plan tests => 4;
64
65
    $schema->storage->txn_begin;
66
67
    t::lib::Mocks::mock_preference( 'marcflavour', 'MARC21' );
68
69
    my @mappings = (
70
        {
71
            name => 'author',
72
            type => 'string',
73
            facet => 1,
74
            suggestible => 1,
75
            sort => '~',
76
            marc_type => 'marc21',
77
            marc_field => '100a',
78
        },
79
        {
80
            name => 'author',
81
            type => 'string',
82
            facet => 1,
83
            suggestible => 1,
84
            sort => '~',
85
            marc_type => 'marc21',
86
            marc_field => '110a',
87
        },
88
    );
89
90
91
    my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
92
    $se->mock( '_foreach_mapping', sub {
93
        my ($self, $sub ) = @_;
94
95
        foreach my $map ( @mappings ) {
96
            $sub->(
97
                $map->{name},
98
                $map->{type},
99
                $map->{facet},
100
                $map->{suggestible},
101
                $map->{sort},
102
                $map->{marc_type},
103
                $map->{marc_field}
104
            );
105
        }
106
    });
107
108
    my $marc_record = MARC::Record->new();
109
    $marc_record->append_fields(
110
        MARC::Field->new( '001', '1234567' ),
111
        MARC::Field->new( '020', '', '', 'a' => '1234567890123' ),
112
        MARC::Field->new( '100', '', '', 'a' => 'Author' ),
113
        MARC::Field->new( '110', '', '', 'a' => 'Corp Author' ),
114
        MARC::Field->new( '245', '', '', 'a' => 'Title' ),
115
    );
116
    my $marc_record_2 = MARC::Record->new();
117
    $marc_record_2->append_fields(
118
        MARC::Field->new( '001', '1234567' ),
119
        MARC::Field->new( '020', '', '', 'a' => '1234567890123' ),
120
        MARC::Field->new( '100', '', '', 'a' => 'Author' ),
121
        MARC::Field->new( '245', '', '', 'a' => 'Title' ),
122
    );
123
    my @records = ( $marc_record, $marc_record_2 );
124
125
    my $importer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => 'biblios' })->_convert_marc_to_json( \@records );
126
    my $conv = $importer->next();
127
    is( $conv->{author}[0], "Author", "First mapped author should be 100a");
128
    is( $conv->{author}[1], "Corp Author", "Second mapped author should be 110a");
129
130
    $conv = $importer->next();
131
    is( $conv->{author}[0], "Author", "First mapped author should be 100a");
132
    is( scalar @{$conv->{author}} , 1, "We should map field only if exists, shouldn't add extra nulls");
133
134
    $schema->storage->txn_rollback;
135
};
136
- 

Return to bug 19893