View | Details | Raw Unified | Return to bug 19893
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-29 / +205 lines)
Lines 34-39 use Search::Elasticsearch; Link Here
34
use Try::Tiny;
34
use Try::Tiny;
35
use YAML::Syck;
35
use YAML::Syck;
36
36
37
use List::Util qw( sum0 reduce );
38
use Search::Elasticsearch;
39
use MARC::File::XML;
40
use MIME::Base64;
41
use Encode qw(encode);
42
37
__PACKAGE__->mk_ro_accessors(qw( index ));
43
__PACKAGE__->mk_ro_accessors(qw( index ));
38
__PACKAGE__->mk_accessors(qw( sort_fields ));
44
__PACKAGE__->mk_accessors(qw( sort_fields ));
39
45
Lines 67-72 sub new { Link Here
67
    return $self;
73
    return $self;
68
}
74
}
69
75
76
sub get_elasticsearch {
77
    my $self = shift @_;
78
    unless (defined $self->{elasticsearch}) {
79
        my $conf = $self->get_elasticsearch_params();
80
        $self->{elasticsearch} = Search::Elasticsearch->new(
81
            client => "5_0::Direct",
82
            nodes => $conf->{nodes},
83
            cxn_pool => 'Sniff'
84
        );
85
    }
86
    return $self->{elasticsearch};
87
}
88
70
=head2 get_elasticsearch_params
89
=head2 get_elasticsearch_params
71
90
72
    my $params = $self->get_elasticsearch_params();
91
    my $params = $self->get_elasticsearch_params();
Lines 281-328 sub sort_fields { Link Here
281
    return $self->_sort_fields_accessor();
300
    return $self->_sort_fields_accessor();
282
}
301
}
283
302
284
# Provides the rules for data conversion.
303
sub marc_records_to_documents {
285
sub get_fixer_rules {
304
    my ($self, $records) = @_;
305
    my $rules = $self->get_marc_mapping_rules();
306
    my $control_fields_rules = $rules->{control_fields};
307
    my $data_fields_rules = $rules->{data_fields};
308
    my $marcflavour = lc C4::Context->preference('marcflavour');
309
    my $serialization_format = C4::Context->preference('ElasticsearchMARCSerializationFormat');
310
311
    my @record_documents;
312
313
    sub _process_mappings {
314
        my ($mappings, $data, $record_document) = @_;
315
        foreach my $mapping (@{$mappings}) {
316
            my ($target, $options) = @{$mapping};
317
            # Copy (scalar) data since can have multiple targets
318
            # with differing options for (possibly) mutating data
319
            # so need a different copy for each
320
            my $_data = $data;
321
            $record_document->{$target} //= [];
322
            if (defined $options->{substr}) {
323
                my ($start, $length) = @{$options->{substr}};
324
                $_data = length($data) > $start ? substr $data, $start, $length : '';
325
            }
326
            if (defined $options->{value_callbacks}) {
327
                $_data = reduce { $b->($a) } ($_data, @{$options->{value_callbacks}});
328
            }
329
            if (defined $options->{property}) {
330
                $_data = {
331
                    $options->{property} => $_data
332
                }
333
            }
334
            push @{$record_document->{$target}}, $_data;
335
        }
336
    }
337
    foreach my $record (@{$records}) {
338
        my $record_document = {};
339
        my $mappings = $rules->{leader};
340
        if ($mappings) {
341
            _process_mappings($mappings, $record->leader(), $record_document);
342
        }
343
        foreach my $field ($record->fields()) {
344
            if($field->is_control_field()) {
345
                my $mappings = $control_fields_rules->{$field->tag()};
346
                if ($mappings) {
347
                    _process_mappings($mappings, $field->data(), $record_document);
348
                }
349
            }
350
            else {
351
                my $subfields_mappings = $data_fields_rules->{$field->tag()};
352
                if ($subfields_mappings) {
353
                    my $wildcard_mappings = $subfields_mappings->{'*'};
354
                    foreach my $subfield ($field->subfields()) {
355
                        my ($code, $data) = @{$subfield};
356
                        my $mappings = $subfields_mappings->{$code} // [];
357
                        if ($wildcard_mappings) {
358
                            $mappings = [@{$mappings}, @{$wildcard_mappings}];
359
                        }
360
                        if (@{$mappings}) {
361
                            _process_mappings($mappings, $data, $record_document);
362
                        }
363
                    }
364
                }
365
            }
366
        }
367
        foreach my $field (keys %{$rules->{defaults}}) {
368
            unless (defined $record_document->{$field}) {
369
                $record_document->{$field} = $rules->{defaults}->{$field};
370
            }
371
        }
372
        foreach my $field (@{$rules->{sum}}) {
373
            if (defined $record_document->{$field}) {
374
                # TODO: validate numeric? filter?
375
                # TODO: Or should only accept fields without nested values?
376
                # TODO: Quick and dirty, improve if needed
377
                $record_document->{$field} = sum0(grep { !ref($_) && m/\d+(\.\d+)?/} @{$record_document->{$field}});
378
            }
379
        }
380
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
381
        $record->encoding('UTF-8');
382
        if ($serialization_format eq 'base64ISO2709') {
383
            $record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));
384
        }
385
        else {
386
            $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
387
        }
388
        my $id = $record->subfield('999', 'c');
389
        push @record_documents, [$id, $record_document];
390
    }
391
    return \@record_documents;
392
}
393
394
# Provides the rules for marc to Elasticsearch JSON document conversion.
395
sub get_marc_mapping_rules {
286
    my ($self) = @_;
396
    my ($self) = @_;
287
397
288
    my $marcflavour = lc C4::Context->preference('marcflavour');
398
    my $marcflavour = lc C4::Context->preference('marcflavour');
289
    my @rules;
399
    my @rules;
290
400
291
    $self->_foreach_mapping(
401
    sub _field_mappings {
292
        sub {
402
        my ($facet, $suggestible, $sort, $target_name, $target_type, $range) = @_;
293
            my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_;
403
        my %mapping_defaults = ();
294
            return if $marc_type ne $marcflavour;
404
        my @mappings;
295
            my $options ='';
405
406
        my $substr_args = undef;
407
        if ($range) {
408
            # TODO: use value_callback instead?
409
            my ($start, $end) = map(int, split /-/, $range, 2);
410
            $substr_args = [$start];
411
            push @{$substr_args}, (defined $end ? $end - $start + 1 : 1);
412
        }
413
        my $default_options = {};
414
        if ($substr_args) {
415
            $default_options->{substr} = $substr_args;
416
        }
417
418
        # TODO: Should probably have per type value callback/hook
419
        # but hard code for now
420
        if ($target_type eq 'boolean') {
421
            $default_options->{value_callbacks} //= [];
422
            push @{$default_options->{value_callbacks}}, sub {
423
                my ($value) = @_;
424
                # Trim whitespace at both ends
425
                $value =~ s/^\s+|\s+$//g;
426
                return $value ? 'true' : 'false';
427
            };
428
        }
296
429
297
            push @rules, "marc_map('$marc_field','${name}.\$append', $options)";
430
        my $mapping = [$target_name, $default_options];
298
            if ($facet) {
431
        push @mappings, $mapping;
299
                push @rules, "marc_map('$marc_field','${name}__facet.\$append', $options)";
432
433
        my @suffixes = ();
434
        push @suffixes, 'facet' if $facet;
435
        push @suffixes, 'suggestion' if $suggestible;
436
        push @suffixes, 'sort' if !defined $sort || $sort;
437
438
        foreach my $suffix (@suffixes) {
439
            my $mapping = ["${target_name}__$suffix"];
440
            # Hack, fix later in less hideous manner
441
            if ($suffix eq 'suggestion') {
442
                push @{$mapping}, {%{$default_options}, property => 'input'};
300
            }
443
            }
301
            if ($suggestible) {
444
            else {
302
                push @rules,
445
                push @{$mapping}, $default_options;
303
                    #"marc_map('$marc_field','${name}__suggestion.input.\$append', '')"; #must not have nested data structures in .input
304
                    "marc_map('$marc_field','${name}__suggestion.input.\$append')";
305
            }
446
            }
306
            if ( $type eq 'boolean' ) {
447
            push @mappings, $mapping;
448
        }
449
        return @mappings;
450
    };
451
    my $field_spec_regexp = qr/^([0-9]{3})([0-9a-z]+)?(?:_\/(\d+(?:-\d+)?))?$/;
452
    my $leader_regexp = qr/^leader(?:_\/(\d+(?:-\d+)?))?$/;
453
    my $rules = {
454
        'leader' => [],
455
        'control_fields' => {},
456
        'data_fields' => {},
457
        'sum' => [],
458
        'defaults' => {}
459
    };
460
461
    $self->_foreach_mapping(sub {
462
        my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_;
463
        return if $marc_type ne $marcflavour;
464
465
        if ($type eq 'sum') {
466
            push @{$rules->{sum}}, $name;
467
        }
468
        elsif($type eq 'boolean') {
469
            # boolean gets special handling, if value doesn't exist for a field,
470
            # it is set to false
471
            $rules->{defaults}->{$name} = 'false';
472
        }
307
473
308
                # boolean gets special handling, basically if it doesn't exist,
474
        if ($marc_field =~ $field_spec_regexp) {
309
                # it's added and set to false. Otherwise we can't query it.
475
            my $field_tag = $1;
310
                push @rules,
476
            my $subfields = defined $2 ? $2 : '*';
311
                  "unless exists('$name') add_field('$name', 0) end";
477
            my $range = defined $3 ? $3 : undef;
312
            }
478
            if ($field_tag < 10) {
313
            if ($type eq 'sum' ) {
479
                $rules->{control_fields}->{$field_tag} //= [];
314
                push @rules, "sum('$name')";
480
                my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
481
                push @{$rules->{control_fields}->{$field_tag}}, @mappings;
315
            }
482
            }
316
            if ($self->sort_fields()->{$name}) {
483
            else {
317
                if ($sort || !defined $sort) {
484
                $rules->{data_fields}->{$field_tag} //= {};
318
                    push @rules, "marc_map('$marc_field','${name}__sort.\$append', $options)";
485
                foreach my $subfield (split //, $subfields) {
486
                    $rules->{data_fields}->{$field_tag}->{$subfield} //= [];
487
                    my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
488
                    push @{$rules->{data_fields}->{$field_tag}->{$subfield}}, @mappings;
319
                }
489
                }
320
            }
490
            }
321
        }
491
        }
322
    );
492
        elsif ($marc_field =~ $leader_regexp) {
323
493
            my $range = defined $1 ? $1 : undef;
324
    push @rules, "move_field(_id,es_id)"; #Also you must set the Catmandu::Store::ElasticSearch->new(key_prefix: 'es_');
494
            my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
325
    return \@rules;
495
            push @{$rules->{leader}}, @mappings;
496
        }
497
        else {
498
            die("Invalid marc field: $marc_field");
499
        }
500
    });
501
    return $rules;
326
}
502
}
327
503
328
=head2 _foreach_mapping
504
=head2 _foreach_mapping
(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-60 / +81 lines)
Lines 66-89 sub update_index { Link Here
66
        $self->_sanitise_records($biblionums, $records);
66
        $self->_sanitise_records($biblionums, $records);
67
    }
67
    }
68
68
69
    my $from    = $self->_convert_marc_to_json($records);
69
    $self->ensure_mappings_updated();
70
    if ( !$self->store ) {
70
    $self->bulk_index($records);
71
        my $params  = $self->get_elasticsearch_params();
71
    return 1;
72
        $self->store(
72
}
73
            Catmandu::Store::ElasticSearch->new(
73
74
                %$params,
74
sub bulk_index {
75
                index_settings => $self->get_elasticsearch_settings(),
75
    my ($self, $records) = @_;
76
                index_mappings => $self->get_elasticsearch_mappings(),
76
    my $conf = $self->get_elasticsearch_params();
77
            )
77
    my $elasticsearch = $self->get_elasticsearch();
78
    my $documents = $self->marc_records_to_documents($records);
79
    my @body;
80
81
    foreach my $document_info (@{$documents}) {
82
        my ($id, $document) = @{$document_info};
83
        push @body, {
84
            index => {
85
                _id => $id
86
            }
87
        };
88
        push @body, $document;
89
    }
90
    if (@body) {
91
        my $response = $elasticsearch->bulk(
92
            index => $conf->{index_name},
93
            type => 'data', # is just hard coded in Indexer.pm?
94
            body => \@body
78
        );
95
        );
79
    }
96
    }
80
97
    # TODO: handle response
81
    #print Data::Dumper::Dumper( $from->to_array );
82
    $self->store->bag->add_many($from);
83
    $self->store->bag->commit;
84
    return 1;
98
    return 1;
85
}
99
}
86
100
101
sub ensure_mappings_updated {
102
    my ($self) = @_;
103
    unless ($self->{_mappings_updated}) {
104
        $self->update_mappings();
105
    }
106
}
107
108
sub update_mappings {
109
    my ($self) = @_;
110
    my $conf = $self->get_elasticsearch_params();
111
    my $elasticsearch = $self->get_elasticsearch();
112
    my $mappings = $self->get_elasticsearch_mappings();
113
114
    foreach my $type (keys %{$mappings}) {
115
        my $response = $elasticsearch->indices->put_mapping(
116
            index => $conf->{index_name},
117
            type => $type,
118
            body => {
119
                $type => $mappings->{$type}
120
            }
121
        );
122
        # TODO: process response, produce errors etc
123
    }
124
    $self->{_mappings_updated} = 1;
125
}
126
87
=head2 $indexer->update_index_background($biblionums, $records)
127
=head2 $indexer->update_index_background($biblionums, $records)
88
128
89
This has exactly the same API as C<update_index_background> however it'll
129
This has exactly the same API as C<update_index_background> however it'll
Lines 139-166 sub delete_index_background { Link Here
139
    $self->delete_index(@_);
179
    $self->delete_index(@_);
140
}
180
}
141
181
142
=head2 $indexer->create_index();
143
144
Create an index on the Elasticsearch server.
145
146
=cut
147
148
sub create_index {
149
    my ($self) = @_;
150
151
    if (!$self->store) {
152
        my $params  = $self->get_elasticsearch_params();
153
        $self->store(
154
            Catmandu::Store::ElasticSearch->new(
155
                %$params,
156
                index_settings => $self->get_elasticsearch_settings(),
157
                index_mappings => $self->get_elasticsearch_mappings(),
158
            )
159
        );
160
    }
161
    $self->store->bag->commit;
162
}
163
164
=head2 $indexer->drop_index();
182
=head2 $indexer->drop_index();
165
183
166
Drops the index from the elasticsearch server. Calling C<update_index>
184
Drops the index from the elasticsearch server. Calling C<update_index>
Lines 170-191 after this will recreate it again. Link Here
170
188
171
sub drop_index {
189
sub drop_index {
172
    my ($self) = @_;
190
    my ($self) = @_;
173
191
    if ($self->index_exists) {
174
    if (!$self->store) {
192
        my $conf = $self->get_elasticsearch_params();
175
        # If this index doesn't exist, this will create it. Then it'll be
193
        my $elasticsearch = $self->get_elasticsearch();
176
        # deleted. That's not the end of the world however.
194
        my $response = $elasticsearch->indices->delete(index => $conf->{index_name});
177
        my $params  = $self->get_elasticsearch_params();
195
        # TODO: Handle response? Convert errors to exceptions/die
178
        $self->store(
179
            Catmandu::Store::ElasticSearch->new(
180
                %$params,
181
                index_settings => $self->get_elasticsearch_settings(),
182
                index_mappings => $self->get_elasticsearch_mappings(),
183
            )
184
        );
185
    }
196
    }
186
    my $store = $self->store;
197
}
187
    $self->store(undef);
198
188
    $store->drop();
199
sub create_index {
200
    my ($self) = @_;
201
    my $conf = $self->get_elasticsearch_params();
202
    my $settings = $self->get_elasticsearch_settings();
203
    my $elasticsearch = $self->get_elasticsearch();
204
    my $response = $elasticsearch->indices->create(
205
        index => $conf->{index_name},
206
        body => {
207
            settings => $settings
208
        }
209
    );
210
    # TODO: Handle response? Convert errors to exceptions/die
211
}
212
213
sub index_exists {
214
    my ($self) = @_;
215
    my $conf = $self->get_elasticsearch_params();
216
    my $elasticsearch = $self->get_elasticsearch();
217
    return $elasticsearch->indices->exists(
218
        index => $conf->{index_name},
219
    );
189
}
220
}
190
221
191
sub _sanitise_records {
222
sub _sanitise_records {
Lines 209-224 sub _sanitise_records { Link Here
209
    }
240
    }
210
}
241
}
211
242
212
sub _convert_marc_to_json {
213
    my $self    = shift;
214
    my $records = shift;
215
    my $importer =
216
      Catmandu::Importer::MARC->new( records => $records, id => '999c' );
217
    my $fixer = Catmandu::Fix->new( fixes => $self->get_fixer_rules() );
218
    $importer = $fixer->fix($importer);
219
    return $importer;
220
}
221
222
1;
243
1;
223
244
224
__END__
245
__END__
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-47 / +24 lines)
Lines 49-57 use Koha::SearchEngine::QueryBuilder; Link Here
49
use Koha::SearchEngine::Search;
49
use Koha::SearchEngine::Search;
50
use MARC::Record;
50
use MARC::Record;
51
use Catmandu::Store::ElasticSearch;
51
use Catmandu::Store::ElasticSearch;
52
52
use MARC::File::XML;
53
use Data::Dumper; #TODO remove
53
use Data::Dumper; #TODO remove
54
use Carp qw(cluck);
54
use Carp qw(cluck);
55
use MIME::Base64;
55
56
56
Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store ));
57
Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store ));
57
58
Lines 157-171 sub search_compat { Link Here
157
    my $results = $self->search($query, undef, $results_per_page, %options);
158
    my $results = $self->search($query, undef, $results_per_page, %options);
158
159
159
    # Convert each result into a MARC::Record
160
    # Convert each result into a MARC::Record
160
    my (@records, $index);
161
    my @records;
161
    $index = $offset; # opac-search expects results to be put in the
162
    # opac-search expects results to be put in the
162
        # right place in the array, according to $offset
163
    # right place in the array, according to $offset
164
    my $index = $offset;
163
    $results->each(sub {
165
    $results->each(sub {
164
            # The results come in an array for some reason
166
        $records[$index++] = $self->decode_record_from_result(@_);
165
            my $marc_json = $_[0]->{record};
167
    });
166
            my $marc = $self->json2marc($marc_json);
167
            $records[$index++] = $marc;
168
        });
169
    # consumers of this expect a name-spaced result, we provide the default
168
    # consumers of this expect a name-spaced result, we provide the default
170
    # configuration.
169
    # configuration.
171
    my %result;
170
    my %result;
Lines 196-209 sub search_auth_compat { Link Here
196
    $res->each(
195
    $res->each(
197
        sub {
196
        sub {
198
            my %result;
197
            my %result;
199
            my $record    = $_[0];
200
            my $marc_json = $record->{record};
201
198
202
            # I wonder if these should be real values defined in the mapping
199
            # I wonder if these should be real values defined in the mapping
203
            # rather than hard-coded conversions.
200
            # rather than hard-coded conversions.
201
            my $record    = $_[0];
204
            # Handle legacy nested arrays indexed with splitting enabled.
202
            # Handle legacy nested arrays indexed with splitting enabled.
205
            my $authid = $record->{ 'Local-number' }[0];
203
            my $authid = $record->{ 'Local-number' }[0];
206
            $authid = @$authid[0] if (ref $authid eq 'ARRAY');
204
            $authid = @$authid[0] if (ref $authid eq 'ARRAY');
205
207
            $result{authid} = $authid;
206
            $result{authid} = $authid;
208
207
209
            # TODO put all this info into the record at index time so we
208
            # TODO put all this info into the record at index time so we
Lines 219-225 sub search_auth_compat { Link Here
219
            # it's not reproduced here yet.
218
            # it's not reproduced here yet.
220
            my $authtype           = $rs->single;
219
            my $authtype           = $rs->single;
221
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
220
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
222
            my $marc               = $self->json2marc($marc_json);
221
            my $marc               = $self->decode_record_from_result(@_);
223
            my $mainentry          = $marc->field($auth_tag_to_report);
222
            my $mainentry          = $marc->field($auth_tag_to_report);
224
            my $reported_tag;
223
            my $reported_tag;
225
            if ($mainentry) {
224
            if ($mainentry) {
Lines 338-346 sub simple_search_compat { Link Here
338
    my $results = $self->search($query, undef, $max_results, %options);
337
    my $results = $self->search($query, undef, $max_results, %options);
339
    my @records;
338
    my @records;
340
    $results->each(sub {
339
    $results->each(sub {
341
            # The results come in an array for some reason
340
            my $marc = $self->decode_record_from_result(@_);
342
            my $marc_json = $_[0]->{record};
343
            my $marc = $self->json2marc($marc_json);
344
            push @records, $marc;
341
            push @records, $marc;
345
        });
342
        });
346
    return (undef, \@records, $results->total);
343
    return (undef, \@records, $results->total);
Lines 361-403 sub extract_biblionumber { Link Here
361
    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
358
    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
362
}
359
}
363
360
364
=head2 json2marc
361
=head2 decode_record_from_result
365
362
    my $marc_record = $self->decode_record_from_result(@result);
366
    my $marc = $self->json2marc($marc_json);
367
363
368
Converts the form of marc (based on its JSON, but as a Perl structure) that
364
Extracts marc data from Elasticsearch result and decodes to MARC::Record object
369
Catmandu stores into a MARC::Record object.
370
365
371
=cut
366
=cut
372
367
373
sub json2marc {
368
sub decode_record_from_result {
374
    my ( $self, $marcjson ) = @_;
369
    # Result is passed in as array, will get flattened
375
370
    # and first element will be $result
376
    my $marc = MARC::Record->new();
371
    my ( $self, $result ) = @_;
377
    $marc->encoding('UTF-8');
372
    if (C4::Context->preference('ElasticsearchMARCSerializationFormat') eq 'MARCXML') {
378
373
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
379
    # fields are like:
374
    }
380
    # [ '245', '1', '2', 'a' => 'Title', 'b' => 'Subtitle' ]
375
    else {
381
    # or
376
        return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
382
    # [ '001', undef, undef, '_', 'a value' ]
383
    # conveniently, this is the form that MARC::Field->new() likes
384
    foreach my $field (@$marcjson) {
385
        next if @$field < 5;
386
        if ( $field->[0] eq 'LDR' ) {
387
            $marc->leader( $field->[4] );
388
        }
389
        else {
390
            my $tag = $field->[0];
391
            my $marc_field;
392
            if ( MARC::Field->is_controlfield_tag( $field->[0] ) ) {
393
                $marc_field = MARC::Field->new($field->[0], $field->[4]);
394
            } else {
395
                $marc_field = MARC::Field->new(@$field);
396
            }
397
            $marc->append_fields($marc_field);
398
        }
399
    }
377
    }
400
    return $marc;
401
}
378
}
402
379
403
=head2 max_result_window
380
=head2 max_result_window
(-)a/admin/searchengine/elasticsearch/field_config.yaml (-1 / +3 lines)
Lines 5-13 general: Link Here
5
    type: string
5
    type: string
6
    analyzer: analyser_standard
6
    analyzer: analyser_standard
7
  properties:
7
  properties:
8
    record:
8
    marc_data:
9
      store: true
9
      store: true
10
      type: text
10
      type: text
11
      analyzer: keyword
12
      index: false
11
# Search fields
13
# Search fields
12
search:
14
search:
13
  boolean:
15
  boolean:
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref (+8 lines)
Lines 425-427 Administration: Link Here
425
              choices:
425
              choices:
426
                Zebra: Zebra
426
                Zebra: Zebra
427
                Elasticsearch: Elasticsearch
427
                Elasticsearch: Elasticsearch
428
        -
429
            - "Use"
430
            - pref: ElasticsearchMARCSerializationFormat
431
              default: MARCXML
432
              choices:
433
                MARCXML: MARCXML
434
                base64ISO2709: base64ISO2709
435
            - "as serialization format for MARC records stored in Elasticsearch index. base64ISO2709 is faster and will use less space but have a maximum record length which could cause issues with very large records."
(-)a/misc/search_tools/rebuild_elastic_search.pl (-1 / +6 lines)
Lines 160-167 sub do_reindex { Link Here
160
    my ( $next, $index_name ) = @_;
160
    my ( $next, $index_name ) = @_;
161
161
162
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
162
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
163
163
    if ($delete) {
164
    if ($delete) {
164
        $indexer->drop_index();
165
        $indexer->drop_index() if $indexer->index_exists();
166
        $indexer->create_index();
167
    }
168
    elsif (!$indexer->index_exists()) {
169
        # Create index if does not exist
165
        $indexer->create_index();
170
        $indexer->create_index();
166
    }
171
    }
167
172
(-)a/t/Koha/SearchEngine/Elasticsearch.t (-1 / +239 lines)
Lines 17-27 Link Here
17
17
18
use Modern::Perl;
18
use Modern::Perl;
19
19
20
use Test::More tests => 3;
20
use Test::More tests => 4;
21
use Test::Exception;
21
use Test::Exception;
22
22
23
use t::lib::Mocks;
23
use t::lib::Mocks;
24
24
25
use Test::MockModule;
26
27
use MARC::Record;
28
25
use Koha::SearchEngine::Elasticsearch;
29
use Koha::SearchEngine::Elasticsearch;
26
30
27
subtest '_read_configuration() tests' => sub {
31
subtest '_read_configuration() tests' => sub {
Lines 108-110 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
108
    $mappings = $es->get_elasticsearch_mappings();
112
    $mappings = $es->get_elasticsearch_mappings();
109
    is( $mappings->{data}{_all}{type}, 'string', 'Field mappings parsed correctly' );
113
    is( $mappings->{data}{_all}{type}, 'string', 'Field mappings parsed correctly' );
110
};
114
};
115
116
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
117
118
    plan tests => 29;
119
120
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
121
122
    my @mappings = (
123
        {
124
            name => 'author',
125
            type => 'string',
126
            facet => 1,
127
            suggestible => 1,
128
            sort => undef,
129
            marc_type => 'marc21',
130
            marc_field => '100a',
131
        },
132
        {
133
            name => 'author',
134
            type => 'string',
135
            facet => 1,
136
            suggestible => 1,
137
            sort => 1,
138
            marc_type => 'marc21',
139
            marc_field => '110a',
140
        },
141
        {
142
            name => 'title',
143
            type => 'string',
144
            facet => 0,
145
            suggestible => 1,
146
            sort => 1,
147
            marc_type => 'marc21',
148
            marc_field => '245a',
149
        },
150
        {
151
            name => 'unimarc_title',
152
            type => 'string',
153
            facet => 0,
154
            suggestible => 1,
155
            sort => 1,
156
            marc_type => 'unimarc',
157
            marc_field => '245a',
158
        },
159
        {
160
            name => 'title',
161
            type => 'string',
162
            facet => 0,
163
            suggestible => undef,
164
            sort => 0,
165
            marc_type => 'marc21',
166
            marc_field => '220',
167
        },
168
        {
169
            name => 'sum_item_price',
170
            type => 'sum',
171
            facet => 0,
172
            suggestible => 0,
173
            sort => 0,
174
            marc_type => 'marc21',
175
            marc_field => '952g',
176
        },
177
        {
178
            name => 'items_withdrawn_status',
179
            type => 'boolean',
180
            facet => 0,
181
            suggestible => 0,
182
            sort => 0,
183
            marc_type => 'marc21',
184
            marc_field => '9520',
185
        },
186
        {
187
            name => 'type_of_record',
188
            type => 'string',
189
            facet => 0,
190
            suggestible => 0,
191
            sort => 0,
192
            marc_type => 'marc21',
193
            marc_field => 'leader_/6',
194
        },
195
        {
196
            name => 'type_of_record_and_bib_level',
197
            type => 'string',
198
            facet => 0,
199
            suggestible => 0,
200
            sort => 0,
201
            marc_type => 'marc21',
202
            marc_field => 'leader_/6-7',
203
        },
204
    );
205
206
    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
207
    $se->mock('_foreach_mapping', sub {
208
        my ($self, $sub) = @_;
209
210
        foreach my $map (@mappings) {
211
            $sub->(
212
                $map->{name},
213
                $map->{type},
214
                $map->{facet},
215
                $map->{suggestible},
216
                $map->{sort},
217
                $map->{marc_type},
218
                $map->{marc_field}
219
            );
220
        }
221
    });
222
223
    my $see = Koha::SearchEngine::Elasticsearch->new({ index => 'biblios' });
224
225
    my $marc_record_1 = MARC::Record->new();
226
    $marc_record_1->leader('     cam  22      a 4500');
227
    $marc_record_1->append_fields(
228
        MARC::Field->new('100', '', '', a => 'Author 1'),
229
        MARC::Field->new('110', '', '', a => 'Corp Author'),
230
        MARC::Field->new('210', '', '', a => 'Title 1'),
231
        MARC::Field->new('245', '', '', a => 'Title: first record'),
232
        MARC::Field->new('999', '', '', c => '1234567'),
233
        # '  ' for testing trimming of white space in boolean value callback:
234
        MARC::Field->new('952', '', '', 0 => '  ', g => '123.30'),
235
        MARC::Field->new('952', '', '', 0 => 0, g => '127.20'),
236
    );
237
    my $marc_record_2 = MARC::Record->new();
238
    $marc_record_2->leader('     cam  22      a 4500');
239
    $marc_record_2->append_fields(
240
        MARC::Field->new('100', '', '', a => 'Author 2'),
241
        # MARC::Field->new('210', '', '', a => 'Title 2'),
242
        # MARC::Field->new('245', '', '', a => 'Title: second record'),
243
        MARC::Field->new('999', '', '', c => '1234568'),
244
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),
245
    );
246
    my $records = [$marc_record_1, $marc_record_2];
247
248
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
249
250
    my $docs = $see->marc_records_to_documents($records);
251
252
    # First record:
253
254
    is(scalar @{$docs}, 2, 'Two records converted to documents');
255
256
    is($docs->[0][0], '1234567', 'First document biblionumber should be set as first element in document touple');
257
258
    is(scalar @{$docs->[0][1]->{author}}, 2, 'First document author field should contain two values');
259
    is_deeply($docs->[0][1]->{author}, ['Author 1', 'Corp Author'], 'First document author field should be set correctly');
260
261
    is(scalar @{$docs->[0][1]->{author__sort}}, 2, 'First document author__sort field should have two values');
262
    is_deeply($docs->[0][1]->{author__sort}, ['Author 1', 'Corp Author'], 'First document author__sort field should be set correctly');
263
264
    is(scalar @{$docs->[0][1]->{title__sort}}, 1, 'First document title__sort field should have one value');
265
    is_deeply($docs->[0][1]->{title__sort}, ['Title: first record'], 'First document title__sort field should be set correctly');
266
267
    is(scalar @{$docs->[0][1]->{author__suggestion}}, 2, 'First document author__suggestion field should contain two values');
268
    is_deeply(
269
        $docs->[0][1]->{author__suggestion},
270
        [
271
            {
272
                'input' => 'Author 1'
273
            },
274
            {
275
                'input' => 'Corp Author'
276
            }
277
        ],
278
        'First document author__suggestion field should be set correctly'
279
    );
280
281
    is(scalar @{$docs->[0][1]->{title__suggestion}}, 1, 'First document title__suggestion field should contain one value');
282
    is_deeply(
283
        $docs->[0][1]->{title__suggestion},
284
        [{ 'input' => 'Title: first record' }],
285
        'First document title__suggestion field should be set correctly'
286
    );
287
288
    ok(!(defined $docs->[0][1]->{title__facet}), 'First document should have no title__facet field');
289
290
    is(scalar @{$docs->[0][1]->{author__facet}}, 2, 'First document author__facet field should have two values');
291
    is_deeply(
292
        $docs->[0][1]->{author__facet},
293
        ['Author 1', 'Corp Author'],
294
        'First document author__facet field should be set correctly'
295
    );
296
297
    is(scalar @{$docs->[0][1]->{items_withdrawn_status}}, 2, 'First document items_withdrawn_status field should have two values');
298
    is_deeply(
299
        $docs->[0][1]->{items_withdrawn_status},
300
        ['false', 'false'],
301
        'First document items_withdrawn_status field should be set correctly'
302
    );
303
304
    is(
305
        $docs->[0][1]->{sum_item_price},
306
        '250.5',
307
        'First document sum_item_price field should be set correctly'
308
    );
309
310
    ok(defined $docs->[0][1]->{marc_xml}, 'First document marc_xml field should be set');
311
312
    is(scalar @{$docs->[0][1]->{type_of_record}}, 1, 'First document type_of_record field should have one value');
313
    is_deeply(
314
        $docs->[0][1]->{type_of_record},
315
        ['a'],
316
        'First document type_of_record field should be set correctly'
317
    );
318
319
    is(scalar @{$docs->[0][1]->{type_of_record_and_bib_level}}, 1, 'First document type_of_record_and_bib_level field should have one value');
320
    is_deeply(
321
        $docs->[0][1]->{type_of_record_and_bib_level},
322
        ['am'],
323
        'First document type_of_record_and_bib_level field should be set correctly'
324
    );
325
326
    # Second record:
327
328
    is(scalar @{$docs->[1][1]->{author}}, 1, 'Second document author field should contain one value');
329
    is_deeply($docs->[1][1]->{author}, ['Author 2'], 'Second document author field should be set correctly');
330
331
    is(scalar @{$docs->[1][1]->{items_withdrawn_status}}, 1, 'Second document items_withdrawn_status field should have one value');
332
    is_deeply(
333
        $docs->[1][1]->{items_withdrawn_status},
334
        ['true'],
335
        'Second document items_withdrawn_status field should be set correctly'
336
    );
337
338
    is(
339
        $docs->[1][1]->{sum_item_price},
340
        0,
341
        'Second document sum_item_price field should be set correctly'
342
    );
343
344
    # Mappings marc_type:
345
346
    ok(!(defined $docs->[0][1]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
347
348
};
(-)a/t/db_dependent/Koha_Elasticsearch.t (-164 lines)
Lines 1-164 Link Here
1
# Copyright 2015 Catalyst IT
2
#
3
# This file is part of Koha.
4
#
5
# Koha is free software; you can redistribute it and/or modify it
6
# under the terms of the GNU General Public License as published by
7
# the Free Software Foundation; either version 3 of the License, or
8
# (at your option) any later version.
9
#
10
# Koha is distributed in the hope that it will be useful, but
11
# WITHOUT ANY WARRANTY; without even the implied warranty of
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
# GNU General Public License for more details.
14
#
15
# You should have received a copy of the GNU General Public License
16
# along with Koha; if not, see <http://www.gnu.org/licenses>
17
18
use Modern::Perl;
19
20
use Test::More tests => 2;
21
use Test::MockModule;
22
23
use t::lib::Mocks;
24
use MARC::Record;
25
26
my $schema = Koha::Database->schema;
27
28
use_ok('Koha::SearchEngine::Elasticsearch');
29
30
subtest 'get_fixer_rules() tests' => sub {
31
32
    plan tests => 49;
33
34
    $schema->storage->txn_begin;
35
36
    t::lib::Mocks::mock_preference( 'marcflavour', 'MARC21' );
37
38
    my @mappings;
39
40
    my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
41
    $se->mock( '_foreach_mapping', sub {
42
        my ($self, $sub ) = @_;
43
44
        foreach my $map ( @mappings ) {
45
            $sub->(
46
                $map->{name},
47
                $map->{type},
48
                $map->{facet},
49
                $map->{suggestible},
50
                $map->{sort},
51
                $map->{marc_type},
52
                $map->{marc_field}
53
            );
54
        }
55
    });
56
57
    my $see = Koha::SearchEngine::Elasticsearch->new({ index => 'biblios' });
58
59
    @mappings = (
60
        {
61
            name => 'author',
62
            type => 'string',
63
            facet => 1,
64
            suggestible => 1,
65
            sort => undef,
66
            marc_type => 'marc21',
67
            marc_field => '100a',
68
        },
69
        {
70
            name => 'author',
71
            type => 'string',
72
            facet => 1,
73
            suggestible => 1,
74
            sort => 1,
75
            marc_type => 'marc21',
76
            marc_field => '110a',
77
        },
78
    );
79
80
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
81
    my $result = $see->get_fixer_rules();
82
    is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
83
    is( $result->[1], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__facet.$append', )});
84
    is( $result->[2], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__suggestion.input.$append')});
85
    is( $result->[3], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__sort.$append', )});
86
    is( $result->[4], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
87
    is( $result->[5], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__facet.$append', )});
88
    is( $result->[6], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__suggestion.input.$append')});
89
    is( $result->[7], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
90
    is( $result->[8], q{move_field(_id,es_id)});
91
92
    $mappings[0]->{type}  = 'boolean';
93
    $mappings[1]->{type}  = 'boolean';
94
    $result = $see->get_fixer_rules();
95
    is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
96
    is( $result->[1], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__facet.$append', )});
97
    is( $result->[2], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__suggestion.input.$append')});
98
    is( $result->[3], q{unless exists('} . $mappings[0]->{name} . q{') add_field('} . $mappings[0]->{name} . q{', 0) end});
99
    is( $result->[4], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__sort.$append', )});
100
    is( $result->[5], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
101
    is( $result->[6], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__facet.$append', )});
102
    is( $result->[7], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__suggestion.input.$append')});
103
    is( $result->[8], q{unless exists('} . $mappings[1]->{name} . q{') add_field('} . $mappings[1]->{name} . q{', 0) end});
104
    is( $result->[9], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
105
    is( $result->[10], q{move_field(_id,es_id)});
106
107
    $mappings[0]->{type}  = 'sum';
108
    $mappings[1]->{type}  = 'sum';
109
    $result = $see->get_fixer_rules();
110
    is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
111
    is( $result->[1], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__facet.$append', )});
112
    is( $result->[2], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__suggestion.input.$append')});
113
    is( $result->[3], q{sum('} . $mappings[0]->{name} . q{')});
114
    is( $result->[4], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__sort.$append', )});
115
    is( $result->[5], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
116
    is( $result->[6], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__facet.$append', )});
117
    is( $result->[7], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__suggestion.input.$append')});
118
    is( $result->[8], q{sum('} . $mappings[1]->{name} . q{')});
119
    is( $result->[9], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
120
    is( $result->[10], q{move_field(_id,es_id)});
121
122
    $mappings[0]->{type}  = 'string';
123
    $mappings[0]->{facet} = 0;
124
    $mappings[1]->{type}  = 'string';
125
    $mappings[1]->{facet} = 0;
126
127
    $result = $see->get_fixer_rules();
128
    is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
129
    is( $result->[1], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__suggestion.input.$append')});
130
    is( $result->[2], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__sort.$append', )});
131
    is( $result->[3], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
132
    is( $result->[4], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__suggestion.input.$append')});
133
    is( $result->[5], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
134
    is( $result->[6], q{move_field(_id,es_id)});
135
136
    $mappings[0]->{suggestible}  = 0;
137
    $mappings[1]->{suggestible}  = 0;
138
139
    $result = $see->get_fixer_rules();
140
    is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
141
    is( $result->[1], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__sort.$append', )});
142
    is( $result->[2], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
143
    is( $result->[3], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
144
    is( $result->[4], q{move_field(_id,es_id)});
145
146
    $mappings[0]->{sort}  = 0;
147
    $mappings[1]->{sort}  = undef;
148
149
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
150
    $result = $see->get_fixer_rules();
151
    is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
152
    is( $result->[1], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
153
    is( $result->[2], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
154
    is( $result->[3], q{move_field(_id,es_id)});
155
156
    t::lib::Mocks::mock_preference( 'marcflavour', 'UNIMARC' );
157
158
    $result = $see->get_fixer_rules();
159
    is( $result->[0], q{move_field(_id,es_id)});
160
    is( $result->[1], undef, q{No mapping when marc_type doesn't match marchflavour} );
161
162
    $schema->storage->txn_rollback;
163
164
};
(-)a/t/db_dependent/Koha_Elasticsearch_Indexer.t (-117 / +5 lines)
Lines 17-23 Link Here
17
17
18
use Modern::Perl;
18
use Modern::Perl;
19
19
20
use Test::More tests => 7;
20
use Test::More tests => 3;
21
use Test::MockModule;
21
use Test::MockModule;
22
use t::lib::Mocks;
22
use t::lib::Mocks;
23
23
Lines 37-52 ok( Link Here
37
37
38
my $marc_record = MARC::Record->new();
38
my $marc_record = MARC::Record->new();
39
$marc_record->append_fields(
39
$marc_record->append_fields(
40
    MARC::Field->new( '001', '1234567' ),
40
	MARC::Field->new( '001', '1234567' ),
41
    MARC::Field->new( '020', '', '', 'a' => '1234567890123' ),
41
	MARC::Field->new( '020', '', '', 'a' => '1234567890123' ),
42
    MARC::Field->new( '245', '', '', 'a' => 'Title' )
42
	MARC::Field->new( '245', '', '', 'a' => 'Title' )
43
);
43
);
44
45
my $records = [$marc_record];
44
my $records = [$marc_record];
46
ok( my $converted = $indexer->_convert_marc_to_json($records),
47
    'Convert some records' );
48
49
is( $converted->count, 1, 'One converted record' );
50
45
51
SKIP: {
46
SKIP: {
52
47
Lines 55-165 SKIP: { Link Here
55
    skip 'Elasticsearch configuration not available', 1
50
    skip 'Elasticsearch configuration not available', 1
56
        if $@;
51
        if $@;
57
52
58
    ok( $indexer->update_index(undef,$records), 'Update Index' );
53
    ok( $indexer->update_index(undef, $records), 'Update Index' );
59
}
54
}
60
61
subtest 'create_index() tests' => sub {
62
63
    plan tests => 3;
64
65
    my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
66
    $se->mock( 'get_elasticsearch_params', sub {
67
        my ($self, $sub ) = @_;
68
69
        my $method = $se->original( 'get_elasticsearch_params' );
70
        my $params = $method->( $self );
71
        $params->{index_name} .= '__test';
72
        return $params;
73
    });
74
75
    my $indexer;
76
    ok(
77
        $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ 'index' => 'biblios' }),
78
        'Creating a new indexer object'
79
    );
80
    ok(
81
        $indexer->create_index(),
82
        'Creating an index'
83
    );
84
    $indexer->drop_index();
85
    ok(
86
        $indexer->drop_index(),
87
        'Dropping the index'
88
    );
89
};
90
91
subtest '_convert_marc_to_json() tests' => sub {
92
93
    plan tests => 4;
94
95
    $schema->storage->txn_begin;
96
97
    t::lib::Mocks::mock_preference( 'marcflavour', 'MARC21' );
98
99
    my @mappings = (
100
        {
101
            name => 'author',
102
            type => 'string',
103
            facet => 1,
104
            suggestible => 1,
105
            sort => '~',
106
            marc_type => 'marc21',
107
            marc_field => '100a',
108
        },
109
        {
110
            name => 'author',
111
            type => 'string',
112
            facet => 1,
113
            suggestible => 1,
114
            sort => '~',
115
            marc_type => 'marc21',
116
            marc_field => '110a',
117
        },
118
    );
119
120
121
    my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
122
    $se->mock( '_foreach_mapping', sub {
123
        my ($self, $sub ) = @_;
124
125
        foreach my $map ( @mappings ) {
126
            $sub->(
127
                $map->{name},
128
                $map->{type},
129
                $map->{facet},
130
                $map->{suggestible},
131
                $map->{sort},
132
                $map->{marc_type},
133
                $map->{marc_field}
134
            );
135
        }
136
    });
137
138
    my $marc_record = MARC::Record->new();
139
    $marc_record->append_fields(
140
        MARC::Field->new( '001', '1234567' ),
141
        MARC::Field->new( '020', '', '', 'a' => '1234567890123' ),
142
        MARC::Field->new( '100', '', '', 'a' => 'Author' ),
143
        MARC::Field->new( '110', '', '', 'a' => 'Corp Author' ),
144
        MARC::Field->new( '245', '', '', 'a' => 'Title' ),
145
    );
146
    my $marc_record_2 = MARC::Record->new();
147
    $marc_record_2->append_fields(
148
        MARC::Field->new( '001', '1234567' ),
149
        MARC::Field->new( '020', '', '', 'a' => '1234567890123' ),
150
        MARC::Field->new( '100', '', '', 'a' => 'Author' ),
151
        MARC::Field->new( '245', '', '', 'a' => 'Title' ),
152
    );
153
    my @records = ( $marc_record, $marc_record_2 );
154
155
    my $importer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => 'biblios' })->_convert_marc_to_json( \@records );
156
    my $conv = $importer->next();
157
    is( $conv->{author}[0], "Author", "First mapped author should be 100a");
158
    is( $conv->{author}[1], "Corp Author", "Second mapped author should be 110a");
159
160
    $conv = $importer->next();
161
    is( $conv->{author}[0], "Author", "First mapped author should be 100a");
162
    is( scalar @{$conv->{author}} , 1, "We should map field only if exists, shouldn't add extra nulls");
163
164
    $schema->storage->txn_rollback;
165
};
166
- 

Return to bug 19893