View | Details | Raw Unified | Return to bug 22771
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-11 / +93 lines)
Lines 41-46 use MARC::File::XML; Link Here
41
use MIME::Base64;
41
use MIME::Base64;
42
use Encode qw(encode);
42
use Encode qw(encode);
43
use Business::ISBN;
43
use Business::ISBN;
44
use Scalar::Util qw(looks_like_number);
44
45
45
__PACKAGE__->mk_ro_accessors(qw( index ));
46
__PACKAGE__->mk_ro_accessors(qw( index ));
46
__PACKAGE__->mk_accessors(qw( sort_fields ));
47
__PACKAGE__->mk_accessors(qw( sort_fields ));
Lines 346-352 sub sort_fields { Link Here
346
    return $self->_sort_fields_accessor();
347
    return $self->_sort_fields_accessor();
347
}
348
}
348
349
349
=head2 _process_mappings($mappings, $data, $record_document, $altscript)
350
=head2 _process_mappings($mappings, $data, $record_document, $meta)
350
351
351
    $self->_process_mappings($mappings, $marc_field_data, $record_document, 0)
352
    $self->_process_mappings($mappings, $marc_field_data, $record_document, 0)
352
353
Lines 374-396 The source data from a MARC record field. Link Here
374
Hashref representing the Elasticsearch document on which mappings should be
375
Hashref representing the Elasticsearch document on which mappings should be
375
applied.
376
applied.
376
377
377
=item C<$altscript>
378
=item C<$meta>
378
379
379
A boolean value indicating whether an alternate script presentation is being
380
A hashref containing metadata useful for enforcing per mapping rules. For
381
example for providing extra context for mapping options, or treating mapping
382
targets differently depending on type (sort, search, facet etc). Combining
383
this metadata with the mapping options and metadata allows us to mutate the
384
data per mapping, or even replace it with other data retrieved from the
385
metadata context.
386
387
Current properties are:
388
389
C<altscript>: A boolean value indicating whether an alternate script presentation is being
380
processed.
390
processed.
381
391
392
C<data_source>: The source of the $<data> argument. Possible values are: 'leader', 'control_field',
393
'subfield' or 'subfields_group'.
394
395
C<code>: The code of the subfield C<$data> was retrieved, if C<data_source> is 'subfield'.
396
397
C<codes>: Subfield codes of the subfields group from which C<$data> was retrieved, if C<data_source>
398
is 'subfields_group'.
399
400
C<field>: The original marc field object.
401
382
=back
402
=back
383
403
384
=cut
404
=cut
385
405
386
sub _process_mappings {
406
sub _process_mappings {
387
    my ($_self, $mappings, $data, $record_document, $altscript) = @_;
407
    my ($_self, $mappings, $data, $record_document, $meta) = @_;
388
    foreach my $mapping (@{$mappings}) {
408
    foreach my $mapping (@{$mappings}) {
389
        my ($target, $options) = @{$mapping};
409
        my ($target, $options) = @{$mapping};
390
410
391
        # Don't process sort fields for alternate scripts
411
        # Don't process sort fields for alternate scripts
392
        my $sort = $target =~ /__sort$/;
412
        my $sort = $target =~ /__sort$/;
393
        if ($sort && $altscript) {
413
        if ($sort && $meta->{altscript}) {
394
            next;
414
            next;
395
        }
415
        }
396
416
Lines 411-416 sub _process_mappings { Link Here
411
                $options->{property} => $_data
431
                $options->{property} => $_data
412
            }
432
            }
413
        }
433
        }
434
        if (defined $options->{nonfiling_characters_indicator}) {
435
            my $nonfiling_chars = $meta->{field}->indicator($options->{nonfiling_characters_indicator});
436
            $nonfiling_chars = looks_like_number($nonfiling_chars) ? int($nonfiling_chars) : 0;
437
            if ($nonfiling_chars) {
438
                $_data = substr $_data, $nonfiling_chars;
439
            }
440
        }
414
        push @{$record_document->{$target}}, $_data;
441
        push @{$record_document->{$target}}, $_data;
415
    }
442
    }
416
}
443
}
Lines 448-460 sub marc_records_to_documents { Link Here
448
        my $record_document = {};
475
        my $record_document = {};
449
        my $mappings = $rules->{leader};
476
        my $mappings = $rules->{leader};
450
        if ($mappings) {
477
        if ($mappings) {
451
            $self->_process_mappings($mappings, $record->leader(), $record_document, 0);
478
            $self->_process_mappings($mappings, $record->leader(), $record_document, {
479
                    altscript => 0,
480
                    data_source => 'leader'
481
                }
482
            );
452
        }
483
        }
453
        foreach my $field ($record->fields()) {
484
        foreach my $field ($record->fields()) {
454
            if ($field->is_control_field()) {
485
            if ($field->is_control_field()) {
455
                my $mappings = $control_fields_rules->{$field->tag()};
486
                my $mappings = $control_fields_rules->{$field->tag()};
456
                if ($mappings) {
487
                if ($mappings) {
457
                    $self->_process_mappings($mappings, $field->data(), $record_document, 0);
488
                    $self->_process_mappings($mappings, $field->data(), $record_document, {
489
                            altscript => 0,
490
                            data_source => 'control_field',
491
                            field => $field
492
                        }
493
                    );
458
                }
494
                }
459
            }
495
            }
460
            else {
496
            else {
Lines 481-487 sub marc_records_to_documents { Link Here
481
                            $mappings = [@{$mappings}, @{$wildcard_mappings}];
517
                            $mappings = [@{$mappings}, @{$wildcard_mappings}];
482
                        }
518
                        }
483
                        if (@{$mappings}) {
519
                        if (@{$mappings}) {
484
                            $self->_process_mappings($mappings, $data, $record_document, $altscript);
520
                            $self->_process_mappings($mappings, $data, $record_document, {
521
                                    altscript => $altscript,
522
                                    data_source => 'subfield',
523
                                    code => $code,
524
                                    field => $field
525
                                }
526
                            );
485
                        }
527
                        }
486
                    }
528
                    }
487
529
Lines 497-503 sub marc_records_to_documents { Link Here
497
                                )
539
                                )
498
                            );
540
                            );
499
                            if ($data) {
541
                            if ($data) {
500
                                $self->_process_mappings($subfields_join_mappings->{$subfields_group}, $data, $record_document, $altscript);
542
                                $self->_process_mappings($subfields_join_mappings->{$subfields_group}, $data, $record_document, {
543
                                        altscript => $altscript,
544
                                        data_source => 'subfields_group',
545
                                        codes => $subfields_group,
546
                                        field => $field
547
                                    }
548
                                );
501
                            }
549
                            }
502
                        }
550
                        }
503
                    }
551
                    }
Lines 786-792 sub _field_mappings { Link Here
786
            push @{$mapping}, {%{$default_options}, property => 'input'};
834
            push @{$mapping}, {%{$default_options}, property => 'input'};
787
        }
835
        }
788
        else {
836
        else {
789
            push @{$mapping}, $default_options;
837
            # Important! Make shallow clone, or we end up with the same hashref
838
            # shared by all mappings
839
            push @{$mapping}, {%{$default_options}};
790
        }
840
        }
791
        push @mappings, $mapping;
841
        push @mappings, $mapping;
792
    }
842
    }
Lines 905-911 sub _get_marc_mapping_rules { Link Here
905
                    push @{$rules->{data_fields}->{$field_tag}->{subfields}->{$subfield}}, @mappings;
955
                    push @{$rules->{data_fields}->{$field_tag}->{subfields}->{$subfield}}, @mappings;
906
                }
956
                }
907
                foreach my $subfield_group (@subfield_groups) {
957
                foreach my $subfield_group (@subfield_groups) {
908
                    $rules->{data_fields}->{$field_tag}->{subfields_join}->{$subfield_group} //= [];
958
                   $rules->{data_fields}->{$field_tag}->{subfields_join}->{$subfield_group} //= [];
909
                    push @{$rules->{data_fields}->{$field_tag}->{subfields_join}->{$subfield_group}}, @mappings;
959
                    push @{$rules->{data_fields}->{$field_tag}->{subfields_join}->{$subfield_group}}, @mappings;
910
                }
960
                }
911
            }
961
            }
Lines 921-926 sub _get_marc_mapping_rules { Link Here
921
            );
971
            );
922
        }
972
        }
923
    });
973
    });
974
975
    # Marc-flavour specific rule tweaks, could/should also provide hook for this
976
    if ($marcflavour eq 'marc21') {
977
        # Nonfiling characters processing for sort fields
978
        my %title_fields;
979
        if ($self->index eq $Koha::SearchEngine::BIBLIOS_INDEX) {
980
            # Format is: nonfiling characters indicator => field names list
981
            %title_fields = (
982
                1 => [630, 730, 740],
983
                2 => [130, 222, 240, 242, 243, 440, 830]
984
            );
985
        }
986
        elsif ($self->index eq $Koha::SearchEngine::AUTHORITIES_INDEX) {
987
            %title_fields = (
988
                1 => [730],
989
                2 => [130, 430, 530]
990
            );
991
        }
992
        foreach my $indicator (keys %title_fields) {
993
            foreach my $field_tag (@{$title_fields{$indicator}}) {
994
                my $mappings = $rules->{data_fields}->{$field_tag}->{subfields}->{a} // [];
995
                foreach my $mapping (@{$mappings}) {
996
                    if ($mapping->[0] =~ /__sort$/) {
997
                        # Mark this as to be processed for nonfiling characters indicator
998
                        # later on in _process_mappings
999
                        $mapping->[1]->{nonfiling_characters_indicator} = $indicator;
1000
                    }
1001
                }
1002
            }
1003
        }
1004
    }
1005
924
    return $rules;
1006
    return $rules;
925
}
1007
}
926
1008
(-)a/t/Koha/SearchEngine/Elasticsearch.t (-2 / +24 lines)
Lines 117-123 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
117
117
118
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
118
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
119
119
120
    plan tests => 51;
120
    plan tests => 53;
121
121
122
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
122
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
123
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
123
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
Lines 193-198 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
193
            marc_type => 'marc21',
193
            marc_type => 'marc21',
194
            marc_field => '220',
194
            marc_field => '220',
195
        },
195
        },
196
        {
197
            name => 'uniform_title',
198
            type => 'string',
199
            facet => 0,
200
            suggestible => 0,
201
            searchable => 1,
202
            sort => 1,
203
            marc_type => 'marc21',
204
            marc_field => '240a',
205
        },
196
        {
206
        {
197
            name => 'title_wildcard',
207
            name => 'title_wildcard',
198
            type => 'string',
208
            type => 'string',
Lines 308-313 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
308
        MARC::Field->new('100', '', '', a => 'Author 1'),
318
        MARC::Field->new('100', '', '', a => 'Author 1'),
309
        MARC::Field->new('110', '', '', a => 'Corp Author'),
319
        MARC::Field->new('110', '', '', a => 'Corp Author'),
310
        MARC::Field->new('210', '', '', a => 'Title 1'),
320
        MARC::Field->new('210', '', '', a => 'Title 1'),
321
        MARC::Field->new('240', '', '4', a => 'The uniform title with nonfiling indicator'),
311
        MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
322
        MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
312
        MARC::Field->new('999', '', '', c => '1234567'),
323
        MARC::Field->new('999', '', '', c => '1234567'),
313
        # '  ' for testing trimming of white space in boolean value callback:
324
        # '  ' for testing trimming of white space in boolean value callback:
Lines 432-437 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
432
        'First document local_classification field should be set correctly'
443
        'First document local_classification field should be set correctly'
433
    );
444
    );
434
445
446
    # Nonfiling characters for sort fields
447
    is_deeply(
448
        $docs->[0]->{uniform_title},
449
        ['The uniform title with nonfiling indicator'],
450
        'First document uniform_title field should contain the title verbatim'
451
    );
452
    is_deeply(
453
        $docs->[0]->{uniform_title__sort},
454
        ['uniform title with nonfiling indicator'],
455
        'First document uniform_title__sort field should contain the title with the first four initial characters removed'
456
    );
457
435
    # Second record:
458
    # Second record:
436
459
437
    is(scalar @{$docs->[1]->{author}}, 1, 'Second document author field should contain one value');
460
    is(scalar @{$docs->[1]->{author}}, 1, 'Second document author field should contain one value');
438
- 

Return to bug 22771