View | Details | Raw Unified | Return to bug 22771
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-10 / +92 lines)
Lines 42-47 use MARC::File::XML; Link Here
42
use MIME::Base64;
42
use MIME::Base64;
43
use Encode qw(encode);
43
use Encode qw(encode);
44
use Business::ISBN;
44
use Business::ISBN;
45
use Scalar::Util qw(looks_like_number);
45
46
46
__PACKAGE__->mk_ro_accessors(qw( index ));
47
__PACKAGE__->mk_ro_accessors(qw( index ));
47
__PACKAGE__->mk_accessors(qw( sort_fields ));
48
__PACKAGE__->mk_accessors(qw( sort_fields ));
Lines 347-353 sub sort_fields { Link Here
347
    return $self->_sort_fields_accessor();
348
    return $self->_sort_fields_accessor();
348
}
349
}
349
350
350
=head2 _process_mappings($mappings, $data, $record_document, $altscript)
351
=head2 _process_mappings($mappings, $data, $record_document, $meta)
351
352
352
    $self->_process_mappings($mappings, $marc_field_data, $record_document, 0)
353
    $self->_process_mappings($mappings, $marc_field_data, $record_document, 0)
353
354
Lines 375-397 The source data from a MARC record field. Link Here
375
Hashref representing the Elasticsearch document on which mappings should be
376
Hashref representing the Elasticsearch document on which mappings should be
376
applied.
377
applied.
377
378
378
=item C<$altscript>
379
=item C<$meta>
379
380
380
A boolean value indicating whether an alternate script presentation is being
381
A hashref containing metadata useful for enforcing per mapping rules. For
382
example for providing extra context for mapping options, or treating mapping
383
targets differently depending on type (sort, search, facet etc). Combining
384
this metadata with the mapping options and metadata allows us to mutate the
385
data per mapping, or even replace it with other data retrieved from the
386
metadata context.
387
388
Current properties are:
389
390
C<altscript>: A boolean value indicating whether an alternate script presentation is being
381
processed.
391
processed.
382
392
393
C<data_source>: The source of the $<data> argument. Possible values are: 'leader', 'control_field',
394
'subfield' or 'subfields_group'.
395
396
C<code>: The code of the subfield C<$data> was retrieved, if C<data_source> is 'subfield'.
397
398
C<codes>: Subfield codes of the subfields group from which C<$data> was retrieved, if C<data_source>
399
is 'subfields_group'.
400
401
C<field>: The original C<MARC::Record> object.
402
383
=back
403
=back
384
404
385
=cut
405
=cut
386
406
387
sub _process_mappings {
407
sub _process_mappings {
388
    my ($_self, $mappings, $data, $record_document, $altscript) = @_;
408
    my ($_self, $mappings, $data, $record_document, $meta) = @_;
389
    foreach my $mapping (@{$mappings}) {
409
    foreach my $mapping (@{$mappings}) {
390
        my ($target, $options) = @{$mapping};
410
        my ($target, $options) = @{$mapping};
391
411
392
        # Don't process sort fields for alternate scripts
412
        # Don't process sort fields for alternate scripts
393
        my $sort = $target =~ /__sort$/;
413
        my $sort = $target =~ /__sort$/;
394
        if ($sort && $altscript) {
414
        if ($sort && $meta->{altscript}) {
395
            next;
415
            next;
396
        }
416
        }
397
417
Lines 412-417 sub _process_mappings { Link Here
412
                $options->{property} => $_data
432
                $options->{property} => $_data
413
            }
433
            }
414
        }
434
        }
435
        if (defined $options->{nonfiling_characters_indicator}) {
436
            my $nonfiling_chars = $meta->{field}->indicator($options->{nonfiling_characters_indicator});
437
            $nonfiling_chars = looks_like_number($nonfiling_chars) ? int($nonfiling_chars) : 0;
438
            if ($nonfiling_chars) {
439
                $_data = substr $_data, $nonfiling_chars;
440
            }
441
        }
415
        push @{$record_document->{$target}}, $_data;
442
        push @{$record_document->{$target}}, $_data;
416
    }
443
    }
417
}
444
}
Lines 449-461 sub marc_records_to_documents { Link Here
449
        my $record_document = {};
476
        my $record_document = {};
450
        my $mappings = $rules->{leader};
477
        my $mappings = $rules->{leader};
451
        if ($mappings) {
478
        if ($mappings) {
452
            $self->_process_mappings($mappings, $record->leader(), $record_document, 0);
479
            $self->_process_mappings($mappings, $record->leader(), $record_document, {
480
                    altscript => 0,
481
                    data_source => 'leader'
482
                }
483
            );
453
        }
484
        }
454
        foreach my $field ($record->fields()) {
485
        foreach my $field ($record->fields()) {
455
            if ($field->is_control_field()) {
486
            if ($field->is_control_field()) {
456
                my $mappings = $control_fields_rules->{$field->tag()};
487
                my $mappings = $control_fields_rules->{$field->tag()};
457
                if ($mappings) {
488
                if ($mappings) {
458
                    $self->_process_mappings($mappings, $field->data(), $record_document, 0);
489
                    $self->_process_mappings($mappings, $field->data(), $record_document, {
490
                            altscript => 0,
491
                            data_source => 'control_field',
492
                            field => $field
493
                        }
494
                    );
459
                }
495
                }
460
            }
496
            }
461
            else {
497
            else {
Lines 481-487 sub marc_records_to_documents { Link Here
481
                            $mappings = [@{$mappings}, @{$wildcard_mappings}];
517
                            $mappings = [@{$mappings}, @{$wildcard_mappings}];
482
                        }
518
                        }
483
                        if (@{$mappings}) {
519
                        if (@{$mappings}) {
484
                            $self->_process_mappings($mappings, $data, $record_document, $altscript);
520
                            $self->_process_mappings($mappings, $data, $record_document, {
521
                                    altscript => $altscript,
522
                                    data_source => 'subfield',
523
                                    code => $code,
524
                                    field => $field
525
                                }
526
                            );
485
                        }
527
                        }
486
                        if ( defined @{$mappings}[0] && grep /match-heading/, @{@{$mappings}[0]} ){
528
                        if ( defined @{$mappings}[0] && grep /match-heading/, @{@{$mappings}[0]} ){
487
                            # Used by the authority linker the match-heading field requires a specific syntax
529
                            # Used by the authority linker the match-heading field requires a specific syntax
Lines 504-510 sub marc_records_to_documents { Link Here
504
                                )
546
                                )
505
                            );
547
                            );
506
                            if ($data) {
548
                            if ($data) {
507
                                $self->_process_mappings($subfields_join_mappings->{$subfields_group}, $data, $record_document, $altscript);
549
                                $self->_process_mappings($subfields_join_mappings->{$subfields_group}, $data, $record_document, {
550
                                        altscript => $altscript,
551
                                        data_source => 'subfields_group',
552
                                        codes => $subfields_group,
553
                                        field => $field
554
                                    }
555
                                );
508
                            }
556
                            }
509
                            if ( grep { $_->[0] eq 'match-heading' } @{$subfields_join_mappings->{$subfields_group}} ){
557
                            if ( grep { $_->[0] eq 'match-heading' } @{$subfields_join_mappings->{$subfields_group}} ){
510
                                # Used by the authority linker the match-heading field requires a specific syntax
558
                                # Used by the authority linker the match-heading field requires a specific syntax
Lines 800-806 sub _field_mappings { Link Here
800
            push @{$mapping}, {%{$default_options}, property => 'input'};
848
            push @{$mapping}, {%{$default_options}, property => 'input'};
801
        }
849
        }
802
        else {
850
        else {
803
            push @{$mapping}, $default_options;
851
            # Important! Make shallow clone, or we end up with the same hashref
852
            # shared by all mappings
853
            push @{$mapping}, {%{$default_options}};
804
        }
854
        }
805
        push @mappings, $mapping;
855
        push @mappings, $mapping;
806
    }
856
    }
Lines 935-940 sub _get_marc_mapping_rules { Link Here
935
            );
985
            );
936
        }
986
        }
937
    });
987
    });
988
989
    # Marc-flavour specific rule tweaks, could/should also provide hook for this
990
    if ($marcflavour eq 'marc21') {
991
        # Nonfiling characters processing for sort fields
992
        my %title_fields;
993
        if ($self->index eq $Koha::SearchEngine::BIBLIOS_INDEX) {
994
            # Format is: nonfiling characters indicator => field names list
995
            %title_fields = (
996
                1 => [630, 730, 740],
997
                2 => [130, 222, 240, 242, 243, 440, 830]
998
            );
999
        }
1000
        elsif ($self->index eq $Koha::SearchEngine::AUTHORITIES_INDEX) {
1001
            %title_fields = (
1002
                1 => [730],
1003
                2 => [130, 430, 530]
1004
            );
1005
        }
1006
        foreach my $indicator (keys %title_fields) {
1007
            foreach my $field_tag (@{$title_fields{$indicator}}) {
1008
                my $mappings = $rules->{data_fields}->{$field_tag}->{subfields}->{a} // [];
1009
                foreach my $mapping (@{$mappings}) {
1010
                    if ($mapping->[0] =~ /__sort$/) {
1011
                        # Mark this as to be processed for nonfiling characters indicator
1012
                        # later on in _process_mappings
1013
                        $mapping->[1]->{nonfiling_characters_indicator} = $indicator;
1014
                    }
1015
                }
1016
            }
1017
        }
1018
    }
1019
938
    return $rules;
1020
    return $rules;
939
}
1021
}
940
1022
(-)a/t/Koha/SearchEngine/Elasticsearch.t (-2 / +24 lines)
Lines 118-124 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
118
118
119
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
119
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
120
120
121
    plan tests => 51;
121
    plan tests => 53;
122
122
123
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
123
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
124
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
124
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
Lines 195-200 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
195
            marc_field => '220',
195
            marc_field => '220',
196
        },
196
        },
197
        {
197
        {
198
            name => 'uniform_title',
199
            type => 'string',
200
            facet => 0,
201
            suggestible => 0,
202
            searchable => 1,
203
            sort => 1,
204
            marc_type => 'marc21',
205
            marc_field => '240a',
206
        },
207
        {
198
            name => 'title_wildcard',
208
            name => 'title_wildcard',
199
            type => 'string',
209
            type => 'string',
200
            facet => 0,
210
            facet => 0,
Lines 309-314 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
309
        MARC::Field->new('100', '', '', a => 'Author 1'),
319
        MARC::Field->new('100', '', '', a => 'Author 1'),
310
        MARC::Field->new('110', '', '', a => 'Corp Author'),
320
        MARC::Field->new('110', '', '', a => 'Corp Author'),
311
        MARC::Field->new('210', '', '', a => 'Title 1'),
321
        MARC::Field->new('210', '', '', a => 'Title 1'),
322
        MARC::Field->new('240', '', '4', a => 'The uniform title with nonfiling indicator'),
312
        MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
323
        MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
313
        MARC::Field->new('999', '', '', c => '1234567'),
324
        MARC::Field->new('999', '', '', c => '1234567'),
314
        # '  ' for testing trimming of white space in boolean value callback:
325
        # '  ' for testing trimming of white space in boolean value callback:
Lines 433-438 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
433
        'First document local_classification field should be set correctly'
444
        'First document local_classification field should be set correctly'
434
    );
445
    );
435
446
447
    # Nonfiling characters for sort fields
448
    is_deeply(
449
        $docs->[0]->{uniform_title},
450
        ['The uniform title with nonfiling indicator'],
451
        'First document uniform_title field should contain the title verbatim'
452
    );
453
    is_deeply(
454
        $docs->[0]->{uniform_title__sort},
455
        ['uniform title with nonfiling indicator'],
456
        'First document uniform_title__sort field should contain the title with the first four initial characters removed'
457
    );
458
436
    # Second record:
459
    # Second record:
437
460
438
    is(scalar @{$docs->[1]->{author}}, 1, 'Second document author field should contain one value');
461
    is(scalar @{$docs->[1]->{author}}, 1, 'Second document author field should contain one value');
439
- 

Return to bug 22771