View | Details | Raw Unified | Return to bug 22771
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-10 / +92 lines)
Lines 42-47 use MARC::File::XML; Link Here
42
use MIME::Base64;
42
use MIME::Base64;
43
use Encode qw(encode);
43
use Encode qw(encode);
44
use Business::ISBN;
44
use Business::ISBN;
45
use Scalar::Util qw(looks_like_number);
45
46
46
__PACKAGE__->mk_ro_accessors(qw( index ));
47
__PACKAGE__->mk_ro_accessors(qw( index ));
47
__PACKAGE__->mk_accessors(qw( sort_fields ));
48
__PACKAGE__->mk_accessors(qw( sort_fields ));
Lines 346-352 sub sort_fields { Link Here
346
    return $self->_sort_fields_accessor();
347
    return $self->_sort_fields_accessor();
347
}
348
}
348
349
349
=head2 _process_mappings($mappings, $data, $record_document, $altscript)
350
=head2 _process_mappings($mappings, $data, $record_document, $meta)
350
351
351
    $self->_process_mappings($mappings, $marc_field_data, $record_document, 0)
352
    $self->_process_mappings($mappings, $marc_field_data, $record_document, 0)
352
353
Lines 374-396 The source data from a MARC record field. Link Here
374
Hashref representing the Elasticsearch document on which mappings should be
375
Hashref representing the Elasticsearch document on which mappings should be
375
applied.
376
applied.
376
377
377
=item C<$altscript>
378
=item C<$meta>
378
379
379
A boolean value indicating whether an alternate script presentation is being
380
A hashref containing metadata useful for enforcing per mapping rules. For
381
example for providing extra context for mapping options, or treating mapping
382
targets differently depending on type (sort, search, facet etc). Combining
383
this metadata with the mapping options and metadata allows us to mutate the
384
data per mapping, or even replace it with other data retrieved from the
385
metadata context.
386
387
Current properties are:
388
389
C<altscript>: A boolean value indicating whether an alternate script presentation is being
380
processed.
390
processed.
381
391
392
C<data_source>: The source of the $<data> argument. Possible values are: 'leader', 'control_field',
393
'subfield' or 'subfields_group'.
394
395
C<code>: The code of the subfield C<$data> was retrieved, if C<data_source> is 'subfield'.
396
397
C<codes>: Subfield codes of the subfields group from which C<$data> was retrieved, if C<data_source>
398
is 'subfields_group'.
399
400
C<field>: The original C<MARC::Record> object.
401
382
=back
402
=back
383
403
384
=cut
404
=cut
385
405
386
sub _process_mappings {
406
sub _process_mappings {
387
    my ($_self, $mappings, $data, $record_document, $altscript) = @_;
407
    my ($_self, $mappings, $data, $record_document, $meta) = @_;
388
    foreach my $mapping (@{$mappings}) {
408
    foreach my $mapping (@{$mappings}) {
389
        my ($target, $options) = @{$mapping};
409
        my ($target, $options) = @{$mapping};
390
410
391
        # Don't process sort fields for alternate scripts
411
        # Don't process sort fields for alternate scripts
392
        my $sort = $target =~ /__sort$/;
412
        my $sort = $target =~ /__sort$/;
393
        if ($sort && $altscript) {
413
        if ($sort && $meta->{altscript}) {
394
            next;
414
            next;
395
        }
415
        }
396
416
Lines 411-416 sub _process_mappings { Link Here
411
                $options->{property} => $_data
431
                $options->{property} => $_data
412
            }
432
            }
413
        }
433
        }
434
        if (defined $options->{nonfiling_characters_indicator}) {
435
            my $nonfiling_chars = $meta->{field}->indicator($options->{nonfiling_characters_indicator});
436
            $nonfiling_chars = looks_like_number($nonfiling_chars) ? int($nonfiling_chars) : 0;
437
            if ($nonfiling_chars) {
438
                $_data = substr $_data, $nonfiling_chars;
439
            }
440
        }
414
        push @{$record_document->{$target}}, $_data;
441
        push @{$record_document->{$target}}, $_data;
415
    }
442
    }
416
}
443
}
Lines 448-460 sub marc_records_to_documents { Link Here
448
        my $record_document = {};
475
        my $record_document = {};
449
        my $mappings = $rules->{leader};
476
        my $mappings = $rules->{leader};
450
        if ($mappings) {
477
        if ($mappings) {
451
            $self->_process_mappings($mappings, $record->leader(), $record_document, 0);
478
            $self->_process_mappings($mappings, $record->leader(), $record_document, {
479
                    altscript => 0,
480
                    data_source => 'leader'
481
                }
482
            );
452
        }
483
        }
453
        foreach my $field ($record->fields()) {
484
        foreach my $field ($record->fields()) {
454
            if ($field->is_control_field()) {
485
            if ($field->is_control_field()) {
455
                my $mappings = $control_fields_rules->{$field->tag()};
486
                my $mappings = $control_fields_rules->{$field->tag()};
456
                if ($mappings) {
487
                if ($mappings) {
457
                    $self->_process_mappings($mappings, $field->data(), $record_document, 0);
488
                    $self->_process_mappings($mappings, $field->data(), $record_document, {
489
                            altscript => 0,
490
                            data_source => 'control_field',
491
                            field => $field
492
                        }
493
                    );
458
                }
494
                }
459
            }
495
            }
460
            else {
496
            else {
Lines 480-486 sub marc_records_to_documents { Link Here
480
                            $mappings = [@{$mappings}, @{$wildcard_mappings}];
516
                            $mappings = [@{$mappings}, @{$wildcard_mappings}];
481
                        }
517
                        }
482
                        if (@{$mappings}) {
518
                        if (@{$mappings}) {
483
                            $self->_process_mappings($mappings, $data, $record_document, $altscript);
519
                            $self->_process_mappings($mappings, $data, $record_document, {
520
                                    altscript => $altscript,
521
                                    data_source => 'subfield',
522
                                    code => $code,
523
                                    field => $field
524
                                }
525
                            );
484
                        }
526
                        }
485
                        if ( defined @{$mappings}[0] && grep /match-heading/, @{@{$mappings}[0]} ){
527
                        if ( defined @{$mappings}[0] && grep /match-heading/, @{@{$mappings}[0]} ){
486
                            # Used by the authority linker the match-heading field requires a specific syntax
528
                            # Used by the authority linker the match-heading field requires a specific syntax
Lines 503-509 sub marc_records_to_documents { Link Here
503
                                )
545
                                )
504
                            );
546
                            );
505
                            if ($data) {
547
                            if ($data) {
506
                                $self->_process_mappings($subfields_join_mappings->{$subfields_group}, $data, $record_document, $altscript);
548
                                $self->_process_mappings($subfields_join_mappings->{$subfields_group}, $data, $record_document, {
549
                                        altscript => $altscript,
550
                                        data_source => 'subfields_group',
551
                                        codes => $subfields_group,
552
                                        field => $field
553
                                    }
554
                                );
507
                            }
555
                            }
508
                            if ( grep { $_->[0] eq 'match-heading' } @{$subfields_join_mappings->{$subfields_group}} ){
556
                            if ( grep { $_->[0] eq 'match-heading' } @{$subfields_join_mappings->{$subfields_group}} ){
509
                                # Used by the authority linker the match-heading field requires a specific syntax
557
                                # Used by the authority linker the match-heading field requires a specific syntax
Lines 799-805 sub _field_mappings { Link Here
799
            push @{$mapping}, {%{$default_options}, property => 'input'};
847
            push @{$mapping}, {%{$default_options}, property => 'input'};
800
        }
848
        }
801
        else {
849
        else {
802
            push @{$mapping}, $default_options;
850
            # Important! Make shallow clone, or we end up with the same hashref
851
            # shared by all mappings
852
            push @{$mapping}, {%{$default_options}};
803
        }
853
        }
804
        push @mappings, $mapping;
854
        push @mappings, $mapping;
805
    }
855
    }
Lines 934-939 sub _get_marc_mapping_rules { Link Here
934
            );
984
            );
935
        }
985
        }
936
    });
986
    });
987
988
    # Marc-flavour specific rule tweaks, could/should also provide hook for this
989
    if ($marcflavour eq 'marc21') {
990
        # Nonfiling characters processing for sort fields
991
        my %title_fields;
992
        if ($self->index eq $Koha::SearchEngine::BIBLIOS_INDEX) {
993
            # Format is: nonfiling characters indicator => field names list
994
            %title_fields = (
995
                1 => [630, 730, 740],
996
                2 => [130, 222, 240, 242, 243, 440, 830]
997
            );
998
        }
999
        elsif ($self->index eq $Koha::SearchEngine::AUTHORITIES_INDEX) {
1000
            %title_fields = (
1001
                1 => [730],
1002
                2 => [130, 430, 530]
1003
            );
1004
        }
1005
        foreach my $indicator (keys %title_fields) {
1006
            foreach my $field_tag (@{$title_fields{$indicator}}) {
1007
                my $mappings = $rules->{data_fields}->{$field_tag}->{subfields}->{a} // [];
1008
                foreach my $mapping (@{$mappings}) {
1009
                    if ($mapping->[0] =~ /__sort$/) {
1010
                        # Mark this as to be processed for nonfiling characters indicator
1011
                        # later on in _process_mappings
1012
                        $mapping->[1]->{nonfiling_characters_indicator} = $indicator;
1013
                    }
1014
                }
1015
            }
1016
        }
1017
    }
1018
937
    return $rules;
1019
    return $rules;
938
}
1020
}
939
1021
(-)a/t/Koha/SearchEngine/Elasticsearch.t (-2 / +24 lines)
Lines 118-124 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
118
118
119
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
119
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
120
120
121
    plan tests => 51;
121
    plan tests => 53;
122
122
123
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
123
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
124
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
124
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
Lines 195-200 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
195
            marc_field => '220',
195
            marc_field => '220',
196
        },
196
        },
197
        {
197
        {
198
            name => 'uniform_title',
199
            type => 'string',
200
            facet => 0,
201
            suggestible => 0,
202
            searchable => 1,
203
            sort => 1,
204
            marc_type => 'marc21',
205
            marc_field => '240a',
206
        },
207
        {
198
            name => 'title_wildcard',
208
            name => 'title_wildcard',
199
            type => 'string',
209
            type => 'string',
200
            facet => 0,
210
            facet => 0,
Lines 309-314 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
309
        MARC::Field->new('100', '', '', a => 'Author 1'),
319
        MARC::Field->new('100', '', '', a => 'Author 1'),
310
        MARC::Field->new('110', '', '', a => 'Corp Author'),
320
        MARC::Field->new('110', '', '', a => 'Corp Author'),
311
        MARC::Field->new('210', '', '', a => 'Title 1'),
321
        MARC::Field->new('210', '', '', a => 'Title 1'),
322
        MARC::Field->new('240', '', '4', a => 'The uniform title with nonfiling indicator'),
312
        MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
323
        MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
313
        MARC::Field->new('999', '', '', c => '1234567'),
324
        MARC::Field->new('999', '', '', c => '1234567'),
314
        # '  ' for testing trimming of white space in boolean value callback:
325
        # '  ' for testing trimming of white space in boolean value callback:
Lines 433-438 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
433
        'First document local_classification field should be set correctly'
444
        'First document local_classification field should be set correctly'
434
    );
445
    );
435
446
447
    # Nonfiling characters for sort fields
448
    is_deeply(
449
        $docs->[0]->{uniform_title},
450
        ['The uniform title with nonfiling indicator'],
451
        'First document uniform_title field should contain the title verbatim'
452
    );
453
    is_deeply(
454
        $docs->[0]->{uniform_title__sort},
455
        ['uniform title with nonfiling indicator'],
456
        'First document uniform_title__sort field should contain the title with the first four initial characters removed'
457
    );
458
436
    # Second record:
459
    # Second record:
437
460
438
    is(scalar @{$docs->[1]->{author}}, 1, 'Second document author field should contain one value');
461
    is(scalar @{$docs->[1]->{author}}, 1, 'Second document author field should contain one value');
439
- 

Return to bug 22771