View | Details | Raw Unified | Return to bug 24807
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-25 / +48 lines)
Lines 479-515 sub _process_mappings { Link Here
479
        # Copy (scalar) data since can have multiple targets
479
        # Copy (scalar) data since can have multiple targets
480
        # with differing options for (possibly) mutating data
480
        # with differing options for (possibly) mutating data
481
        # so need a different copy for each
481
        # so need a different copy for each
482
        my $_data = $data;
482
        my $data_copy = $data;
483
        if (defined $options->{substr}) {
483
        if (defined $options->{substr}) {
484
            my ($start, $length) = @{$options->{substr}};
484
            my ($start, $length) = @{$options->{substr}};
485
            $_data = length($data) > $start ? substr $data, $start, $length : '';
485
            $data_copy = length($data) > $start ? substr $data_copy, $start, $length : '';
486
        }
487
488
        # Add data to tokens array for callbacks processing
489
        my $tokens = [$data_copy];
490
491
        # Tokenize callbacks takes as token (possibly tokenized subfield data)
492
        # as argument, and returns a possibly different list of tokens.
493
        # Note that this list also might be empty.
494
        if (defined $options->{tokenize_callbacks}) {
495
            foreach my $callback (@{$options->{tokenize_callbacks}}) {
496
                # Pass each token to current callback which returns a list
497
                # (scalar is fine too) resulting either in a list or
498
                # a list of lists that will be flattened by perl.
499
                # The next callback will recieve the possibly expanded list of tokens.
500
                $tokens = [ map { $callback->($_) } @{$tokens} ];
501
            }
486
        }
502
        }
487
        if (defined $options->{value_callbacks}) {
503
        if (defined $options->{value_callbacks}) {
488
            $_data = reduce { $b->($a) } ($_data, @{$options->{value_callbacks}});
504
            $tokens = [ map { reduce { $b->($a) } ($_, @{$options->{value_callbacks}}) } @{$tokens} ];
489
        }
505
        }
490
        if (defined $options->{filter_callbacks}) {
506
        if (defined $options->{filter_callbacks}) {
491
            # Skip mapping unless all filter callbacks return true
507
            my @tokens_filtered;
492
            next unless all { $_data = $_->($_data) } @{$options->{filter_callbacks}};
508
            foreach my $_data (@{$tokens}) {
509
                if ( all { $_->($_data) } @{$options->{filter_callbacks}} ) {
510
                    push @tokens_filtered, $_data;
511
                }
512
            }
513
            # Overwrite $tokens with filtered values
514
            $tokens = \@tokens_filtered;
493
        }
515
        }
516
        # Skip mapping if all values has been removed
517
        next unless @{$tokens};
518
494
        if (defined $options->{property}) {
519
        if (defined $options->{property}) {
495
            $_data = {
520
            $tokens = [ map { { $options->{property} => $_ } } @{$tokens} ];
496
                $options->{property} => $_data
497
            }
498
        }
521
        }
499
        if (defined $options->{nonfiling_characters_indicator}) {
522
        if (defined $options->{nonfiling_characters_indicator}) {
500
            my $nonfiling_chars = $meta->{field}->indicator($options->{nonfiling_characters_indicator});
523
            my $nonfiling_chars = $meta->{field}->indicator($options->{nonfiling_characters_indicator});
501
            $nonfiling_chars = looks_like_number($nonfiling_chars) ? int($nonfiling_chars) : 0;
524
            $nonfiling_chars = looks_like_number($nonfiling_chars) ? int($nonfiling_chars) : 0;
502
            if ($nonfiling_chars) {
525
            # Nonfiling chars does not make sense for multiple tokens
503
                $_data = substr $_data, $nonfiling_chars;
526
            # Only apply on first element
504
            }
527
            $tokens->[0] = substr $tokens->[0], $nonfiling_chars;
505
        }
528
        }
506
529
507
        $record_document->{$target} //= [];
530
        $record_document->{$target} //= [];
508
        if( ref $_data eq 'ARRAY' ){
531
        push @{$record_document->{$target}}, @{$tokens};
509
            push @{$record_document->{$target}}, @{$_data};
510
        } else {
511
            push @{$record_document->{$target}}, $_data;
512
        }
513
    }
532
    }
514
}
533
}
515
534
Lines 901-916 sub _field_mappings { Link Here
901
        };
920
        };
902
    }
921
    }
903
    elsif ($target_type eq 'year') {
922
    elsif ($target_type eq 'year') {
904
        $default_options->{filter_callbacks} //= [];
923
        $default_options->{tokenize_callbacks} //= [];
905
        push @{$default_options->{filter_callbacks}}, sub {
924
        # Only accept years containing digits and "u"
925
        push @{$default_options->{tokenize_callbacks}}, sub {
906
            my ($value) = @_;
926
            my ($value) = @_;
907
            my @years = ();
927
            my @years = ( $value =~ /[0-9u]{4}/g );
908
            my @field_years = ( $value =~ /[0-9u]{4}/g );
928
            return @years;
909
            foreach my $year (@field_years){
929
        };
910
                $year =~ s/[u]/0/g;
930
911
                push @years, $year;
931
        $default_options->{value_callbacks} //= [];
912
            }
932
        # Replace "u" with "0" for sorting
913
            return \@years;
933
        push @{$default_options->{value_callbacks}}, sub {
934
            my ($value) = @_;
935
            $value =~ s/[u]/0/g;
936
            return $value;
914
        };
937
        };
915
    }
938
    }
916
939
(-)a/t/Koha/SearchEngine/Elasticsearch.t (-3 / +2 lines)
Lines 357-362 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
357
        MARC::Field->new('100', '', '', a => 'Author 2'),
357
        MARC::Field->new('100', '', '', a => 'Author 2'),
358
        # MARC::Field->new('210', '', '', a => 'Title 2'),
358
        # MARC::Field->new('210', '', '', a => 'Title 2'),
359
        # MARC::Field->new('245', '', '', a => 'Title: second record'),
359
        # MARC::Field->new('245', '', '', a => 'Title: second record'),
360
        MARC::Field->new('260', '', '', a => 'New York :', b => 'Ace ,', c => '1963-2003'),
360
        MARC::Field->new('999', '', '', c => '1234568'),
361
        MARC::Field->new('999', '', '', c => '1234568'),
361
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno),
362
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno),
362
    );
363
    );
Lines 517-523 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
517
    # Tests for 'year' type
518
    # Tests for 'year' type
518
    is_deeply(
519
    is_deeply(
519
      $docs->[1]->{'copydate'},
520
      $docs->[1]->{'copydate'},
520
      ['1963','2003'],
521
      ['1963', '2003'],
521
      'Second document copydate field should be set correctly'
522
      'Second document copydate field should be set correctly'
522
    );
523
    );
523
    is_deeply(
524
    is_deeply(
Lines 659-665 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t Link Here
659
        MARC::Field->new('100', '', '', a => 'Author 2'),
660
        MARC::Field->new('100', '', '', a => 'Author 2'),
660
        # MARC::Field->new('210', '', '', a => 'Title 2'),
661
        # MARC::Field->new('210', '', '', a => 'Title 2'),
661
        # MARC::Field->new('245', '', '', a => 'Title: second record'),
662
        # MARC::Field->new('245', '', '', a => 'Title: second record'),
662
        MARC::Field->new('260', '', '', a => 'New York :', b => 'Ace ,', c => '1963-2003'),
663
        MARC::Field->new('999', '', '', c => '1234568'),
663
        MARC::Field->new('999', '', '', c => '1234568'),
664
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),
664
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),
665
    );
665
    );
666
- 

Return to bug 24807