View | Details | Raw Unified | Return to bug 24807
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-12 / +39 lines)
Lines 39-45 use Search::Elasticsearch; Link Here
39
use Try::Tiny;
39
use Try::Tiny;
40
use YAML::Syck;
40
use YAML::Syck;
41
41
42
use List::Util qw( sum0 reduce );
42
use List::Util qw( sum0 reduce all );
43
use MARC::File::XML;
43
use MARC::File::XML;
44
use MIME::Base64;
44
use MIME::Base64;
45
use Encode qw(encode);
45
use Encode qw(encode);
Lines 209-214 sub get_elasticsearch_mappings { Link Here
209
                    $es_type = 'integer';
209
                    $es_type = 'integer';
210
                } elsif ($type eq 'isbn' || $type eq 'stdno') {
210
                } elsif ($type eq 'isbn' || $type eq 'stdno') {
211
                    $es_type = 'stdno';
211
                    $es_type = 'stdno';
212
                } elsif ($type eq 'year') {
213
                    $es_type = 'year';
212
                }
214
                }
213
215
214
                if ($search) {
216
                if ($search) {
Lines 469-496 sub _process_mappings { Link Here
469
        # Copy (scalar) data since can have multiple targets
471
        # Copy (scalar) data since can have multiple targets
470
        # with differing options for (possibly) mutating data
472
        # with differing options for (possibly) mutating data
471
        # so need a different copy for each
473
        # so need a different copy for each
472
        my $_data = $data;
474
        my $data_copy = $data;
473
        $record_document->{$target} //= [];
474
        if (defined $options->{substr}) {
475
        if (defined $options->{substr}) {
475
            my ($start, $length) = @{$options->{substr}};
476
            my ($start, $length) = @{$options->{substr}};
476
            $_data = length($data) > $start ? substr $data, $start, $length : '';
477
            $data_copy = length($data) > $start ? substr $data_copy, $start, $length : '';
477
        }
478
        }
479
480
        # Add data to values array for callbacks processing
481
        my $values = [$data_copy];
482
483
        # Value callbacks takes subfield data (or values from previous
484
        # callbacks) as argument, and returns a possibly different list of values.
485
        # Note that the returned list may also be empty.
478
        if (defined $options->{value_callbacks}) {
486
        if (defined $options->{value_callbacks}) {
479
            $_data = reduce { $b->($a) } ($_data, @{$options->{value_callbacks}});
487
            foreach my $callback (@{$options->{value_callbacks}}) {
488
                # Pass each value to current callback which returns a list
489
                # (scalar is fine too) resulting either in a list or
490
                # a list of lists that will be flattened by perl.
491
                # The next callback will receive the possibly expanded list of values.
492
                $values = [ map { $callback->($_) } @{$values} ];
493
            }
480
        }
494
        }
495
496
        # Skip mapping if all values has been removed
497
        next unless @{$values};
498
481
        if (defined $options->{property}) {
499
        if (defined $options->{property}) {
482
            $_data = {
500
            $values = [ map { { $options->{property} => $_ } } @{$values} ];
483
                $options->{property} => $_data
484
            }
485
        }
501
        }
486
        if (defined $options->{nonfiling_characters_indicator}) {
502
        if (defined $options->{nonfiling_characters_indicator}) {
487
            my $nonfiling_chars = $meta->{field}->indicator($options->{nonfiling_characters_indicator});
503
            my $nonfiling_chars = $meta->{field}->indicator($options->{nonfiling_characters_indicator});
488
            $nonfiling_chars = looks_like_number($nonfiling_chars) ? int($nonfiling_chars) : 0;
504
            $nonfiling_chars = looks_like_number($nonfiling_chars) ? int($nonfiling_chars) : 0;
489
            if ($nonfiling_chars) {
505
            # Nonfiling chars does not make sense for multiple values
490
                $_data = substr $_data, $nonfiling_chars;
506
            # Only apply on first element
491
            }
507
            $values->[0] = substr $values->[0], $nonfiling_chars;
492
        }
508
        }
493
        push @{$record_document->{$target}}, $_data;
509
510
        $record_document->{$target} //= [];
511
        push @{$record_document->{$target}}, @{$values};
494
    }
512
    }
495
}
513
}
496
514
Lines 885-890 sub _field_mappings { Link Here
885
            return $value ? 'true' : 'false';
903
            return $value ? 'true' : 'false';
886
        };
904
        };
887
    }
905
    }
906
    elsif ($target_type eq 'year') {
907
        $default_options->{value_callbacks} //= [];
908
        # Only accept years containing digits and "u"
909
        push @{$default_options->{value_callbacks}}, sub {
910
            my ($value) = @_;
911
            # Replace "u" with "0" for sorting
912
            return map { s/[u\s]/0/gr } ( $value =~ /[0-9u\s]{4}/g );
913
        };
914
    }
888
915
889
    if ($search) {
916
    if ($search) {
890
        my $mapping = [$target_name, $default_options];
917
        my $mapping = [$target_name, $default_options];
(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (+3 lines)
Lines 119-124 sub update_index { Link Here
119
            type => 'data', # is just hard coded in Indexer.pm?
119
            type => 'data', # is just hard coded in Indexer.pm?
120
            body => \@body
120
            body => \@body
121
        );
121
        );
122
        if ($response->{errors}) {
123
            carp "One or more ElasticSearch errors occurred when indexing documents";
124
        }
122
    }
125
    }
123
    return $response;
126
    return $response;
124
}
127
}
(-)a/admin/searchengine/elasticsearch/field_config.yaml (+2 lines)
Lines 25-30 search: Link Here
25
    type: integer
25
    type: integer
26
    null_value: 0
26
    null_value: 0
27
    ignore_malformed: true
27
    ignore_malformed: true
28
  year:
29
    type: short
28
  stdno:
30
  stdno:
29
    type: text
31
    type: text
30
    analyzer: analyzer_stdno
32
    analyzer: analyzer_stdno
(-)a/installer/data/mysql/atomicupdate/bug_24807-add-year-search-field-type.perl (+7 lines)
Line 0 Link Here
1
$DBversion = 'XXX'; # will be replaced by the RM
2
if( CheckVersion( $DBversion ) ) {
3
    $dbh->do( "ALTER TABLE `search_field` MODIFY COLUMN `type` enum('','string','date','number','boolean','sum','isbn','stdno','year') NOT NULL" );
4
    $dbh->do( "UPDATE `search_field` SET type = 'year' WHERE name = 'date-of-publication'" );
5
    SetVersion( $DBversion );
6
    print "Upgrade to $DBversion done (Bug 14957 - Add 'year' type to improve sorting behaviour)\n";
7
}
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt (+5 lines)
Lines 196-201 a.add, a.delete { Link Here
196
                          [% ELSE %]
196
                          [% ELSE %]
197
                            <option value="date">Date</option>
197
                            <option value="date">Date</option>
198
                          [% END %]
198
                          [% END %]
199
                          [% IF search_field.type == "year" %]
200
                            <option value="year" selected="selected">Year</option>
201
                          [% ELSE %]
202
                            <option value="year">Year</option>
203
                          [% END %]
199
                          [% IF search_field.type == "number" %]
204
                          [% IF search_field.type == "number" %]
200
                            <option value="number" selected="selected">Number</option>
205
                            <option value="number" selected="selected">Number</option>
201
                          [% ELSE %]
206
                          [% ELSE %]
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t (-3 / +70 lines)
Lines 132-138 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
132
132
133
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
133
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
134
134
135
    plan tests => 53;
135
    plan tests => 59;
136
136
137
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
137
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
138
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
138
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
Lines 297-303 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
297
            sort => 1,
297
            sort => 1,
298
            marc_type => 'marc21',
298
            marc_type => 'marc21',
299
            marc_field => '952l',
299
            marc_field => '952l',
300
          },
301
          {
302
            name => 'copydate',
303
            type => 'year',
304
            facet => 0,
305
            suggestible => 0,
306
            searchable => 1,
307
            sort => 1,
308
            marc_type => 'marc21',
309
            marc_field => '260c',
310
          },
311
          {
312
            name => 'date-of-publication',
313
            type => 'year',
314
            facet => 0,
315
            suggestible => 0,
316
            searchable => 1,
317
            sort => 1,
318
            marc_type => 'marc21',
319
            marc_field => '008_/7-10',
300
        },
320
        },
321
301
    );
322
    );
302
323
303
    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
324
    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
Lines 329-340 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
329
    $marc_record_1->append_fields(
350
    $marc_record_1->append_fields(
330
        MARC::Field->new('001', '123'),
351
        MARC::Field->new('001', '123'),
331
        MARC::Field->new('007', 'ku'),
352
        MARC::Field->new('007', 'ku'),
353
        MARC::Field->new('008', '901111s1962 xxk|||| |00| ||eng c'),
332
        MARC::Field->new('020', '', '', a => '1-56619-909-3'),
354
        MARC::Field->new('020', '', '', a => '1-56619-909-3'),
333
        MARC::Field->new('100', '', '', a => 'Author 1'),
355
        MARC::Field->new('100', '', '', a => 'Author 1'),
334
        MARC::Field->new('110', '', '', a => 'Corp Author'),
356
        MARC::Field->new('110', '', '', a => 'Corp Author'),
335
        MARC::Field->new('210', '', '', a => 'Title 1'),
357
        MARC::Field->new('210', '', '', a => 'Title 1'),
336
        MARC::Field->new('240', '', '4', a => 'The uniform title with nonfiling indicator'),
358
        MARC::Field->new('240', '', '4', a => 'The uniform title with nonfiling indicator'),
337
        MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
359
        MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
360
        MARC::Field->new('260', '', '', a => 'New York :', b => 'Ace ,', c => 'c1962'),
338
        MARC::Field->new('999', '', '', c => '1234567'),
361
        MARC::Field->new('999', '', '', c => '1234567'),
339
        # '  ' for testing trimming of white space in boolean value callback:
362
        # '  ' for testing trimming of white space in boolean value callback:
340
        MARC::Field->new('952', '', '', 0 => '  ', g => '123.30', o => $callno, l => 3),
363
        MARC::Field->new('952', '', '', 0 => '  ', g => '123.30', o => $callno, l => 3),
Lines 344-363 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
344
    my $marc_record_2 = MARC::Record->new();
367
    my $marc_record_2 = MARC::Record->new();
345
    $marc_record_2->leader('     cam  22      a 4500');
368
    $marc_record_2->leader('     cam  22      a 4500');
346
    $marc_record_2->append_fields(
369
    $marc_record_2->append_fields(
370
        MARC::Field->new('008', '901111s19uu xxk|||| |00| ||eng c'),
347
        MARC::Field->new('100', '', '', a => 'Author 2'),
371
        MARC::Field->new('100', '', '', a => 'Author 2'),
348
        # MARC::Field->new('210', '', '', a => 'Title 2'),
372
        # MARC::Field->new('210', '', '', a => 'Title 2'),
349
        # MARC::Field->new('245', '', '', a => 'Title: second record'),
373
        # MARC::Field->new('245', '', '', a => 'Title: second record'),
374
        MARC::Field->new('260', '', '', a => 'New York :', b => 'Ace ,', c => '1963-2003'),
350
        MARC::Field->new('999', '', '', c => '1234568'),
375
        MARC::Field->new('999', '', '', c => '1234568'),
351
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno),
376
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno),
352
    );
377
    );
353
    my $records = [ $marc_record_1, $marc_record_2 ];
378
379
    my $marc_record_3 = MARC::Record->new();
380
    $marc_record_3->leader('     cam  22      a 4500');
381
    $marc_record_3->append_fields(
382
        MARC::Field->new('008', '901111s19uu xxk|||| |00| ||eng c'),
383
        MARC::Field->new('100', '', '', a => 'Author 2'),
384
        # MARC::Field->new('210', '', '', a => 'Title 3'),
385
        # MARC::Field->new('245', '', '', a => 'Title: third record'),
386
        MARC::Field->new('260', '', '', a => 'New York :', b => 'Ace ,', c => ' 89 '),
387
        MARC::Field->new('999', '', '', c => '1234568'),
388
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno),
389
    );
390
    my $records = [$marc_record_1, $marc_record_2, $marc_record_3];
354
391
355
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
392
    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
356
393
357
    my $docs = $see->marc_records_to_documents($records);
394
    my $docs = $see->marc_records_to_documents($records);
358
395
359
    # First record:
396
    # First record:
360
    is(scalar @{$docs}, 2, 'Two records converted to documents');
397
    is(scalar @{$docs}, 3, 'Two records converted to documents');
361
398
362
    is_deeply($docs->[0]->{control_number}, ['123'], 'First record control number should be set correctly');
399
    is_deeply($docs->[0]->{control_number}, ['123'], 'First record control number should be set correctly');
363
400
Lines 470-475 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
470
        'First document uniform_title__sort field should contain the title with the first four initial characters removed'
507
        'First document uniform_title__sort field should contain the title with the first four initial characters removed'
471
    );
508
    );
472
509
510
    # Tests for 'year' type
511
    is(scalar @{$docs->[0]->{'date-of-publication'}}, 1, 'First document date-of-publication field should contain one value');
512
    is_deeply($docs->[0]->{'date-of-publication'}, ['1962'], 'First document date-of-publication field should be set correctly');
513
514
    is_deeply(
515
      $docs->[0]->{'copydate'},
516
      ['1962'],
517
      'First document copydate field should be set correctly'
518
    );
519
473
    # Second record:
520
    # Second record:
474
521
475
    is(scalar @{$docs->[1]->{author}}, 1, 'Second document author field should contain one value');
522
    is(scalar @{$docs->[1]->{author}}, 1, 'Second document author field should contain one value');
Lines 494-499 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
494
        'Second document local_classification__sort field should be set correctly'
541
        'Second document local_classification__sort field should be set correctly'
495
    );
542
    );
496
543
544
    # Tests for 'year' type
545
    is_deeply(
546
      $docs->[1]->{'copydate'},
547
      ['1963', '2003'],
548
      'Second document copydate field should be set correctly'
549
    );
550
    is_deeply(
551
      $docs->[1]->{'date-of-publication'},
552
      ['1900'],
553
      'Second document date-of-publication field should be set correctly'
554
    );
555
556
    # Third record:
557
558
    is_deeply(
559
      $docs->[2]->{'copydate'},
560
      ['0890'],
561
      'Third document copydate field should be set correctly'
562
    );
563
497
    # Mappings marc_type:
564
    # Mappings marc_type:
498
565
499
    ok(!(defined $docs->[0]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
566
    ok(!(defined $docs->[0]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch/Indexer.t (-30 / +4 lines)
Lines 19-25 Link Here
19
19
20
use Modern::Perl;
20
use Modern::Perl;
21
21
22
use Test::More tests => 3;
22
use Test::More tests => 2;
23
use Test::MockModule;
23
use Test::MockModule;
24
use t::lib::Mocks;
24
use t::lib::Mocks;
25
25
Lines 35-45 SKIP: { Link Here
35
35
36
    eval { Koha::SearchEngine::Elasticsearch->get_elasticsearch_params; };
36
    eval { Koha::SearchEngine::Elasticsearch->get_elasticsearch_params; };
37
37
38
    skip 'Elasticsearch configuration not available', 2
38
    skip 'Elasticsearch configuration not available', 1
39
        if $@;
39
        if $@;
40
40
41
subtest 'create_index() tests' => sub {
41
subtest 'create_index() tests' => sub {
42
    plan tests => 5;
42
    plan tests => 6;
43
    my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
43
    my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
44
    $se->mock( '_read_configuration', sub {
44
    $se->mock( '_read_configuration', sub {
45
            my ($self, $sub ) = @_;
45
            my ($self, $sub ) = @_;
Lines 72-77 subtest 'create_index() tests' => sub { Link Here
72
    my $response = $indexer->update_index([1], $records);
72
    my $response = $indexer->update_index([1], $records);
73
    is( $response->{errors}, 0, "no error on update_index" );
73
    is( $response->{errors}, 0, "no error on update_index" );
74
    is( scalar(@{$response->{items}}), 1, "1 item indexed" );
74
    is( scalar(@{$response->{items}}), 1, "1 item indexed" );
75
    is( $response->{items}[0]->{index}->{_id},"1", "We should get a string matching the bibnumber passed in");
75
76
76
    is(
77
    is(
77
        $indexer->drop_index(),
78
        $indexer->drop_index(),
Lines 79-108 subtest 'create_index() tests' => sub { Link Here
79
        'Dropping the index'
80
        'Dropping the index'
80
    );
81
    );
81
};
82
};
82
83
subtest 'update_index() tests' => sub {
84
    plan tests => 2;
85
    my $kse = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
86
    $kse->mock( 'marc_records_to_documents', sub {
87
            my ($self, $params ) = @_;
88
            return [1];
89
    });
90
91
    my $indexer;
92
    ok(
93
        $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ 'index' => 'biblios' }),
94
        'Creating a new indexer object'
95
    );
96
97
    my $searcher = $indexer->get_elasticsearch();
98
    my $se = Test::MockModule->new( ref $searcher );
99
    $se->mock( 'bulk', sub {
100
            my ($self, %params ) = @_;
101
            return $params{body};
102
    });
103
104
    my $bibnumber_array = $indexer->update_index([13],["faked"]);
105
    is( $bibnumber_array->[0]->{index}->{_id},"13", "We should get a string matching the bibnumber");
106
};
107
108
}
83
}
109
- 

Return to bug 24807