View | Details | Raw Unified | Return to bug 22246
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-6 / +15 lines)
Lines 375-386 sub _process_mappings { Link Here
375
                $options->{property} => $_data
375
                $options->{property} => $_data
376
            }
376
            }
377
        }
377
        }
378
        # For sort fields, index only a single field with concatenated values
378
        push @{$record_document->{$target}}, $_data;
379
        if ($sort && @{$record_document->{$target}}) {
380
            @{$record_document->{$target}}[0] .= " $_data";
381
        } else {
382
            push @{$record_document->{$target}}, $_data;
383
        }
384
    }
379
    }
385
}
380
}
386
381
Lines 512-517 sub marc_records_to_documents { Link Here
512
            }
507
            }
513
        }
508
        }
514
509
510
        # Remove duplicate values and collapse sort fields
511
        foreach my $field (keys %{$record_document}) {
512
            if (ref($record_document->{$field}) eq 'ARRAY') {
513
                @{$record_document->{$field}} = do {
514
                    my %seen;
515
                    grep { !$seen{ref($_) eq 'HASH' && defined $_->{input} ? $_->{input} : $_}++ } @{$record_document->{$field}};
516
                };
517
                if ($field =~ /__sort$/) {
518
                    # Make sure to keep the sort field length sensible. 255 was chosen as a nice round value.
519
                    $record_document->{$field} = [substr(join(' ', @{$record_document->{$field}}), 0, 255)];
520
                }
521
            }
522
        }
523
515
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
524
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
516
        $record->encoding('UTF-8');
525
        $record->encoding('UTF-8');
517
        my @warnings;
526
        my @warnings;
(-)a/admin/searchengine/elasticsearch/mappings.yaml (-27 / +27 lines)
Lines 1967-1983 biblios: Link Here
1967
      - facet: '1'
1967
      - facet: '1'
1968
        marc_field: 952b
1968
        marc_field: 952b
1969
        marc_type: marc21
1969
        marc_type: marc21
1970
        sort: ~
1970
        sort: 0
1971
        suggestible: ''
1971
        suggestible: ''
1972
      - facet: '1'
1972
      - facet: '1'
1973
        marc_field: 952b
1973
        marc_field: 952b
1974
        marc_type: normarc
1974
        marc_type: normarc
1975
        sort: ~
1975
        sort: 0
1976
        suggestible: ''
1976
        suggestible: ''
1977
      - facet: ''
1977
      - facet: ''
1978
        marc_field: 995c
1978
        marc_field: 995c
1979
        marc_type: unimarc
1979
        marc_type: unimarc
1980
        sort: ~
1980
        sort: 0
1981
        suggestible: ''
1981
        suggestible: ''
1982
    type: string
1982
    type: string
1983
  homebranch:
1983
  homebranch:
Lines 1986-2002 biblios: Link Here
1986
      - facet: '1'
1986
      - facet: '1'
1987
        marc_field: 952a
1987
        marc_field: 952a
1988
        marc_type: marc21
1988
        marc_type: marc21
1989
        sort: ~
1989
        sort: 0
1990
        suggestible: ''
1990
        suggestible: ''
1991
      - facet: '1'
1991
      - facet: '1'
1992
        marc_field: 952a
1992
        marc_field: 952a
1993
        marc_type: normarc
1993
        marc_type: normarc
1994
        sort: ~
1994
        sort: 0
1995
        suggestible: ''
1995
        suggestible: ''
1996
      - facet: '1'
1996
      - facet: '1'
1997
        marc_field: 995b
1997
        marc_field: 995b
1998
        marc_type: unimarc
1998
        marc_type: unimarc
1999
        sort: ~
1999
        sort: 0
2000
        suggestible: ''
2000
        suggestible: ''
2001
    type: string
2001
    type: string
2002
  host-item:
2002
  host-item:
Lines 2217-2233 biblios: Link Here
2217
      - facet: ''
2217
      - facet: ''
2218
        marc_field: '9529'
2218
        marc_field: '9529'
2219
        marc_type: marc21
2219
        marc_type: marc21
2220
        sort: ~
2220
        sort: 0
2221
        suggestible: ''
2221
        suggestible: ''
2222
      - facet: ''
2222
      - facet: ''
2223
        marc_field: '9529'
2223
        marc_field: '9529'
2224
        marc_type: normarc
2224
        marc_type: normarc
2225
        sort: ~
2225
        sort: 0
2226
        suggestible: ''
2226
        suggestible: ''
2227
      - facet: ''
2227
      - facet: ''
2228
        marc_field: '9959'
2228
        marc_field: '9959'
2229
        marc_type: unimarc
2229
        marc_type: unimarc
2230
        sort: ~
2230
        sort: 0
2231
        suggestible: ''
2231
        suggestible: ''
2232
    type: number
2232
    type: number
2233
  itemtype:
2233
  itemtype:
Lines 2634-2655 biblios: Link Here
2634
      - facet: ''
2634
      - facet: ''
2635
        marc_field: 952o
2635
        marc_field: 952o
2636
        marc_type: marc21
2636
        marc_type: marc21
2637
        sort: ~
2637
        sort: 0
2638
        suggestible: '1'
2638
        suggestible: '1'
2639
      - facet: ''
2639
      - facet: ''
2640
        marc_field: 952o
2640
        marc_field: 952o
2641
        marc_type: normarc
2641
        marc_type: normarc
2642
        sort: ~
2642
        sort: 0
2643
        suggestible: '1'
2643
        suggestible: '1'
2644
      - facet: ''
2644
      - facet: ''
2645
        marc_field: '686'
2645
        marc_field: '686'
2646
        marc_type: unimarc
2646
        marc_type: unimarc
2647
        sort: ~
2647
        sort: 0
2648
        suggestible: ''
2648
        suggestible: ''
2649
      - facet: ''
2649
      - facet: ''
2650
        marc_field: 995k
2650
        marc_field: 995k
2651
        marc_type: unimarc
2651
        marc_type: unimarc
2652
        sort: ~
2652
        sort: 0
2653
        suggestible: '1'
2653
        suggestible: '1'
2654
    type: ''
2654
    type: ''
2655
  local-number:
2655
  local-number:
Lines 2658-2674 biblios: Link Here
2658
      - facet: ''
2658
      - facet: ''
2659
        marc_field: 999c
2659
        marc_field: 999c
2660
        marc_type: marc21
2660
        marc_type: marc21
2661
        sort: ~
2661
        sort: 0
2662
        suggestible: ''
2662
        suggestible: ''
2663
      - facet: ''
2663
      - facet: ''
2664
        marc_field: 999c
2664
        marc_field: 999c
2665
        marc_type: normarc
2665
        marc_type: normarc
2666
        sort: ~
2666
        sort: 0
2667
        suggestible: ''
2667
        suggestible: ''
2668
      - facet: ''
2668
      - facet: ''
2669
        marc_field: '001'
2669
        marc_field: '001'
2670
        marc_type: unimarc
2670
        marc_type: unimarc
2671
        sort: ~
2671
        sort: 0
2672
        suggestible: ''
2672
        suggestible: ''
2673
    type: string
2673
    type: string
2674
  location:
2674
  location:
Lines 2677-2693 biblios: Link Here
2677
      - facet: '1'
2677
      - facet: '1'
2678
        marc_field: 952c
2678
        marc_field: 952c
2679
        marc_type: marc21
2679
        marc_type: marc21
2680
        sort: ~
2680
        sort: 0
2681
        suggestible: ''
2681
        suggestible: ''
2682
      - facet: '1'
2682
      - facet: '1'
2683
        marc_field: 952c
2683
        marc_field: 952c
2684
        marc_type: normarc
2684
        marc_type: normarc
2685
        sort: ~
2685
        sort: 0
2686
        suggestible: ''
2686
        suggestible: ''
2687
      - facet: '1'
2687
      - facet: '1'
2688
        marc_field: 995e
2688
        marc_field: 995e
2689
        marc_type: unimarc
2689
        marc_type: unimarc
2690
        sort: ~
2690
        sort: 0
2691
        suggestible: ''
2691
        suggestible: ''
2692
    type: ''
2692
    type: ''
2693
  lost:
2693
  lost:
Lines 2829-2835 biblios: Link Here
2829
      - facet: ''
2829
      - facet: ''
2830
        marc_field: '700'
2830
        marc_field: '700'
2831
        marc_type: marc21
2831
        marc_type: marc21
2832
        sort: ~
2832
        sort: 0
2833
        suggestible: ''
2833
        suggestible: ''
2834
      - facet: ''
2834
      - facet: ''
2835
        marc_field: '710'
2835
        marc_field: '710'
Lines 2956-2962 biblios: Link Here
2956
      - facet: ''
2956
      - facet: ''
2957
        marc_field: '060'
2957
        marc_field: '060'
2958
        marc_type: marc21
2958
        marc_type: marc21
2959
        sort: ~
2959
        sort: 0
2960
        suggestible: ''
2960
        suggestible: ''
2961
    type: ''
2961
    type: ''
2962
  not-onloan-count:
2962
  not-onloan-count:
Lines 2998-3014 biblios: Link Here
2998
      - facet: 0
2998
      - facet: 0
2999
        marc_field: '9527'
2999
        marc_field: '9527'
3000
        marc_type: marc21
3000
        marc_type: marc21
3001
        sort: ~
3001
        sort: 0
3002
        suggestible: 0
3002
        suggestible: 0
3003
      - facet: 0
3003
      - facet: 0
3004
        marc_field: '9527'
3004
        marc_field: '9527'
3005
        marc_type: normarc
3005
        marc_type: normarc
3006
        sort: ~
3006
        sort: 0
3007
        suggestible: 0
3007
        suggestible: 0
3008
      - facet: 0
3008
      - facet: 0
3009
        marc_field: 995o
3009
        marc_field: 995o
3010
        marc_type: unimarc
3010
        marc_type: unimarc
3011
        sort: ~
3011
        sort: 0
3012
        suggestible: 0
3012
        suggestible: 0
3013
    type: number
3013
    type: number
3014
  number-db:
3014
  number-db:
Lines 3072-3088 biblios: Link Here
3072
      - facet: ''
3072
      - facet: ''
3073
        marc_field: 952q
3073
        marc_field: 952q
3074
        marc_type: marc21
3074
        marc_type: marc21
3075
        sort: ~
3075
        sort: 0
3076
        suggestible: ''
3076
        suggestible: ''
3077
      - facet: ''
3077
      - facet: ''
3078
        marc_field: 952q
3078
        marc_field: 952q
3079
        marc_type: normarc
3079
        marc_type: normarc
3080
        sort: ~
3080
        sort: 0
3081
        suggestible: ''
3081
        suggestible: ''
3082
      - facet: ''
3082
      - facet: ''
3083
        marc_field: 995n
3083
        marc_field: 995n
3084
        marc_type: unimarc
3084
        marc_type: unimarc
3085
        sort: ~
3085
        sort: 0
3086
        suggestible: ''
3086
        suggestible: ''
3087
    type: boolean
3087
    type: boolean
3088
  other-control-number:
3088
  other-control-number:
(-)a/t/Koha/SearchEngine/Elasticsearch.t (-6 / +31 lines)
Lines 117-123 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
117
117
118
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
118
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
119
119
120
    plan tests => 47;
120
    plan tests => 49;
121
121
122
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
122
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
123
123
Lines 212-217 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
212
            marc_type => 'marc21',
212
            marc_type => 'marc21',
213
            marc_field => '9520',
213
            marc_field => '9520',
214
        },
214
        },
215
        {
216
            name => 'local_classification',
217
            type => 'string',
218
            facet => 0,
219
            suggestible => 0,
220
            sort => 1,
221
            marc_type => 'marc21',
222
            marc_field => '952o',
223
        },
215
        {
224
        {
216
            name => 'type_of_record',
225
            name => 'type_of_record',
217
            type => 'string',
226
            type => 'string',
Lines 251-256 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
251
260
252
    my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });
261
    my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });
253
262
263
    my $callno = 'ABC123';
264
    my $callno2 = 'ABC456';
265
    my $long_callno = '1234567890' x 30;
266
254
    my $marc_record_1 = MARC::Record->new();
267
    my $marc_record_1 = MARC::Record->new();
255
    $marc_record_1->leader('     cam  22      a 4500');
268
    $marc_record_1->leader('     cam  22      a 4500');
256
    $marc_record_1->append_fields(
269
    $marc_record_1->append_fields(
Lines 262-269 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
262
        MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
275
        MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
263
        MARC::Field->new('999', '', '', c => '1234567'),
276
        MARC::Field->new('999', '', '', c => '1234567'),
264
        # '  ' for testing trimming of white space in boolean value callback:
277
        # '  ' for testing trimming of white space in boolean value callback:
265
        MARC::Field->new('952', '', '', 0 => '  ', g => '123.30'),
278
        MARC::Field->new('952', '', '', 0 => '  ', g => '123.30', o => $callno),
266
        MARC::Field->new('952', '', '', 0 => 0, g => '127.20'),
279
        MARC::Field->new('952', '', '', 0 => 0, g => '127.20', o => $callno2),
280
        MARC::Field->new('952', '', '', 0 => 1, g => '0.00', o => $long_callno),
267
    );
281
    );
268
    my $marc_record_2 = MARC::Record->new();
282
    my $marc_record_2 = MARC::Record->new();
269
    $marc_record_2->leader('     cam  22      a 4500');
283
    $marc_record_2->leader('     cam  22      a 4500');
Lines 272-278 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
272
        # MARC::Field->new('210', '', '', a => 'Title 2'),
286
        # MARC::Field->new('210', '', '', a => 'Title 2'),
273
        # MARC::Field->new('245', '', '', a => 'Title: second record'),
287
        # MARC::Field->new('245', '', '', a => 'Title: second record'),
274
        MARC::Field->new('999', '', '', c => '1234568'),
288
        MARC::Field->new('999', '', '', c => '1234568'),
275
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),
289
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno),
276
    );
290
    );
277
    my $records = [$marc_record_1, $marc_record_2];
291
    my $records = [$marc_record_1, $marc_record_2];
278
292
Lines 336-342 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
336
    is(scalar @{$docs->[0][1]->{items_withdrawn_status}}, 2, 'First document items_withdrawn_status field should have two values');
350
    is(scalar @{$docs->[0][1]->{items_withdrawn_status}}, 2, 'First document items_withdrawn_status field should have two values');
337
    is_deeply(
351
    is_deeply(
338
        $docs->[0][1]->{items_withdrawn_status},
352
        $docs->[0][1]->{items_withdrawn_status},
339
        ['false', 'false'],
353
        ['false', 'true'],
340
        'First document items_withdrawn_status field should be set correctly'
354
        'First document items_withdrawn_status field should be set correctly'
341
    );
355
    );
342
356
Lines 372-377 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
372
    is(scalar @{$docs->[0][1]->{isbn}}, 4, 'First document isbn field should contain four values');
386
    is(scalar @{$docs->[0][1]->{isbn}}, 4, 'First document isbn field should contain four values');
373
    is_deeply($docs->[0][1]->{isbn}, ['978-1-56619-909-4', '9781566199094', '1-56619-909-3', '1566199093'], 'First document isbn field should be set correctly');
387
    is_deeply($docs->[0][1]->{isbn}, ['978-1-56619-909-4', '9781566199094', '1-56619-909-3', '1566199093'], 'First document isbn field should be set correctly');
374
388
389
    is_deeply(
390
        $docs->[0][1]->{'local_classification'},
391
        [$callno, $callno2, $long_callno],
392
        'First document local_classification field should be set correctly'
393
    );
394
375
    # Second record:
395
    # Second record:
376
396
377
    is(scalar @{$docs->[1][1]->{author}}, 1, 'Second document author field should contain one value');
397
    is(scalar @{$docs->[1][1]->{author}}, 1, 'Second document author field should contain one value');
Lines 390-395 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
390
        'Second document sum_item_price field should be set correctly'
410
        'Second document sum_item_price field should be set correctly'
391
    );
411
    );
392
412
413
    is_deeply(
414
        $docs->[1][1]->{local_classification__sort},
415
        [substr($long_callno, 0, 255)],
416
        'Second document local_classification__sort field should be set correctly'
417
    );
418
393
    # Mappings marc_type:
419
    # Mappings marc_type:
394
420
395
    ok(!(defined $docs->[0][1]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
421
    ok(!(defined $docs->[0][1]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
396
- 

Return to bug 22246