View | Details | Raw Unified | Return to bug 22246
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-6 / +15 lines)
Lines 375-386 sub _process_mappings { Link Here
375
                $options->{property} => $_data
375
                $options->{property} => $_data
376
            }
376
            }
377
        }
377
        }
378
        # For sort fields, index only a single field with concatenated values
378
        push @{$record_document->{$target}}, $_data;
379
        if ($sort && @{$record_document->{$target}}) {
380
            @{$record_document->{$target}}[0] .= " $_data";
381
        } else {
382
            push @{$record_document->{$target}}, $_data;
383
        }
384
    }
379
    }
385
}
380
}
386
381
Lines 512-517 sub marc_records_to_documents { Link Here
512
            }
507
            }
513
        }
508
        }
514
509
510
        # Remove duplicate values and collapse sort fields
511
        foreach my $field (keys %{$record_document}) {
512
            if (ref($record_document->{$field}) eq 'ARRAY') {
513
                @{$record_document->{$field}} = do {
514
                    my %seen;
515
                    grep { !$seen{ref($_) eq 'HASH' && defined $_->{input} ? $_->{input} : $_}++ } @{$record_document->{$field}};
516
                };
517
                if ($field =~ /__sort$/) {
518
                    # Make sure to keep the sort field length sensible. 255 was chosen as a nice round value.
519
                    $record_document->{$field} = [substr(join(' ', @{$record_document->{$field}}), 0, 255)];
520
                }
521
            }
522
        }
523
515
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
524
        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
516
        $record->encoding('UTF-8');
525
        $record->encoding('UTF-8');
517
        my @warnings;
526
        my @warnings;
(-)a/admin/searchengine/elasticsearch/mappings.yaml (-24 / +24 lines)
Lines 2063-2079 biblios: Link Here
2063
      - facet: '1'
2063
      - facet: '1'
2064
        marc_field: 952b
2064
        marc_field: 952b
2065
        marc_type: marc21
2065
        marc_type: marc21
2066
        sort: ~
2066
        sort: 0
2067
        suggestible: ''
2067
        suggestible: ''
2068
      - facet: '1'
2068
      - facet: '1'
2069
        marc_field: 952b
2069
        marc_field: 952b
2070
        marc_type: normarc
2070
        marc_type: normarc
2071
        sort: ~
2071
        sort: 0
2072
        suggestible: ''
2072
        suggestible: ''
2073
      - facet: ''
2073
      - facet: ''
2074
        marc_field: 995c
2074
        marc_field: 995c
2075
        marc_type: unimarc
2075
        marc_type: unimarc
2076
        sort: ~
2076
        sort: 0
2077
        suggestible: ''
2077
        suggestible: ''
2078
    type: string
2078
    type: string
2079
  homebranch:
2079
  homebranch:
Lines 2082-2098 biblios: Link Here
2082
      - facet: '1'
2082
      - facet: '1'
2083
        marc_field: 952a
2083
        marc_field: 952a
2084
        marc_type: marc21
2084
        marc_type: marc21
2085
        sort: ~
2085
        sort: 0
2086
        suggestible: ''
2086
        suggestible: ''
2087
      - facet: '1'
2087
      - facet: '1'
2088
        marc_field: 952a
2088
        marc_field: 952a
2089
        marc_type: normarc
2089
        marc_type: normarc
2090
        sort: ~
2090
        sort: 0
2091
        suggestible: ''
2091
        suggestible: ''
2092
      - facet: '1'
2092
      - facet: '1'
2093
        marc_field: 995b
2093
        marc_field: 995b
2094
        marc_type: unimarc
2094
        marc_type: unimarc
2095
        sort: ~
2095
        sort: 0
2096
        suggestible: ''
2096
        suggestible: ''
2097
    type: string
2097
    type: string
2098
  identifier-standard:
2098
  identifier-standard:
Lines 2222-2238 biblios: Link Here
2222
      - facet: ''
2222
      - facet: ''
2223
        marc_field: '9529'
2223
        marc_field: '9529'
2224
        marc_type: marc21
2224
        marc_type: marc21
2225
        sort: ~
2225
        sort: 0
2226
        suggestible: ''
2226
        suggestible: ''
2227
      - facet: ''
2227
      - facet: ''
2228
        marc_field: '9529'
2228
        marc_field: '9529'
2229
        marc_type: normarc
2229
        marc_type: normarc
2230
        sort: ~
2230
        sort: 0
2231
        suggestible: ''
2231
        suggestible: ''
2232
      - facet: ''
2232
      - facet: ''
2233
        marc_field: '9959'
2233
        marc_field: '9959'
2234
        marc_type: unimarc
2234
        marc_type: unimarc
2235
        sort: ~
2235
        sort: 0
2236
        suggestible: ''
2236
        suggestible: ''
2237
    type: number
2237
    type: number
2238
  itype:
2238
  itype:
Lines 2316-2327 biblios: Link Here
2316
    label: llength
2316
    label: llength
2317
    mappings:
2317
    mappings:
2318
      - facet: ''
2318
      - facet: ''
2319
        marc_field: leader_/1-5
2319
        marc_field: leader_/0-4
2320
        marc_type: marc21
2320
        marc_type: marc21
2321
        sort: ~
2321
        sort: ~
2322
        suggestible: ''
2322
        suggestible: ''
2323
      - facet: ''
2323
      - facet: ''
2324
        marc_field: leader_/1-5
2324
        marc_field: leader_/0-4
2325
        marc_type: normarc
2325
        marc_type: normarc
2326
        sort: ~
2326
        sort: ~
2327
        suggestible: ''
2327
        suggestible: ''
Lines 2351-2372 biblios: Link Here
2351
      - facet: ''
2351
      - facet: ''
2352
        marc_field: 952o
2352
        marc_field: 952o
2353
        marc_type: marc21
2353
        marc_type: marc21
2354
        sort: ~
2354
        sort: 0
2355
        suggestible: '1'
2355
        suggestible: '1'
2356
      - facet: ''
2356
      - facet: ''
2357
        marc_field: 952o
2357
        marc_field: 952o
2358
        marc_type: normarc
2358
        marc_type: normarc
2359
        sort: ~
2359
        sort: 0
2360
        suggestible: '1'
2360
        suggestible: '1'
2361
      - facet: ''
2361
      - facet: ''
2362
        marc_field: '686'
2362
        marc_field: '686'
2363
        marc_type: unimarc
2363
        marc_type: unimarc
2364
        sort: ~
2364
        sort: 0
2365
        suggestible: ''
2365
        suggestible: ''
2366
      - facet: ''
2366
      - facet: ''
2367
        marc_field: 995k
2367
        marc_field: 995k
2368
        marc_type: unimarc
2368
        marc_type: unimarc
2369
        sort: ~
2369
        sort: 0
2370
        suggestible: '1'
2370
        suggestible: '1'
2371
    type: ''
2371
    type: ''
2372
  location:
2372
  location:
Lines 2375-2391 biblios: Link Here
2375
      - facet: '1'
2375
      - facet: '1'
2376
        marc_field: 952c
2376
        marc_field: 952c
2377
        marc_type: marc21
2377
        marc_type: marc21
2378
        sort: ~
2378
        sort: 0
2379
        suggestible: ''
2379
        suggestible: ''
2380
      - facet: '1'
2380
      - facet: '1'
2381
        marc_field: 952c
2381
        marc_field: 952c
2382
        marc_type: normarc
2382
        marc_type: normarc
2383
        sort: ~
2383
        sort: 0
2384
        suggestible: ''
2384
        suggestible: ''
2385
      - facet: '1'
2385
      - facet: '1'
2386
        marc_field: 995e
2386
        marc_field: 995e
2387
        marc_type: unimarc
2387
        marc_type: unimarc
2388
        sort: ~
2388
        sort: 0
2389
        suggestible: ''
2389
        suggestible: ''
2390
    type: ''
2390
    type: ''
2391
  material-type:
2391
  material-type:
Lines 2422-2438 biblios: Link Here
2422
      - facet: 0
2422
      - facet: 0
2423
        marc_field: '9527'
2423
        marc_field: '9527'
2424
        marc_type: marc21
2424
        marc_type: marc21
2425
        sort: ~
2425
        sort: 0
2426
        suggestible: 0
2426
        suggestible: 0
2427
      - facet: 0
2427
      - facet: 0
2428
        marc_field: '9527'
2428
        marc_field: '9527'
2429
        marc_type: normarc
2429
        marc_type: normarc
2430
        sort: ~
2430
        sort: 0
2431
        suggestible: 0
2431
        suggestible: 0
2432
      - facet: 0
2432
      - facet: 0
2433
        marc_field: 995o
2433
        marc_field: 995o
2434
        marc_type: unimarc
2434
        marc_type: unimarc
2435
        sort: ~
2435
        sort: 0
2436
        suggestible: 0
2436
        suggestible: 0
2437
    type: number
2437
    type: number
2438
  number-db:
2438
  number-db:
Lines 2478-2494 biblios: Link Here
2478
      - facet: ''
2478
      - facet: ''
2479
        marc_field: 952q
2479
        marc_field: 952q
2480
        marc_type: marc21
2480
        marc_type: marc21
2481
        sort: ~
2481
        sort: 0
2482
        suggestible: ''
2482
        suggestible: ''
2483
      - facet: ''
2483
      - facet: ''
2484
        marc_field: 952q
2484
        marc_field: 952q
2485
        marc_type: normarc
2485
        marc_type: normarc
2486
        sort: ~
2486
        sort: 0
2487
        suggestible: ''
2487
        suggestible: ''
2488
      - facet: ''
2488
      - facet: ''
2489
        marc_field: 995n
2489
        marc_field: 995n
2490
        marc_type: unimarc
2490
        marc_type: unimarc
2491
        sort: ~
2491
        sort: 0
2492
        suggestible: ''
2492
        suggestible: ''
2493
    type: boolean
2493
    type: boolean
2494
  pl:
2494
  pl:
(-)a/t/Koha/SearchEngine/Elasticsearch.t (-6 / +31 lines)
Lines 117-123 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
117
117
118
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
118
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
119
119
120
    plan tests => 47;
120
    plan tests => 49;
121
121
122
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
122
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
123
123
Lines 212-217 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
212
            marc_type => 'marc21',
212
            marc_type => 'marc21',
213
            marc_field => '9520',
213
            marc_field => '9520',
214
        },
214
        },
215
        {
216
            name => 'local_classification',
217
            type => 'string',
218
            facet => 0,
219
            suggestible => 0,
220
            sort => 1,
221
            marc_type => 'marc21',
222
            marc_field => '952o',
223
        },
215
        {
224
        {
216
            name => 'type_of_record',
225
            name => 'type_of_record',
217
            type => 'string',
226
            type => 'string',
Lines 251-256 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
251
260
252
    my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });
261
    my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });
253
262
263
    my $callno = 'ABC123';
264
    my $callno2 = 'ABC456';
265
    my $long_callno = '1234567890' x 30;
266
254
    my $marc_record_1 = MARC::Record->new();
267
    my $marc_record_1 = MARC::Record->new();
255
    $marc_record_1->leader('     cam  22      a 4500');
268
    $marc_record_1->leader('     cam  22      a 4500');
256
    $marc_record_1->append_fields(
269
    $marc_record_1->append_fields(
Lines 262-269 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
262
        MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
275
        MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
263
        MARC::Field->new('999', '', '', c => '1234567'),
276
        MARC::Field->new('999', '', '', c => '1234567'),
264
        # '  ' for testing trimming of white space in boolean value callback:
277
        # '  ' for testing trimming of white space in boolean value callback:
265
        MARC::Field->new('952', '', '', 0 => '  ', g => '123.30'),
278
        MARC::Field->new('952', '', '', 0 => '  ', g => '123.30', o => $callno),
266
        MARC::Field->new('952', '', '', 0 => 0, g => '127.20'),
279
        MARC::Field->new('952', '', '', 0 => 0, g => '127.20', o => $callno2),
280
        MARC::Field->new('952', '', '', 0 => 1, g => '0.00', o => $long_callno),
267
    );
281
    );
268
    my $marc_record_2 = MARC::Record->new();
282
    my $marc_record_2 = MARC::Record->new();
269
    $marc_record_2->leader('     cam  22      a 4500');
283
    $marc_record_2->leader('     cam  22      a 4500');
Lines 272-278 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
272
        # MARC::Field->new('210', '', '', a => 'Title 2'),
286
        # MARC::Field->new('210', '', '', a => 'Title 2'),
273
        # MARC::Field->new('245', '', '', a => 'Title: second record'),
287
        # MARC::Field->new('245', '', '', a => 'Title: second record'),
274
        MARC::Field->new('999', '', '', c => '1234568'),
288
        MARC::Field->new('999', '', '', c => '1234568'),
275
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),
289
        MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno),
276
    );
290
    );
277
    my $records = [$marc_record_1, $marc_record_2];
291
    my $records = [$marc_record_1, $marc_record_2];
278
292
Lines 336-342 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
336
    is(scalar @{$docs->[0][1]->{items_withdrawn_status}}, 2, 'First document items_withdrawn_status field should have two values');
350
    is(scalar @{$docs->[0][1]->{items_withdrawn_status}}, 2, 'First document items_withdrawn_status field should have two values');
337
    is_deeply(
351
    is_deeply(
338
        $docs->[0][1]->{items_withdrawn_status},
352
        $docs->[0][1]->{items_withdrawn_status},
339
        ['false', 'false'],
353
        ['false', 'true'],
340
        'First document items_withdrawn_status field should be set correctly'
354
        'First document items_withdrawn_status field should be set correctly'
341
    );
355
    );
342
356
Lines 372-377 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
372
    is(scalar @{$docs->[0][1]->{isbn}}, 4, 'First document isbn field should contain four values');
386
    is(scalar @{$docs->[0][1]->{isbn}}, 4, 'First document isbn field should contain four values');
373
    is_deeply($docs->[0][1]->{isbn}, ['978-1-56619-909-4', '9781566199094', '1-56619-909-3', '1566199093'], 'First document isbn field should be set correctly');
387
    is_deeply($docs->[0][1]->{isbn}, ['978-1-56619-909-4', '9781566199094', '1-56619-909-3', '1566199093'], 'First document isbn field should be set correctly');
374
388
389
    is_deeply(
390
        $docs->[0][1]->{'local_classification'},
391
        [$callno, $callno2, $long_callno],
392
        'First document local_classification field should be set correctly'
393
    );
394
375
    # Second record:
395
    # Second record:
376
396
377
    is(scalar @{$docs->[1][1]->{author}}, 1, 'Second document author field should contain one value');
397
    is(scalar @{$docs->[1][1]->{author}}, 1, 'Second document author field should contain one value');
Lines 390-395 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
390
        'Second document sum_item_price field should be set correctly'
410
        'Second document sum_item_price field should be set correctly'
391
    );
411
    );
392
412
413
    is_deeply(
414
        $docs->[1][1]->{local_classification__sort},
415
        [substr($long_callno, 0, 255)],
416
        'Second document local_classification__sort field should be set correctly'
417
    );
418
393
    # Mappings marc_type:
419
    # Mappings marc_type:
394
420
395
    ok(!(defined $docs->[0][1]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
421
    ok(!(defined $docs->[0][1]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
396
- 

Return to bug 22246