Attachment #84607 for bug #22246

View | Details | Raw Unified | Return to bug 22246
Collapse All | Expand All




                $options->{property} => $_data
            }
        }
        push @{$record_document->{$target}}, $_data;
        if ($sort && @{$record_document->{$target}}) {
            @{$record_document->{$target}}[0] .= " $_data";
        } else {
            push @{$record_document->{$target}}, $_data;
        }
    }
}


            }
        }

        # Remove duplicate values and collapse sort fields
        foreach my $field (keys %{$record_document}) {
            if (ref($record_document->{$field}) eq 'ARRAY') {
                @{$record_document->{$field}} = do {
                    my %seen;
                    grep { !$seen{ref($_) eq 'HASH' && defined $_->{input} ? $_->{input} : $_}++ } @{$record_document->{$field}};
                };
                if ($field =~ /__sort$/) {
                    # Make sure to keep the sort field length sensible. 255 was chosen as a nice round value.
                    $record_document->{$field} = [substr(join(' ', @{$record_document->{$field}}), 0, 255)];
                }
            }
        }

        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
        $record->encoding('UTF-8');
        my @warnings;




      - facet: '1'
        marc_field: 952b
        marc_type: marc21
        sort: 0
        suggestible: ''
      - facet: '1'
        marc_field: 952b
        marc_type: normarc
        sort: 0
        suggestible: ''
      - facet: ''
        marc_field: 995c
        marc_type: unimarc
        sort: 0
        suggestible: ''
    type: string
  homebranch:

      - facet: '1'
        marc_field: 952a
        marc_type: marc21
        sort: 0
        suggestible: ''
      - facet: '1'
        marc_field: 952a
        marc_type: normarc
        sort: 0
        suggestible: ''
      - facet: '1'
        marc_field: 995b
        marc_type: unimarc
        sort: 0
        suggestible: ''
    type: string
  identifier-standard:

      - facet: ''
        marc_field: '9529'
        marc_type: marc21
        sort: 0
        suggestible: ''
      - facet: ''
        marc_field: '9529'
        marc_type: normarc
        sort: 0
        suggestible: ''
      - facet: ''
        marc_field: '9959'
        marc_type: unimarc
        sort: 0
        suggestible: ''
    type: number
  itype:

    label: llength
    mappings:
      - facet: ''
        marc_field: leader_/0-4
        marc_type: marc21
        sort: ~
        suggestible: ''
      - facet: ''
        marc_field: leader_/0-4
        marc_type: normarc
        sort: ~
        suggestible: ''

      - facet: ''
        marc_field: 952o
        marc_type: marc21
        sort: 0
        suggestible: '1'
      - facet: ''
        marc_field: 952o
        marc_type: normarc
        sort: 0
        suggestible: '1'
      - facet: ''
        marc_field: '686'
        marc_type: unimarc
        sort: 0
        suggestible: ''
      - facet: ''
        marc_field: 995k
        marc_type: unimarc
        sort: 0
        suggestible: '1'
    type: ''
  location:

      - facet: '1'
        marc_field: 952c
        marc_type: marc21
        sort: 0
        suggestible: ''
      - facet: '1'
        marc_field: 952c
        marc_type: normarc
        sort: 0
        suggestible: ''
      - facet: '1'
        marc_field: 995e
        marc_type: unimarc
        sort: 0
        suggestible: ''
    type: ''
  material-type:

      - facet: 0
        marc_field: '9527'
        marc_type: marc21
        sort: 0
        suggestible: 0
      - facet: 0
        marc_field: '9527'
        marc_type: normarc
        sort: 0
        suggestible: 0
      - facet: 0
        marc_field: 995o
        marc_type: unimarc
        sort: 0
        suggestible: 0
    type: number
  number-db:

      - facet: ''
        marc_field: 952q
        marc_type: marc21
        sort: 0
        suggestible: ''
      - facet: ''
        marc_field: 952q
        marc_type: normarc
        sort: 0
        suggestible: ''
      - facet: ''
        marc_field: 995n
        marc_type: unimarc
        sort: 0
        suggestible: ''
    type: boolean
  pl:

Lines 117-123 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here

(-)a/t/Koha/SearchEngine/Elasticsearch.t (-6 / +31 lines)
117		117
118	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {	118	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
119		119
120	plan tests => 47;	120	plan tests => 49;
121		121
122	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');	122	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
123		123
Lines 212-217 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
212	marc_type => 'marc21',	212	marc_type => 'marc21',
213	marc_field => '9520',	213	marc_field => '9520',
214	},	214	},
		215	{
		216	name => 'local_classification',
		217	type => 'string',
		218	facet => 0,
		219	suggestible => 0,
		220	sort => 1,
		221	marc_type => 'marc21',
		222	marc_field => '952o',
		223	},
215	{	224	{
216	name => 'type_of_record',	225	name => 'type_of_record',
217	type => 'string',	226	type => 'string',
Lines 251-256 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
251		260
252	my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });	261	my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });
253		262
		263	my $callno = 'ABC123';
		264	my $callno2 = 'ABC456';
		265	my $long_callno = '1234567890' x 30;
		266
254	my $marc_record_1 = MARC::Record->new();	267	my $marc_record_1 = MARC::Record->new();
255	$marc_record_1->leader(' cam 22 a 4500');	268	$marc_record_1->leader(' cam 22 a 4500');
256	$marc_record_1->append_fields(	269	$marc_record_1->append_fields(
Lines 262-269 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
262	MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),	275	MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
263	MARC::Field->new('999', '', '', c => '1234567'),	276	MARC::Field->new('999', '', '', c => '1234567'),
264	# ' ' for testing trimming of white space in boolean value callback:	277	# ' ' for testing trimming of white space in boolean value callback:
265	MARC::Field->new('952', '', '', 0 => ' ', g => '123.30'),	278	MARC::Field->new('952', '', '', 0 => ' ', g => '123.30', o => $callno),
266	MARC::Field->new('952', '', '', 0 => 0, g => '127.20'),	279	MARC::Field->new('952', '', '', 0 => 0, g => '127.20', o => $callno2),
		280	MARC::Field->new('952', '', '', 0 => 1, g => '0.00', o => $long_callno),
267	);	281	);
268	my $marc_record_2 = MARC::Record->new();	282	my $marc_record_2 = MARC::Record->new();
269	$marc_record_2->leader(' cam 22 a 4500');	283	$marc_record_2->leader(' cam 22 a 4500');
Lines 272-278 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
272	# MARC::Field->new('210', '', '', a => 'Title 2'),	286	# MARC::Field->new('210', '', '', a => 'Title 2'),
273	# MARC::Field->new('245', '', '', a => 'Title: second record'),	287	# MARC::Field->new('245', '', '', a => 'Title: second record'),
274	MARC::Field->new('999', '', '', c => '1234568'),	288	MARC::Field->new('999', '', '', c => '1234568'),
275	MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),	289	MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno),
276	);	290	);
277	my $records = [$marc_record_1, $marc_record_2];	291	my $records = [$marc_record_1, $marc_record_2];
278		292
Lines 336-342 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
336	is(scalar @{$docs->[0][1]->{items_withdrawn_status}}, 2, 'First document items_withdrawn_status field should have two values');	350	is(scalar @{$docs->[0][1]->{items_withdrawn_status}}, 2, 'First document items_withdrawn_status field should have two values');
337	is_deeply(	351	is_deeply(
338	$docs->[0][1]->{items_withdrawn_status},	352	$docs->[0][1]->{items_withdrawn_status},
339	['false', 'false'],	353	['false', 'true'],
340	'First document items_withdrawn_status field should be set correctly'	354	'First document items_withdrawn_status field should be set correctly'
341	);	355	);
342		356
Lines 372-377 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
372	is(scalar @{$docs->[0][1]->{isbn}}, 4, 'First document isbn field should contain four values');	386	is(scalar @{$docs->[0][1]->{isbn}}, 4, 'First document isbn field should contain four values');
373	is_deeply($docs->[0][1]->{isbn}, ['978-1-56619-909-4', '9781566199094', '1-56619-909-3', '1566199093'], 'First document isbn field should be set correctly');	387	is_deeply($docs->[0][1]->{isbn}, ['978-1-56619-909-4', '9781566199094', '1-56619-909-3', '1566199093'], 'First document isbn field should be set correctly');
374		388
		389	is_deeply(
		390	$docs->[0][1]->{'local_classification'},
		391	[$callno, $callno2, $long_callno],
		392	'First document local_classification field should be set correctly'
		393	);
		394
375	# Second record:	395	# Second record:
376		396
377	is(scalar @{$docs->[1][1]->{author}}, 1, 'Second document author field should contain one value');	397	is(scalar @{$docs->[1][1]->{author}}, 1, 'Second document author field should contain one value');
Lines 390-395 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
390	'Second document sum_item_price field should be set correctly'	410	'Second document sum_item_price field should be set correctly'
391	);	411	);
392		412
		413	is_deeply(
		414	$docs->[1][1]->{local_classification__sort},
		415	[substr($long_callno, 0, 255)],
		416	'Second document local_classification__sort field should be set correctly'
		417	);
		418
393	# Mappings marc_type:	419	# Mappings marc_type:
394		420
395	ok(!(defined $docs->[0][1]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");	421	ok(!(defined $docs->[0][1]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
396	-

Return to bug 22246

Lines 375-386 sub _process_mappings { Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (-6 / +15 lines)
375	$options->{property} => $_data	375	$options->{property} => $_data
376	}	376	}
377	}	377	}
378	# For sort fields, index only a single field with concatenated values	378	push @{$record_document->{$target}}, $_data;
379	if ($sort && @{$record_document->{$target}}) {
380	@{$record_document->{$target}}[0] .= " $_data";
381	} else {
382	push @{$record_document->{$target}}, $_data;
383	}
384	}	379	}
385	}	380	}
386		381
Lines 512-517 sub marc_records_to_documents { Link Here
512	}	507	}
513	}	508	}
514		509
		510	# Remove duplicate values and collapse sort fields
		511	foreach my $field (keys %{$record_document}) {
		512	if (ref($record_document->{$field}) eq 'ARRAY') {
		513	@{$record_document->{$field}} = do {
		514	my %seen;
		515	grep { !$seen{ref($_) eq 'HASH' && defined $_->{input} ? $_->{input} : $_}++ } @{$record_document->{$field}};
		516	};
		517	if ($field =~ /__sort$/) {
		518	# Make sure to keep the sort field length sensible. 255 was chosen as a nice round value.
		519	$record_document->{$field} = [substr(join(' ', @{$record_document->{$field}}), 0, 255)];
		520	}
		521	}
		522	}
		523
515	# TODO: Perhaps should check if $records_document non empty, but really should never be the case	524	# TODO: Perhaps should check if $records_document non empty, but really should never be the case
516	$record->encoding('UTF-8');	525	$record->encoding('UTF-8');
517	my @warnings;	526	my @warnings;

Lines 2063-2079 biblios: Link Here

(-)a/admin/searchengine/elasticsearch/mappings.yaml (-24 / +24 lines)
2063	- facet: '1'	2063	- facet: '1'
2064	marc_field: 952b	2064	marc_field: 952b
2065	marc_type: marc21	2065	marc_type: marc21
2066	sort: ~	2066	sort: 0
2067	suggestible: ''	2067	suggestible: ''
2068	- facet: '1'	2068	- facet: '1'
2069	marc_field: 952b	2069	marc_field: 952b
2070	marc_type: normarc	2070	marc_type: normarc
2071	sort: ~	2071	sort: 0
2072	suggestible: ''	2072	suggestible: ''
2073	- facet: ''	2073	- facet: ''
2074	marc_field: 995c	2074	marc_field: 995c
2075	marc_type: unimarc	2075	marc_type: unimarc
2076	sort: ~	2076	sort: 0
2077	suggestible: ''	2077	suggestible: ''
2078	type: string	2078	type: string
2079	homebranch:	2079	homebranch:
Lines 2082-2098 biblios: Link Here
2082	- facet: '1'	2082	- facet: '1'
2083	marc_field: 952a	2083	marc_field: 952a
2084	marc_type: marc21	2084	marc_type: marc21
2085	sort: ~	2085	sort: 0
2086	suggestible: ''	2086	suggestible: ''
2087	- facet: '1'	2087	- facet: '1'
2088	marc_field: 952a	2088	marc_field: 952a
2089	marc_type: normarc	2089	marc_type: normarc
2090	sort: ~	2090	sort: 0
2091	suggestible: ''	2091	suggestible: ''
2092	- facet: '1'	2092	- facet: '1'
2093	marc_field: 995b	2093	marc_field: 995b
2094	marc_type: unimarc	2094	marc_type: unimarc
2095	sort: ~	2095	sort: 0
2096	suggestible: ''	2096	suggestible: ''
2097	type: string	2097	type: string
2098	identifier-standard:	2098	identifier-standard:
Lines 2222-2238 biblios: Link Here
2222	- facet: ''	2222	- facet: ''
2223	marc_field: '9529'	2223	marc_field: '9529'
2224	marc_type: marc21	2224	marc_type: marc21
2225	sort: ~	2225	sort: 0
2226	suggestible: ''	2226	suggestible: ''
2227	- facet: ''	2227	- facet: ''
2228	marc_field: '9529'	2228	marc_field: '9529'
2229	marc_type: normarc	2229	marc_type: normarc
2230	sort: ~	2230	sort: 0
2231	suggestible: ''	2231	suggestible: ''
2232	- facet: ''	2232	- facet: ''
2233	marc_field: '9959'	2233	marc_field: '9959'
2234	marc_type: unimarc	2234	marc_type: unimarc
2235	sort: ~	2235	sort: 0
2236	suggestible: ''	2236	suggestible: ''
2237	type: number	2237	type: number
2238	itype:	2238	itype:
Lines 2316-2327 biblios: Link Here
2316	label: llength	2316	label: llength
2317	mappings:	2317	mappings:
2318	- facet: ''	2318	- facet: ''
2319	marc_field: leader_/1-5	2319	marc_field: leader_/0-4
2320	marc_type: marc21	2320	marc_type: marc21
2321	sort: ~	2321	sort: ~
2322	suggestible: ''	2322	suggestible: ''
2323	- facet: ''	2323	- facet: ''
2324	marc_field: leader_/1-5	2324	marc_field: leader_/0-4
2325	marc_type: normarc	2325	marc_type: normarc
2326	sort: ~	2326	sort: ~
2327	suggestible: ''	2327	suggestible: ''
Lines 2351-2372 biblios: Link Here
2351	- facet: ''	2351	- facet: ''
2352	marc_field: 952o	2352	marc_field: 952o
2353	marc_type: marc21	2353	marc_type: marc21
2354	sort: ~	2354	sort: 0
2355	suggestible: '1'	2355	suggestible: '1'
2356	- facet: ''	2356	- facet: ''
2357	marc_field: 952o	2357	marc_field: 952o
2358	marc_type: normarc	2358	marc_type: normarc
2359	sort: ~	2359	sort: 0
2360	suggestible: '1'	2360	suggestible: '1'
2361	- facet: ''	2361	- facet: ''
2362	marc_field: '686'	2362	marc_field: '686'
2363	marc_type: unimarc	2363	marc_type: unimarc
2364	sort: ~	2364	sort: 0
2365	suggestible: ''	2365	suggestible: ''
2366	- facet: ''	2366	- facet: ''
2367	marc_field: 995k	2367	marc_field: 995k
2368	marc_type: unimarc	2368	marc_type: unimarc
2369	sort: ~	2369	sort: 0
2370	suggestible: '1'	2370	suggestible: '1'
2371	type: ''	2371	type: ''
2372	location:	2372	location:
Lines 2375-2391 biblios: Link Here
2375	- facet: '1'	2375	- facet: '1'
2376	marc_field: 952c	2376	marc_field: 952c
2377	marc_type: marc21	2377	marc_type: marc21
2378	sort: ~	2378	sort: 0
2379	suggestible: ''	2379	suggestible: ''
2380	- facet: '1'	2380	- facet: '1'
2381	marc_field: 952c	2381	marc_field: 952c
2382	marc_type: normarc	2382	marc_type: normarc
2383	sort: ~	2383	sort: 0
2384	suggestible: ''	2384	suggestible: ''
2385	- facet: '1'	2385	- facet: '1'
2386	marc_field: 995e	2386	marc_field: 995e
2387	marc_type: unimarc	2387	marc_type: unimarc
2388	sort: ~	2388	sort: 0
2389	suggestible: ''	2389	suggestible: ''
2390	type: ''	2390	type: ''
2391	material-type:	2391	material-type:
Lines 2422-2438 biblios: Link Here
2422	- facet: 0	2422	- facet: 0
2423	marc_field: '9527'	2423	marc_field: '9527'
2424	marc_type: marc21	2424	marc_type: marc21
2425	sort: ~	2425	sort: 0
2426	suggestible: 0	2426	suggestible: 0
2427	- facet: 0	2427	- facet: 0
2428	marc_field: '9527'	2428	marc_field: '9527'
2429	marc_type: normarc	2429	marc_type: normarc
2430	sort: ~	2430	sort: 0
2431	suggestible: 0	2431	suggestible: 0
2432	- facet: 0	2432	- facet: 0
2433	marc_field: 995o	2433	marc_field: 995o
2434	marc_type: unimarc	2434	marc_type: unimarc
2435	sort: ~	2435	sort: 0
2436	suggestible: 0	2436	suggestible: 0
2437	type: number	2437	type: number
2438	number-db:	2438	number-db:
Lines 2478-2494 biblios: Link Here
2478	- facet: ''	2478	- facet: ''
2479	marc_field: 952q	2479	marc_field: 952q
2480	marc_type: marc21	2480	marc_type: marc21
2481	sort: ~	2481	sort: 0
2482	suggestible: ''	2482	suggestible: ''
2483	- facet: ''	2483	- facet: ''
2484	marc_field: 952q	2484	marc_field: 952q
2485	marc_type: normarc	2485	marc_type: normarc
2486	sort: ~	2486	sort: 0
2487	suggestible: ''	2487	suggestible: ''
2488	- facet: ''	2488	- facet: ''
2489	marc_field: 995n	2489	marc_field: 995n
2490	marc_type: unimarc	2490	marc_type: unimarc
2491	sort: ~	2491	sort: 0
2492	suggestible: ''	2492	suggestible: ''
2493	type: boolean	2493	type: boolean
2494	pl:	2494	pl: