From ff6f6c1f7afe80d07c0157da65a25efb5183eff9 Mon Sep 17 00:00:00 2001 From: Ere Maijala Date: Tue, 3 Jul 2018 14:14:45 +0300 Subject: [PATCH] Bug 20244: Improve indexing so that multiple subfields are joined together https://bugs.koha-community.org/show_bug.cgi?id=20244 Also adds 245b to the title field. Joining the subfields in a single field affects relevancy, exact phrase search etc. It is especially important for authorities where working phrase search is required for finding matches. Test plan: 1. Add a personal name authority record with 100 $aTester $b1999 $eaut 2. Add a personal name authority record with 100 $aTester $baut $c1999 2. Make sure you can find the first and only the first record by doing a "Search main heading" search for Personal name with "starts with" option (N.B. the exact match option may still have issues). --- Koha/SearchEngine/Elasticsearch.pm | 29 +++++++++++++------------- admin/searchengine/elasticsearch/mappings.yaml | 2 +- t/Koha/SearchEngine/Elasticsearch.t | 9 ++++---- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm index 5ee978d..d2d48d0 100644 --- a/Koha/SearchEngine/Elasticsearch.pm +++ b/Koha/SearchEngine/Elasticsearch.pm @@ -373,15 +373,17 @@ sub marc_records_to_documents { } my $subfields_mappings = $data_fields_rules->{$tag}; if ($subfields_mappings) { - my $wildcard_mappings = $subfields_mappings->{'*'}; - foreach my $subfield ($field->subfields()) { - my ($code, $data) = @{$subfield}; - my $mappings = $subfields_mappings->{$code} // []; - if ($wildcard_mappings) { - $mappings = [@{$mappings}, @{$wildcard_mappings}]; + foreach my $rule (@{$subfields_mappings}) { + my @selected = (); + foreach my $subfield ($field->subfields()) { + my ($code, $data) = @{$subfield}; + if (!%{$rule->{codes}} || defined $rule->{codes}{$code}) { + $data =~ s/\s+$//; + push @selected, $data; + } } - if (@{$mappings}) { - _process_mappings($mappings, $data, $record_document); + if (@selected) { + _process_mappings($rule->{mappings}, join(' ', @selected), $record_document, $altscript); } } } @@ -542,7 +544,7 @@ sub get_marc_mapping_rules { if ($marc_field =~ $field_spec_regexp) { my $field_tag = $1; - my $subfields = defined $2 ? $2 : '*'; + my %subfields = defined $2 ? map { $_ => 1 } split(//, $2) : (); my $range = defined $3 ? $3 : undef; if ($field_tag < 10) { $rules->{control_fields}->{$field_tag} //= []; @@ -550,12 +552,9 @@ sub get_marc_mapping_rules { push @{$rules->{control_fields}->{$field_tag}}, @mappings; } else { - $rules->{data_fields}->{$field_tag} //= {}; - foreach my $subfield (split //, $subfields) { - $rules->{data_fields}->{$field_tag}->{$subfield} //= []; - my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range); - push @{$rules->{data_fields}->{$field_tag}->{$subfield}}, @mappings; - } + my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range); + $rules->{data_fields}->{$field_tag} //= []; + push @{$rules->{data_fields}->{$field_tag}}, {codes => \%subfields, mappings => \@mappings}; } } elsif ($marc_field =~ $leader_regexp) { diff --git a/admin/searchengine/elasticsearch/mappings.yaml b/admin/searchengine/elasticsearch/mappings.yaml index 18e63fa..f4e51a5 100644 --- a/admin/searchengine/elasticsearch/mappings.yaml +++ b/admin/searchengine/elasticsearch/mappings.yaml @@ -2939,7 +2939,7 @@ biblios: sort: ~ suggestible: '' - facet: '' - marc_field: 245a + marc_field: 245ab marc_type: marc21 sort: 1 suggestible: '1' diff --git a/t/Koha/SearchEngine/Elasticsearch.t b/t/Koha/SearchEngine/Elasticsearch.t index a595784..f7d9eda 100644 --- a/t/Koha/SearchEngine/Elasticsearch.t +++ b/t/Koha/SearchEngine/Elasticsearch.t @@ -154,7 +154,7 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' suggestible => 1, sort => 1, marc_type => 'marc21', - marc_field => '245a', + marc_field => '245ab', }, { name => 'unimarc_title', @@ -238,7 +238,8 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' MARC::Field->new('100', '', '', a => 'Author 1'), MARC::Field->new('110', '', '', a => 'Corp Author'), MARC::Field->new('210', '', '', a => 'Title 1'), - MARC::Field->new('245', '', '', a => 'Title: first record'), + MARC::Field->new('245', '', '', a => 'Title: first record : ', b => 'Subtitle', 'c' => 'Author'), + MARC::Field->new('246', '', '', a => 'Another title : first record : ', b => 'Subtitle'), MARC::Field->new('999', '', '', c => '1234567'), # ' ' for testing trimming of white space in boolean value callback: MARC::Field->new('952', '', '', 0 => ' ', g => '123.30'), @@ -271,7 +272,7 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' is_deeply($docs->[0][1]->{author__sort}, ['Author 1 Corp Author'], 'First document author__sort field should be set correctly'); is(scalar @{$docs->[0][1]->{title__sort}}, 1, 'First document title__sort field should have one value'); - is_deeply($docs->[0][1]->{title__sort}, ['Title: first record'], 'First document title__sort field should be set correctly'); + is_deeply($docs->[0][1]->{title__sort}, ['Title: first record : Subtitle'], 'First document title__sort field should be set correctly'); is(scalar @{$docs->[0][1]->{author__suggestion}}, 2, 'First document author__suggestion field should contain two values'); is_deeply( @@ -290,7 +291,7 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' is(scalar @{$docs->[0][1]->{title__suggestion}}, 1, 'First document title__suggestion field should contain one value'); is_deeply( $docs->[0][1]->{title__suggestion}, - [{ 'input' => 'Title: first record' }], + [{ 'input' => 'Title: first record : Subtitle' }], 'First document title__suggestion field should be set correctly' ); -- 2.7.4