Attachment 76649 Details for Bug 20244 – Bug 20244: Improve indexing so that multiple subfields are joined together

[patch] Bug 20244: Improve indexing so that multiple subfields are joined together

Bug-20244-Improve-indexing-so-that-multiple-subfie.patch (text/plain), 6.94 KB, created by Ere Maijala on 2018-07-03 11:16:42 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Ere Maijala

Created: 2018-07-03 11:16:42 UTC

Size: 6.94 KB

patch

obsolete

>From ff6f6c1f7afe80d07c0157da65a25efb5183eff9 Mon Sep 17 00:00:00 2001
>From: Ere Maijala <ere.maijala@helsinki.fi>
>Date: Tue, 3 Jul 2018 14:14:45 +0300
>Subject: [PATCH] Bug 20244: Improve indexing so that multiple subfields are
> joined together
>
>https://bugs.koha-community.org/show_bug.cgi?id=20244
>
>Also adds 245b to the title field. Joining the subfields in a single field affects
>relevancy, exact phrase search etc. It is especially important for authorities where
>working phrase search is required for finding matches.
>
>Test plan:
>
>1. Add a personal name authority record with 100 $aTester $b1999 $eaut
>2. Add a personal name authority record with 100 $aTester $baut $c1999
>2. Make sure you can find the first and only the first record by doing a "Search main heading" search for Personal name with "starts with" option (N.B. the exact match option may still have issues).
>---
> Koha/SearchEngine/Elasticsearch.pm             | 29 +++++++++++++-------------
> admin/searchengine/elasticsearch/mappings.yaml |  2 +-
> t/Koha/SearchEngine/Elasticsearch.t            |  9 ++++----
> 3 files changed, 20 insertions(+), 20 deletions(-)
>
>diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm
>index 5ee978d..d2d48d0 100644
>--- a/Koha/SearchEngine/Elasticsearch.pm
>+++ b/Koha/SearchEngine/Elasticsearch.pm
>@@ -373,15 +373,17 @@ sub marc_records_to_documents {
>                 }
>                 my $subfields_mappings = $data_fields_rules->{$tag};
>                 if ($subfields_mappings) {
>-                    my $wildcard_mappings = $subfields_mappings->{'*'};
>-                    foreach my $subfield ($field->subfields()) {
>-                        my ($code, $data) = @{$subfield};
>-                        my $mappings = $subfields_mappings->{$code} // [];
>-                        if ($wildcard_mappings) {
>-                            $mappings = [@{$mappings}, @{$wildcard_mappings}];
>+                    foreach my $rule (@{$subfields_mappings}) {
>+                        my @selected = ();
>+                        foreach my $subfield ($field->subfields()) {
>+                            my ($code, $data) = @{$subfield};
>+                            if (!%{$rule->{codes}} || defined $rule->{codes}{$code}) {
>+                                $data =~ s/\s+$//;
>+                                push @selected, $data;
>+                            }
>                         }
>-                        if (@{$mappings}) {
>-                            _process_mappings($mappings, $data, $record_document);
>+                        if (@selected) {
>+                            _process_mappings($rule->{mappings}, join(' ', @selected), $record_document, $altscript);
>                         }
>                     }
>                 }
>@@ -542,7 +544,7 @@ sub get_marc_mapping_rules {
> 
>         if ($marc_field =~ $field_spec_regexp) {
>             my $field_tag = $1;
>-            my $subfields = defined $2 ? $2 : '*';
>+            my %subfields = defined $2 ? map { $_ => 1 } split(//, $2) : ();
>             my $range = defined $3 ? $3 : undef;
>             if ($field_tag < 10) {
>                 $rules->{control_fields}->{$field_tag} //= [];
>@@ -550,12 +552,9 @@ sub get_marc_mapping_rules {
>                 push @{$rules->{control_fields}->{$field_tag}}, @mappings;
>             }
>             else {
>-                $rules->{data_fields}->{$field_tag} //= {};
>-                foreach my $subfield (split //, $subfields) {
>-                    $rules->{data_fields}->{$field_tag}->{$subfield} //= [];
>-                    my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
>-                    push @{$rules->{data_fields}->{$field_tag}->{$subfield}}, @mappings;
>-                }
>+                my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
>+                $rules->{data_fields}->{$field_tag} //= [];
>+                push @{$rules->{data_fields}->{$field_tag}}, {codes => \%subfields, mappings => \@mappings};
>             }
>         }
>         elsif ($marc_field =~ $leader_regexp) {
>diff --git a/admin/searchengine/elasticsearch/mappings.yaml b/admin/searchengine/elasticsearch/mappings.yaml
>index 18e63fa..f4e51a5 100644
>--- a/admin/searchengine/elasticsearch/mappings.yaml
>+++ b/admin/searchengine/elasticsearch/mappings.yaml
>@@ -2939,7 +2939,7 @@ biblios:
>         sort: ~
>         suggestible: ''
>       - facet: ''
>-        marc_field: 245a
>+        marc_field: 245ab
>         marc_type: marc21
>         sort: 1
>         suggestible: '1'
>diff --git a/t/Koha/SearchEngine/Elasticsearch.t b/t/Koha/SearchEngine/Elasticsearch.t
>index a595784..f7d9eda 100644
>--- a/t/Koha/SearchEngine/Elasticsearch.t
>+++ b/t/Koha/SearchEngine/Elasticsearch.t
>@@ -154,7 +154,7 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
>             suggestible => 1,
>             sort => 1,
>             marc_type => 'marc21',
>-            marc_field => '245a',
>+            marc_field => '245ab',
>         },
>         {
>             name => 'unimarc_title',
>@@ -238,7 +238,8 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
>         MARC::Field->new('100', '', '', a => 'Author 1'),
>         MARC::Field->new('110', '', '', a => 'Corp Author'),
>         MARC::Field->new('210', '', '', a => 'Title 1'),
>-        MARC::Field->new('245', '', '', a => 'Title: first record'),
>+        MARC::Field->new('245', '', '', a => 'Title: first record : ', b => 'Subtitle', 'c' => 'Author'),
>+        MARC::Field->new('246', '', '', a => 'Another title : first record : ', b => 'Subtitle'),
>         MARC::Field->new('999', '', '', c => '1234567'),
>         # '  ' for testing trimming of white space in boolean value callback:
>         MARC::Field->new('952', '', '', 0 => '  ', g => '123.30'),
>@@ -271,7 +272,7 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
>     is_deeply($docs->[0][1]->{author__sort}, ['Author 1 Corp Author'], 'First document author__sort field should be set correctly');
> 
>     is(scalar @{$docs->[0][1]->{title__sort}}, 1, 'First document title__sort field should have one value');
>-    is_deeply($docs->[0][1]->{title__sort}, ['Title: first record'], 'First document title__sort field should be set correctly');
>+    is_deeply($docs->[0][1]->{title__sort}, ['Title: first record : Subtitle'], 'First document title__sort field should be set correctly');
> 
>     is(scalar @{$docs->[0][1]->{author__suggestion}}, 2, 'First document author__suggestion field should contain two values');
>     is_deeply(
>@@ -290,7 +291,7 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
>     is(scalar @{$docs->[0][1]->{title__suggestion}}, 1, 'First document title__suggestion field should contain one value');
>     is_deeply(
>         $docs->[0][1]->{title__suggestion},
>-        [{ 'input' => 'Title: first record' }],
>+        [{ 'input' => 'Title: first record : Subtitle' }],
>         'First document title__suggestion field should be set correctly'
>     );
> 
>-- 
>2.7.4

Actions: View | Diff | Splinter Review

Attachments on bug 20244: 71919 | 73372 | 73373 | 74321 | 76646 | 76647 | 76648 | 76649 | 78727 | 78728 | 78729 | 79012 | 79013 | 79014 | 82160 | 82161 | 82162 | 82272 | 82273