Attachment 82273 Details for Bug 20244 – Bug 20244: Add alt script indexing and fix sort field indexing

[patch] Bug 20244: Add alt script indexing and fix sort field indexing

Bug-20244-Add-alt-script-indexing-and-fix-sort-fie.patch (text/plain), 7.39 KB, created by Martin Renvoize (ashimema) on 2018-11-13 13:24:07 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Martin Renvoize (ashimema)

Created: 2018-11-13 13:24:07 UTC

Size: 7.39 KB

patch

obsolete

>From 1bab0f836a34faf292f131cedc7be7d25d1cb4f6 Mon Sep 17 00:00:00 2001
>From: Ere Maijala <ere.maijala@helsinki.fi>
>Date: Mon, 2 Jul 2018 16:04:05 +0300
>Subject: [PATCH] Bug 20244: Add alt script indexing and fix sort field
> indexing
>
>https://bugs.koha-community.org/show_bug.cgi?id=20244
>
>Test plan:
>
>1. Add a record with alternate script fields in 880 (sample attached in the bug).
>2. Make sure the the record can be found e.g. with the alternate script title.
>3. Add a record with multiple authors.
>4. Check that in the index there is only a single author__sort field.
>
>Signed-off-by: Brendan Gallagher <brendan@bywatersolutions.com>
>Signed-off-by: Martin Renvoize <martin.renvoize@ptfs-europe.com>
>---
> Koha/SearchEngine/Elasticsearch.pm  | 48 +++++++++++++++++++++++------
> t/Koha/SearchEngine/Elasticsearch.t | 10 +++---
> 2 files changed, 43 insertions(+), 15 deletions(-)
>
>diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm
>index 64ea156625..77580a715f 100644
>--- a/Koha/SearchEngine/Elasticsearch.pm
>+++ b/Koha/SearchEngine/Elasticsearch.pm
>@@ -310,9 +310,9 @@ sub sort_fields {
>     return $self->_sort_fields_accessor();
> }
> 
>-=head2 _process_mappings($mappings, $data, $record_document)
>+=head2 _process_mappings($mappings, $data, $record_document, $altscript)
> 
>-    $self->_process_mappings($mappings, $marc_field_data, $record_document)
>+    $self->_process_mappings($mappings, $marc_field_data, $record_document, 0)
> 
> Process all C<$mappings> targets operating on a specific MARC field C<$data>.
> Since we group all mappings by MARC field targets C<$mappings> will contain
>@@ -338,14 +338,26 @@ The source data from a MARC record field.
> Hashref representing the Elasticsearch document on which mappings should be
> applied.
> 
>+=item C<$altscript>
>+
>+A boolean value indicating whether an alternate script presentation is being
>+processed.
>+
> =back
> 
> =cut
> 
> sub _process_mappings {
>-    my ($_self, $mappings, $data, $record_document) = @_;
>+    my ($_self, $mappings, $data, $record_document, $altscript) = @_;
>     foreach my $mapping (@{$mappings}) {
>         my ($target, $options) = @{$mapping};
>+
>+        # Don't process sort fields for alternate scripts
>+        my $sort = $target =~ /__sort$/;
>+        if ($sort && $altscript) {
>+            next;
>+        }
>+
>         # Copy (scalar) data since can have multiple targets
>         # with differing options for (possibly) mutating data
>         # so need a different copy for each
>@@ -363,7 +375,12 @@ sub _process_mappings {
>                 $options->{property} => $_data
>             }
>         }
>-        push @{$record_document->{$target}}, $_data;
>+        # For sort fields, index only a single field with concatenated values
>+        if ($sort && @{$record_document->{$target}}) {
>+            @{$record_document->{$target}}[0] .= " $_data";
>+        } else {
>+            push @{$record_document->{$target}}, $_data;
>+        }
>     }
> }
> 
>@@ -399,17 +416,28 @@ sub marc_records_to_documents {
>         my $record_document = {};
>         my $mappings = $rules->{leader};
>         if ($mappings) {
>-            $self->_process_mappings($mappings, $record->leader(), $record_document);
>+            $self->_process_mappings($mappings, $record->leader(), $record_document, 0);
>         }
>         foreach my $field ($record->fields()) {
>-            if($field->is_control_field()) {
>+            if ($field->is_control_field()) {
>                 my $mappings = $control_fields_rules->{$field->tag()};
>                 if ($mappings) {
>-                    $self->_process_mappings($mappings, $field->data(), $record_document);
>+                    $self->_process_mappings($mappings, $field->data(), $record_document, 0);
>                 }
>             }
>             else {
>-                my $data_field_rules = $data_fields_rules->{$field->tag()};
>+                my $tag = $field->tag();
>+                # Handle alternate scripts in MARC 21
>+                my $altscript = 0;
>+                if ($marcflavour eq 'marc21' && $tag eq '880') {
>+                    my $sub6 = $field->subfield('6');
>+                    if ($sub6 =~ /^(...)-\d+/) {
>+                        $tag = $1;
>+                        $altscript = 1;
>+                    }
>+                }
>+
>+                my $data_field_rules = $data_fields_rules->{$tag};
> 
>                 if ($data_field_rules) {
>                     my $subfields_mappings = $data_field_rules->{subfields};
>@@ -421,7 +449,7 @@ sub marc_records_to_documents {
>                             $mappings = [@{$mappings}, @{$wildcard_mappings}];
>                         }
>                         if (@{$mappings}) {
>-                            $self->_process_mappings($mappings, $data, $record_document);
>+                            $self->_process_mappings($mappings, $data, $record_document, $altscript);
>                         }
>                     }
> 
>@@ -437,7 +465,7 @@ sub marc_records_to_documents {
>                                 )
>                             );
>                             if ($data) {
>-                                $self->_process_mappings($subfields_join_mappings->{$subfields_group}, $data, $record_document);
>+                                $self->_process_mappings($subfields_join_mappings->{$subfields_group}, $data, $record_document, $altscript);
>                             }
>                         }
>                     }
>diff --git a/t/Koha/SearchEngine/Elasticsearch.t b/t/Koha/SearchEngine/Elasticsearch.t
>index 933fdfb644..a4fc6b561b 100644
>--- a/t/Koha/SearchEngine/Elasticsearch.t
>+++ b/t/Koha/SearchEngine/Elasticsearch.t
>@@ -117,7 +117,7 @@ subtest 'get_elasticsearch_mappings() tests' => sub {
> 
> subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
> 
>-    plan tests => 32;
>+    plan tests => 47;
> 
>     t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
> 
>@@ -290,11 +290,11 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
>     is(scalar @{$docs->[0][1]->{author}}, 2, 'First document author field should contain two values');
>     is_deeply($docs->[0][1]->{author}, ['Author 1', 'Corp Author'], 'First document author field should be set correctly');
> 
>-    is(scalar @{$docs->[0][1]->{author__sort}}, 2, 'First document author__sort field should have two values');
>-    is_deeply($docs->[0][1]->{author__sort}, ['Author 1', 'Corp Author'], 'First document author__sort field should be set correctly');
>+    is(scalar @{$docs->[0][1]->{author__sort}}, 1, 'First document author__sort field should have a single value');
>+    is_deeply($docs->[0][1]->{author__sort}, ['Author 1 Corp Author'], 'First document author__sort field should be set correctly');
> 
>-    is(scalar @{$docs->[0][1]->{title__sort}}, 3, 'First document title__sort field should have three values');
>-    is_deeply($docs->[0][1]->{title__sort}, ['Title:', 'first record', 'Title: first record'], 'First document title__sort field should be set correctly');
>+    is(scalar @{$docs->[0][1]->{title__sort}}, 1, 'First document title__sort field should have a single');
>+    is_deeply($docs->[0][1]->{title__sort}, ['Title: first record Title: first record'], 'First document title__sort field should be set correctly');
> 
>     is(scalar @{$docs->[0][1]->{title_wildcard}}, 2, 'First document title_wildcard field should have two values');
>     is_deeply($docs->[0][1]->{title_wildcard}, ['Title:', 'first record'], 'First document title_wildcard field should be set correctly');
>-- 
>2.19.1

Actions: View | Diff | Splinter Review

Attachments on bug 20244: 71919 | 73372 | 73373 | 74321 | 76646 | 76647 | 76648 | 76649 | 78727 | 78728 | 78729 | 79012 | 79013 | 79014 | 82160 | 82161 | 82162 | 82272 | 82273