Attachment 99626 Details for Bug 24559 – Bug 24559: Add 'year' type to Elasticsearch

[patch] Bug 24559: Add 'year' type to Elasticsearch

Bug-24559-Add-year-type-to-Elasticsearch.patch (text/plain), 7.98 KB, created by Nick Clemens (kidclamp) on 2020-02-26 02:31:35 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Nick Clemens (kidclamp)

Created: 2020-02-26 02:31:35 UTC

Size: 7.98 KB

patch

obsolete

>From 4c8eb48a009ec35ebcee61325c6442d1a4557ced Mon Sep 17 00:00:00 2001
>From: Nick Clemens <nick@bywatersolutions.com>
>Date: Wed, 26 Feb 2020 02:24:24 +0000
>Subject: [PATCH] Bug 24559: Add 'year' type to Elasticsearch
>
>When indexing fields like 260c or 008_/7-10 we need a way to format
>these 'date' fields into years. Removing punctuation and coverting
>unknown values to 0
>
>To test:
> 1 - Edit some records to have dates of publication in 008 like
>    202u
>    2021
>    203u
>    2039
> 2 - Also add to title for all records a unique word like 'octoparrot'
> 3 - Search for 'octoparrot'
> 4 - Sort records by publication/copyrightdate
> 5 - Confirm sorting is not as expected
> 6 - Apply patch
> 7 - Set copydate to 'year' type
> 8 - Reindex
> 9 - Repeat search
>10 - Confirm sorting works as expected
>---
> Koha/SearchEngine/Elasticsearch.pm                 | 18 ++++++++
> .../admin/searchengine/elasticsearch/mappings.tt   |  5 +++
> t/Koha/SearchEngine/Elasticsearch.t                | 51 +++++++++++++++++++++-
> 3 files changed, 73 insertions(+), 1 deletion(-)
>
>diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm
>index 7ba3e14f70..1965a0c6b1 100644
>--- a/Koha/SearchEngine/Elasticsearch.pm
>+++ b/Koha/SearchEngine/Elasticsearch.pm
>@@ -557,6 +557,19 @@ sub marc_records_to_documents {
>                 $record_document->{$field} = \@isbns;
>             }
>         }
>+        foreach my $field (@{$rules->{year}}) {
>+            if (defined $record_document->{$field}) {
>+                my @years = ();
>+                foreach my $input_year (@{$record_document->{$field}}) {
>+                    my @field_years = ( $input_year =~ /[0-9ux]{4}/g );
>+                    foreach my $year (@field_years){
>+                        $year =~ s/[ux]/0/g;
>+                        push @years, $year;
>+                    }
>+                }
>+                $record_document->{$field} = \@years;
>+            }
>+        }
> 
>         # Remove duplicate values and collapse sort fields
>         foreach my $field (keys %{$record_document}) {
>@@ -839,6 +852,7 @@ sub _get_marc_mapping_rules {
>         'data_fields' => {},
>         'sum' => [],
>         'isbn' => [],
>+        'year' => [],
>         'defaults' => {}
>     };
> 
>@@ -853,6 +867,10 @@ sub _get_marc_mapping_rules {
>         elsif ($type eq 'isbn') {
>             push @{$rules->{isbn}}, $name;
>         }
>+        elsif ($type eq 'year') {
>+            push @{$rules->{year}}, $name;
>+            push @{$rules->{year}}, $name."__sort" if $sort;
>+        }
>         elsif ($type eq 'boolean') {
>             # boolean gets special handling, if value doesn't exist for a field,
>             # it is set to false
>diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt
>index e3a207db96..23b47ccbf4 100644
>--- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt
>+++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt
>@@ -192,6 +192,11 @@ a.add, a.delete {
>                           [% ELSE %]
>                             <option value="date">Date</option>
>                           [% END %]
>+                          [% IF search_field.type == "year" %]
>+                            <option value="year" selected="selected">Year</option>
>+                          [% ELSE %]
>+                            <option value="year">Year</option>
>+                          [% END %]
>                           [% IF search_field.type == "number" %]
>                             <option value="number" selected="selected">Number</option>
>                           [% ELSE %]
>diff --git a/t/Koha/SearchEngine/Elasticsearch.t b/t/Koha/SearchEngine/Elasticsearch.t
>index 0d76dd17d9..133f26e268 100644
>--- a/t/Koha/SearchEngine/Elasticsearch.t
>+++ b/t/Koha/SearchEngine/Elasticsearch.t
>@@ -118,7 +118,7 @@ subtest 'get_elasticsearch_mappings() tests' => sub {
> 
> subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
> 
>-    plan tests => 51;
>+    plan tests => 55;
> 
>     t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
>     t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
>@@ -274,6 +274,26 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
>             marc_type => 'marc21',
>             marc_field => '952l',
>         },
>+        {
>+            name => 'copydate',
>+            type => 'year',
>+            facet => 0,
>+            suggestible => 0,
>+            searchable => 1,
>+            sort => 1,
>+            marc_type => 'marc21',
>+            marc_field => '260c',
>+        },
>+        {
>+            name => 'date-of-publication',
>+            type => 'year',
>+            facet => 0,
>+            suggestible => 0,
>+            searchable => 1,
>+            sort => 1,
>+            marc_type => 'marc21',
>+            marc_field => '008_/7-10',
>+        },
>     );
> 
>     my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
>@@ -305,11 +325,13 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
>     $marc_record_1->append_fields(
>         MARC::Field->new('001', '123'),
>         MARC::Field->new('007', 'ku'),
>+        MARC::Field->new('008', '980514s19uu    gw            000 0 lat  '),
>         MARC::Field->new('020', '', '', a => '1-56619-909-3'),
>         MARC::Field->new('100', '', '', a => 'Author 1'),
>         MARC::Field->new('110', '', '', a => 'Corp Author'),
>         MARC::Field->new('210', '', '', a => 'Title 1'),
>         MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
>+        MARC::Field->new('260', '', '', a => 'New York :', b => 'Ace ,', 'c' => 'c1962'),
>         MARC::Field->new('999', '', '', c => '1234567'),
>         # '  ' for testing trimming of white space in boolean value callback:
>         MARC::Field->new('952', '', '', 0 => '  ', g => '123.30', o => $callno, l => 3),
>@@ -319,9 +341,11 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
>     my $marc_record_2 = MARC::Record->new();
>     $marc_record_2->leader('     cam  22      a 4500');
>     $marc_record_2->append_fields(
>+        MARC::Field->new('008', '980514s195u    gw            000 0 lat  '),
>         MARC::Field->new('100', '', '', a => 'Author 2'),
>         # MARC::Field->new('210', '', '', a => 'Title 2'),
>         # MARC::Field->new('245', '', '', a => 'Title: second record'),
>+        MARC::Field->new('260', '', '', a => 'New York :', b => 'Ace ,', 'c' => '1963-2003'),
>         MARC::Field->new('999', '', '', c => '1234568'),
>         MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno),
>     );
>@@ -433,6 +457,18 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
>         'First document local_classification field should be set correctly'
>     );
> 
>+    # Test year type
>+    is_deeply(
>+        $docs->[0]->{'copydate'},
>+        ['1962'],
>+        'First document copydate field should be set correctly'
>+    );
>+    is_deeply(
>+        $docs->[0]->{'date-of-publication'},
>+        ['1900'],
>+        'First document date-of-publication field should be set correctly'
>+    );
>+
>     # Second record:
> 
>     is(scalar @{$docs->[1]->{author}}, 1, 'Second document author field should contain one value');
>@@ -457,6 +493,19 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
>         'Second document local_classification__sort field should be set correctly'
>     );
> 
>+    # Test year type
>+    is_deeply(
>+        $docs->[1]->{'copydate'},
>+        ['1963','2003'],
>+        'Second document copydate field should be set correctly'
>+    );
>+    is_deeply(
>+        $docs->[1]->{'date-of-publication'},
>+        ['1950'],
>+        'Second document date-of-publication field should be set correctly'
>+    );
>+
>+
>     # Mappings marc_type:
> 
>     ok(!(defined $docs->[0]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
>-- 
>2.11.0

Actions: View | Diff | Splinter Review

Attachments on bug 24559: 99626 | 99785 | 99791 | 99792