From 4c8eb48a009ec35ebcee61325c6442d1a4557ced Mon Sep 17 00:00:00 2001 From: Nick Clemens Date: Wed, 26 Feb 2020 02:24:24 +0000 Subject: [PATCH] Bug 24559: Add 'year' type to Elasticsearch When indexing fields like 260c or 008_/7-10 we need a way to format these 'date' fields into years. Removing punctuation and coverting unknown values to 0 To test: 1 - Edit some records to have dates of publication in 008 like 202u 2021 203u 2039 2 - Also add to title for all records a unique word like 'octoparrot' 3 - Search for 'octoparrot' 4 - Sort records by publication/copyrightdate 5 - Confirm sorting is not as expected 6 - Apply patch 7 - Set copydate to 'year' type 8 - Reindex 9 - Repeat search 10 - Confirm sorting works as expected --- Koha/SearchEngine/Elasticsearch.pm | 18 ++++++++ .../admin/searchengine/elasticsearch/mappings.tt | 5 +++ t/Koha/SearchEngine/Elasticsearch.t | 51 +++++++++++++++++++++- 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm index 7ba3e14f70..1965a0c6b1 100644 --- a/Koha/SearchEngine/Elasticsearch.pm +++ b/Koha/SearchEngine/Elasticsearch.pm @@ -557,6 +557,19 @@ sub marc_records_to_documents { $record_document->{$field} = \@isbns; } } + foreach my $field (@{$rules->{year}}) { + if (defined $record_document->{$field}) { + my @years = (); + foreach my $input_year (@{$record_document->{$field}}) { + my @field_years = ( $input_year =~ /[0-9ux]{4}/g ); + foreach my $year (@field_years){ + $year =~ s/[ux]/0/g; + push @years, $year; + } + } + $record_document->{$field} = \@years; + } + } # Remove duplicate values and collapse sort fields foreach my $field (keys %{$record_document}) { @@ -839,6 +852,7 @@ sub _get_marc_mapping_rules { 'data_fields' => {}, 'sum' => [], 'isbn' => [], + 'year' => [], 'defaults' => {} }; @@ -853,6 +867,10 @@ sub _get_marc_mapping_rules { elsif ($type eq 'isbn') { push @{$rules->{isbn}}, $name; } + elsif ($type eq 'year') { + push @{$rules->{year}}, $name; + push @{$rules->{year}}, $name."__sort" if $sort; + } elsif ($type eq 'boolean') { # boolean gets special handling, if value doesn't exist for a field, # it is set to false diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt index e3a207db96..23b47ccbf4 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt @@ -192,6 +192,11 @@ a.add, a.delete { [% ELSE %] [% END %] + [% IF search_field.type == "year" %] + + [% ELSE %] + + [% END %] [% IF search_field.type == "number" %] [% ELSE %] diff --git a/t/Koha/SearchEngine/Elasticsearch.t b/t/Koha/SearchEngine/Elasticsearch.t index 0d76dd17d9..133f26e268 100644 --- a/t/Koha/SearchEngine/Elasticsearch.t +++ b/t/Koha/SearchEngine/Elasticsearch.t @@ -118,7 +118,7 @@ subtest 'get_elasticsearch_mappings() tests' => sub { subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub { - plan tests => 51; + plan tests => 55; t::lib::Mocks::mock_preference('marcflavour', 'MARC21'); t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709'); @@ -274,6 +274,26 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' marc_type => 'marc21', marc_field => '952l', }, + { + name => 'copydate', + type => 'year', + facet => 0, + suggestible => 0, + searchable => 1, + sort => 1, + marc_type => 'marc21', + marc_field => '260c', + }, + { + name => 'date-of-publication', + type => 'year', + facet => 0, + suggestible => 0, + searchable => 1, + sort => 1, + marc_type => 'marc21', + marc_field => '008_/7-10', + }, ); my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch'); @@ -305,11 +325,13 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' $marc_record_1->append_fields( MARC::Field->new('001', '123'), MARC::Field->new('007', 'ku'), + MARC::Field->new('008', '980514s19uu gw 000 0 lat '), MARC::Field->new('020', '', '', a => '1-56619-909-3'), MARC::Field->new('100', '', '', a => 'Author 1'), MARC::Field->new('110', '', '', a => 'Corp Author'), MARC::Field->new('210', '', '', a => 'Title 1'), MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'), + MARC::Field->new('260', '', '', a => 'New York :', b => 'Ace ,', 'c' => 'c1962'), MARC::Field->new('999', '', '', c => '1234567'), # ' ' for testing trimming of white space in boolean value callback: MARC::Field->new('952', '', '', 0 => ' ', g => '123.30', o => $callno, l => 3), @@ -319,9 +341,11 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' my $marc_record_2 = MARC::Record->new(); $marc_record_2->leader(' cam 22 a 4500'); $marc_record_2->append_fields( + MARC::Field->new('008', '980514s195u gw 000 0 lat '), MARC::Field->new('100', '', '', a => 'Author 2'), # MARC::Field->new('210', '', '', a => 'Title 2'), # MARC::Field->new('245', '', '', a => 'Title: second record'), + MARC::Field->new('260', '', '', a => 'New York :', b => 'Ace ,', 'c' => '1963-2003'), MARC::Field->new('999', '', '', c => '1234568'), MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno), ); @@ -433,6 +457,18 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' 'First document local_classification field should be set correctly' ); + # Test year type + is_deeply( + $docs->[0]->{'copydate'}, + ['1962'], + 'First document copydate field should be set correctly' + ); + is_deeply( + $docs->[0]->{'date-of-publication'}, + ['1900'], + 'First document date-of-publication field should be set correctly' + ); + # Second record: is(scalar @{$docs->[1]->{author}}, 1, 'Second document author field should contain one value'); @@ -457,6 +493,19 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' 'Second document local_classification__sort field should be set correctly' ); + # Test year type + is_deeply( + $docs->[1]->{'copydate'}, + ['1963','2003'], + 'Second document copydate field should be set correctly' + ); + is_deeply( + $docs->[1]->{'date-of-publication'}, + ['1950'], + 'Second document date-of-publication field should be set correctly' + ); + + # Mappings marc_type: ok(!(defined $docs->[0]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour"); -- 2.11.0