@@ -, +, @@ --- Koha/SearchEngine/Elasticsearch.pm | 18 +++++++++++++-- Koha/SearchEngine/Elasticsearch/Indexer.pm | 3 +++ .../elasticsearch/field_config.yaml | 2 ++ .../searchengine/elasticsearch/mappings.tt | 5 +++++ t/Koha/SearchEngine/Elasticsearch.t | 22 ++++++++++++++++++- 5 files changed, 47 insertions(+), 3 deletions(-) --- a/Koha/SearchEngine/Elasticsearch.pm +++ a/Koha/SearchEngine/Elasticsearch.pm @@ -36,7 +36,7 @@ use Search::Elasticsearch; use Try::Tiny; use YAML::Syck; -use List::Util qw( sum0 reduce ); +use List::Util qw( sum0 reduce all ); use MARC::File::XML; use MIME::Base64; use Encode qw(encode); @@ -215,6 +215,8 @@ sub get_elasticsearch_mappings { $es_type = 'integer'; } elsif ($type eq 'isbn' || $type eq 'stdno') { $es_type = 'stdno'; + } elsif ($type eq 'year') { + $es_type = 'year'; } if ($search) { @@ -398,7 +400,6 @@ sub _process_mappings { # with differing options for (possibly) mutating data # so need a different copy for each my $_data = $data; - $record_document->{$target} //= []; if (defined $options->{substr}) { my ($start, $length) = @{$options->{substr}}; $_data = length($data) > $start ? substr $data, $start, $length : ''; @@ -406,11 +407,17 @@ sub _process_mappings { if (defined $options->{value_callbacks}) { $_data = reduce { $b->($a) } ($_data, @{$options->{value_callbacks}}); } + if (defined $options->{filter_callbacks}) { + # Skip mapping unless all filter callbacks return true + next unless all { $_->($_data) } @{$options->{filter_callbacks}}; + } if (defined $options->{property}) { $_data = { $options->{property} => $_data } } + + $record_document->{$target} //= []; push @{$record_document->{$target}}, $_data; } } @@ -768,6 +775,13 @@ sub _field_mappings { return $value ? 'true' : 'false'; }; } + elsif ($target_type eq 'year') { + $default_options->{filter_callbacks} //= []; + push @{$default_options->{filter_callbacks}}, sub { + my ($value) = @_; + return $value =~ /^\d+$/; + }; + } if ($search) { my $mapping = [$target_name, $default_options]; --- a/Koha/SearchEngine/Elasticsearch/Indexer.pm +++ a/Koha/SearchEngine/Elasticsearch/Indexer.pm @@ -123,6 +123,9 @@ sub update_index { type => 'data', # is just hard coded in Indexer.pm? body => \@body ); + if ($response->{errors}) { + carp "One or more ElasticSearch errors occured when indexing documents"; + } } # TODO: handle response return 1; --- a/admin/searchengine/elasticsearch/field_config.yaml +++ a/admin/searchengine/elasticsearch/field_config.yaml @@ -24,6 +24,8 @@ search: integer: type: integer null_value: 0 + year: + type: short stdno: type: text analyzer: analyzer_stdno --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt +++ a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt @@ -192,6 +192,11 @@ a.add, a.delete { [% ELSE %] [% END %] + [% IF search_field.type == "year" %] + + [% ELSE %] + + [% END %] [% IF search_field.type == "number" %] [% ELSE %] --- a/t/Koha/SearchEngine/Elasticsearch.t +++ a/t/Koha/SearchEngine/Elasticsearch.t @@ -117,7 +117,7 @@ subtest 'get_elasticsearch_mappings() tests' => sub { subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub { - plan tests => 51; + plan tests => 54; t::lib::Mocks::mock_preference('marcflavour', 'MARC21'); t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709'); @@ -273,6 +273,17 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' marc_type => 'marc21', marc_field => '952l', }, + { + name => 'date-of-publication', + type => 'year', + facet => 0, + suggestible => 0, + searchable => 1, + sort => 1, + marc_type => 'marc21', + marc_field => '008_/7-10', + }, + ); my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch'); @@ -304,6 +315,7 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' $marc_record_1->append_fields( MARC::Field->new('001', '123'), MARC::Field->new('007', 'ku'), + MARC::Field->new('008', '901111s1962 xxk|||| |00| ||eng c'), MARC::Field->new('020', '', '', a => '1-56619-909-3'), MARC::Field->new('100', '', '', a => 'Author 1'), MARC::Field->new('110', '', '', a => 'Corp Author'), @@ -318,6 +330,7 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' my $marc_record_2 = MARC::Record->new(); $marc_record_2->leader(' cam 22 a 4500'); $marc_record_2->append_fields( + MARC::Field->new('008', '901111s19uu xxk|||| |00| ||eng c'), MARC::Field->new('100', '', '', a => 'Author 2'), # MARC::Field->new('210', '', '', a => 'Title 2'), # MARC::Field->new('245', '', '', a => 'Title: second record'), @@ -432,6 +445,10 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' 'First document local_classification field should be set correctly' ); + # Tests for 'year' type and 'filter_callbacks' + is(scalar @{$docs->[0]->{'date-of-publication'}}, 1, 'First document date-of-publication field should contain one value'); + is_deeply($docs->[0]->{'date-of-publication'}, ['1962'], 'First document date-of-publication field should be set correctly'); + # Second record: is(scalar @{$docs->[1]->{author}}, 1, 'Second document author field should contain one value'); @@ -456,6 +473,9 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' 'Second document local_classification__sort field should be set correctly' ); + # Tests for 'year' type and 'filter_callbacks' + ok(!(defined $docs->[1]->{'date-of-publication'}), "Second document invalid date-of-publication value should have been removed"); + # Mappings marc_type: ok(!(defined $docs->[0]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour"); --