From b37a1ac98d1d3412f4f04df649a5ea45ecb0f053 Mon Sep 17 00:00:00 2001 From: Ere Maijala Date: Mon, 19 Feb 2018 10:31:50 +0200 Subject: [PATCH] Bug 20244: Improve ES indexing: don't split everything, handle linked fields and index ISBN versions. https://bugs.koha-community.org/show_bug.cgi?id=20244 Test plan: 1. Make sure you have recent Catmandu::Identifiers module installed. 2. Add records with alternate scripts (e.g. the attached sample record). 3. Make sure the records can be found also with the alternate script. 4. Add a record with an ISBN-10 or ISBN-13 that can be converted to ISBN-10 (e.g. 1-56619-909-3). 5. Verify that the record can be found by searching for "1-56619-909-3", "1566199093", "978-1-56619-909-4" and "9781566199094". --- Koha/SearchEngine/Elasticsearch.pm | 36 ++++++++++++++++++++++--------- Koha/SearchEngine/Elasticsearch/Search.pm | 6 +++--- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm index d859a39..8875658 100644 --- a/Koha/SearchEngine/Elasticsearch.pm +++ b/Koha/SearchEngine/Elasticsearch.pm @@ -34,8 +34,6 @@ use Search::Elasticsearch; use Try::Tiny; use YAML::Syck; -use Data::Dumper; # TODO remove - __PACKAGE__->mk_ro_accessors(qw( index )); __PACKAGE__->mk_accessors(qw( sort_fields )); @@ -331,28 +329,46 @@ sub get_fixer_rules { # really a big deal, ES doesn't mind. $options = '-split => 1' unless $marc_field =~ m|_/| || $type eq 'sum'; push @rules, "marc_map('$marc_field','${name}.\$append', $options)"; + if ($marc_field !~ m|_/| && ($type eq '' || $type eq 'string')) { + # Handle linked fields + my $tag = substr($marc_field, 0, 3); + my $subfields = substr($marc_field, 3); + $subfields = 'abcdefghijklmnopqrstuvwxyz' unless $subfields; + my $rule = "{\$6/0-2/=\\$tag}"; + # Add dollars and rules to subfields + $subfields =~ s/(.)/\$$1$rule/g; + # Create a marc_spec rule to select correct 880 fields + push @rules, "marc_spec('880${subfields}','${name}.\$append', $options)"; + } if ($facet) { push @rules, "marc_map('$marc_field','${name}__facet.\$append', $options)"; } if ($suggestible) { push @rules, - #"marc_map('$marc_field','${name}__suggestion.input.\$append', $options)"; #must not have nested data structures in .input - "marc_map('$marc_field','${name}__suggestion.input.\$append')"; + "marc_map('$marc_field','${name}__suggestion.input.\$append', $options)"; } if ( $type eq 'boolean' ) { - # boolean gets special handling, basically if it doesn't exist, # it's added and set to false. Otherwise we can't query it. push @rules, - "unless exists('$name') add_field('$name', 0) end"; + "unless exists('$name') add_field('$name', false) end"; } if ($type eq 'sum' ) { push @rules, "sum('$name')"; + } elsif ($type eq 'isbn') { + push @rules, qq{ +do list(path:isbn, var:c) + copy_field(c, isbn_tmp.\$append) + isbn_versions(c) + copy_field(c, isbn_tmp.\$append) +end +move_field(isbn_tmp, isbn) +}; } - if ($self->sort_fields()->{$name}) { - if ($sort || !defined $sort) { - push @rules, "marc_map('$marc_field','${name}__sort.\$append', $options)"; - } + my $sort_fields = $self->sort_fields(); + if (defined $sort_fields->{$name} && $sort_fields->{$name}) { + push @rules, "marc_map('$marc_field','${name}__sort.\$append')"; + push @rules, "join_field('${name}__sort',' ')"; } } ); diff --git a/Koha/SearchEngine/Elasticsearch/Search.pm b/Koha/SearchEngine/Elasticsearch/Search.pm index 4d584f7..459e485 100644 --- a/Koha/SearchEngine/Elasticsearch/Search.pm +++ b/Koha/SearchEngine/Elasticsearch/Search.pm @@ -200,9 +200,9 @@ sub search_auth_compat { # I wonder if these should be real values defined in the mapping # rather than hard-coded conversions. - # Our results often come through as nested arrays, to fix this - # requires changes in catmandu. - my $authid = $record->{ 'Local-number' }[0][0]; + # Handle legacy nested arrays indexed with splitting enabled. + my $authid = $record->{ 'Local-number' }[0]; + $authid = @$authid[0] if (ref $authid eq 'ARRAY'); $result{authid} = $authid; # TODO put all this info into the record at index time so we -- 2.7.4