From 6e98c0558811f92c4287c3299e72ce2d5c330160 Mon Sep 17 00:00:00 2001 From: Ere Maijala Date: Mon, 19 Feb 2018 10:31:50 +0200 Subject: [PATCH] Improve ES indexing: don't split everything, handle linked fields and index ISBN versions. https://bugs.koha-community.org/show_bug.cgi?id=20244 --- Koha/SearchEngine/Elasticsearch.pm | 42 ++++++++++++++++++++----------- Koha/SearchEngine/Elasticsearch/Search.pm | 6 ++--- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm index d859a39..d997771 100644 --- a/Koha/SearchEngine/Elasticsearch.pm +++ b/Koha/SearchEngine/Elasticsearch.pm @@ -34,8 +34,6 @@ use Search::Elasticsearch; use Try::Tiny; use YAML::Syck; -use Data::Dumper; # TODO remove - __PACKAGE__->mk_ro_accessors(qw( index )); __PACKAGE__->mk_accessors(qw( sort_fields )); @@ -325,34 +323,48 @@ sub get_fixer_rules { return if $marc_type ne $marcflavour; my $options = ''; - # There's a bug when using 'split' with something that - # selects a range - # The split makes everything into nested arrays, but that's not - # really a big deal, ES doesn't mind. - $options = '-split => 1' unless $marc_field =~ m|_/| || $type eq 'sum'; + $options = "join:' '" unless $marc_field =~ m|_/| || $type eq 'sum'; push @rules, "marc_map('$marc_field','${name}.\$append', $options)"; + if ($marc_field !~ m|_/| && ($type eq '' || $type eq 'string')) { + # Handle linked fields + my $tag = substr($marc_field, 0, 3); + my $subfields = substr($marc_field, 3); + $subfields = 'abcdefghijklmnopqrstuvwxyz' unless $subfields; + my $rule = "{\$6/0-2/=\\$tag}"; + # Add dollars and rules to subfields + $subfields =~ s/(.)/\$$1$rule/g; + # Create a marc_spec rule to select correct 880 fields + push @rules, "marc_spec('880${subfields}','${name}.\$append', $options)"; + } if ($facet) { push @rules, "marc_map('$marc_field','${name}__facet.\$append', $options)"; } if ($suggestible) { push @rules, - #"marc_map('$marc_field','${name}__suggestion.input.\$append', $options)"; #must not have nested data structures in .input - "marc_map('$marc_field','${name}__suggestion.input.\$append')"; + "marc_map('$marc_field','${name}__suggestion.input.\$append', $options)"; } if ( $type eq 'boolean' ) { - # boolean gets special handling, basically if it doesn't exist, # it's added and set to false. Otherwise we can't query it. push @rules, - "unless exists('$name') add_field('$name', 0) end"; + "unless exists('$name') add_field('$name', false) end"; } if ($type eq 'sum' ) { push @rules, "sum('$name')"; + } elsif ($type eq 'isbn') { + push @rules, qq{ +do list(path:isbn, var:c) + copy_field(c, isbn_tmp.\$append) + isbn_versions(c) + copy_field(c, isbn_tmp.\$append) +end +move_field(isbn_tmp, isbn) +}; } - if ($self->sort_fields()->{$name}) { - if ($sort || !defined $sort) { - push @rules, "marc_map('$marc_field','${name}__sort.\$append', $options)"; - } + my $sort_fields = $self->sort_fields(); + if (defined $sort_fields->{$name} && $sort_fields->{$name}) { + push @rules, "marc_map('$marc_field','${name}__sort.\$append')"; + push @rules, "join_field('${name}__sort',' ')"; } } ); diff --git a/Koha/SearchEngine/Elasticsearch/Search.pm b/Koha/SearchEngine/Elasticsearch/Search.pm index 4d584f7..459e485 100644 --- a/Koha/SearchEngine/Elasticsearch/Search.pm +++ b/Koha/SearchEngine/Elasticsearch/Search.pm @@ -200,9 +200,9 @@ sub search_auth_compat { # I wonder if these should be real values defined in the mapping # rather than hard-coded conversions. - # Our results often come through as nested arrays, to fix this - # requires changes in catmandu. - my $authid = $record->{ 'Local-number' }[0][0]; + # Handle legacy nested arrays indexed with splitting enabled. + my $authid = $record->{ 'Local-number' }[0]; + $authid = @$authid[0] if (ref $authid eq 'ARRAY'); $result{authid} = $authid; # TODO put all this info into the record at index time so we -- 2.7.4