From f3dc85f9c84eb0dacfea2a0f51d3d760763245ef Mon Sep 17 00:00:00 2001 From: Ere Maijala Date: Mon, 19 Feb 2018 10:31:50 +0200 Subject: [PATCH] Bug 20244: Improve ES indexing: join subfields with at least a space, handle linked fields, index ISBN versions and create single-valued sort fields. https://bugs.koha-community.org/show_bug.cgi?id=20244 Test plan (ISBN): 1. Make sure you have recent Catmandu::Identifiers module installed. 2. Add records with alternate scripts (e.g. the attached sample record). 3. Make sure the records can be found also with the alternate script. 4. Add a record with an ISBN-10 or ISBN-13 that can be converted to ISBN-10 (e.g. 1-56619-909-3). 5. Verify that the record can be found by searching for "1-56619-909-3", "1566199093", "978-1-56619-909-4" and "9781566199094". Test plan (joining fields): 1. Add a record that has a 650 field with multiple subfields and no punctuation between fields (e.g. record 19 from https://github.com/joubu/koha-misc4dev/tree/master/data/sql/marc21/1611/after_17196). 2. Verify that you can find the record by entering all terms from the 650 field. Test plan (sorting): 1. Add three records with "Novels" as 240a field and "A", "B" and "C" as 245a field. 2. Search for "Novels", sort by title and make sure the results are in correct order. --- Koha/SearchEngine/Elasticsearch.pm | 43 +++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm index fe2e0ea..d898263 100644 --- a/Koha/SearchEngine/Elasticsearch.pm +++ b/Koha/SearchEngine/Elasticsearch.pm @@ -34,8 +34,6 @@ use Search::Elasticsearch; use Try::Tiny; use YAML::Syck; -use Data::Dumper; # TODO remove - __PACKAGE__->mk_ro_accessors(qw( index )); __PACKAGE__->mk_accessors(qw( sort_fields )); @@ -319,38 +317,59 @@ sub get_fixer_rules { my $marcflavour = lc C4::Context->preference('marcflavour'); my @rules; + my $sort_fields = $self->sort_fields(); $self->_foreach_mapping( sub { my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_; return if $marc_type ne $marcflavour; - my $options =''; + my $options = ($marc_field =~ m|_/| || $type eq 'sum') ? '' : "join:' '"; push @rules, "marc_map('$marc_field','${name}.\$append', $options)"; + if ($marc_field !~ m|_/| && ($type eq '' || $type eq 'string')) { + # Handle linked fields + my $tag = substr($marc_field, 0, 3); + my $subfields = substr($marc_field, 3); + $subfields = 'abcdefghijklmnopqrstuvwxyz' unless $subfields; + my $rule = "{\$6/0-2/=\\$tag}"; + # Add dollars and rules to subfields + $subfields =~ s/(.)/\$$1$rule/g; + # Create a marc_spec rule to select correct 880 fields + push @rules, "marc_spec('880${subfields}','${name}.\$append', $options)"; + } if ($facet) { push @rules, "marc_map('$marc_field','${name}__facet.\$append', $options)"; } if ($suggestible) { - push @rules, - #"marc_map('$marc_field','${name}__suggestion.input.\$append', '')"; #must not have nested data structures in .input - "marc_map('$marc_field','${name}__suggestion.input.\$append')"; + push @rules, "marc_map('$marc_field','${name}__suggestion.input.\$append', $options)"; } if ( $type eq 'boolean' ) { - # boolean gets special handling, basically if it doesn't exist, # it's added and set to false. Otherwise we can't query it. push @rules, - "unless exists('$name') add_field('$name', 0) end"; + "unless exists('$name') add_field('$name', false) end"; } if ($type eq 'sum' ) { push @rules, "sum('$name')"; + } elsif ($type eq 'isbn') { + push @rules, qq{ +do list(path:isbn, var:c) + copy_field(c, isbn_tmp.\$append) + isbn_versions(c) + copy_field(c, isbn_tmp.\$append) +end +move_field(isbn_tmp, isbn) +}; } - if ($self->sort_fields()->{$name}) { - if ($sort || !defined $sort) { - push @rules, "marc_map('$marc_field','${name}__sort.\$append', $options)"; - } + + if (defined $sort_fields->{$name}) { + push @rules, "marc_map('$marc_field','${name}__sort.\$append', $options)"; } } ); + # Merge sort fields into a single entry + foreach my $field (keys $sort_fields) { + push @rules, "join_field('${field}__sort', ' ')"; + } push @rules, "move_field(_id,es_id)"; #Also you must set the Catmandu::Store::ElasticSearch->new(key_prefix: 'es_'); return \@rules; -- 2.7.4