From b37a1ac98d1d3412f4f04df649a5ea45ecb0f053 Mon Sep 17 00:00:00 2001
From: Ere Maijala <ere.maijala@helsinki.fi>
Date: Mon, 19 Feb 2018 10:31:50 +0200
Subject: [PATCH] Bug 20244: Improve ES indexing: don't split everything,
 handle linked fields and index ISBN versions.

https://bugs.koha-community.org/show_bug.cgi?id=20244

Test plan:

1. Make sure you have recent Catmandu::Identifiers module installed.
2. Add records with alternate scripts (e.g. the attached sample record).
3. Make sure the records can be found also with the alternate script.
4. Add a record with an ISBN-10 or ISBN-13 that can be converted to ISBN-10 (e.g. 1-56619-909-3).
5. Verify that the record can be found by searching for "1-56619-909-3", "1566199093", "978-1-56619-909-4" and "9781566199094".
---
 Koha/SearchEngine/Elasticsearch.pm        | 36 ++++++++++++++++++++++---------
 Koha/SearchEngine/Elasticsearch/Search.pm |  6 +++---
 2 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm
index d859a39..8875658 100644
--- a/Koha/SearchEngine/Elasticsearch.pm
+++ b/Koha/SearchEngine/Elasticsearch.pm
@@ -34,8 +34,6 @@ use Search::Elasticsearch;
 use Try::Tiny;
 use YAML::Syck;
 
-use Data::Dumper;    # TODO remove
-
 __PACKAGE__->mk_ro_accessors(qw( index ));
 __PACKAGE__->mk_accessors(qw( sort_fields ));
 
@@ -331,28 +329,46 @@ sub get_fixer_rules {
             # really a big deal, ES doesn't mind.
             $options = '-split => 1' unless $marc_field =~ m|_/| || $type eq 'sum';
             push @rules, "marc_map('$marc_field','${name}.\$append', $options)";
+            if ($marc_field !~ m|_/| && ($type eq '' || $type eq 'string')) {
+                # Handle linked fields
+                my $tag = substr($marc_field, 0, 3);
+                my $subfields = substr($marc_field, 3);
+                $subfields = 'abcdefghijklmnopqrstuvwxyz' unless $subfields;
+                my $rule = "{\$6/0-2/=\\$tag}";
+                # Add dollars and rules to subfields
+                $subfields =~ s/(.)/\$$1$rule/g;
+                # Create a marc_spec rule to select correct 880 fields
+                push @rules, "marc_spec('880${subfields}','${name}.\$append', $options)";
+            }
             if ($facet) {
                 push @rules, "marc_map('$marc_field','${name}__facet.\$append', $options)";
             }
             if ($suggestible) {
                 push @rules,
-                    #"marc_map('$marc_field','${name}__suggestion.input.\$append', $options)"; #must not have nested data structures in .input
-                    "marc_map('$marc_field','${name}__suggestion.input.\$append')";
+                    "marc_map('$marc_field','${name}__suggestion.input.\$append', $options)";
             }
             if ( $type eq 'boolean' ) {
-
                 # boolean gets special handling, basically if it doesn't exist,
                 # it's added and set to false. Otherwise we can't query it.
                 push @rules,
-                  "unless exists('$name') add_field('$name', 0) end";
+                  "unless exists('$name') add_field('$name', false) end";
             }
             if ($type eq 'sum' ) {
                 push @rules, "sum('$name')";
+            } elsif ($type eq 'isbn') {
+                push @rules, qq{
+do list(path:isbn, var:c)
+    copy_field(c, isbn_tmp.\$append)
+    isbn_versions(c)
+    copy_field(c, isbn_tmp.\$append)
+end
+move_field(isbn_tmp, isbn)
+};
             }
-            if ($self->sort_fields()->{$name}) {
-                if ($sort || !defined $sort) {
-                    push @rules, "marc_map('$marc_field','${name}__sort.\$append', $options)";
-                }
+            my $sort_fields = $self->sort_fields();
+            if (defined $sort_fields->{$name} && $sort_fields->{$name}) {
+                push @rules, "marc_map('$marc_field','${name}__sort.\$append')";
+                push @rules, "join_field('${name}__sort',' ')";
             }
         }
     );
diff --git a/Koha/SearchEngine/Elasticsearch/Search.pm b/Koha/SearchEngine/Elasticsearch/Search.pm
index 4d584f7..459e485 100644
--- a/Koha/SearchEngine/Elasticsearch/Search.pm
+++ b/Koha/SearchEngine/Elasticsearch/Search.pm
@@ -200,9 +200,9 @@ sub search_auth_compat {
 
             # I wonder if these should be real values defined in the mapping
             # rather than hard-coded conversions.
-            # Our results often come through as nested arrays, to fix this
-            # requires changes in catmandu.
-            my $authid = $record->{ 'Local-number' }[0][0];
+            # Handle legacy nested arrays indexed with splitting enabled.
+            my $authid = $record->{ 'Local-number' }[0];
+            $authid = @$authid[0] if (ref $authid eq 'ARRAY');
             $result{authid} = $authid;
 
             # TODO put all this info into the record at index time so we
-- 
2.7.4