From e733d81c05345a6ce99f056599642d9d2f999a29 Mon Sep 17 00:00:00 2001
From: Ere Maijala <ere.maijala@helsinki.fi>
Date: Thu, 28 Mar 2019 13:37:13 +0200
Subject: [PATCH] Bug 22592: Add index scan emulation to Elasticsearch

Adds support for using the "scan indexes" action in advanced search by using faceting with a prefix filter. Requires that the field be set as facetable for anything to be found.

Test plan:
1. Apply patch
2. Go to advanced search and click "More options"
3. Select author as the search field, enter a last name and check "Scan indexes"
4. Perform search and observe the result list resembling scan results

Signed-off-by: Michal Denar <black23@gmail.com>

Signed-off-by: Michal Denar <black23@gmail.com>
---
 Koha/SearchEngine/Elasticsearch/QueryBuilder.pm    | 199 ++++++++++++---------
 Koha/SearchEngine/Elasticsearch/Search.pm          |  61 ++++++-
 .../Koha/SearchEngine/Elasticsearch/QueryBuilder.t |  22 ++-
 3 files changed, 196 insertions(+), 86 deletions(-)

diff --git a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm
index 084d5ad3db..8f4f0d9ca1 100644
--- a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm
+++ b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm
@@ -136,45 +136,6 @@ sub build_query {
     return $res;
 }
 
-=head2 build_browse_query
-
-    my $browse_query = $builder->build_browse_query($field, $query);
-
-This performs a "starts with" style query on a particular field. The field
-to be searched must have been indexed with an appropriate mapping as a
-"phrase" subfield, which pretty much everything has.
-
-=cut
-
-# XXX this isn't really a browse query like we want in the end
-sub build_browse_query {
-    my ( $self, $field, $query ) = @_;
-
-    my $fuzzy_enabled = C4::Context->preference("QueryFuzzy") || 0;
-
-    return { query => '*' } if !defined $query;
-
-    # TODO this should come from Koha::SearchEngine::Elasticsearch
-    my %field_whitelist = (
-        title  => 1,
-        author => 1,
-    );
-    $field = 'title' if !exists $field_whitelist{$field};
-    my $sort = $self->_sort_field($field);
-    my $res = {
-        query => {
-            match_phrase_prefix => {
-                "$field.phrase" => {
-                    query     => $query,
-                    operator  => 'or',
-                    fuzziness => $fuzzy_enabled ? 'auto' : '0',
-                }
-            }
-        },
-        sort => [ { $sort => { order => "asc" } } ],
-    };
-}
-
 =head2 build_query_compat
 
     my (
@@ -200,55 +161,63 @@ sub build_query_compat {
         $lang, $params )
       = @_;
 
-#die Dumper ( $self, $operators, $operands, $indexes, $orig_limits, $sort_by, $scan, $lang );
-    my @sort_params  = $self->_convert_sort_fields(@$sort_by);
-    my @index_params = $self->_convert_index_fields(@$indexes);
-    my $limits       = $self->_fix_limit_special_cases($orig_limits);
-    if ( $params->{suppress} ) { push @$limits, "suppress:0"; }
-
-    # Merge the indexes in with the search terms and the operands so that
-    # each search thing is a handy unit.
-    unshift @$operators, undef;    # The first one can't have an op
-    my @search_params;
-    my $truncate = C4::Context->preference("QueryAutoTruncate") || 0;
-    my $ea = each_array( @$operands, @$operators, @index_params );
-    while ( my ( $oand, $otor, $index ) = $ea->() ) {
-        next if ( !defined($oand) || $oand eq '' );
-        $oand = $self->_clean_search_term($oand);
-        $oand = $self->_truncate_terms($oand) if ($truncate);
-        push @search_params, {
-            operand => $oand,      # the search terms
-            operator => defined($otor) ? uc $otor : undef,    # AND and so on
-            $index ? %$index : (),
-        };
-    }
+    my $query;
+    my $query_str = '';
+    my $search_param_query_str = '';
+    my $limits = ();
+    if ( $scan ) {
+        ($query, $query_str) = $self->_build_scan_query( $operands, $indexes );
+        $search_param_query_str = $query_str;
+    } else {
+        my @sort_params  = $self->_convert_sort_fields(@$sort_by);
+        my @index_params = $self->_convert_index_fields(@$indexes);
+        my $limits       = $self->_fix_limit_special_cases($orig_limits);
+        if ( $params->{suppress} ) { push @$limits, "suppress:0"; }
+
+        # Merge the indexes in with the search terms and the operands so that
+        # each search thing is a handy unit.
+        unshift @$operators, undef;    # The first one can't have an op
+        my @search_params;
+        my $truncate = C4::Context->preference("QueryAutoTruncate") || 0;
+        my $ea = each_array( @$operands, @$operators, @index_params );
+        while ( my ( $oand, $otor, $index ) = $ea->() ) {
+            next if ( !defined($oand) || $oand eq '' );
+            $oand = $self->_clean_search_term($oand);
+            $oand = $self->_truncate_terms($oand) if ($truncate);
+            push @search_params, {
+                operand => $oand,      # the search terms
+                operator => defined($otor) ? uc $otor : undef,    # AND and so on
+                $index ? %$index : (),
+            };
+        }
 
-    # We build a string query from limits and the queries. An alternative
-    # would be to pass them separately into build_query and let it build
-    # them into a structured ES query itself. Maybe later, though that'd be
-    # more robust.
-    my $search_param_query_str = join( ' ', $self->_create_query_string(@search_params) );
-    my $query_str = join( ' AND ',
-        $search_param_query_str || (),
-        $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () );
-
-    my @fields = '_all';
-    if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) {
-        push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields;
-    }
+        # We build a string query from limits and the queries. An alternative
+        # would be to pass them separately into build_query and let it build
+        # them into a structured ES query itself. Maybe later, though that'd be
+        # more robust.
+        $search_param_query_str = join( ' ', $self->_create_query_string(@search_params) );
+        my $query_str = join( ' AND ',
+            $search_param_query_str || (),
+            $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () );
+
+        my @fields = '_all';
+        if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) {
+            push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields;
+        }
 
-    # If there's no query on the left, let's remove the junk left behind
-    $query_str =~ s/^ AND //;
-    my %options;
-    $options{fields} = \@fields;
-    $options{sort} = \@sort_params;
-    $options{expanded_facet} = $params->{expanded_facet};
-    my $query = $self->build_query( $query_str, %options );
+        # If there's no query on the left, let's remove the junk left behind
+        $query_str =~ s/^ AND //;
+        my %options;
+        $options{fields} = \@fields;
+        $options{sort} = \@sort_params;
+        $options{expanded_facet} = $params->{expanded_facet};
+        $query = $self->build_query( $query_str, %options );
+    }
 
     # We roughly emulate the CGI parameters of the zebra query builder
     my $query_cgi = '';
     shift @$operators; # Shift out the one we unshifted before
-    $ea = each_array( @$operands, @$operators, @$indexes );
+    my $ea = each_array( @$operands, @$operators, @$indexes );
     while ( my ( $oand, $otor, $index ) = $ea->() ) {
         $query_cgi .= '&' if $query_cgi;
         $query_cgi .= 'idx=' . uri_escape_utf8( $index // '') . '&q=' . uri_escape_utf8( $oand );
@@ -482,6 +451,70 @@ sub build_authorities_query_compat {
     return $query;
 }
 
+=head2 _build_scan_query
+
+    my ($query, $query_str) = $builder->build_scan_query(\@operands, \@indexes)
+
+This will build an aggregation scan query that can be issued to elasticsearch from
+the provided string input.
+
+=cut
+
+our %scan_field_convert = (
+    'ti' => 'title',
+    'au' => 'author',
+    'su' => 'subject',
+    'se' => 'title-series',
+    'pb' => 'publisher',
+);
+
+sub _build_scan_query {
+    my ( $self, $operands, $indexes ) = @_;
+
+    my $term = scalar( @$operands ) == 0 ? '' : $operands->[0];
+    my $index = scalar( @$indexes ) == 0 ? 'title' : $indexes->[0];
+
+    my ( $f, $d ) = split( /,/, $index);
+    $index = $scan_field_convert{$f} || $f;
+
+    my $res;
+    $res->{query} = {
+        query_string => {
+            query => '*'
+        }
+    };
+    $res->{aggregations} = {
+        $index => {
+            terms => {
+                field => $index . '__facet',
+                order => { '_term' => 'asc' },
+                include => $self->_create_regex_filter($self->_clean_search_term($term)) . '.*'
+            }
+        }
+    };
+    return ($res, $term);
+}
+
+=head2 _create_regex_filter
+
+    my $filter = $builder->create_regex_filter('term')
+
+This will create a regex filter that can be used with an aggregation query.
+
+=cut
+
+sub _create_regex_filter {
+    my ($self, $term) = @_;
+
+    my $result = '';
+    foreach my $c (split(//, quotemeta($term))) {
+        my $lc = lc($c);
+        my $uc = uc($c);
+        $result .= $lc ne $uc ? '[' . $lc . $uc . ']' : $c;
+    }
+    return $result;
+}
+
 =head2 _convert_sort_fields
 
     my @sort_params = _convert_sort_fields(@sort_by)
@@ -728,7 +761,7 @@ sub _modify_string_by_type {
     return $str unless $str;    # Empty or undef, we can't use it.
 
     $str .= '*' if $type eq 'right-truncate';
-    $str = '"' . $str . '"' if $type eq 'phrase';
+    $str = '"' . $str . '"' if $type eq 'phrase' && $str !~ /^".*"$/;
     return $str;
 }
 
diff --git a/Koha/SearchEngine/Elasticsearch/Search.pm b/Koha/SearchEngine/Elasticsearch/Search.pm
index a4b4e84cae..7268bb7236 100644
--- a/Koha/SearchEngine/Elasticsearch/Search.pm
+++ b/Koha/SearchEngine/Elasticsearch/Search.pm
@@ -85,7 +85,7 @@ sub search {
 
     my $params = $self->get_elasticsearch_params();
     # 20 is the default number of results per page
-    $query->{size} = $count || 20;
+    $query->{size} = $count // 20;
     # ES doesn't want pages, it wants a record to start from.
     if (exists $options{offset}) {
         $query->{from} = $options{offset};
@@ -146,8 +146,13 @@ sub search_compat {
     my (
         $self,     $query,            $simple_query, $sort_by,
         $servers,  $results_per_page, $offset,       $expanded_facet,
-        $branches, $query_type,       $scan
+        $branches, $item_types,       $query_type,   $scan
     ) = @_;
+
+    if ( $scan ) {
+        return $self->_aggregation_scan( $query, $results_per_page, $offset );
+    }
+
     my %options;
     if ( !defined $offset or $offset < 0 ) {
         $offset = 0;
@@ -513,4 +518,56 @@ sub _convert_facets {
     return \@facets;
 }
 
+=head2 _aggregation_scan
+
+    my $result = $self->_aggregration_scan($query, 10, 0);
+
+Perform an aggregation request for scan purposes.
+
+=cut
+
+sub _aggregation_scan {
+    my ($self, $query, $results_per_page, $offset) = @_;
+
+    if (!scalar(keys %{$query->{aggregations}})) {
+        my %result = {
+            biblioserver => {
+                hits => 0,
+                RECORDS => undef
+            }
+        };
+        return (undef, \%result, undef);
+    }
+    my ($field) = keys %{$query->{aggregations}};
+    $query->{aggregations}{$field}{terms}{size} = 1000;
+    my $results = $self->search($query, 1, 0);
+
+    # Convert each result into a MARC::Record
+    my (@records, $index);
+    # opac-search expects results to be put in the
+    # right place in the array, according to $offset
+    $index = $offset - 1;
+
+    my $count = scalar(@{$results->{aggregations}{$field}{buckets}});
+    for (my $index = $offset; $index - $offset < $results_per_page && $index < $count; $index++) {
+        my $bucket = $results->{aggregations}{$field}{buckets}->[$index];
+        # Scan values are expressed as 100 a (count) and 245a (term)
+        my $marc = MARC::Record->new;
+        $marc->encoding('UTF-8');
+        $marc->append_fields(
+            MARC::Field->new((100, ' ',  ' ', 'a' => $bucket->{doc_count}))
+        );
+        $marc->append_fields(
+            MARC::Field->new((245, ' ',  ' ', 'a' => $bucket->{key}))
+        );
+        $records[$index] = $marc->as_usmarc();
+    };
+    # consumers of this expect a namespaced result, we provide the default
+    # configuration.
+    my %result;
+    $result{biblioserver}{hits} = $count;
+    $result{biblioserver}{RECORDS} = \@records;
+    return (undef, \%result, undef);
+}
+
 1;
diff --git a/t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t b/t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t
index df2f323383..becb3f5a77 100644
--- a/t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t
+++ b/t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t
@@ -45,7 +45,8 @@ $se->mock( 'get_elasticsearch_mappings', sub {
                     type => 'text'
                 },
                 subject => {
-                    type => 'text'
+                    type => 'text',
+                    facet => 1
                 },
                 itemnumber => {
                     type => 'integer'
@@ -375,6 +376,25 @@ subtest 'build_query tests' => sub {
     );
     is($query_cgi, 'idx=&q=title%3A%22donald%20duck%22', 'query cgi');
     is($query_desc, 'title:"donald duck"', 'query desc ok');
+
+    # Scan queries
+    ( undef, $query, $simple_query, $query_cgi, $query_desc ) = $qb->build_query_compat( undef, ['new'], ['su'], undef, undef, 1 );
+    is(
+        $query->{query}{query_string}{query},
+        '*',
+        "scan query is properly formed"
+    );
+    is_deeply(
+        $query->{aggregations}{'subject'}{'terms'},
+        {
+            field => 'subject__facet',
+            order => { '_term' => 'asc' },
+            include => '[nN][eE][wW].*'
+        },
+        "scan aggregation request is properly formed"
+    );
+    is($query_cgi, 'idx=su&q=new&scan=1', 'query cgi');
+    is($query_desc, 'new', 'query desc ok');
 };
 
 
-- 
2.11.0