From 04ed4f50592693ecf0b3a30ec4eedacba9f4d9b5 Mon Sep 17 00:00:00 2001 From: Ere Maijala Date: Thu, 28 Mar 2019 13:37:13 +0200 Subject: [PATCH] Bug 22592: Add index scan emulation to Elasticsearch Adds support for using the "scan indexes" action in advanced search by using faceting with a prefix filter. Requires that the field be set as facetable for anything to be found. Test plan: 1. Apply patch 2. Go to advanced search and click "More options" 3. Select author as the search field, enter a last name and check "Scan indexes" 4. Perform search and observe the result list resembling scan results --- .../Elasticsearch/QueryBuilder.pm | 195 ++++++++++-------- Koha/SearchEngine/Elasticsearch/Search.pm | 61 +++++- .../SearchEngine/Elasticsearch/QueryBuilder.t | 24 ++- 3 files changed, 194 insertions(+), 86 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm index c257a0ed1c..b104f9f154 100644 --- a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm +++ b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm @@ -136,45 +136,6 @@ sub build_query { return $res; } -=head2 build_browse_query - - my $browse_query = $builder->build_browse_query($field, $query); - -This performs a "starts with" style query on a particular field. The field -to be searched must have been indexed with an appropriate mapping as a -"phrase" subfield, which pretty much everything has. - -=cut - -# XXX this isn't really a browse query like we want in the end -sub build_browse_query { - my ( $self, $field, $query ) = @_; - - my $fuzzy_enabled = C4::Context->preference("QueryFuzzy") || 0; - - return { query => '*' } if !defined $query; - - # TODO this should come from Koha::SearchEngine::Elasticsearch - my %field_whitelist = ( - title => 1, - author => 1, - ); - $field = 'title' if !exists $field_whitelist{$field}; - my $sort = $self->_sort_field($field); - my $res = { - query => { - match_phrase_prefix => { - "$field.phrase" => { - query => $query, - operator => 'or', - fuzziness => $fuzzy_enabled ? 'auto' : '0', - } - } - }, - sort => [ { $sort => { order => "asc" } } ], - }; -} - =head2 build_query_compat my ( @@ -200,54 +161,60 @@ sub build_query_compat { $lang, $params ) = @_; -#die Dumper ( $self, $operators, $operands, $indexes, $orig_limits, $sort_by, $scan, $lang ); - my @sort_params = $self->_convert_sort_fields(@$sort_by); - my @index_params = $self->_convert_index_fields(@$indexes); - my $limits = $self->_fix_limit_special_cases($orig_limits); - if ( $params->{suppress} ) { push @$limits, "suppress:0"; } - - # Merge the indexes in with the search terms and the operands so that - # each search thing is a handy unit. - unshift @$operators, undef; # The first one can't have an op - my @search_params; - my $truncate = C4::Context->preference("QueryAutoTruncate") || 0; - my $ea = each_array( @$operands, @$operators, @index_params ); - while ( my ( $oand, $otor, $index ) = $ea->() ) { - next if ( !defined($oand) || $oand eq '' ); - $oand = $self->_clean_search_term($oand); - $oand = $self->_truncate_terms($oand) if ($truncate); - push @search_params, { - operand => $oand, # the search terms - operator => defined($otor) ? uc $otor : undef, # AND and so on - $index ? %$index : (), - }; - } + my $query; + my $query_str = ''; + my $limits = (); + if ( $scan ) { + ($query, $query_str) = $self->_build_scan_query( $operands, $indexes ); + } else { + my @sort_params = $self->_convert_sort_fields(@$sort_by); + my @index_params = $self->_convert_index_fields(@$indexes); + my $limits = $self->_fix_limit_special_cases($orig_limits); + if ( $params->{suppress} ) { push @$limits, "suppress:0"; } + + # Merge the indexes in with the search terms and the operands so that + # each search thing is a handy unit. + unshift @$operators, undef; # The first one can't have an op + my @search_params; + my $truncate = C4::Context->preference("QueryAutoTruncate") || 0; + my $ea = each_array( @$operands, @$operators, @index_params ); + while ( my ( $oand, $otor, $index ) = $ea->() ) { + next if ( !defined($oand) || $oand eq '' ); + $oand = $self->_clean_search_term($oand); + $oand = $self->_truncate_terms($oand) if ($truncate); + push @search_params, { + operand => $oand, # the search terms + operator => defined($otor) ? uc $otor : undef, # AND and so on + $index ? %$index : (), + }; + } - # We build a string query from limits and the queries. An alternative - # would be to pass them separately into build_query and let it build - # them into a structured ES query itself. Maybe later, though that'd be - # more robust. - my $query_str = join( ' AND ', - join( ' ', $self->_create_query_string(@search_params) ) || (), - $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () ); - - my @fields = '_all'; - if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) { - push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields; - } + # We build a string query from limits and the queries. An alternative + # would be to pass them separately into build_query and let it build + # them into a structured ES query itself. Maybe later, though that'd be + # more robust. + $query_str = join( ' AND ', + join( ' ', $self->_create_query_string(@search_params) ) || (), + $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () ); + + my @fields = '_all'; + if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) { + push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields; + } - # If there's no query on the left, let's remove the junk left behind - $query_str =~ s/^ AND //; - my %options; - $options{fields} = \@fields; - $options{sort} = \@sort_params; - $options{expanded_facet} = $params->{expanded_facet}; - my $query = $self->build_query( $query_str, %options ); + # If there's no query on the left, let's remove the junk left behind + $query_str =~ s/^ AND //; + my %options; + $options{fields} = \@fields; + $options{sort} = \@sort_params; + $options{expanded_facet} = $params->{expanded_facet}; + $query = $self->build_query( $query_str, %options ); + } # We roughly emulate the CGI parameters of the zebra query builder my $query_cgi = ''; shift @$operators; # Shift out the one we unshifted before - $ea = each_array( @$operands, @$operators, @$indexes ); + my $ea = each_array( @$operands, @$operators, @$indexes ); while ( my ( $oand, $otor, $index ) = $ea->() ) { $query_cgi .= '&' if $query_cgi; $query_cgi .= 'idx=' . uri_escape_utf8( $index // '') . '&q=' . uri_escape_utf8( $oand ); @@ -481,6 +448,70 @@ sub build_authorities_query_compat { return $query; } +=head2 _build_scan_query + + my ($query, $query_str) = $builder->build_scan_query(\@operands, \@indexes) + +This will build an aggregation scan query that can be issued to elasticsearch from +the provided string input. + +=cut + +our %scan_field_convert = ( + 'ti' => 'title', + 'au' => 'author', + 'su' => 'subject', + 'se' => 'title-series', + 'pb' => 'publisher', +); + +sub _build_scan_query { + my ( $self, $operands, $indexes ) = @_; + + my $term = scalar( @$operands ) == 0 ? '' : $operands->[0]; + my $index = scalar( @$indexes ) == 0 ? 'title' : $indexes->[0]; + + my ( $f, $d ) = split( /,/, $index); + $index = $scan_field_convert{$f} || $f; + + my $res; + $res->{query} = { + query_string => { + query => '*' + } + }; + $res->{aggregations} = { + $index => { + terms => { + field => $index . '__facet', + order => { '_term' => 'asc' }, + include => $self->_create_regex_filter($self->_clean_search_term($term)) . '.*' + } + } + }; + return ($res, $term); +} + +=head2 _create_regex_filter + + my $filter = $builder->create_regex_filter('term') + +This will create a regex filter that can be used with an aggregation query. + +=cut + +sub _create_regex_filter { + my ($self, $term) = @_; + + my $result = ''; + foreach my $c (split(//, quotemeta($term))) { + my $lc = lc($c); + my $uc = uc($c); + $result .= $lc ne $uc ? '[' . $lc . $uc . ']' : $c; + } + return $result; +} + =head2 _convert_sort_fields my @sort_params = _convert_sort_fields(@sort_by) @@ -727,7 +758,7 @@ sub _modify_string_by_type { return $str unless $str; # Empty or undef, we can't use it. $str .= '*' if $type eq 'right-truncate'; - $str = '"' . $str . '"' if $type eq 'phrase'; + $str = '"' . $str . '"' if $type eq 'phrase' && $str !~ /^".*"$/; return $str; } diff --git a/Koha/SearchEngine/Elasticsearch/Search.pm b/Koha/SearchEngine/Elasticsearch/Search.pm index a4b4e84cae..7268bb7236 100644 --- a/Koha/SearchEngine/Elasticsearch/Search.pm +++ b/Koha/SearchEngine/Elasticsearch/Search.pm @@ -85,7 +85,7 @@ sub search { my $params = $self->get_elasticsearch_params(); # 20 is the default number of results per page - $query->{size} = $count || 20; + $query->{size} = $count // 20; # ES doesn't want pages, it wants a record to start from. if (exists $options{offset}) { $query->{from} = $options{offset}; @@ -146,8 +146,13 @@ sub search_compat { my ( $self, $query, $simple_query, $sort_by, $servers, $results_per_page, $offset, $expanded_facet, - $branches, $query_type, $scan + $branches, $item_types, $query_type, $scan ) = @_; + + if ( $scan ) { + return $self->_aggregation_scan( $query, $results_per_page, $offset ); + } + my %options; if ( !defined $offset or $offset < 0 ) { $offset = 0; @@ -513,4 +518,56 @@ sub _convert_facets { return \@facets; } +=head2 _aggregation_scan + + my $result = $self->_aggregration_scan($query, 10, 0); + +Perform an aggregation request for scan purposes. + +=cut + +sub _aggregation_scan { + my ($self, $query, $results_per_page, $offset) = @_; + + if (!scalar(keys %{$query->{aggregations}})) { + my %result = { + biblioserver => { + hits => 0, + RECORDS => undef + } + }; + return (undef, \%result, undef); + } + my ($field) = keys %{$query->{aggregations}}; + $query->{aggregations}{$field}{terms}{size} = 1000; + my $results = $self->search($query, 1, 0); + + # Convert each result into a MARC::Record + my (@records, $index); + # opac-search expects results to be put in the + # right place in the array, according to $offset + $index = $offset - 1; + + my $count = scalar(@{$results->{aggregations}{$field}{buckets}}); + for (my $index = $offset; $index - $offset < $results_per_page && $index < $count; $index++) { + my $bucket = $results->{aggregations}{$field}{buckets}->[$index]; + # Scan values are expressed as 100 a (count) and 245a (term) + my $marc = MARC::Record->new; + $marc->encoding('UTF-8'); + $marc->append_fields( + MARC::Field->new((100, ' ', ' ', 'a' => $bucket->{doc_count})) + ); + $marc->append_fields( + MARC::Field->new((245, ' ', ' ', 'a' => $bucket->{key})) + ); + $records[$index] = $marc->as_usmarc(); + }; + # consumers of this expect a namespaced result, we provide the default + # configuration. + my %result; + $result{biblioserver}{hits} = $count; + $result{biblioserver}{RECORDS} = \@records; + return (undef, \%result, undef); +} + 1; diff --git a/t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t b/t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t index 3ae6004563..5f9cf7a9ce 100644 --- a/t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t +++ b/t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t @@ -45,7 +45,8 @@ $se->mock( 'get_elasticsearch_mappings', sub { type => 'text' }, subject => { - type => 'text' + type => 'text', + facet => 1 }, itemnumber => { type => 'integer' @@ -169,7 +170,7 @@ subtest 'build_authorities_query_compat() tests' => sub { }; subtest 'build_query tests' => sub { - plan tests => 29; + plan tests => 33; my $qb; @@ -347,6 +348,25 @@ subtest 'build_query tests' => sub { ); is($query_cgi, 'idx=&q=title%3A%22donald%20duck%22', 'query cgi'); is($query_desc, 'title:"donald duck"', 'query desc ok'); + + # Scan queries + ( undef, $query, $simple_query, $query_cgi, $query_desc ) = $qb->build_query_compat( undef, ['new'], ['su'], undef, undef, 1 ); + is( + $query->{query}{query_string}{query}, + '*', + "scan query is properly formed" + ); + is_deeply( + $query->{aggregations}{'subject'}{'terms'}, + { + field => 'subject__facet', + order => { '_term' => 'asc' }, + include => '[nN][eE][wW].*' + }, + "scan aggregation request is properly formed" + ); + is($query_cgi, 'idx=su&q=new&scan=1', 'query cgi'); + is($query_desc, 'new', 'query desc ok'); }; -- 2.17.1