Bugzilla – Attachment 88807 Details for
Bug 22592
Elasticsearch - Support for index scan
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 22592: Add index scan emulation to Elasticsearch
Bug-22592-Add-index-scan-emulation-to-Elasticsearc.patch (text/plain), 14.01 KB, created by
Michal Denar
on 2019-04-25 18:52:26 UTC
(
hide
)
Description:
Bug 22592: Add index scan emulation to Elasticsearch
Filename:
MIME Type:
Creator:
Michal Denar
Created:
2019-04-25 18:52:26 UTC
Size:
14.01 KB
patch
obsolete
>From e733d81c05345a6ce99f056599642d9d2f999a29 Mon Sep 17 00:00:00 2001 >From: Ere Maijala <ere.maijala@helsinki.fi> >Date: Thu, 28 Mar 2019 13:37:13 +0200 >Subject: [PATCH] Bug 22592: Add index scan emulation to Elasticsearch > >Adds support for using the "scan indexes" action in advanced search by using faceting with a prefix filter. Requires that the field be set as facetable for anything to be found. > >Test plan: >1. Apply patch >2. Go to advanced search and click "More options" >3. Select author as the search field, enter a last name and check "Scan indexes" >4. Perform search and observe the result list resembling scan results > >Signed-off-by: Michal Denar <black23@gmail.com> > >Signed-off-by: Michal Denar <black23@gmail.com> >--- > Koha/SearchEngine/Elasticsearch/QueryBuilder.pm | 199 ++++++++++++--------- > Koha/SearchEngine/Elasticsearch/Search.pm | 61 ++++++- > .../Koha/SearchEngine/Elasticsearch/QueryBuilder.t | 22 ++- > 3 files changed, 196 insertions(+), 86 deletions(-) > >diff --git a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm >index 084d5ad3db..8f4f0d9ca1 100644 >--- a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm >+++ b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm >@@ -136,45 +136,6 @@ sub build_query { > return $res; > } > >-=head2 build_browse_query >- >- my $browse_query = $builder->build_browse_query($field, $query); >- >-This performs a "starts with" style query on a particular field. The field >-to be searched must have been indexed with an appropriate mapping as a >-"phrase" subfield, which pretty much everything has. >- >-=cut >- >-# XXX this isn't really a browse query like we want in the end >-sub build_browse_query { >- my ( $self, $field, $query ) = @_; >- >- my $fuzzy_enabled = C4::Context->preference("QueryFuzzy") || 0; >- >- return { query => '*' } if !defined $query; >- >- # TODO this should come from Koha::SearchEngine::Elasticsearch >- my %field_whitelist = ( >- title => 1, >- author => 1, >- ); >- $field = 'title' if !exists $field_whitelist{$field}; >- my $sort = $self->_sort_field($field); >- my $res = { >- query => { >- match_phrase_prefix => { >- "$field.phrase" => { >- query => $query, >- operator => 'or', >- fuzziness => $fuzzy_enabled ? 'auto' : '0', >- } >- } >- }, >- sort => [ { $sort => { order => "asc" } } ], >- }; >-} >- > =head2 build_query_compat > > my ( >@@ -200,55 +161,63 @@ sub build_query_compat { > $lang, $params ) > = @_; > >-#die Dumper ( $self, $operators, $operands, $indexes, $orig_limits, $sort_by, $scan, $lang ); >- my @sort_params = $self->_convert_sort_fields(@$sort_by); >- my @index_params = $self->_convert_index_fields(@$indexes); >- my $limits = $self->_fix_limit_special_cases($orig_limits); >- if ( $params->{suppress} ) { push @$limits, "suppress:0"; } >- >- # Merge the indexes in with the search terms and the operands so that >- # each search thing is a handy unit. >- unshift @$operators, undef; # The first one can't have an op >- my @search_params; >- my $truncate = C4::Context->preference("QueryAutoTruncate") || 0; >- my $ea = each_array( @$operands, @$operators, @index_params ); >- while ( my ( $oand, $otor, $index ) = $ea->() ) { >- next if ( !defined($oand) || $oand eq '' ); >- $oand = $self->_clean_search_term($oand); >- $oand = $self->_truncate_terms($oand) if ($truncate); >- push @search_params, { >- operand => $oand, # the search terms >- operator => defined($otor) ? uc $otor : undef, # AND and so on >- $index ? %$index : (), >- }; >- } >+ my $query; >+ my $query_str = ''; >+ my $search_param_query_str = ''; >+ my $limits = (); >+ if ( $scan ) { >+ ($query, $query_str) = $self->_build_scan_query( $operands, $indexes ); >+ $search_param_query_str = $query_str; >+ } else { >+ my @sort_params = $self->_convert_sort_fields(@$sort_by); >+ my @index_params = $self->_convert_index_fields(@$indexes); >+ my $limits = $self->_fix_limit_special_cases($orig_limits); >+ if ( $params->{suppress} ) { push @$limits, "suppress:0"; } >+ >+ # Merge the indexes in with the search terms and the operands so that >+ # each search thing is a handy unit. >+ unshift @$operators, undef; # The first one can't have an op >+ my @search_params; >+ my $truncate = C4::Context->preference("QueryAutoTruncate") || 0; >+ my $ea = each_array( @$operands, @$operators, @index_params ); >+ while ( my ( $oand, $otor, $index ) = $ea->() ) { >+ next if ( !defined($oand) || $oand eq '' ); >+ $oand = $self->_clean_search_term($oand); >+ $oand = $self->_truncate_terms($oand) if ($truncate); >+ push @search_params, { >+ operand => $oand, # the search terms >+ operator => defined($otor) ? uc $otor : undef, # AND and so on >+ $index ? %$index : (), >+ }; >+ } > >- # We build a string query from limits and the queries. An alternative >- # would be to pass them separately into build_query and let it build >- # them into a structured ES query itself. Maybe later, though that'd be >- # more robust. >- my $search_param_query_str = join( ' ', $self->_create_query_string(@search_params) ); >- my $query_str = join( ' AND ', >- $search_param_query_str || (), >- $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () ); >- >- my @fields = '_all'; >- if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) { >- push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields; >- } >+ # We build a string query from limits and the queries. An alternative >+ # would be to pass them separately into build_query and let it build >+ # them into a structured ES query itself. Maybe later, though that'd be >+ # more robust. >+ $search_param_query_str = join( ' ', $self->_create_query_string(@search_params) ); >+ my $query_str = join( ' AND ', >+ $search_param_query_str || (), >+ $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () ); >+ >+ my @fields = '_all'; >+ if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) { >+ push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields; >+ } > >- # If there's no query on the left, let's remove the junk left behind >- $query_str =~ s/^ AND //; >- my %options; >- $options{fields} = \@fields; >- $options{sort} = \@sort_params; >- $options{expanded_facet} = $params->{expanded_facet}; >- my $query = $self->build_query( $query_str, %options ); >+ # If there's no query on the left, let's remove the junk left behind >+ $query_str =~ s/^ AND //; >+ my %options; >+ $options{fields} = \@fields; >+ $options{sort} = \@sort_params; >+ $options{expanded_facet} = $params->{expanded_facet}; >+ $query = $self->build_query( $query_str, %options ); >+ } > > # We roughly emulate the CGI parameters of the zebra query builder > my $query_cgi = ''; > shift @$operators; # Shift out the one we unshifted before >- $ea = each_array( @$operands, @$operators, @$indexes ); >+ my $ea = each_array( @$operands, @$operators, @$indexes ); > while ( my ( $oand, $otor, $index ) = $ea->() ) { > $query_cgi .= '&' if $query_cgi; > $query_cgi .= 'idx=' . uri_escape_utf8( $index // '') . '&q=' . uri_escape_utf8( $oand ); >@@ -482,6 +451,70 @@ sub build_authorities_query_compat { > return $query; > } > >+=head2 _build_scan_query >+ >+ my ($query, $query_str) = $builder->build_scan_query(\@operands, \@indexes) >+ >+This will build an aggregation scan query that can be issued to elasticsearch from >+the provided string input. >+ >+=cut >+ >+our %scan_field_convert = ( >+ 'ti' => 'title', >+ 'au' => 'author', >+ 'su' => 'subject', >+ 'se' => 'title-series', >+ 'pb' => 'publisher', >+); >+ >+sub _build_scan_query { >+ my ( $self, $operands, $indexes ) = @_; >+ >+ my $term = scalar( @$operands ) == 0 ? '' : $operands->[0]; >+ my $index = scalar( @$indexes ) == 0 ? 'title' : $indexes->[0]; >+ >+ my ( $f, $d ) = split( /,/, $index); >+ $index = $scan_field_convert{$f} || $f; >+ >+ my $res; >+ $res->{query} = { >+ query_string => { >+ query => '*' >+ } >+ }; >+ $res->{aggregations} = { >+ $index => { >+ terms => { >+ field => $index . '__facet', >+ order => { '_term' => 'asc' }, >+ include => $self->_create_regex_filter($self->_clean_search_term($term)) . '.*' >+ } >+ } >+ }; >+ return ($res, $term); >+} >+ >+=head2 _create_regex_filter >+ >+ my $filter = $builder->create_regex_filter('term') >+ >+This will create a regex filter that can be used with an aggregation query. >+ >+=cut >+ >+sub _create_regex_filter { >+ my ($self, $term) = @_; >+ >+ my $result = ''; >+ foreach my $c (split(//, quotemeta($term))) { >+ my $lc = lc($c); >+ my $uc = uc($c); >+ $result .= $lc ne $uc ? '[' . $lc . $uc . ']' : $c; >+ } >+ return $result; >+} >+ > =head2 _convert_sort_fields > > my @sort_params = _convert_sort_fields(@sort_by) >@@ -728,7 +761,7 @@ sub _modify_string_by_type { > return $str unless $str; # Empty or undef, we can't use it. > > $str .= '*' if $type eq 'right-truncate'; >- $str = '"' . $str . '"' if $type eq 'phrase'; >+ $str = '"' . $str . '"' if $type eq 'phrase' && $str !~ /^".*"$/; > return $str; > } > >diff --git a/Koha/SearchEngine/Elasticsearch/Search.pm b/Koha/SearchEngine/Elasticsearch/Search.pm >index a4b4e84cae..7268bb7236 100644 >--- a/Koha/SearchEngine/Elasticsearch/Search.pm >+++ b/Koha/SearchEngine/Elasticsearch/Search.pm >@@ -85,7 +85,7 @@ sub search { > > my $params = $self->get_elasticsearch_params(); > # 20 is the default number of results per page >- $query->{size} = $count || 20; >+ $query->{size} = $count // 20; > # ES doesn't want pages, it wants a record to start from. > if (exists $options{offset}) { > $query->{from} = $options{offset}; >@@ -146,8 +146,13 @@ sub search_compat { > my ( > $self, $query, $simple_query, $sort_by, > $servers, $results_per_page, $offset, $expanded_facet, >- $branches, $query_type, $scan >+ $branches, $item_types, $query_type, $scan > ) = @_; >+ >+ if ( $scan ) { >+ return $self->_aggregation_scan( $query, $results_per_page, $offset ); >+ } >+ > my %options; > if ( !defined $offset or $offset < 0 ) { > $offset = 0; >@@ -513,4 +518,56 @@ sub _convert_facets { > return \@facets; > } > >+=head2 _aggregation_scan >+ >+ my $result = $self->_aggregration_scan($query, 10, 0); >+ >+Perform an aggregation request for scan purposes. >+ >+=cut >+ >+sub _aggregation_scan { >+ my ($self, $query, $results_per_page, $offset) = @_; >+ >+ if (!scalar(keys %{$query->{aggregations}})) { >+ my %result = { >+ biblioserver => { >+ hits => 0, >+ RECORDS => undef >+ } >+ }; >+ return (undef, \%result, undef); >+ } >+ my ($field) = keys %{$query->{aggregations}}; >+ $query->{aggregations}{$field}{terms}{size} = 1000; >+ my $results = $self->search($query, 1, 0); >+ >+ # Convert each result into a MARC::Record >+ my (@records, $index); >+ # opac-search expects results to be put in the >+ # right place in the array, according to $offset >+ $index = $offset - 1; >+ >+ my $count = scalar(@{$results->{aggregations}{$field}{buckets}}); >+ for (my $index = $offset; $index - $offset < $results_per_page && $index < $count; $index++) { >+ my $bucket = $results->{aggregations}{$field}{buckets}->[$index]; >+ # Scan values are expressed as 100 a (count) and 245a (term) >+ my $marc = MARC::Record->new; >+ $marc->encoding('UTF-8'); >+ $marc->append_fields( >+ MARC::Field->new((100, ' ', ' ', 'a' => $bucket->{doc_count})) >+ ); >+ $marc->append_fields( >+ MARC::Field->new((245, ' ', ' ', 'a' => $bucket->{key})) >+ ); >+ $records[$index] = $marc->as_usmarc(); >+ }; >+ # consumers of this expect a namespaced result, we provide the default >+ # configuration. >+ my %result; >+ $result{biblioserver}{hits} = $count; >+ $result{biblioserver}{RECORDS} = \@records; >+ return (undef, \%result, undef); >+} >+ > 1; >diff --git a/t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t b/t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t >index df2f323383..becb3f5a77 100644 >--- a/t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t >+++ b/t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t >@@ -45,7 +45,8 @@ $se->mock( 'get_elasticsearch_mappings', sub { > type => 'text' > }, > subject => { >- type => 'text' >+ type => 'text', >+ facet => 1 > }, > itemnumber => { > type => 'integer' >@@ -375,6 +376,25 @@ subtest 'build_query tests' => sub { > ); > is($query_cgi, 'idx=&q=title%3A%22donald%20duck%22', 'query cgi'); > is($query_desc, 'title:"donald duck"', 'query desc ok'); >+ >+ # Scan queries >+ ( undef, $query, $simple_query, $query_cgi, $query_desc ) = $qb->build_query_compat( undef, ['new'], ['su'], undef, undef, 1 ); >+ is( >+ $query->{query}{query_string}{query}, >+ '*', >+ "scan query is properly formed" >+ ); >+ is_deeply( >+ $query->{aggregations}{'subject'}{'terms'}, >+ { >+ field => 'subject__facet', >+ order => { '_term' => 'asc' }, >+ include => '[nN][eE][wW].*' >+ }, >+ "scan aggregation request is properly formed" >+ ); >+ is($query_cgi, 'idx=su&q=new&scan=1', 'query cgi'); >+ is($query_desc, 'new', 'query desc ok'); > }; > > >-- >2.11.0
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 22592
:
87135
|
87136
|
87344
|
87345
|
87668
|
87669
|
88777
|
88778
|
88807
|
88808
|
92391
|
92392
|
92393
|
93743
|
93745
|
94962