From 60db6163979bdb6f9d0f54a656034a57275378dc Mon Sep 17 00:00:00 2001 From: Julian Maurice Date: Thu, 30 Jan 2020 12:28:14 +0100 Subject: [PATCH] Bug 21357: Allow to use multiple ES language analyzers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Test plan: 1. Apply this patch 2. If you have a custom `field_config.yaml` report the diff in your custom config 3. Create biblios in english and french 4. misc/search_tools/rebuild_elasticsearch.pl 5. Test biblio search at intranet with stemming and ellision in mind Examples of test queries: - "journal actualité" => should return results with "Journaux d'actualités" in it - "lord lady" => should return results with "Lords and Ladies" in it --- .../Elasticsearch/QueryBuilder.pm | 124 ++++++++++++------ .../elasticsearch/field_config.yaml | 6 + 2 files changed, 88 insertions(+), 42 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm index c1011ecc1e..2f5a9c517c 100644 --- a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm +++ b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm @@ -78,30 +78,7 @@ according to these values. Valid values for C are 'asc' and 'desc'. sub build_query { my ( $self, $query, %options ) = @_; - my $stemming = C4::Context->preference("QueryStemming") || 0; - my $auto_truncation = C4::Context->preference("QueryAutoTruncate") || 0; - my $fuzzy_enabled = C4::Context->preference("QueryFuzzy") || 0; - - $query = '*' unless defined $query; - - my $res; - my $fields = $self->_search_fields({ - is_opac => $options{is_opac}, - weighted_fields => $options{weighted_fields}, - }); - if ($options{whole_record}) { - push @$fields, 'marc_data_array.*'; - } - $res->{query} = { - query_string => { - query => $query, - fuzziness => $fuzzy_enabled ? 'auto' : '0', - default_operator => 'AND', - fields => $fields, - lenient => JSON::true, - analyze_wildcard => JSON::true, - } - }; + my $res = { query => $query }; if ( $options{sort} ) { foreach my $sort ( @{ $options{sort} } ) { @@ -195,23 +172,78 @@ sub build_query_compat { }; } - # We build a string query from limits and the queries. An alternative - # would be to pass them separately into build_query and let it build - # them into a structured ES query itself. Maybe later, though that'd be - # more robust. - $search_param_query_str = join( ' ', $self->_create_query_string(@search_params) ); - $query_str = join( ' AND ', - $search_param_query_str || (), - $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () ); - - # If there's no query on the left, let's remove the junk left behind - $query_str =~ s/^ AND //; - my %options; - $options{sort} = \@sort_params; - $options{is_opac} = $params->{is_opac}; - $options{weighted_fields} = $params->{weighted_fields}; - $options{whole_record} = $params->{whole_record}; - $query = $self->build_query( $query_str, %options ); + my $fuzzy_enabled = C4::Context->preference("QueryFuzzy") || 0; + my $default_fields = $self->_search_fields({ + is_opac => $params->{is_opac}, + weighted_fields => $params->{weighted_fields}, + }); + if ($params->{whole_record}) { + push @$default_fields, 'marc_data_array.*'; + } + + my $es_query = { + bool => { + minimum_should_match => 1, + should => [ + { + bool => { + must => [], + must_not => [], + }, + } + ], + }, + }; + + foreach my $search_param (@search_params) { + my $operand = $search_param->{operand}; + my $operator = $search_param->{operator} // ''; + my $field = $search_param->{field}; + + my @fields; + if ($field) { + @fields = ($field, "$field.lang_*"); + } else { + @fields = @$default_fields; + } + + my $qs = { + query_string => { + query => $operand, + fuzziness => $fuzzy_enabled ? 'auto' : 0, + default_operator => 'AND', + fields => \@fields, + lenient => JSON::true, + analyze_wildcard => JSON::true, + }, + }; + + if ($operator eq 'OR') { + push @{ $es_query->{bool}->{should} }, { + bool => { + must => [$qs], + must_not => [], + }, + }; + } elsif ($operator eq 'NOT') { + push @{ $es_query->{bool}->{should}->[-1]->{bool}->{must_not} }, $qs; + } else { + push @{ $es_query->{bool}->{should}->[-1]->{bool}->{must} }, $qs; + } + } + + my $limit_query_string = $self->_join_queries($self->_convert_index_strings(@$limits)); + if ($limit_query_string) { + $es_query->{bool}{filter} = { + query_string => { + query => $limit_query_string, + default_operator => 'AND', + lenient => JSON::true, + }, + }; + } + + $query = $self->build_query( $es_query, sort => \@sort_params ); } # We roughly emulate the CGI parameters of the zebra query builder @@ -1172,10 +1204,18 @@ sub _search_fields { # Copy values to avoid mutating cached # data (since unsafe is used) my ($field, $weight) = @{$_}; - ["${field}.${subfield}", $weight]; + ["${field}.${subfield}", $weight ? $weight : ()]; } @{$search_fields} ]; + } else { + $search_fields = [ + map { + my ($field, $weight) = @{$_}; + $_, ["$field.lang_*", $weight // ()]; + } @$search_fields + ]; } + if ($params->{weighted_fields}) { return [map { join('^', @{$_}) } @{$search_fields}]; } diff --git a/admin/searchengine/elasticsearch/field_config.yaml b/admin/searchengine/elasticsearch/field_config.yaml index e48cde7f57..b7077cb794 100644 --- a/admin/searchengine/elasticsearch/field_config.yaml +++ b/admin/searchengine/elasticsearch/field_config.yaml @@ -50,6 +50,12 @@ search: ci_raw: type: keyword normalizer: icu_folding_normalizer + lang_en: + type: text + analyzer: english + lang_fr: + type: text + analyzer: french # Facets facet: default: -- 2.20.1