From d8b73bf02224b5316201cf69f29d554e1985c1b0 Mon Sep 17 00:00:00 2001 From: Nick Clemens Date: Thu, 12 Dec 2024 21:42:22 +0000 Subject: [PATCH] Bug 38694: Add ESBoostFieldMatch option to Elasticsearch What this patch does is: 1 - Wraps the existing search code in a a "bool" compound query as a "must". This should not affect relevancy or results of the existing searches. 2 - Before we clean/truncate terms, loop through the passed in search terms and indexes to build a new 'should' query, using the 'match' on the specified index/field that is added to the 'bool' query from above. This means that if a result from the original query is also returned here, that item will be boosted in the result. For searches on 'keyword' or 'title', or if no index is set, we use 'title-cover' as the most narrow form of title This query isn't going to help when users enter CCL (i.e. ti:To die for) and it won't boost titles from 505, series, etc when doing a general search. Nor will it have a detrimental effect, it will only boost field matches To test: 1 - Add a record with 245 $a novel 2 - Add a record with 245 $a A novel : $b about things / $c by me 3 - Search for: novel - Above records are returned lower in the list (results #6 and #14 for me) 4 - Search for: a novel - Above records are returned lower in the list (results #6 and #14 for me) 5 - Apply patch, restart all, enable ESBoostFieldMatch option 6 - Repeat searches, note exact titles are boosted - For 'novel' record from #1 is #1 result and other is second - For 'a novel' record form #2 is #1 result and other is third 7 - Disable the new pref and repeat steps 3 and 4 and get results as before patch 8 - Experiment with other searches, turning pref on and off to verify relevant titles are boosted when enabled 9 - Search results when disabled should return as before the patch Signed-off-by: Matthias Meusburger --- .../Elasticsearch/QueryBuilder.pm | 63 ++++++++++++++----- installer/data/mysql/atomicupdate/bug38694.pl | 21 +++++++ installer/data/mysql/mandatory/sysprefs.sql | 1 + .../modules/admin/preferences/searching.pref | 8 +++ 4 files changed, 77 insertions(+), 16 deletions(-) create mode 100755 installer/data/mysql/atomicupdate/bug38694.pl diff --git a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm index 0e9d964727b..f454498bacd 100644 --- a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm +++ b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm @@ -203,17 +203,19 @@ sub build_query { if ( $options{whole_record} ) { push @$fields, 'marc_data_array.*'; } - $res->{query} = { - query_string => { - query => $query, - fuzziness => $fuzzy_enabled ? 'auto' : '0', - default_operator => 'AND', - fields => $fields, - lenient => JSON::true, - analyze_wildcard => JSON::true, - } + my $query_string = { + query => $query, + fuzziness => $fuzzy_enabled ? 'auto' : '0', + default_operator => 'AND', + fields => $fields, + lenient => JSON::true, + analyze_wildcard => JSON::true, }; - $res->{query}->{query_string}->{type} = 'cross_fields' if C4::Context->preference('ElasticsearchCrossFields'); + $query_string->{type} = 'cross_fields' if C4::Context->preference('ElasticsearchCrossFields'); + + $res->{query} = { bool => { must => [ { query_string => $query_string } ] } }; + + $res->{query}->{bool}->{should} = $options{field_match_boost_query} if $options{field_match_boost_query}; if ( $options{sort} ) { foreach my $sort ( @{ $options{sort} } ) { @@ -280,6 +282,10 @@ sub build_query_compat { } else { my @sort_params = $self->_convert_sort_fields(@$sort_by); my @index_params = $self->_convert_index_fields(@$indexes); + my $field_match_boost_query = + C4::Context->preference('ESBoostFieldMatch') + ? $self->_build_field_match_boost_query( { operands => $operands, indexes => \@index_params } ) + : []; $limits = $self->_fix_limit_special_cases($orig_limits); if ( $params->{suppress} ) { push @$limits, "suppress:false"; } @@ -320,12 +326,13 @@ sub build_query_compat { # If there's no query on the left, let's remove the junk left behind $query_str =~ s/^ AND //; my %options; - $options{sort} = \@sort_params; - $options{is_opac} = $params->{is_opac}; - $options{weighted_fields} = $params->{weighted_fields}; - $options{whole_record} = $params->{whole_record}; - $options{skip_facets} = $params->{skip_facets}; - $query = $self->build_query( $query_str, %options ); + $options{sort} = \@sort_params; + $options{is_opac} = $params->{is_opac}; + $options{weighted_fields} = $params->{weighted_fields}; + $options{whole_record} = $params->{whole_record}; + $options{skip_facets} = $params->{skip_facets}; + $options{field_match_boost_query} = $field_match_boost_query if @$field_match_boost_query; + $query = $self->build_query( $query_str, %options ); } # We roughly emulate the CGI parameters of the zebra query builder @@ -361,6 +368,30 @@ sub build_query_compat { ); } +=head2 _build_field_match_boost_query + + my ($query, $query_str) = $builder->_build_field_match_boost_query({ operands => \@operands, indexes => \@indexes) + +This will build an array of match queries for terms and indexes passed in and return a reference to the array. + +=cut + +sub _build_field_match_boost_query { + my ( $self, $params ) = @_; + my $indexes = $params->{indexes}; + my $operands = $params->{operands}; + + my @boost_query; + my $ea = each_array( @$operands, @$indexes ); + while ( my ( $operand, $index ) = $ea->() ) { + next unless $operand; + $index = $index->{field} if ref $index eq 'HASH'; + $index = 'title-cover' if ( !$index || $index eq 'kw' || $index eq 'ti' || $index eq 'title' ); + push @boost_query, { match => { $index => { query => $operand } } }; + } + return \@boost_query; +} + =head2 build_authorities_query my $query = $builder->build_authorities_query(\%search); diff --git a/installer/data/mysql/atomicupdate/bug38694.pl b/installer/data/mysql/atomicupdate/bug38694.pl new file mode 100755 index 00000000000..e8a0846f93f --- /dev/null +++ b/installer/data/mysql/atomicupdate/bug38694.pl @@ -0,0 +1,21 @@ +use Modern::Perl; +use Koha::Installer::Output qw(say_warning say_success say_info); + +return { + bug_number => "38694", + description => "Add ESBoostFieldMatch system preference", + up => sub { + my ($args) = @_; + my ( $dbh, $out ) = @$args{qw(dbh out)}; + + # Do you stuffs here + $dbh->do( + q{ + INSERT IGNORE INTO systempreferences ( `variable`, `value`, `options`, `explanation`, `type` ) VALUES + ('ESBoostFieldMatch', '0', NULL, 'Add a "match" query to es when searching, will follow indexes chosen in advanced search, or use title-cover for generic keyword or title index search', 'YesNo') + } + ); + + say_success( $out, "Added new system preference 'ESBoostFieldMatch'" ); + }, +}; diff --git a/installer/data/mysql/mandatory/sysprefs.sql b/installer/data/mysql/mandatory/sysprefs.sql index 2bf9d0b6486..764eed4f4d1 100644 --- a/installer/data/mysql/mandatory/sysprefs.sql +++ b/installer/data/mysql/mandatory/sysprefs.sql @@ -260,6 +260,7 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` ('ERMProviderEbscoApiKey', '', '', 'API key for EBSCO', 'free'), ('ERMProviderEbscoCustomerID', '', '', 'Customer ID for EBSCO', 'free'), ('ERMProviders', 'local', 'local|ebsco', 'Set the providers for the ERM module', 'Choice'), +('ESBoostFieldMatch', '0', NULL, 'Add a "match" query to es when searching, will follow indexes chosen in advanced search, or use title-cover for generic keyword or title index search', 'YesNo'), ('ESPreventAutoTruncate', 'barcode|control-number|control-number-identifier|date-of-acquisition|date-of-publication|date-time-last-modified|identifier-standard|isbn|issn|itype|lc-card-number|number-local-acquisition|other-control-number|record-control-number', NULL, 'List of searchfields (separated by | or ,) that should not be autotruncated by Elasticsearch even if QueryAutoTruncate is set to Yes', 'free'), ('ExcludeHolidaysFromMaxPickUpDelay', '0', NULL, 'If ON, reserves max pickup delay takes into accountthe closed days.', 'YesNo'), ('expandedSearchOption','0',NULL,'If ON, set advanced search to be expanded by default','YesNo'), diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref index b9bbf800838..cf78ddd362f 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref @@ -95,6 +95,14 @@ Searching: 0: Disable - "the cross_fields option for Elasticsearch searches, supported in Elasticsearch 6.X and above." - See the Elasticsearch cross_fields documentation. + - + - pref: ESBoostFieldMatch + default: 0 + choices: + 1: Enable + 0: Disable + - "adding a second 'match' field search to the ES query to boost relevancy. Keyword and title fields will be boosted with title-cover, other fields will boost directly." + - "This will not boost CCL style searches, only standard or advanced searches" - - pref: SavedSearchFilters default: 0 -- 2.39.5