From 1ec8e27143f6c066aed48901e65c99a0260379b4 Mon Sep 17 00:00:00 2001 From: Nick Clemens Date: Thu, 12 Dec 2024 21:42:22 +0000 Subject: [PATCH] Bug 38694: Add ESBoostFieldMatch option to Elasticsearch What this patch does is: 1 - Wraps the existing search code in a a "bool" compound query as a "must". This should not affect relevancy or results of the existing searches. 2 - Before we clean/truncate terms, loop through the passed in search terms and indexes to build a new 'should' query, using the 'match' on the specified index/field that is added to the 'bool' query from above. This means that if a result from the original query is also returned here, that item will be boosted in the result. For searches on 'keyword' or 'title', or if no index is set, we use 'title-cover' as the most narrow form of title This query isn't going to help when users enter CCL (i.e. ti:To die for) and it won't boost titles from 505, series, etc when doing a general search. Nor will it have a detrimental effect, it will only boost field matches To test: 1 - Add a record with 245 $a novel 2 - Add a record with 245 $a A novel : $b about things / $c by me 3 - Search for: novel 4 - Search for: a novel 5 - Apply patch, restart all, enable ESBoostFieldMatch option 6 - Repeat searches, note exact titles are boosted 7 - Experiment with other searches, turniung pref on and off to verify relevant titles are boosted when enabled 8 - Search results when disabled should not change --- .../Elasticsearch/QueryBuilder.pm | 44 ++++++++++++++++--- installer/data/mysql/atomicupdate/bug38694.pl | 21 +++++++++ installer/data/mysql/mandatory/sysprefs.sql | 1 + .../modules/admin/preferences/searching.pref | 8 ++++ 4 files changed, 68 insertions(+), 6 deletions(-) create mode 100755 installer/data/mysql/atomicupdate/bug38694.pl diff --git a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm index ad240bdde60..932288653ee 100644 --- a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm +++ b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm @@ -200,17 +200,19 @@ sub build_query { if ($options{whole_record}) { push @$fields, 'marc_data_array.*'; } - $res->{query} = { - query_string => { + my $query_string = { query => $query, fuzziness => $fuzzy_enabled ? 'auto' : '0', default_operator => 'AND', fields => $fields, lenient => JSON::true, analyze_wildcard => JSON::true, - } - }; - $res->{query}->{query_string}->{type} = 'cross_fields' if C4::Context->preference('ElasticsearchCrossFields'); + }; + $query_string->{type} = 'cross_fields' if C4::Context->preference('ElasticsearchCrossFields'); + + $res->{query} = { bool => { must => [ { query_string => $query_string } ] } }; + + $res->{query}->{bool}->{should} = $options{field_match_boost_query} if $options{field_match_boost_query}; if ( $options{sort} ) { foreach my $sort ( @{ $options{sort} } ) { @@ -262,6 +264,7 @@ sub build_query_compat { $lang, $params ) = @_; + my $query; my $query_str = ''; my $search_param_query_str = ''; @@ -272,7 +275,11 @@ sub build_query_compat { } else { my @sort_params = $self->_convert_sort_fields(@$sort_by); my @index_params = $self->_convert_index_fields(@$indexes); - $limits = $self->_fix_limit_special_cases($orig_limits); + my $field_match_boost_query = + C4::Context->preference('ESBoostFieldMatch') + ? $self->_build_field_match_boost_query( { operands => $operands, indexes => \@index_params } ) + : []; + $limits = $self->_fix_limit_special_cases($orig_limits); if ( $params->{suppress} ) { push @$limits, "suppress:false"; } # Merge the indexes in with the search terms and the operands so that # each search thing is a handy unit. @@ -314,6 +321,7 @@ sub build_query_compat { $options{is_opac} = $params->{is_opac}; $options{weighted_fields} = $params->{weighted_fields}; $options{whole_record} = $params->{whole_record}; + $options{field_match_boost_query} = $field_match_boost_query if @$field_match_boost_query; $query = $self->build_query( $query_str, %options ); } @@ -349,6 +357,30 @@ sub build_query_compat { ); } +=head2 _build_field_match_boost_query + + my ($query, $query_str) = $builder->_build_field_match_boost_query({ operands => \@operands, indexes => \@indexes) + +This will build an array of match queries for terms and indexes passed in and return a reference to the array. + +=cut + +sub _build_field_match_boost_query { + my ( $self, $params ) = @_; + my $indexes = $params->{indexes}; + my $operands = $params->{operands}; + + my @boost_query; + my $ea = each_array( @$operands, @$indexes ); + while ( my ( $operand, $index ) = $ea->() ) { + next unless $operand; + $index = $index->{field} if ref $index eq 'HASH'; + $index = 'title-cover' if ( !$index || $index eq 'kw' || $index eq 'ti' || $index eq 'title' ); + push @boost_query, { match => { $index => { query => $operand } } }; + } + return \@boost_query; +} + =head2 build_authorities_query my $query = $builder->build_authorities_query(\%search); diff --git a/installer/data/mysql/atomicupdate/bug38694.pl b/installer/data/mysql/atomicupdate/bug38694.pl new file mode 100755 index 00000000000..e8a0846f93f --- /dev/null +++ b/installer/data/mysql/atomicupdate/bug38694.pl @@ -0,0 +1,21 @@ +use Modern::Perl; +use Koha::Installer::Output qw(say_warning say_success say_info); + +return { + bug_number => "38694", + description => "Add ESBoostFieldMatch system preference", + up => sub { + my ($args) = @_; + my ( $dbh, $out ) = @$args{qw(dbh out)}; + + # Do you stuffs here + $dbh->do( + q{ + INSERT IGNORE INTO systempreferences ( `variable`, `value`, `options`, `explanation`, `type` ) VALUES + ('ESBoostFieldMatch', '0', NULL, 'Add a "match" query to es when searching, will follow indexes chosen in advanced search, or use title-cover for generic keyword or title index search', 'YesNo') + } + ); + + say_success( $out, "Added new system preference 'ESBoostFieldMatch'" ); + }, +}; diff --git a/installer/data/mysql/mandatory/sysprefs.sql b/installer/data/mysql/mandatory/sysprefs.sql index f49557be9c7..d3d70e877ee 100644 --- a/installer/data/mysql/mandatory/sysprefs.sql +++ b/installer/data/mysql/mandatory/sysprefs.sql @@ -258,6 +258,7 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` ('ERMProviderEbscoApiKey', '', '', 'API key for EBSCO', 'free'), ('ERMProviderEbscoCustomerID', '', '', 'Customer ID for EBSCO', 'free'), ('ERMProviders', 'local', 'local|ebsco', 'Set the providers for the ERM module', 'Choice'), +('ESBoostFieldMatch', '0', NULL, 'Add a "match" query to es when searching, will follow indexes chosen in advanced search, or use title-cover for generic keyword or title index search', 'YesNo'), ('ESPreventAutoTruncate', 'barcode|control-number|control-number-identifier|date-of-acquisition|date-of-publication|date-time-last-modified|identifier-standard|isbn|issn|itype|lc-card-number|number-local-acquisition|other-control-number|record-control-number', NULL, 'List of searchfields (separated by | or ,) that should not be autotruncated by Elasticsearch even if QueryAutoTruncate is set to Yes', 'free'), ('ExcludeHolidaysFromMaxPickUpDelay', '0', NULL, 'If ON, reserves max pickup delay takes into accountthe closed days.', 'YesNo'), ('expandedSearchOption','0',NULL,'If ON, set advanced search to be expanded by default','YesNo'), diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref index 1f334e2da10..7e9c7fccfb4 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref @@ -95,6 +95,14 @@ Searching: 0: Disable - "the cross_fields option for Elasticsearch searches, supported in Elasticsearch 6.X and above." - "See documentation at https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-multi-match-query.html#type-cross-fields" + - + - pref: ESBoostFieldMatch + default: 0 + choices: + 1: Enable + 0: Disable + - "adding a second 'match' field search to the ES query to boost relevancy. Keyword and title fields will be boosted with title-cover, other fields will boost directly." + - "This will not boost CCL style searches, only standard or advanced searches" - - pref: SavedSearchFilters default: 0 -- 2.39.5