From d8c54ee24e84d7ed1bd528574662fb8d84b4c95f Mon Sep 17 00:00:00 2001 From: Nick Clemens Date: Wed, 30 Apr 2025 15:40:31 +0000 Subject: [PATCH] Bug 39790: Use constant score queries for matching To reduce some overhead in calculating relevancy here, we can use a constant_score query to simplify the search passed to elastic. This patch adds a new build_biblio_match_query routine in Zebra and ES. For ES we update to use the new queries. Zebra is simply a pass through to the previous code. To test: 1 - Export osme records from Koha 2 - Setup a matching rule on title, author, and isbn, score fo 200 each 3 - Stage the file and match, confirm all match 4 - Apply patch, restart all 5 - Change the matching to 'none' and 'Apply different matching rules' 6 - Change matching back to your rule and apply again 7 - Should have the same matches as before 8 - Change your search engine and unmatch/match again 9 - Should have same matches 10 - Test with other matching rules as desired 11 - Sign off! --- C4/Matcher.pm | 17 ++-------- .../Elasticsearch/QueryBuilder.pm | 32 ++++++++++++++++++ Koha/SearchEngine/Zebra/QueryBuilder.pm | 33 +++++++++++++++++++ 3 files changed, 67 insertions(+), 15 deletions(-) diff --git a/C4/Matcher.pm b/C4/Matcher.pm index 1e6d78da82e..90ceecdb9cd 100644 --- a/C4/Matcher.pm +++ b/C4/Matcher.pm @@ -686,26 +686,13 @@ sub get_matches { my $total_hits; if ( $self->{'record_type'} eq 'biblio' ) { - my $phr = - ( C4::Context->preference('AggressiveMatchOnISBN') || C4::Context->preference('AggressiveMatchOnISSN') ) - ? ',phr' - : q{}; - $query = join( - " OR ", - map { "$matchpoint->{'index'}$phr=\"$_\"" } @source_keys - ); - - #NOTE: double-quote the values so you don't get a "Embedded truncation not supported" error when a term has a ? in it. - # Use state variables to avoid recreating the objects every time. # With Elasticsearch this also avoids creating a massive amount of # ES connectors that would eventually run out of file descriptors. state $query_builder = Koha::SearchEngine::QueryBuilder->new( { index => $Koha::SearchEngine::BIBLIOS_INDEX } ); - ( undef, $query ) = $query_builder->build_query_compat( - undef, [$query], undef, undef, undef, undef, undef, - { skip_facets => 1 } - ); + $query = $query_builder->build_biblio_match_query_compat( + { matchpoint => $matchpoint, source_keys => \@source_keys } ); state $searcher = Koha::SearchEngine::Search->new( { index => $Koha::SearchEngine::BIBLIOS_INDEX } ); ( $error, $searchresults, $total_hits ) = $searcher->simple_search_compat( $query, 0, $max_matches, undef, skip_normalize => 1 ); diff --git a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm index 253df42d692..de3baa2096f 100644 --- a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm +++ b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm @@ -254,6 +254,38 @@ sub build_query { return $res; } +=head2 build_biblio_match_query_compat + + Used only for record matching. + Skip normal query build and generate a constant score terms query, just returning the simple query + so that Matcher can process. + This should reduce some searchign overhead since relevancy doesn't much matter here. + + + my $query = + build_biblio_match_query_compat({ matchpoint => $matchpoint, source_keys => $source_keys}) + +=cut + +sub build_biblio_match_query_compat { + + my $self = shift; + my ($params) = @_; + + my $query; + + # The 'terms' query doesn't work well on 'text' fields that are analyzed, and we + # aren't analyzing the fields from incoming records, so we use the 'raw' field + # FIXME: We can probably combine the searches for all matchpoints into a single query + $query->{query}->{constant_score}->{filter}->{terms} = { + $params->{matchpoint}->{index} . '.raw' => $params->{source_keys}, + boost => $params->{matchpoint}->{score}, + }; + + return $query; + +} + =head2 build_query_compat my ( diff --git a/Koha/SearchEngine/Zebra/QueryBuilder.pm b/Koha/SearchEngine/Zebra/QueryBuilder.pm index 33736005761..8ea25cf7c1b 100644 --- a/Koha/SearchEngine/Zebra/QueryBuilder.pm +++ b/Koha/SearchEngine/Zebra/QueryBuilder.pm @@ -55,6 +55,39 @@ sub build_query { C4::Search::buildQuery @_; } +=head2 build_biblio_match_query_compat + + Used only for record matching. + Handle some isbn processing and pass through build_query_compat, just returning the simple query + so that Matcher can process. + + my $query = + build_biblio_match_query_compat({ matchpoint => $matchpoint, source_keys => $source_keys}) + +=cut + +sub build_biblio_match_query_compat { + my $self = shift; + my ($params) = @_; + + my $phr = + ( C4::Context->preference('AggressiveMatchOnISBN') || C4::Context->preference('AggressiveMatchOnISSN') ) + ? ',phr' + : q{}; + my $joined_query = join( + " OR ", + map { "$params->{matchpoint}->{'index'}$phr=\"$_\"" } @{ $params->{source_keys} } + ); + + my ( undef, $query ) = $self->build_query_compat( + undef, [$joined_query], undef, undef, undef, undef, undef, + { skip_facets => 1 } + ); + + return $query; + +} + =head2 build_query_compat my ($error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type) = -- 2.39.5