From bbbde0883a25a6eaa384945d89cf6ec6299c1bbb Mon Sep 17 00:00:00 2001 From: David Cook Date: Tue, 19 Jan 2016 16:37:09 +1100 Subject: [PATCH] Bug 15541 - Prevent normalization during matching/import process This patch allows you to use the "qualifier,qualifier" syntax in the Record Matching Rules "Search Index" when using the QueryParser. While QueryParser doesn't support this syntax, it will now fallback correctly to non-QueryParser functionality. Without the patch, your search will just fail silently. This patch also adds a "skip_normalize" flag to C4::Search::SimpleSearch(), and sets the flag during C4::Matcher::get_matches. This prevents the s/:/=/g and s/=/:/g normalization which happens in a heavy-handed way to provide correct query syntax. However, get_matches() already uses the correct syntax, so this normalization is unneeded. The normalization also mangles URLs which causes failures when using the "url" (ie "u") register in Zebra (see bug 15555). This patch also creates "raw" and "none" normalizers for the Record Matching Rules, which prevents the stripping of spaces and punctuation by Koha prior to sending queries to Zebra. This is important for a number of reasons. First and foremost, Zebra does normalization better than Koha. ICU and CHR normalize strings differently, so it's better not to try to outsmart Zebra with pre-normalization of punctuation and spaces, as it will lead to search problems. Second, when using the "u" register in Zebra, you don't want to normalize the value, since it's stored "as is" in the Zebra database. Normalization causes search failures. _TEST PLAN_ 1) Apply patches from http://bugs.koha-community.org/bugzilla3/show_bug.cgi?id=15555 2) Create a record in Koha with a 024 $a http://koha-community.org/test $2 uri 3) Do a full re-index of Zebra 4) Create a Record Matching Rule in Koha with the following details: Description: Test Match threshold: 100 Record type: Bibliographic Match point 1: Search index: id-other,st-urx Score: 100 Tag: 024 Subfields: a Normalization rule: raw 5) Download your record from Koha as a .mrc file (ie isomarc, binary marc, etc) 6) Go to "Stage MARC records for import" 7) Upload your .marc file. 8) Change your "Record matching rule" to "Test" 9) Click Stage for import 10) It should say "1 records with at least one match in catalogu per matching rule "Test". Signed-off-by: Bouzid Fergani --- C4/Matcher.pm | 14 ++++++++++---- C4/Search.pm | 6 +++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/C4/Matcher.pm b/C4/Matcher.pm index 0dd8782..85474d2 100644 --- a/C4/Matcher.pm +++ b/C4/Matcher.pm @@ -649,7 +649,9 @@ sub get_matches { my $total_hits; if ( $self->{'record_type'} eq 'biblio' ) { - if ($QParser) { + #NOTE: The QueryParser can't handle the CCL syntax of 'qualifier','qualifier', so fallback to non-QueryParser. + #NOTE: You can see this in C4::Search::SimpleSearch() as well in a different way. + if ($QParser && $matchpoint->{'index'} !~ m/\w,\w/) { $query = join( " || ", map { "$matchpoint->{'index'}:$_" } @source_keys ); } @@ -660,9 +662,8 @@ sub get_matches { } require C4::Search; - ( $error, $searchresults, $total_hits ) = - C4::Search::SimpleSearch( $query, 0, $max_matches ); + C4::Search::SimpleSearch( $query, 0, $max_matches, undef, "skip_normalize" ); } elsif ( $self->{'record_type'} eq 'authority' ) { my $authresults; @@ -822,7 +823,12 @@ sub _get_match_keys { } elsif ($component->{'offset'}) { $string= substr($string, $component->{'offset'}); } - $key = _normalize($string); + $key = $string; + my $norms = $component->{'norms'}; + my $normalizer = $norms->[0]; #Matching rules, currently, can only have one normalizer. It will always be defined, even if it's only as ''. + if ($normalizer !~ /(none|raw)/i){ + $key = _normalize($string); + } if ($i == 0) { push @keys, $key if $key; } else { diff --git a/C4/Search.pm b/C4/Search.pm index f52e0ee..825e1b0 100644 --- a/C4/Search.pm +++ b/C4/Search.pm @@ -221,7 +221,7 @@ $template->param(result=>\@results); =cut sub SimpleSearch { - my ( $query, $offset, $max_results, $servers ) = @_; + my ( $query, $offset, $max_results, $servers, $skip_normalize ) = @_; return ( 'No query entered', undef, undef ) unless $query; # FIXME hardcoded value. See catalog/search.pl & opac-search.pl too. @@ -243,12 +243,12 @@ sub SimpleSearch { eval { $zconns[$i] = C4::Context->Zconn( $servers[$i], 1 ); if ($QParser) { - $query =~ s/=/:/g; + $query =~ s/=/:/g unless $skip_normalize; $QParser->parse( $query ); $query = $QParser->target_syntax($servers[$i]); $zoom_queries[$i] = new ZOOM::Query::PQF( $query, $zconns[$i]); } else { - $query =~ s/:/=/g; + $query =~ s/:/=/g unless $skip_normalize; $zoom_queries[$i] = new ZOOM::Query::CCL2RPN( $query, $zconns[$i]); } $tmpresults[$i] = $zconns[$i]->search( $zoom_queries[$i] ); -- 1.9.1