From f242224293561ab259a8b709e604c2b1e44628d6 Mon Sep 17 00:00:00 2001 From: David Gustafsson Date: Fri, 2 Mar 2018 18:16:39 +0100 Subject: [PATCH] Bug 20334 - Option for escaping slashes in search queries Add "QueryRegexEscapeOption" system preference to provide option to escape Elasticsearch regexp delimiters (/) within queries, or alternativly to unescape escaped slashes (\/) while escaping unescaped slashes, in effect making "\/" the new regexp delimiter. How to test: 1) Run tests in ./t/Koha/SearchEngine/ElasticSearch/QueryBuilder.t 2) All tests should succeed --- Koha/SearchEngine/Elasticsearch/QueryBuilder.pm | 25 +++++ ...g_20334-add-syspref-QueryRegexEscapeOptions.sql | 1 + .../en/modules/admin/preferences/searching.pref | 7 ++ .../SearchEngine/ElasticSearch/.QueryBuilder.t.swp | Bin 0 -> 20480 bytes t/Koha/SearchEngine/ElasticSearch/QueryBuilder.t | 125 +++++++++++++++++++++ 5 files changed, 158 insertions(+) create mode 100644 installer/data/mysql/atomicupdate/bug_20334-add-syspref-QueryRegexEscapeOptions.sql create mode 100644 t/Koha/SearchEngine/ElasticSearch/.QueryBuilder.t.swp create mode 100644 t/Koha/SearchEngine/ElasticSearch/QueryBuilder.t diff --git a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm index 95cc86ab78..86d4b1f62e 100644 --- a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm +++ b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm @@ -724,9 +724,34 @@ sub _clean_search_term { $term = $self->_convert_index_strings_freeform($term); $term =~ s/[{}]/"/g; $term = $self->_truncate_terms($term) if ($auto_truncation); + $term = _query_regex_escape_process($term); return $term; } +sub _query_regex_escape_process { + my ($self, $query) = @_; + my $regex_escape_options = C4::Context->preference("QueryRegexEscapeOptions"); + if ($regex_escape_options ne 'dont_escape') { + if ($regex_escape_options eq 'escape') { + # Will escape unescaped slashes (/) while preserving + # unescaped slashes within quotes + # @TODO: assumes quotes are always balanced and will + # not handle escaped qoutes properly, should perhaps be + # replaced with a more general parser solution + # so that this function is ever only provided with unqouted + # query parts + $query =~ s@(?:(?_fix_limit_special_cases($limits); diff --git a/installer/data/mysql/atomicupdate/bug_20334-add-syspref-QueryRegexEscapeOptions.sql b/installer/data/mysql/atomicupdate/bug_20334-add-syspref-QueryRegexEscapeOptions.sql new file mode 100644 index 0000000000..7f70795631 --- /dev/null +++ b/installer/data/mysql/atomicupdate/bug_20334-add-syspref-QueryRegexEscapeOptions.sql @@ -0,0 +1 @@ +INSERT IGNORE INTO systempreferences ( `variable`, `value`, `options`, `explanation`, `type`) VALUES ('QueryRegexEscapeOptions', 'dont_escape', 'dont_escape|escape|unescape_escaped', 'Escape option for regexps delimiters in Elasicsearch queries.', 'Choice'); diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref index 0178468c7b..8233ef6f17 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/searching.pref @@ -30,6 +30,13 @@ Searching: no: Disable - ranking of search results by relevance (REQUIRES ZEBRA). - + - pref: QueryRegexEscapeOptions + choices: + escape: Escape + unescape_escaped: Unescape escaped + dont_escape: Don't escape + - "regular expressions within query strings. If \"Unescape escaped\" is selected this will allow writing regular expressions \"\/like this\/\" while \"/this/\", \"or/this\" will be escaped and interpreted by Elasticsearch as regular strings." + - - pref: OpacGroupResults default: 0 choices: diff --git a/t/Koha/SearchEngine/ElasticSearch/.QueryBuilder.t.swp b/t/Koha/SearchEngine/ElasticSearch/.QueryBuilder.t.swp new file mode 100644 index 0000000000000000000000000000000000000000..301e48e24ad2f0e562e197c2997dcd08dc2278ae GIT binary patch literal 20480 zcmeI4TaVjB6vwBOQf}oc-ng7@DrF04Qd%Uy=~has?1gqOWV5u3+D1-1*;sXKR^WBCw#0K^Aq`;0&cDqdUo&)CU2-gL87sT$F-^jCK5nem+8oEgv9 zM_WJq-pnk$nSX=fc!sgFm%n*`AAP}2Jjg9(rpZ zJ9^~x!{TygY(G8!+|r<&>YO5=2q*%IfFhs>C<2OrBA^KTPYDF>Zgv^AJrvt_EdJe< z`h7S~KNkN^q^|EyrB9~PaZ>##0*Zhlpa>`eihv@Z2q*%IfFhs>C<2PWeITIY1OHiU z-VL$A$M^s4{{PRX8T$kJ1^OBK9GZdlK-ZsQ>?-svREILq-}@N50c}7o5B6JNeBCbN`p=*!g zZ|F1V=3d6upbnIQzI=qS-}f-~8}v2w74$OnJ?#Asx&jH?FNicaT!CJIY`hR@L)Y;l zWCHpT+JH=`MTx`o#}Bxk)8M|(joJqyULi+g@|VuxD0IxAMIK+_YpzU>;c`VLNnUac zgItd{xMA8W#Qlcuaz95wEA-=(k)xLGQ=N00LOaS~P^XnhMma?VCQ8|Q-QvdB!6Fk4 zO*fAwNJL*~S~BiJ)8fDjUj}1nS~BwD(DBHynlz&pwV)PTJ$^tMTNB3AU<}U3#>5%@ zs2Ex!n==g9R%AGnhQ?Q6eD6jnYCT2?ZX-=iSaUbnl0GI;m8BOM6uDvKeD}eJg!fTK z_Iife>y-4AxI-e}fJ^poRJvbhDo5qi{BSbyb9VvmPS!P|+ zW}U`GU(?$75nj^d9=98OJSz>9Q%Tg{^KM`|wx7*WwmSskA!uZ~bq=JM?3Zmu@EuHC zy5(`*z)aYixGdfbFnhHf!)%&7p1x#fjQXr4oH*sY+dzUVXQN>!X8VDYS@zz~gqz&u zV`4-&51Ho%fZnb|a?;q!Sxy|H#-Sb`&j`^>1RpO&zI&U=bCmtK&ywzlPq}Vio+QHb z47wvHO&7ZDv^z>}_@B+P5lcC@vK_fvbmu{kY$u2ocw0ajrr)=m6aAT5aH1RU*@~_o zR?3;8i<@4Qbcfymyo<+};n+d^2q0ej1#qsPN5|j5wS?V*VD5|3>ZZ96w9FO0{1MO%8+h3 zSfn94Q{UDd8$*xao5RdV&}kVYHo?9xXnG?qJ^-ID(G}n4a=aG+h3-p<562!V(>sW zEnnPpwp=+mU7D+wPRyL1sV)<;OwLs2%8QFMIbWd?EtD$NnUhPWOBGsJsw~Vemh-d- zbI=qNKQmLEo?og`X>OU$lqwa}v|N+~oG>7{loxF_+pZ;72D<0zb|7p)VZlgjYA_4} z&#Z?5Y&GEkDA00TaSJ^lGeC8gg%Lzx-VCiAA*VET9GBOf(4)u|4w7!KlcfjTqgC$t zSTG3-MRk)NTD<5qgN_dKC!NqXbg`f+635nNGd~^eQjU@400ea)KQf7$Pgx$>o1!g#+?Jv+fEsXn51Yg;24S>r8wJ^4SX8a!gtWNY0RRvse#DRc^Y2NHY#X=pd}5VQ+o z&_CGw--6Vk2q*%IfFhs>C<2OrBA^H;0*Zhlpa_gUflKkr&eZFzzIUbW=4-C_qi%fp zQvVTsbT#`=4o$_S27OFExRdWLn=kykpMG}mg&1aw{r_HkbDx65{=XaF|8ZiUzk&Vz zE6^Ig-*=#29KoL*`6ahux0V5!S9|i~7^uqcEMd<8gW1}Kr z6T!nG9x??@em;LtB4LG_ZH*+|CVDopXCDxY*9%b@7$X+DZhuS*=$A($xK>7g#{U{` N=K. + +use Modern::Perl; + +use Test::More tests => 2; +use t::lib::Mocks; + +use_ok('Koha::SearchEngine::Elasticsearch::QueryBuilder'); + +subtest 'query_regex_escape_options' => sub { + plan tests => 12; + + t::lib::Mocks::mock_preference('QueryRegexEscapeOptions', 'dont_escape'); + + my $query_with_regexp = "query /with regexp/"; + + my $processed_query = Koha::SearchEngine::Elasticsearch::QueryBuilder->_query_regex_escape_process($query_with_regexp); + is( + $processed_query, + $query_with_regexp, + "Unescaped query regexp has not been escaped when escaping is disabled" + ); + + t::lib::Mocks::mock_preference('QueryRegexEscapeOptions', 'escape'); + + $processed_query = Koha::SearchEngine::Elasticsearch::QueryBuilder->_query_regex_escape_process($query_with_regexp); + is( + $processed_query, + "query \\/with regexp\\/", + "Unescaped query regexp has been escaped when escaping is enabled" + ); + + my $query_with_escaped_regex = "query \\/with regexp\\/"; + $processed_query = Koha::SearchEngine::Elasticsearch::QueryBuilder->_query_regex_escape_process($query_with_escaped_regex); + is( + $processed_query, + $query_with_escaped_regex, + "Escaped query regexp has been left unmodified when escaping is enabled" + ); + + my $query_with_even_preceding_escapes_regex = "query \\\\/with regexp\\\\/"; + $processed_query = Koha::SearchEngine::Elasticsearch::QueryBuilder->_query_regex_escape_process($query_with_even_preceding_escapes_regex); + is( + $processed_query, + "query \\\\\\/with regexp\\\\\\/", + "Query regexp with even preceding escapes, thus unescaped, has been escaped when escaping is enabled" + ); + + my $query_with_odd_preceding_escapes_regex = 'query \\\\\\/with regexp\\\\\\/'; + $processed_query = Koha::SearchEngine::Elasticsearch::QueryBuilder->_query_regex_escape_process($query_with_odd_preceding_escapes_regex); + is( + $processed_query, + $query_with_odd_preceding_escapes_regex, + "Query regexp with odd preceding escapes, thus escaped, has been left unmodified when escaping is enabled" + ); + + my $query_with_quoted_slash = "query with / and \"/ within quotes\""; + $processed_query = Koha::SearchEngine::Elasticsearch::QueryBuilder->_query_regex_escape_process($query_with_quoted_slash); + is( + $processed_query, + "query with \\/ and \"/ within quotes\"", + "Unescaped slash outside of quotes has been escaped while unescaped slash within quotes is left as is when escaping is enabled." + ); + + t::lib::Mocks::mock_preference('QueryRegexEscapeOptions', 'unescape_escaped'); + + $processed_query = Koha::SearchEngine::Elasticsearch::QueryBuilder->_query_regex_escape_process($query_with_regexp); + is( + $processed_query, + "query \\/with regexp\\/", + "Unescaped query regexp has been escaped when unescape escaping is enabled" + ); + + $processed_query = Koha::SearchEngine::Elasticsearch::QueryBuilder->_query_regex_escape_process($query_with_escaped_regex); + is( + $processed_query, + "query /with regexp/", + "Escaped query regexp has been unescaped when unescape escaping is enabled" + ); + + $processed_query = Koha::SearchEngine::Elasticsearch::QueryBuilder->_query_regex_escape_process($query_with_even_preceding_escapes_regex); + is( + $processed_query, + "query \\\\\\/with regexp\\\\\\/", + "Query regexp with even preceding escapes, thus unescaped, has been escaped when unescape escaping is enabled" + ); + + $processed_query = Koha::SearchEngine::Elasticsearch::QueryBuilder->_query_regex_escape_process($query_with_odd_preceding_escapes_regex); + is( + $processed_query, + "query \\\\/with regexp\\\\/", + "Query regexp with odd preceding escapes, thus escaped, has been unescaped when unescape escaping is enabled" + ); + + my $regexp_at_start_of_string_with_odd_preceding_escapes_regex = '\\\\\\/regexp\\\\\\/'; + $processed_query = Koha::SearchEngine::Elasticsearch::QueryBuilder->_query_regex_escape_process($regexp_at_start_of_string_with_odd_preceding_escapes_regex); + is( + $processed_query, + "\\\\/regexp\\\\/", + "Regexp at start of string with odd preceding escapes, thus escaped, has been unescaped when unescape escaping is enabled" + ); + + my $query_with_quoted_escaped_slash = "query with \\/ and \"\\/ within quotes\""; + $processed_query = Koha::SearchEngine::Elasticsearch::QueryBuilder->_query_regex_escape_process($query_with_quoted_escaped_slash); + is( + $processed_query, + "query with / and \"\\/ within quotes\"", + "Escaped slash outside of quotes has been unescaped while escaped slash within quotes is left as is when unescape escaping is enabled." + ); +}; -- 2.11.0