From 1f32600d0a629f2a425db4658468344f011c9ed0 Mon Sep 17 00:00:00 2001 From: Ere Maijala Date: Sat, 15 Sep 2018 23:39:49 +0300 Subject: [PATCH] Bug 21357: Add elision filtering to default ES index config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To test: 1. Rebuild Elasticsearch index with the new config 2. Add a record with "l'avion" in the title 3. Verify that the record can be found with both "l'avion" and "avion" Signed-off-by: Séverine QUEUNE --- admin/searchengine/elasticsearch/index_config.yaml | 27 ++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/admin/searchengine/elasticsearch/index_config.yaml b/admin/searchengine/elasticsearch/index_config.yaml index dce27bc..f47fb22 100644 --- a/admin/searchengine/elasticsearch/index_config.yaml +++ b/admin/searchengine/elasticsearch/index_config.yaml @@ -7,12 +7,16 @@ index: analyser_phrase: tokenizer: keyword filter: + - elision - icu_folding + # Since the char_filter left apostrophes alone, remove them now + - apostrophe char_filter: - punctuation analyser_standard: tokenizer: icu_tokenizer filter: + - elision - icu_folding analyser_stdno: tokenizer: whitespace @@ -24,6 +28,7 @@ index: normalizer_keyword: type: custom filter: + - elision - icu_folding my_normalizer: type: custom @@ -32,6 +37,24 @@ index: # The punctuation filter is used to remove any punctuation chars in fields that don't use icu_tokenizer. punctuation: type: pattern_replace - # The pattern contains all ASCII punctuation characters. - pattern: '([\x00-\x1F,\x21-\x2F,\x3A-\x40,\x5B-\x60,\x7B-\x89,\x8B,\x8D,\x8F,\x90-\x99,\x9B,\x9D,\xA0-\xBF,\xD7,\xF7])' + # The pattern contains all ASCII punctuation characters apart from apostrophe (\x27) since it's needed + # for the elision filter (char_filter always runs before other filters). + pattern: '([\x00-\x1F,\x21-\x26,\x28-\x2F,\x3A-\x40,\x5B-\x60,\x7B-\x89,\x8B,\x8D,\x8F,\x90-\x99,\x9B,\x9D,\xA0-\xBF,\xD7,\xF7])' + replacement: '' + filter: + elision: + type: elision + articles: + - 'c' + - 'd' + - 'j' + - 'l' + - 'm' + - 'n' + - 'qu' + - 's' + - 't' + apostrophe: + type: pattern_replace + pattern: "'" replacement: '' -- 2.1.4