From b139b29db4760ca4f64f1792b2d2e5dac4fcc70e Mon Sep 17 00:00:00 2001 From: Ere Maijala Date: Sat, 15 Sep 2018 23:39:49 +0300 Subject: [PATCH] Bug 21357: Add elision filtering to default ES index config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To test: 1. Rebuild Elasticsearch index with the new config 2. Add a record with "l'avion" in the title 3. Verify that the record can be found with both "l'avion" and "avion" Signed-off-by: Séverine QUEUNE Signed-off-by: Björn Nylén Signed-off-by: Michal Denar Signed-off-by: Bouzid Fergani --- admin/searchengine/elasticsearch/index_config.yaml | 27 ++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/admin/searchengine/elasticsearch/index_config.yaml b/admin/searchengine/elasticsearch/index_config.yaml index 7a8d905..1fcf9f2 100644 --- a/admin/searchengine/elasticsearch/index_config.yaml +++ b/admin/searchengine/elasticsearch/index_config.yaml @@ -7,12 +7,16 @@ index: analyzer_phrase: tokenizer: keyword filter: + - elision - icu_folding + # Since the char_filter left apostrophes alone, remove them now + - apostrophe char_filter: - punctuation analyzer_standard: tokenizer: icu_tokenizer filter: + - elision - icu_folding analyzer_stdno: tokenizer: whitespace @@ -24,6 +28,7 @@ index: icu_folding_normalizer: type: custom filter: + - elision - icu_folding nfkc_cf_normalizer: type: custom @@ -32,7 +37,25 @@ index: # The punctuation filter is used to remove any punctuation chars in fields that don't use icu_tokenizer. punctuation: type: pattern_replace - # The pattern contains all ASCII punctuation characters. - pattern: '([\x00-\x1F,\x21-\x2F,\x3A-\x40,\x5B-\x60,\x7B-\x89,\x8B,\x8D,\x8F,\x90-\x99,\x9B,\x9D,\xA0-\xBF,\xD7,\xF7])' + # The pattern contains all ASCII punctuation characters apart from apostrophe (\x27) since it's needed + # for the elision filter (char_filter always runs before other filters). + pattern: '([\x00-\x1F,\x21-\x26,\x28-\x2F,\x3A-\x40,\x5B-\x60,\x7B-\x89,\x8B,\x8D,\x8F,\x90-\x99,\x9B,\x9D,\xA0-\xBF,\xD7,\xF7])' + replacement: '' + filter: + elision: + type: elision + articles: + - 'c' + - 'd' + - 'j' + - 'l' + - 'm' + - 'n' + - 'qu' + - 's' + - 't' + apostrophe: + type: pattern_replace + pattern: "'" replacement: '' index.mapping.total_fields.limit: 10000 -- 2.7.4