From 20a57d76fd8cf4537cc790a243e33c6d566dc22c Mon Sep 17 00:00:00 2001 From: Nick Clemens Date: Thu, 2 Nov 2017 16:52:53 +0000 Subject: [PATCH] Bug 17661 - Ending punctuation causes duplicate facets To test: 1 - Do search and pull up some records 2 - Edit some of the records to have authors like: Date, C.J. Date, C.j. Date, C.J . 3 - Edit the records to have some series statments like: 830 $aDate, C.J. ;$v5 830 $aDate, C.J. ; $v5 830 $aDate, C.J.; $v5 4 - Search again and note you have 3 facets each for authr and series 5 - Apply patch 6 - Repeat 7 - Now you get 2 facets for author, period not removed when following Upper case immediately, is otherwise 8 - Now you should have a single series facet 9 - Switch search engine to ES (index before applying patch) 10 - Note facets are separate again 11 - Reset mappings and reindex perl misc/search_tools/rebuild_elasticsearch -v -r 12 - Repeat search, facets combined as above --- C4/Search.pm | 8 +++----- admin/searchengine/elasticsearch/field_config.yaml | 1 + admin/searchengine/elasticsearch/index_config.yaml | 6 ++++++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/C4/Search.pm b/C4/Search.pm index a9448e483b..6103af2d04 100644 --- a/C4/Search.pm +++ b/C4/Search.pm @@ -588,7 +588,6 @@ sub getRecords { } } ); - # This sorts the facets into alphabetical order if (@facets_loop) { foreach my $f (@facets_loop) { @@ -626,7 +625,6 @@ sub _get_facets_from_records { my $jmax = $size > $facets_maxrecs ? $facets_maxrecs : $size; - for ( my $j = 0 ; $j < $jmax ; $j++ ) { my $marc_record = new_record_from_zebra ( @@ -661,9 +659,7 @@ facets for Zebra). sub _get_facets_data_from_record { my ( $marc_record, $facets, $facets_counter ) = @_; - for my $facet (@$facets) { - my @used_datas = (); foreach my $tag ( @{ $facet->{ tags } } ) { @@ -680,6 +676,7 @@ sub _get_facets_data_from_record { next if $field->indicator(1) eq 'z'; my $data = $field->as_string( $subfield_letters, $facet->{ sep } ); + $data =~ s/\s*(?textContent; + $facet_value =~ s/\s*(?{ $facet_value } = $term->getAttribute( 'occur' ); + $facets->{ $facet_value } = ( defined $facets->{$facet_value} ) ? $facets->{ $facet_value } + $term->getAttribute( 'occur' ) : $term->getAttribute( 'occur' ); } return $facets; diff --git a/admin/searchengine/elasticsearch/field_config.yaml b/admin/searchengine/elasticsearch/field_config.yaml index 8723a0713c..cbb53fcb70 100644 --- a/admin/searchengine/elasticsearch/field_config.yaml +++ b/admin/searchengine/elasticsearch/field_config.yaml @@ -54,6 +54,7 @@ search: facet: default: type: keyword + normalizer: facet_normalizer # Suggestible suggestible: default: diff --git a/admin/searchengine/elasticsearch/index_config.yaml b/admin/searchengine/elasticsearch/index_config.yaml index 7a8d9052b4..7700986f14 100644 --- a/admin/searchengine/elasticsearch/index_config.yaml +++ b/admin/searchengine/elasticsearch/index_config.yaml @@ -28,6 +28,8 @@ index: nfkc_cf_normalizer: type: custom char_filter: icu_normalizer + facet_normalizer: + char_filter: facet char_filter: # The punctuation filter is used to remove any punctuation chars in fields that don't use icu_tokenizer. punctuation: @@ -35,4 +37,8 @@ index: # The pattern contains all ASCII punctuation characters. pattern: '([\x00-\x1F,\x21-\x2F,\x3A-\x40,\x5B-\x60,\x7B-\x89,\x8B,\x8D,\x8F,\x90-\x99,\x9B,\x9D,\xA0-\xBF,\xD7,\xF7])' replacement: '' + facet: + type: pattern_replace + pattern: '\s*(?