From fdae86da3344be606a7c60eac693f404b14e4262 Mon Sep 17 00:00:00 2001 From: Olli-Antti Kivilahti Date: Tue, 10 Jan 2017 16:28:35 +0200 Subject: [PATCH] Bug 17255 - Upgrade Elastic Search code to work with version 5.1 Builds on top of commit: Bug 17255 - Upgrade Elastic Search code to work with version 2.4+ - rebased wip -Fix data type 'string' to 'keyword' and 'text' -index: not_analyzed deprecated, replaced with type: keyword which is equivalent -store: yes was deprecated, use store: true TODO: Installer bindings to both the debian package install and the raw developer install. A taster in Buug 17851 ZE TEST PLAN 0. Remove existing ES and reinstall ES 5.1 apt-get purge elasticsearch Follow instruction here: https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html 1. Reset Elasticsearch index since facets are hard coded to dynamic search_marc_mappings. 1a. perl -e 'use Koha::SearchEngine::Elasticsearch; Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings();' 1b. If you get trouble, simply DELETE FROM [search_fields|search_marc_to_fields|search_marc_mapping]; and retry 1a. 1c. Destroy elasticsearch index curl -XDELETE localhost:9200/koha_biblios so it can be recreated 2. Recreate the index: perl misc/search_tools/rebuild_elastic_search.pl 2a. Add something to index if your koha.biblio-table is empty 3. Fetch all indexed records and the facet for subject__facet curl -XGET localhost:9200/koha_biblios/data/_search?pretty -d '{ "aggregations": { "my_agg": { "terms": { "field": "subject__facet" } } } }' 4. Run the included tests: perl t/db_dependent/Koha_Elasticsearch_Indexer.t perl t/db_dependent/Koha_Elasticsearch.t perl t/db_dependent/Koha_SearchEngine_Elasticsearch_Search.t Have fun with your new ES 5.1 cluster! --- Koha/SearchEngine/Elasticsearch.pm | 23 ++++++++-------------- Koha/SearchEngine/Elasticsearch/Indexer.pm | 3 ++- Koha/SearchEngine/Elasticsearch/Search.pm | 4 ++-- .../{Koha_Elasticsearch. => Koha_Elasticsearch.t} | 2 +- 4 files changed, 13 insertions(+), 19 deletions(-) rename t/db_dependent/{Koha_Elasticsearch. => Koha_Elasticsearch.t} (94%) diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm index f8b904c..03cc5c3 100644 --- a/Koha/SearchEngine/Elasticsearch.pm +++ b/Koha/SearchEngine/Elasticsearch.pm @@ -148,10 +148,6 @@ sub get_elasticsearch_settings { tokenizer => 'standard', filter => ['lowercase'], }, - default => { - tokenizer => 'keyword', - filter => ['lowercase'], - }, }, } } @@ -176,9 +172,9 @@ sub get_elasticsearch_mappings { data => { properties => { record => { - store => "yes", + store => "true", include_in_all => JSON::false, - type => "string", + type => "text", }, } } @@ -189,7 +185,6 @@ sub get_elasticsearch_mappings { sub { my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_; return if $marc_type ne $marcflavour; - # TODO if this gets any sort of complexity to it, it should # be broken out into its own function. @@ -198,7 +193,7 @@ sub get_elasticsearch_mappings { my $es_type = $type eq 'boolean' ? 'boolean' - : 'string'; + : 'text'; if ($es_type eq 'boolean') { $mappings->{data}{properties}{$name} = _elasticsearch_mapping_for_boolean( $name, $es_type, $facet, $suggestible, $sort, $marc_type ); @@ -209,8 +204,7 @@ sub get_elasticsearch_mappings { if ($facet) { $mappings->{data}{properties}{ $name . '__facet' } = { - type => "string", - index => "not_analyzed", + type => "keyword", }; } if ($suggestible) { @@ -226,13 +220,13 @@ sub get_elasticsearch_mappings { $mappings->{data}{properties}{ $name . '__sort' } = { search_analyzer => "analyser_phrase", analyzer => "analyser_phrase", - type => "string", + type => "text", include_in_all => JSON::false, fields => { phrase => { search_analyzer => "analyser_phrase", analyzer => "analyser_phrase", - type => "string", + type => "text", }, }, }; @@ -272,11 +266,10 @@ sub _elasticsearch_mapping_for_default { phrase => { search_analyzer => "analyser_phrase", analyzer => "analyser_phrase", - type => "string", + type => "text", }, raw => { - type => "string", - index => "not_analyzed", + type => "keyword", } }, }; diff --git a/Koha/SearchEngine/Elasticsearch/Indexer.pm b/Koha/SearchEngine/Elasticsearch/Indexer.pm index 172580a..e5babee 100644 --- a/Koha/SearchEngine/Elasticsearch/Indexer.pm +++ b/Koha/SearchEngine/Elasticsearch/Indexer.pm @@ -77,7 +77,8 @@ sub update_index { ) ); } - my $array_ref = $from->to_array; + + #print Data::Dumper::Dumper( $from->to_array ); $self->store->bag->add_many($from); $self->store->bag->commit; return 1; diff --git a/Koha/SearchEngine/Elasticsearch/Search.pm b/Koha/SearchEngine/Elasticsearch/Search.pm index 6e3ca3b..d3cd294 100644 --- a/Koha/SearchEngine/Elasticsearch/Search.pm +++ b/Koha/SearchEngine/Elasticsearch/Search.pm @@ -257,8 +257,8 @@ sub count_auth_use { my $query = { query => { - filtered => { - query => { match_all => {} }, + bool => { +# query => { match_all => {} }, filter => { term => { an => $authid } } } } diff --git a/t/db_dependent/Koha_Elasticsearch. b/t/db_dependent/Koha_Elasticsearch.t similarity index 94% rename from t/db_dependent/Koha_Elasticsearch. rename to t/db_dependent/Koha_Elasticsearch.t index bad08d2..eb24cbb 100644 --- a/t/db_dependent/Koha_Elasticsearch. +++ b/t/db_dependent/Koha_Elasticsearch.t @@ -20,4 +20,4 @@ use warnings; use Test::More tests => 1; # last test to print -use_ok('Koha::Search::Engine::Elasticsearch'); +use_ok('Koha::SearchEngine::Elasticsearch'); -- 2.7.4