Attachment 59939 Details for Bug 17255 – Bug 17255 - Upgrade Elastic Search code to work with version 5.1

[patch] Bug 17255 - Upgrade Elastic Search code to work with version 5.1

Bug-17255---Upgrade-Elastic-Search-code-to-work-wi.patch (text/plain), 7.12 KB, created by Tomás Cohen Arazi (tcohen) on 2017-02-06 16:51:28 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Tomás Cohen Arazi (tcohen)

Created: 2017-02-06 16:51:28 UTC

Size: 7.12 KB

patch

obsolete

>From 7631bbe87c8fb42b12366516ab69873854e004df Mon Sep 17 00:00:00 2001
>From: Olli-Antti Kivilahti <olli-antti.kivilahti@jns.fi>
>Date: Tue, 10 Jan 2017 16:28:35 +0200
>Subject: [PATCH] Bug 17255 - Upgrade Elastic Search code to work with version
> 5.1
>
>Builds on top of commit:
>    Bug 17255 - Upgrade Elastic Search code to work with version 2.4+ - rebased wip
>
>-Fix data type 'string' to 'keyword' and 'text'
>-index: not_analyzed deprecated, replaced with type: keyword which is equivalent
>-store: yes was deprecated, use store: true
>
>TODO: Installer bindings to both the debian package install and the raw developer install.
>A taster in Buug 17851
>
>ZE TEST PLAN
>
>0. Remove existing ES and reinstall ES 5.1
>   apt-get purge elasticsearch
>   Follow instruction here:
>   https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html
>
>1. Reset Elasticsearch index since facets are hard coded to dynamic search_marc_mappings.
>1a. perl -e 'use Koha::SearchEngine::Elasticsearch; Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings();'
>1b. If you get trouble, simply DELETE FROM [search_fields|search_marc_to_fields|search_marc_mapping];
>    and retry 1a.
>1c. Destroy elasticsearch index
>    curl -XDELETE localhost:9200/koha_biblios
>    so it can be recreated
>
>2. Recreate the index:
>   perl misc/search_tools/rebuild_elastic_search.pl
>2a. Add something to index if your koha.biblio-table is empty
>
>3. Fetch all indexed records and the facet for subject__facet
>
>curl -XGET localhost:9200/koha_biblios/data/_search?pretty -d '{
>  "aggregations": {
>    "my_agg": {
>      "terms": {
>        "field": "subject__facet"
>      }
>    }
>  }
>}'
>
>4. Run the included tests:
>   perl t/db_dependent/Koha_Elasticsearch_Indexer.t
>   perl t/db_dependent/Koha_Elasticsearch.t
>   perl t/db_dependent/Koha_SearchEngine_Elasticsearch_Search.t
>
>Have fun with your new ES 5.1 cluster!
>
>Signed-off-by: Nick Clemens <nick@bywatersolutions.com>
>Signed-off-by: Tomas Cohen Arazi <tomascohen@theke.io>
>---
> Koha/SearchEngine/Elasticsearch.pm                 | 23 ++++++++--------------
> Koha/SearchEngine/Elasticsearch/Indexer.pm         |  3 ++-
> Koha/SearchEngine/Elasticsearch/Search.pm          |  4 ++--
> .../{Koha_Elasticsearch. => Koha_Elasticsearch.t}  |  2 +-
> 4 files changed, 13 insertions(+), 19 deletions(-)
> rename t/db_dependent/{Koha_Elasticsearch. => Koha_Elasticsearch.t} (94%)
>
>diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm
>index f8b904cce7..03cc5c3d68 100644
>--- a/Koha/SearchEngine/Elasticsearch.pm
>+++ b/Koha/SearchEngine/Elasticsearch.pm
>@@ -148,10 +148,6 @@ sub get_elasticsearch_settings {
>                         tokenizer => 'standard',
>                         filter    => ['lowercase'],
>                     },
>-                    default => {
>-                        tokenizer => 'keyword',
>-                        filter    => ['lowercase'],
>-                    },
>                 },
>             }
>         }
>@@ -176,9 +172,9 @@ sub get_elasticsearch_mappings {
>         data => {
>             properties => {
>                 record => {
>-                    store          => "yes",
>+                    store          => "true",
>                     include_in_all => JSON::false,
>-                    type           => "string",
>+                    type           => "text",
>                 },
>             }
>         }
>@@ -189,7 +185,6 @@ sub get_elasticsearch_mappings {
>         sub {
>             my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
>             return if $marc_type ne $marcflavour;
>-
>             # TODO if this gets any sort of complexity to it, it should
>             # be broken out into its own function.
> 
>@@ -198,7 +193,7 @@ sub get_elasticsearch_mappings {
>             my $es_type =
>               $type eq 'boolean'
>               ? 'boolean'
>-              : 'string';
>+              : 'text';
> 
>             if ($es_type eq 'boolean') {
>                 $mappings->{data}{properties}{$name} = _elasticsearch_mapping_for_boolean( $name, $es_type, $facet, $suggestible, $sort, $marc_type );
>@@ -209,8 +204,7 @@ sub get_elasticsearch_mappings {
> 
>             if ($facet) {
>                 $mappings->{data}{properties}{ $name . '__facet' } = {
>-                    type  => "string",
>-                    index => "not_analyzed",
>+                    type  => "keyword",
>                 };
>             }
>             if ($suggestible) {
>@@ -226,13 +220,13 @@ sub get_elasticsearch_mappings {
>                 $mappings->{data}{properties}{ $name . '__sort' } = {
>                     search_analyzer => "analyser_phrase",
>                     analyzer  => "analyser_phrase",
>-                    type            => "string",
>+                    type            => "text",
>                     include_in_all  => JSON::false,
>                     fields          => {
>                         phrase => {
>                             search_analyzer => "analyser_phrase",
>                             analyzer  => "analyser_phrase",
>-                            type            => "string",
>+                            type            => "text",
>                         },
>                     },
>                 };
>@@ -272,11 +266,10 @@ sub _elasticsearch_mapping_for_default {
>             phrase => {
>                 search_analyzer => "analyser_phrase",
>                 analyzer        => "analyser_phrase",
>-                type            => "string",
>+                type            => "text",
>             },
>             raw => {
>-                type    => "string",
>-                index   => "not_analyzed",
>+                type    => "keyword",
>             }
>         },
>     };
>diff --git a/Koha/SearchEngine/Elasticsearch/Indexer.pm b/Koha/SearchEngine/Elasticsearch/Indexer.pm
>index 172580a8c7..e5babee535 100644
>--- a/Koha/SearchEngine/Elasticsearch/Indexer.pm
>+++ b/Koha/SearchEngine/Elasticsearch/Indexer.pm
>@@ -77,7 +77,8 @@ sub update_index {
>             )
>         );
>     }
>-    my $array_ref = $from->to_array;
>+
>+    #print Data::Dumper::Dumper( $from->to_array );
>     $self->store->bag->add_many($from);
>     $self->store->bag->commit;
>     return 1;
>diff --git a/Koha/SearchEngine/Elasticsearch/Search.pm b/Koha/SearchEngine/Elasticsearch/Search.pm
>index 6e3ca3b6a4..d3cd29460d 100644
>--- a/Koha/SearchEngine/Elasticsearch/Search.pm
>+++ b/Koha/SearchEngine/Elasticsearch/Search.pm
>@@ -257,8 +257,8 @@ sub count_auth_use {
> 
>     my $query = {
>         query => {
>-            filtered => {
>-                query  => { match_all => {} },
>+            bool => {
>+#                query  => { match_all => {} },
>                 filter => { term      => { an => $authid } }
>             }
>         }
>diff --git a/t/db_dependent/Koha_Elasticsearch. b/t/db_dependent/Koha_Elasticsearch.t
>similarity index 94%
>rename from t/db_dependent/Koha_Elasticsearch.
>rename to t/db_dependent/Koha_Elasticsearch.t
>index bad08d2495..eb24cbbd96 100644
>--- a/t/db_dependent/Koha_Elasticsearch.
>+++ b/t/db_dependent/Koha_Elasticsearch.t
>@@ -20,4 +20,4 @@ use warnings;
> 
> use Test::More tests => 1;                      # last test to print
> 
>-use_ok('Koha::Search::Engine::Elasticsearch');
>+use_ok('Koha::SearchEngine::Elasticsearch');
>-- 
>2.11.1

Actions: View | Diff | Splinter Review

Attachments on bug 17255: 55205 | 55206 | 58690 | 58757 | 59493 | 59494 | 59938 | 59939