View | Details | Raw Unified | Return to bug 25669
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-8 / +7 lines)
Lines 104-109 sub get_elasticsearch { Link Here
104
        $self->{elasticsearch} = Search::Elasticsearch->new(
104
        $self->{elasticsearch} = Search::Elasticsearch->new(
105
            $self->get_elasticsearch_params()
105
            $self->get_elasticsearch_params()
106
        );
106
        );
107
        $self->{es_version} = $self->{elasticsearch}->info->{version}{number};
107
    }
108
    }
108
    return $self->{elasticsearch};
109
    return $self->{elasticsearch};
109
}
110
}
Lines 189-197 sub get_elasticsearch_mappings { Link Here
189
    if (!defined $all_mappings{$self->index}) {
190
    if (!defined $all_mappings{$self->index}) {
190
        $sort_fields{$self->index} = {};
191
        $sort_fields{$self->index} = {};
191
        # Clone the general mapping to break ties with the original hash
192
        # Clone the general mapping to break ties with the original hash
192
        my $mappings = {
193
        my $mappings = clone(_get_elasticsearch_field_config('general', ''));
193
            data => clone(_get_elasticsearch_field_config('general', ''))
194
        };
195
        my $marcflavour = lc C4::Context->preference('marcflavour');
194
        my $marcflavour = lc C4::Context->preference('marcflavour');
196
        $self->_foreach_mapping(
195
        $self->_foreach_mapping(
197
            sub {
196
            sub {
Lines 214-238 sub get_elasticsearch_mappings { Link Here
214
                }
213
                }
215
214
216
                if ($search) {
215
                if ($search) {
217
                    $mappings->{data}{properties}{$name} = _get_elasticsearch_field_config('search', $es_type);
216
                    $mappings->{properties}{$name} = _get_elasticsearch_field_config('search', $es_type);
218
                }
217
                }
219
218
220
                if ($facet) {
219
                if ($facet) {
221
                    $mappings->{data}{properties}{ $name . '__facet' } = _get_elasticsearch_field_config('facet', $es_type);
220
                    $mappings->{properties}{ $name . '__facet' } = _get_elasticsearch_field_config('facet', $es_type);
222
                }
221
                }
223
                if ($suggestible) {
222
                if ($suggestible) {
224
                    $mappings->{data}{properties}{ $name . '__suggestion' } = _get_elasticsearch_field_config('suggestible', $es_type);
223
                    $mappings->{properties}{ $name . '__suggestion' } = _get_elasticsearch_field_config('suggestible', $es_type);
225
                }
224
                }
226
                # Sort is a bit special as it can be true, false, undef.
225
                # Sort is a bit special as it can be true, false, undef.
227
                # We care about "true" or "undef",
226
                # We care about "true" or "undef",
228
                # "undef" means to do the default thing, which is make it sortable.
227
                # "undef" means to do the default thing, which is make it sortable.
229
                if (!defined $sort || $sort) {
228
                if (!defined $sort || $sort) {
230
                    $mappings->{data}{properties}{ $name . '__sort' } = _get_elasticsearch_field_config('sort', $es_type);
229
                    $mappings->{properties}{ $name . '__sort' } = _get_elasticsearch_field_config('sort', $es_type);
231
                    $sort_fields{$self->index}{$name} = 1;
230
                    $sort_fields{$self->index}{$name} = 1;
232
                }
231
                }
233
            }
232
            }
234
        );
233
        );
235
        $mappings->{data}{properties}{ 'match-heading' } = _get_elasticsearch_field_config('search', 'text') if $self->index eq 'authorities';
234
        $mappings->{properties}{ 'match-heading' } = _get_elasticsearch_field_config('search', 'text') if $self->index eq 'authorities';
236
        $all_mappings{$self->index} = $mappings;
235
        $all_mappings{$self->index} = $mappings;
237
    }
236
    }
238
    $self->sort_fields(\%{$sort_fields{$self->index}});
237
    $self->sort_fields(\%{$sort_fields{$self->index}});
(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-22 / +32 lines)
Lines 21-26 use Carp qw( carp croak ); Link Here
21
use Modern::Perl;
21
use Modern::Perl;
22
use Try::Tiny qw( catch try );
22
use Try::Tiny qw( catch try );
23
use List::Util qw( any );
23
use List::Util qw( any );
24
use version;
24
use base qw(Koha::SearchEngine::Elasticsearch);
25
use base qw(Koha::SearchEngine::Elasticsearch);
25
26
26
use Koha::Exceptions;
27
use Koha::Exceptions;
Lines 115-125 sub update_index { Link Here
115
    if (@body) {
116
    if (@body) {
116
        try{
117
        try{
117
            my $elasticsearch = $self->get_elasticsearch();
118
            my $elasticsearch = $self->get_elasticsearch();
118
            $response = $elasticsearch->bulk(
119
            my %es_bulk = (
119
                index => $self->index_name,
120
                index => $self->index_name,
120
                type => 'data', # is just hard coded in Indexer.pm?
121
                body => \@body
121
                body => \@body
122
            );
122
            );
123
            if (version->parse($self->{es_version}) < version->parse('7.0.0')) {
124
                $es_bulk{'type'} = 'data';
125
            }
126
            $response = $elasticsearch->bulk(%es_bulk);
123
            if ($response->{errors}) {
127
            if ($response->{errors}) {
124
                carp "One or more ElasticSearch errors occurred when indexing documents";
128
                carp "One or more ElasticSearch errors occurred when indexing documents";
125
            }
129
            }
Lines 250-273 sub update_mappings { Link Here
250
    my $elasticsearch = $self->get_elasticsearch();
254
    my $elasticsearch = $self->get_elasticsearch();
251
    my $mappings = $self->get_elasticsearch_mappings();
255
    my $mappings = $self->get_elasticsearch_mappings();
252
256
253
    foreach my $type (keys %{$mappings}) {
257
    try {
254
        try {
258
        my %es_mapping = (
255
            my $response = $elasticsearch->indices->put_mapping(
259
            index => $self->index_name,
256
                index => $self->index_name,
260
            body => $mappings
257
                type => $type,
261
        );
258
                body => {
262
        if (version->parse($self->{es_version}) < version->parse('7.0.0')) {
259
                    $type => $mappings->{$type}
263
            if (version->parse($self->{es_version}) >= version->parse('6.8.0') && version->parse($elasticsearch->VERSION) >= version->parse('6.8.0')) {
260
                }
264
                $es_mapping{'include_type_name'} = \1;
261
            );
265
            }
262
        } catch {
266
            $es_mapping{'type'} = 'data';
263
            $self->set_index_status_recreate_required();
267
            $es_mapping{'body'} = {'data' => $mappings};
264
            my $reason = $_[0]->{vars}->{body}->{error}->{reason};
268
        }
265
            my $index_name = $self->index_name;
269
        my $response = $elasticsearch->indices->put_mapping(%es_mapping);
266
            Koha::Exceptions::Exception->throw(
270
    } catch {
267
                error => "Unable to update mappings for index \"$index_name\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
271
        $self->set_index_status_recreate_required();
268
            );
272
        my $reason = $_[0]->{vars}->{body}->{error}->{reason};
269
        };
273
        my $index_name = $self->index_name;
270
    }
274
        Koha::Exceptions::Exception->throw(
275
            error => "Unable to update mappings for index \"$index_name\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
276
        );
277
    };
271
    $self->set_index_status_ok();
278
    $self->set_index_status_ok();
272
}
279
}
273
280
Lines 346-356 sub delete_index { Link Here
346
353
347
    my $elasticsearch = $self->get_elasticsearch();
354
    my $elasticsearch = $self->get_elasticsearch();
348
    my @body = map { { delete => { _id => "$_" } } } @{$biblionums};
355
    my @body = map { { delete => { _id => "$_" } } } @{$biblionums};
349
    my $result = $elasticsearch->bulk(
356
    my %es_bulk = (
350
        index => $self->index_name,
357
        index => $self->index_name,
351
        type => 'data',
352
        body => \@body,
358
        body => \@body,
353
    );
359
    );
360
    if (version->parse($self->{es_version}) < version->parse('7.0.0')) {
361
        $es_bulk{'type'} = 'data';
362
    }
363
    my $result = $elasticsearch->bulk(%es_bulk);
354
    if ($result->{errors}) {
364
    if ($result->{errors}) {
355
        croak "An Elasticsearch error occurred during bulk delete";
365
        croak "An Elasticsearch error occurred during bulk delete";
356
    }
366
    }
(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-2 / +2 lines)
Lines 577-583 sub build_authorities_query_compat { Link Here
577
577
578
        $m = exists $koha_to_index_name->{$m} ? $koha_to_index_name->{$m} : $m;
578
        $m = exists $koha_to_index_name->{$m} ? $koha_to_index_name->{$m} : $m;
579
        push @indexes, $m;
579
        push @indexes, $m;
580
        warn "Unknown search field $m in marclist" unless (defined $mappings->{data}->{properties}->{$m} || $m eq '' || $m eq 'match-heading');
580
        warn "Unknown search field $m in marclist" unless (defined $mappings->{properties}->{$m} || $m eq '' || $m eq 'match-heading');
581
    }
581
    }
582
    for ( my $i = 0 ; $i < @$value ; $i++ ) {
582
    for ( my $i = 0 ; $i < @$value ; $i++ ) {
583
        next unless $value->[$i]; #clean empty form values, ES doesn't like undefined searches
583
        next unless $value->[$i]; #clean empty form values, ES doesn't like undefined searches
Lines 1123-1129 sub _sort_field { Link Here
1123
    my ($self, $f) = @_;
1123
    my ($self, $f) = @_;
1124
1124
1125
    my $mappings = $self->get_elasticsearch_mappings();
1125
    my $mappings = $self->get_elasticsearch_mappings();
1126
    my $textField = defined $mappings->{data}{properties}{$f}{type} && $mappings->{data}{properties}{$f}{type} eq 'text';
1126
    my $textField = defined $mappings->{properties}{$f}{type} && $mappings->{properties}{$f}{type} eq 'text';
1127
    if (!defined $self->sort_fields()->{$f} || $self->sort_fields()->{$f}) {
1127
    if (!defined $self->sort_fields()->{$f} || $self->sort_fields()->{$f}) {
1128
        $f .= '__sort';
1128
        $f .= '__sort';
1129
    } else {
1129
    } else {
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t (-1 / +1 lines)
Lines 135-141 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
135
    # test reading mappings
135
    # test reading mappings
136
    my $es = Koha::SearchEngine::Elasticsearch->new( {index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX} );
136
    my $es = Koha::SearchEngine::Elasticsearch->new( {index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX} );
137
    $mappings = $es->get_elasticsearch_mappings();
137
    $mappings = $es->get_elasticsearch_mappings();
138
    is( $mappings->{data}{properties}{isbn__sort}{index}, 'false', 'Field mappings parsed correctly' );
138
    is( $mappings->{properties}{isbn__sort}{index}, 'false', 'Field mappings parsed correctly' );
139
};
139
};
140
140
141
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
141
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t (-44 / +42 lines)
Lines 39-88 $se->mock( 'get_elasticsearch_mappings', sub { Link Here
39
    my %all_mappings;
39
    my %all_mappings;
40
40
41
    my $mappings = {
41
    my $mappings = {
42
        data => {
42
        properties => {
43
            properties => {
43
            title => {
44
                title => {
44
                type => 'text'
45
                    type => 'text'
45
            },
46
                },
46
            title__sort => {
47
                title__sort => {
47
                type => 'text'
48
                    type => 'text'
48
            },
49
                },
49
            subject => {
50
                subject => {
50
                type => 'text',
51
                    type => 'text',
51
                facet => 1
52
                    facet => 1
52
            },
53
                },
53
            'subject-heading-thesaurus' => {
54
                'subject-heading-thesaurus' => {
54
                type => 'text',
55
                    type => 'text',
55
                facet => 1
56
                    facet => 1
56
            },
57
                },
57
            itemnumber => {
58
                itemnumber => {
58
                type => 'integer'
59
                    type => 'integer'
59
            },
60
                },
60
            sortablenumber => {
61
                sortablenumber => {
61
                type => 'integer'
62
                    type => 'integer'
62
            },
63
                },
63
            sortablenumber__sort => {
64
                sortablenumber__sort => {
64
                type => 'integer'
65
                    type => 'integer'
65
            },
66
                },
66
            heading => {
67
                heading => {
67
                type => 'text'
68
                    type => 'text'
68
            },
69
                },
69
            'heading-main' => {
70
                'heading-main' => {
70
                type => 'text'
71
                    type => 'text'
71
            },
72
                },
72
            heading__sort => {
73
                heading__sort => {
73
                type => 'text'
74
                    type => 'text'
74
            },
75
                },
75
            match => {
76
                match => {
76
                type => 'text'
77
                    type => 'text'
77
            },
78
                },
78
            'match-heading' => {
79
                'match-heading' => {
79
                type => 'text'
80
                    type => 'text'
80
            },
81
                },
81
            'match-heading-see-from' => {
82
                'match-heading-see-from' => {
82
                type => 'text'
83
                    type => 'text'
83
            },
84
                },
85
            }
86
        }
84
        }
87
    };
85
    };
88
    $all_mappings{$self->index} = $mappings;
86
    $all_mappings{$self->index} = $mappings;
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch/Search.t (-21 / +18 lines)
Lines 31-56 $se->mock( 'get_elasticsearch_mappings', sub { Link Here
31
    my %all_mappings;
31
    my %all_mappings;
32
32
33
    my $mappings = {
33
    my $mappings = {
34
        data => {
34
        properties => {
35
            properties => {
35
            title => {
36
                title => {
36
                type => 'text'
37
                    type => 'text'
37
            },
38
                },
38
            title__sort => {
39
                title__sort => {
39
                type => 'text'
40
                    type => 'text'
40
            },
41
                },
41
            subject => {
42
                subject => {
42
                type => 'text'
43
                    type => 'text'
43
            },
44
                },
44
            itemnumber => {
45
                itemnumber => {
45
                type => 'integer'
46
                    type => 'integer'
46
            },
47
                },
47
            sortablenumber => {
48
                sortablenumber => {
48
                type => 'integer'
49
                    type => 'integer'
49
            },
50
                },
50
            sortablenumber__sort => {
51
                sortablenumber__sort => {
51
                type => 'integer'
52
                    type => 'integer'
53
                }
54
            }
52
            }
55
        }
53
        }
56
    };
54
    };
57
- 

Return to bug 25669