View | Details | Raw Unified | Return to bug 25669
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-8 / +8 lines)
Lines 104-109 sub get_elasticsearch { Link Here
104
        $self->{elasticsearch} = Search::Elasticsearch->new(
104
        $self->{elasticsearch} = Search::Elasticsearch->new(
105
            $self->get_elasticsearch_params()
105
            $self->get_elasticsearch_params()
106
        );
106
        );
107
        # TODO Remove no_type when support for Elasticsearch 6 is dropped
108
        $self->{no_type} = C4::Context->config('elasticsearch_no_type');
107
    }
109
    }
108
    return $self->{elasticsearch};
110
    return $self->{elasticsearch};
109
}
111
}
Lines 189-197 sub get_elasticsearch_mappings { Link Here
189
    if (!defined $all_mappings{$self->index}) {
191
    if (!defined $all_mappings{$self->index}) {
190
        $sort_fields{$self->index} = {};
192
        $sort_fields{$self->index} = {};
191
        # Clone the general mapping to break ties with the original hash
193
        # Clone the general mapping to break ties with the original hash
192
        my $mappings = {
194
        my $mappings = clone(_get_elasticsearch_field_config('general', ''));
193
            data => clone(_get_elasticsearch_field_config('general', ''))
194
        };
195
        my $marcflavour = lc C4::Context->preference('marcflavour');
195
        my $marcflavour = lc C4::Context->preference('marcflavour');
196
        $self->_foreach_mapping(
196
        $self->_foreach_mapping(
197
            sub {
197
            sub {
Lines 214-238 sub get_elasticsearch_mappings { Link Here
214
                }
214
                }
215
215
216
                if ($search) {
216
                if ($search) {
217
                    $mappings->{data}{properties}{$name} = _get_elasticsearch_field_config('search', $es_type);
217
                    $mappings->{properties}{$name} = _get_elasticsearch_field_config('search', $es_type);
218
                }
218
                }
219
219
220
                if ($facet) {
220
                if ($facet) {
221
                    $mappings->{data}{properties}{ $name . '__facet' } = _get_elasticsearch_field_config('facet', $es_type);
221
                    $mappings->{properties}{ $name . '__facet' } = _get_elasticsearch_field_config('facet', $es_type);
222
                }
222
                }
223
                if ($suggestible) {
223
                if ($suggestible) {
224
                    $mappings->{data}{properties}{ $name . '__suggestion' } = _get_elasticsearch_field_config('suggestible', $es_type);
224
                    $mappings->{properties}{ $name . '__suggestion' } = _get_elasticsearch_field_config('suggestible', $es_type);
225
                }
225
                }
226
                # Sort is a bit special as it can be true, false, undef.
226
                # Sort is a bit special as it can be true, false, undef.
227
                # We care about "true" or "undef",
227
                # We care about "true" or "undef",
228
                # "undef" means to do the default thing, which is make it sortable.
228
                # "undef" means to do the default thing, which is make it sortable.
229
                if (!defined $sort || $sort) {
229
                if (!defined $sort || $sort) {
230
                    $mappings->{data}{properties}{ $name . '__sort' } = _get_elasticsearch_field_config('sort', $es_type);
230
                    $mappings->{properties}{ $name . '__sort' } = _get_elasticsearch_field_config('sort', $es_type);
231
                    $sort_fields{$self->index}{$name} = 1;
231
                    $sort_fields{$self->index}{$name} = 1;
232
                }
232
                }
233
            }
233
            }
234
        );
234
        );
235
        $mappings->{data}{properties}{ 'match-heading' } = _get_elasticsearch_field_config('search', 'text') if $self->index eq 'authorities';
235
        $mappings->{properties}{ 'match-heading' } = _get_elasticsearch_field_config('search', 'text') if $self->index eq 'authorities';
236
        $all_mappings{$self->index} = $mappings;
236
        $all_mappings{$self->index} = $mappings;
237
    }
237
    }
238
    $self->sort_fields(\%{$sort_fields{$self->index}});
238
    $self->sort_fields(\%{$sort_fields{$self->index}});
(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-23 / +46 lines)
Lines 115-125 sub update_index { Link Here
115
    if (@body) {
115
    if (@body) {
116
        try{
116
        try{
117
            my $elasticsearch = $self->get_elasticsearch();
117
            my $elasticsearch = $self->get_elasticsearch();
118
            $response = $elasticsearch->bulk(
118
            #TODO Remove check for no_type and use bulk without type when support for Elasticsearch 6 is dropped
119
                index => $self->index_name,
119
            if ($self->{no_type}) {
120
                type => 'data', # is just hard coded in Indexer.pm?
120
                $response = $elasticsearch->bulk(
121
                body => \@body
121
                    index => $self->index_name,
122
            );
122
                    body => \@body
123
                );
124
            }
125
            else {
126
                $response = $elasticsearch->bulk(
127
                    index => $self->index_name,
128
                    type => 'data',
129
                    body => \@body
130
                );
131
            }
123
            if ($response->{errors}) {
132
            if ($response->{errors}) {
124
                carp "One or more ElasticSearch errors occurred when indexing documents";
133
                carp "One or more ElasticSearch errors occurred when indexing documents";
125
            }
134
            }
Lines 250-273 sub update_mappings { Link Here
250
    my $elasticsearch = $self->get_elasticsearch();
259
    my $elasticsearch = $self->get_elasticsearch();
251
    my $mappings = $self->get_elasticsearch_mappings();
260
    my $mappings = $self->get_elasticsearch_mappings();
252
261
253
    foreach my $type (keys %{$mappings}) {
262
    try {
254
        try {
263
        #TODO Remove check for no_type and use put_mapping without type when support for Elasticsearch 6 is dropped
264
        if ($self->{no_type}) {
265
            my $response = $elasticsearch->indices->put_mapping(
266
                index => $self->index_name,
267
                body => $mappings
268
            );
269
        }
270
        else {
255
            my $response = $elasticsearch->indices->put_mapping(
271
            my $response = $elasticsearch->indices->put_mapping(
256
                index => $self->index_name,
272
                index => $self->index_name,
257
                type => $type,
273
                type => 'data',
258
                body => {
274
                body => {
259
                    $type => $mappings->{$type}
275
                    data => $mappings
260
                }
276
                }
261
            );
277
            );
262
        } catch {
278
        }
263
            $self->set_index_status_recreate_required();
279
    } catch {
264
            my $reason = $_[0]->{vars}->{body}->{error}->{reason};
280
        $self->set_index_status_recreate_required();
265
            my $index_name = $self->index_name;
281
        my $reason = $_[0]->{vars}->{body}->{error}->{reason};
266
            Koha::Exception->throw(
282
        my $index_name = $self->index_name;
267
                error => "Unable to update mappings for index \"$index_name\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
283
        Koha::Exception->throw(
268
            );
284
            error => "Unable to update mappings for index \"$index_name\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
269
        };
285
        );
270
    }
286
    };
271
    $self->set_index_status_ok();
287
    $self->set_index_status_ok();
272
}
288
}
273
289
Lines 346-356 sub delete_index { Link Here
346
362
347
    my $elasticsearch = $self->get_elasticsearch();
363
    my $elasticsearch = $self->get_elasticsearch();
348
    my @body = map { { delete => { _id => "$_" } } } @{$biblionums};
364
    my @body = map { { delete => { _id => "$_" } } } @{$biblionums};
349
    my $result = $elasticsearch->bulk(
365
    #TODO Remove check for no_type and use bulk without type when support for Elasticsearch 6 is dropped
350
        index => $self->index_name,
366
    my $result = $self->{no_type} ?
351
        type => 'data',
367
        $elasticsearch->bulk(
352
        body => \@body,
368
            index => $self->index_name,
353
    );
369
            body => \@body,
370
        )
371
        :
372
        $elasticsearch->bulk(
373
            index => $self->index_name,
374
            type => 'data',
375
            body => \@body,
376
        );
354
    if ($result->{errors}) {
377
    if ($result->{errors}) {
355
        croak "An Elasticsearch error occurred during bulk delete";
378
        croak "An Elasticsearch error occurred during bulk delete";
356
    }
379
    }
(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-2 / +2 lines)
Lines 577-583 sub build_authorities_query_compat { Link Here
577
577
578
        $m = exists $koha_to_index_name->{$m} ? $koha_to_index_name->{$m} : $m;
578
        $m = exists $koha_to_index_name->{$m} ? $koha_to_index_name->{$m} : $m;
579
        push @indexes, $m;
579
        push @indexes, $m;
580
        warn "Unknown search field $m in marclist" unless (defined $mappings->{data}->{properties}->{$m} || $m eq '' || $m eq 'match-heading');
580
        warn "Unknown search field $m in marclist" unless (defined $mappings->{properties}->{$m} || $m eq '' || $m eq 'match-heading');
581
    }
581
    }
582
    for ( my $i = 0 ; $i < @$value ; $i++ ) {
582
    for ( my $i = 0 ; $i < @$value ; $i++ ) {
583
        next unless $value->[$i]; #clean empty form values, ES doesn't like undefined searches
583
        next unless $value->[$i]; #clean empty form values, ES doesn't like undefined searches
Lines 1123-1129 sub _sort_field { Link Here
1123
    my ($self, $f) = @_;
1123
    my ($self, $f) = @_;
1124
1124
1125
    my $mappings = $self->get_elasticsearch_mappings();
1125
    my $mappings = $self->get_elasticsearch_mappings();
1126
    my $textField = defined $mappings->{data}{properties}{$f}{type} && $mappings->{data}{properties}{$f}{type} eq 'text';
1126
    my $textField = defined $mappings->{properties}{$f}{type} && $mappings->{properties}{$f}{type} eq 'text';
1127
    if (!defined $self->sort_fields()->{$f} || $self->sort_fields()->{$f}) {
1127
    if (!defined $self->sort_fields()->{$f} || $self->sort_fields()->{$f}) {
1128
        $f .= '__sort';
1128
        $f .= '__sort';
1129
    } else {
1129
    } else {
(-)a/etc/koha-conf.xml (+2 lines)
Lines 177-182 Link Here
177
      Resetting mappings will override any changes made in the Search engine configuration UI.
177
      Resetting mappings will override any changes made in the Search engine configuration UI.
178
 -->
178
 -->
179
 <!-- <elasticsearch_index_mappings>__KOHA_CONF_DIR__/searchengine/elasticsearch/mappings.yaml</elasticsearch_index_mappings> -->
179
 <!-- <elasticsearch_index_mappings>__KOHA_CONF_DIR__/searchengine/elasticsearch/mappings.yaml</elasticsearch_index_mappings> -->
180
 <!-- Uncomment the following line if you have Elasticsearch 7 or higher. TODO Remove when support for Elasticsearch 6 is dropped. -->
181
 <!-- <elasticsearch_no_type>1</elasticsearch_no_type> -->
180
182
181
 <interlibrary_loans>
183
 <interlibrary_loans>
182
     <!-- Path to where Illbackends are located on the system
184
     <!-- Path to where Illbackends are located on the system
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t (-1 / +1 lines)
Lines 135-141 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
135
    # test reading mappings
135
    # test reading mappings
136
    my $es = Koha::SearchEngine::Elasticsearch->new( {index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX} );
136
    my $es = Koha::SearchEngine::Elasticsearch->new( {index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX} );
137
    $mappings = $es->get_elasticsearch_mappings();
137
    $mappings = $es->get_elasticsearch_mappings();
138
    is( $mappings->{data}{properties}{isbn__sort}{index}, 'false', 'Field mappings parsed correctly' );
138
    is( $mappings->{properties}{isbn__sort}{index}, 'false', 'Field mappings parsed correctly' );
139
};
139
};
140
140
141
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
141
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t (-44 / +42 lines)
Lines 39-88 $se->mock( 'get_elasticsearch_mappings', sub { Link Here
39
    my %all_mappings;
39
    my %all_mappings;
40
40
41
    my $mappings = {
41
    my $mappings = {
42
        data => {
42
        properties => {
43
            properties => {
43
            title => {
44
                title => {
44
                type => 'text'
45
                    type => 'text'
45
            },
46
                },
46
            title__sort => {
47
                title__sort => {
47
                type => 'text'
48
                    type => 'text'
48
            },
49
                },
49
            subject => {
50
                subject => {
50
                type => 'text',
51
                    type => 'text',
51
                facet => 1
52
                    facet => 1
52
            },
53
                },
53
            'subject-heading-thesaurus' => {
54
                'subject-heading-thesaurus' => {
54
                type => 'text',
55
                    type => 'text',
55
                facet => 1
56
                    facet => 1
56
            },
57
                },
57
            itemnumber => {
58
                itemnumber => {
58
                type => 'integer'
59
                    type => 'integer'
59
            },
60
                },
60
            sortablenumber => {
61
                sortablenumber => {
61
                type => 'integer'
62
                    type => 'integer'
62
            },
63
                },
63
            sortablenumber__sort => {
64
                sortablenumber__sort => {
64
                type => 'integer'
65
                    type => 'integer'
65
            },
66
                },
66
            heading => {
67
                heading => {
67
                type => 'text'
68
                    type => 'text'
68
            },
69
                },
69
            'heading-main' => {
70
                'heading-main' => {
70
                type => 'text'
71
                    type => 'text'
71
            },
72
                },
72
            heading__sort => {
73
                heading__sort => {
73
                type => 'text'
74
                    type => 'text'
74
            },
75
                },
75
            match => {
76
                match => {
76
                type => 'text'
77
                    type => 'text'
77
            },
78
                },
78
            'match-heading' => {
79
                'match-heading' => {
79
                type => 'text'
80
                    type => 'text'
80
            },
81
                },
81
            'match-heading-see-from' => {
82
                'match-heading-see-from' => {
82
                type => 'text'
83
                    type => 'text'
83
            },
84
                },
85
            }
86
        }
84
        }
87
    };
85
    };
88
    $all_mappings{$self->index} = $mappings;
86
    $all_mappings{$self->index} = $mappings;
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch/Search.t (-21 / +18 lines)
Lines 31-56 $se->mock( 'get_elasticsearch_mappings', sub { Link Here
31
    my %all_mappings;
31
    my %all_mappings;
32
32
33
    my $mappings = {
33
    my $mappings = {
34
        data => {
34
        properties => {
35
            properties => {
35
            title => {
36
                title => {
36
                type => 'text'
37
                    type => 'text'
37
            },
38
                },
38
            title__sort => {
39
                title__sort => {
39
                type => 'text'
40
                    type => 'text'
40
            },
41
                },
41
            subject => {
42
                subject => {
42
                type => 'text'
43
                    type => 'text'
43
            },
44
                },
44
            itemnumber => {
45
                itemnumber => {
45
                type => 'integer'
46
                    type => 'integer'
46
            },
47
                },
47
            sortablenumber => {
48
                sortablenumber => {
48
                type => 'integer'
49
                    type => 'integer'
49
            },
50
                },
50
            sortablenumber__sort => {
51
                sortablenumber__sort => {
51
                type => 'integer'
52
                    type => 'integer'
53
                }
54
            }
52
            }
55
        }
53
        }
56
    };
54
    };
57
- 

Return to bug 25669