View | Details | Raw Unified | Return to bug 19893
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-20 / +70 lines)
Lines 19-24 package Koha::SearchEngine::Elasticsearch::Indexer; Link Here
19
19
20
use Carp;
20
use Carp;
21
use Modern::Perl;
21
use Modern::Perl;
22
use Try::Tiny;
23
use List::Util qw(any);
22
use base qw(Koha::SearchEngine::Elasticsearch);
24
use base qw(Koha::SearchEngine::Elasticsearch);
23
use Data::Dumper;
25
use Data::Dumper;
24
26
Lines 26-31 use Data::Dumper; Link Here
26
use Catmandu::Importer::MARC;
28
use Catmandu::Importer::MARC;
27
use Catmandu::Store::ElasticSearch;
29
use Catmandu::Store::ElasticSearch;
28
30
31
use Koha::Exceptions;
32
use C4::Context;
33
29
Koha::SearchEngine::Elasticsearch::Indexer->mk_accessors(qw( store ));
34
Koha::SearchEngine::Elasticsearch::Indexer->mk_accessors(qw( store ));
30
35
31
=head1 NAME
36
=head1 NAME
Lines 57-62 If that's a problem, clone them first. Link Here
57
62
58
=cut
63
=cut
59
64
65
use constant {
66
    INDEX_STATUS_OK => 0,
67
    INDEX_STATUS_REINDEX_REQUIRED => 1, # Not currently used, but could be useful later, for example if can detect when new field or mapping added
68
    INDEX_STATUS_RECREATE_REQUIRED => 2,
69
};
70
60
sub update_index {
71
sub update_index {
61
    my ($self, $biblionums, $records) = @_;
72
    my ($self, $biblionums, $records) = @_;
62
73
Lines 66-72 sub update_index { Link Here
66
        $self->_sanitise_records($biblionums, $records);
77
        $self->_sanitise_records($biblionums, $records);
67
    }
78
    }
68
79
69
    $self->ensure_mappings_updated();
70
    $self->bulk_index($records);
80
    $self->bulk_index($records);
71
    return 1;
81
    return 1;
72
}
82
}
Lines 98-107 sub bulk_index { Link Here
98
    return 1;
108
    return 1;
99
}
109
}
100
110
101
sub ensure_mappings_updated {
111
sub index_status_ok {
102
    my ($self) = @_;
112
    my ($self, $set) = @_;
103
    unless ($self->{_mappings_updated}) {
113
    return defined $set ?
104
        $self->update_mappings();
114
        $self->index_status(INDEX_STATUS_OK) :
115
        $self->index_status == INDEX_STATUS_OK;
116
}
117
118
sub index_status_reindex_required {
119
    my ($self, $set) = @_;
120
    return defined $set ?
121
        $self->index_status(INDEX_STATUS_REINDEX_REQUIRED) :
122
        $self->index_status == INDEX_STATUS_REINDEX_REQUIRED;
123
}
124
125
sub index_status_recreate_required {
126
    my ($self, $set) = @_;
127
    return defined $set ?
128
        $self->index_status(INDEX_STATUS_RECREATE_REQUIRED) :
129
        $self->index_status == INDEX_STATUS_RECREATE_REQUIRED;
130
}
131
132
sub index_status {
133
    my ($self, $status) = @_;
134
    my $key = 'ElasticsearchIndexStatus_' . $self->index;
135
136
    if (defined $status) {
137
        unless (any { $status == $_ } (
138
                INDEX_STATUS_OK,
139
                INDEX_STATUS_REINDEX_REQUIRED,
140
                INDEX_STATUS_RECREATE_REQUIRED,
141
            )
142
        ) {
143
            Koha::Exceptions::Exception->throw("Invalid index status: $status");
144
        }
145
        C4::Context->set_preference($key, $status);
146
        return $status;
147
    }
148
    else {
149
        return C4::Context->preference($key);
105
    }
150
    }
106
}
151
}
107
152
Lines 112-127 sub update_mappings { Link Here
112
    my $mappings = $self->get_elasticsearch_mappings();
157
    my $mappings = $self->get_elasticsearch_mappings();
113
158
114
    foreach my $type (keys %{$mappings}) {
159
    foreach my $type (keys %{$mappings}) {
115
        my $response = $elasticsearch->indices->put_mapping(
160
        try {
116
            index => $conf->{index_name},
161
            my $response = $elasticsearch->indices->put_mapping(
117
            type => $type,
162
                index => $conf->{index_name},
118
            body => {
163
                type => $type,
119
                $type => $mappings->{$type}
164
                body => {
120
            }
165
                    $type => $mappings->{$type}
121
        );
166
                }
122
        # TODO: process response, produce errors etc
167
            );
168
        } catch {
169
            $self->index_status_recreate_required(1);
170
            my $reason = $_[0]->{vars}->{body}->{error}->{reason};
171
            Koha::Exceptions::Exception->throw(
172
                error => "Unable to update mappings for index \"$conf->{index_name}\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
173
            );
174
        };
123
    }
175
    }
124
    $self->{_mappings_updated} = 1;
176
    $self->index_status_ok(1);
125
}
177
}
126
178
127
=head2 $indexer->update_index_background($biblionums, $records)
179
=head2 $indexer->update_index_background($biblionums, $records)
Lines 181-188 sub delete_index_background { Link Here
181
233
182
=head2 $indexer->drop_index();
234
=head2 $indexer->drop_index();
183
235
184
Drops the index from the elasticsearch server. Calling C<update_index>
236
Drops the index from the elasticsearch server.
185
after this will recreate it again.
186
237
187
=cut
238
=cut
188
239
Lines 191-198 sub drop_index { Link Here
191
    if ($self->index_exists) {
242
    if ($self->index_exists) {
192
        my $conf = $self->get_elasticsearch_params();
243
        my $conf = $self->get_elasticsearch_params();
193
        my $elasticsearch = $self->get_elasticsearch();
244
        my $elasticsearch = $self->get_elasticsearch();
194
        my $response = $elasticsearch->indices->delete(index => $conf->{index_name});
245
        $elasticsearch->indices->delete(index => $conf->{index_name});
195
        # TODO: Handle response? Convert errors to exceptions/die
196
    }
246
    }
197
}
247
}
198
248
Lines 201-213 sub create_index { Link Here
201
    my $conf = $self->get_elasticsearch_params();
251
    my $conf = $self->get_elasticsearch_params();
202
    my $settings = $self->get_elasticsearch_settings();
252
    my $settings = $self->get_elasticsearch_settings();
203
    my $elasticsearch = $self->get_elasticsearch();
253
    my $elasticsearch = $self->get_elasticsearch();
204
    my $response = $elasticsearch->indices->create(
254
    $elasticsearch->indices->create(
205
        index => $conf->{index_name},
255
        index => $conf->{index_name},
206
        body => {
256
        body => {
207
            settings => $settings
257
            settings => $settings
208
        }
258
        }
209
    );
259
    );
210
    # TODO: Handle response? Convert errors to exceptions/die
260
    $self->update_mappings();
211
}
261
}
212
262
213
sub index_exists {
263
sub index_exists {
(-)a/admin/searchengine/elasticsearch/mappings.pl (-1 / +45 lines)
Lines 23-31 use C4::Output; Link Here
23
use C4::Auth;
23
use C4::Auth;
24
24
25
use Koha::SearchEngine::Elasticsearch;
25
use Koha::SearchEngine::Elasticsearch;
26
use Koha::SearchEngine::Elasticsearch::Indexer;
26
use Koha::SearchMarcMaps;
27
use Koha::SearchMarcMaps;
27
use Koha::SearchFields;
28
use Koha::SearchFields;
28
29
30
use Try::Tiny;
31
29
my $input = new CGI;
32
my $input = new CGI;
30
my ( $template, $borrowernumber, $cookie ) = get_template_and_user(
33
my ( $template, $borrowernumber, $cookie ) = get_template_and_user(
31
    {   template_name   => 'admin/searchengine/elasticsearch/mappings.tt',
34
    {   template_name   => 'admin/searchengine/elasticsearch/mappings.tt',
Lines 45-50 my $schema = $database->schema; Link Here
45
48
46
my $marc_type = lc C4::Context->preference('marcflavour');
49
my $marc_type = lc C4::Context->preference('marcflavour');
47
50
51
my @index_names = ('biblios', 'authorities');
52
53
my $update_mappings = sub {
54
    for my $index_name (@index_names) {
55
        my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
56
        try {
57
            $indexer->update_mappings();
58
        } catch {
59
            my $conf = $indexer->get_elasticsearch_params();
60
            push @messages, {
61
                type => 'error',
62
                code => 'error_on_update_es_mappings',
63
                message => $_[0],
64
                index => $conf->{index_name},
65
            };
66
        };
67
    }
68
};
69
48
if ( $op eq 'edit' ) {
70
if ( $op eq 'edit' ) {
49
71
50
    $schema->storage->txn_begin;
72
    $schema->storage->txn_begin;
Lines 110-115 if ( $op eq 'edit' ) { Link Here
110
    } else {
132
    } else {
111
        push @messages, { type => 'message', code => 'success_on_update' };
133
        push @messages, { type => 'message', code => 'success_on_update' };
112
        $schema->storage->txn_commit;
134
        $schema->storage->txn_commit;
135
        $update_mappings->();
113
    }
136
    }
114
}
137
}
115
elsif( $op eq 'reset_confirmed' ) {
138
elsif( $op eq 'reset_confirmed' ) {
Lines 125-131 elsif( $op eq 'reset_confirm' ) { Link Here
125
148
126
my @indexes;
149
my @indexes;
127
150
128
for my $index_name (qw| biblios authorities |) {
151
for my $index_name (@index_names) {
152
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
153
    if (!$indexer->index_status_ok) {
154
        my $conf = $indexer->get_elasticsearch_params();
155
        if ($indexer->index_status_reindex_required) {
156
            push @messages, {
157
                type => 'error',
158
                code => 'reindex_required',
159
                index => $conf->{index_name},
160
            };
161
        }
162
        elsif($indexer->index_status_recreate_required) {
163
            push @messages, {
164
                type => 'error',
165
                code => 'recreate_required',
166
                index => $conf->{index_name},
167
            };
168
        }
169
    }
170
}
171
172
for my $index_name (@index_names) {
129
    my $search_fields = Koha::SearchFields->search(
173
    my $search_fields = Koha::SearchFields->search(
130
        { 'search_marc_map.index_name' => $index_name, 'search_marc_map.marc_type' => $marc_type, },
174
        { 'search_marc_map.index_name' => $index_name, 'search_marc_map.marc_type' => $marc_type, },
131
        {   join => { search_marc_to_fields => 'search_marc_map' },
175
        {   join => { search_marc_to_fields => 'search_marc_map' },
(-)a/installer/data/mysql/atomicupdate/bug_19893_elasticsearch_index_status_sysprefs.sql (+2 lines)
Line 0 Link Here
1
INSERT IGNORE INTO systempreferences (variable, value, explanation, options, type) VALUES ('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL);
2
INSERT IGNORE INTO systempreferences (variable, value, explanation, options, type) VALUES ('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL);
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt (+6 lines)
Lines 81-86 a.add, a.delete { Link Here
81
          (search field [% m.values.field_name | html %] with mapping [% m.values.marc_field | html %].)
81
          (search field [% m.values.field_name | html %] with mapping [% m.values.marc_field | html %].)
82
        [% CASE 'invalid_field_weight' %]
82
        [% CASE 'invalid_field_weight' %]
83
          Invalid field weight "[% m.weight | html %]", must be a positive decimal number.
83
          Invalid field weight "[% m.weight | html %]", must be a positive decimal number.
84
        [% CASE 'error_on_update_es_mappings' %]
85
          An error occurred when updating Elasticsearch index mappings: [% m.message | html %].
86
        [% CASE 'reindex_required' %]
87
          Index "[% m.index | html %]" needs to be reindexed
88
        [% CASE 'recreate_required' %]
89
          Index "[% m.index | html %]" needs to be recreated
84
        [% CASE 'success_on_update' %]
90
        [% CASE 'success_on_update' %]
85
          Mappings updated successfully.
91
          Mappings updated successfully.
86
        [% CASE 'success_on_reset' %]
92
        [% CASE 'success_on_reset' %]
(-)a/misc/search_tools/rebuild_elastic_search.pl (-1 / +5 lines)
Lines 168-173 sub do_reindex { Link Here
168
    elsif (!$indexer->index_exists()) {
168
    elsif (!$indexer->index_exists()) {
169
        # Create index if does not exist
169
        # Create index if does not exist
170
        $indexer->create_index();
170
        $indexer->create_index();
171
    } elsif ($indexer->index_status_ok) {
172
        # Update mapping unless index is some kind of problematic state
173
        $indexer->update_mappings();
174
    } elsif ($indexer->index_status_recreate_required) {
175
        warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;
171
    }
176
    }
172
177
173
    my $count        = 0;
178
    my $count        = 0;
174
- 

Return to bug 19893