View | Details | Raw Unified | Return to bug 19893
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-20 / +70 lines)
Lines 19-24 package Koha::SearchEngine::Elasticsearch::Indexer; Link Here
19
19
20
use Carp;
20
use Carp;
21
use Modern::Perl;
21
use Modern::Perl;
22
use Try::Tiny;
23
use List::Util qw(any);
22
use base qw(Koha::SearchEngine::Elasticsearch);
24
use base qw(Koha::SearchEngine::Elasticsearch);
23
use Data::Dumper;
25
use Data::Dumper;
24
26
Lines 26-31 use Data::Dumper; Link Here
26
use Catmandu::Importer::MARC;
28
use Catmandu::Importer::MARC;
27
use Catmandu::Store::ElasticSearch;
29
use Catmandu::Store::ElasticSearch;
28
30
31
use Koha::Exceptions;
32
use C4::Context;
33
29
Koha::SearchEngine::Elasticsearch::Indexer->mk_accessors(qw( store ));
34
Koha::SearchEngine::Elasticsearch::Indexer->mk_accessors(qw( store ));
30
35
31
=head1 NAME
36
=head1 NAME
Lines 57-62 If that's a problem, clone them first. Link Here
57
62
58
=cut
63
=cut
59
64
65
use constant {
66
    INDEX_STATUS_OK => 0,
67
    INDEX_STATUS_REINDEX_REQUIRED => 1, # Not currently used, but could be useful later, for example if can detect when new field or mapping added
68
    INDEX_STATUS_RECREATE_REQUIRED => 2,
69
};
70
60
sub update_index {
71
sub update_index {
61
    my ($self, $biblionums, $records) = @_;
72
    my ($self, $biblionums, $records) = @_;
62
73
Lines 66-72 sub update_index { Link Here
66
        $self->_sanitise_records($biblionums, $records);
77
        $self->_sanitise_records($biblionums, $records);
67
    }
78
    }
68
79
69
    $self->ensure_mappings_updated();
70
    $self->bulk_index($records);
80
    $self->bulk_index($records);
71
    return 1;
81
    return 1;
72
}
82
}
Lines 96-105 sub bulk_index { Link Here
96
    return 1;
106
    return 1;
97
}
107
}
98
108
99
sub ensure_mappings_updated {
109
sub index_status_ok {
100
    my ($self) = @_;
110
    my ($self, $set) = @_;
101
    unless ($self->{_mappings_updated}) {
111
    return defined $set ?
102
        $self->update_mappings();
112
        $self->index_status(INDEX_STATUS_OK) :
113
        $self->index_status == INDEX_STATUS_OK;
114
}
115
116
sub index_status_reindex_required {
117
    my ($self, $set) = @_;
118
    return defined $set ?
119
        $self->index_status(INDEX_STATUS_REINDEX_REQUIRED) :
120
        $self->index_status == INDEX_STATUS_REINDEX_REQUIRED;
121
}
122
123
sub index_status_recreate_required {
124
    my ($self, $set) = @_;
125
    return defined $set ?
126
        $self->index_status(INDEX_STATUS_RECREATE_REQUIRED) :
127
        $self->index_status == INDEX_STATUS_RECREATE_REQUIRED;
128
}
129
130
sub index_status {
131
    my ($self, $status) = @_;
132
    my $key = 'ElasticsearchIndexStatus_' . $self->index;
133
134
    if (defined $status) {
135
        unless (any { $status == $_ } (
136
                INDEX_STATUS_OK,
137
                INDEX_STATUS_REINDEX_REQUIRED,
138
                INDEX_STATUS_RECREATE_REQUIRED,
139
            )
140
        ) {
141
            Koha::Exceptions::Exception->throw("Invalid index status: $status");
142
        }
143
        C4::Context->set_preference($key, $status);
144
        return $status;
145
    }
146
    else {
147
        return C4::Context->preference($key);
103
    }
148
    }
104
}
149
}
105
150
Lines 110-125 sub update_mappings { Link Here
110
    my $mappings = $self->get_elasticsearch_mappings();
155
    my $mappings = $self->get_elasticsearch_mappings();
111
156
112
    foreach my $type (keys %{$mappings}) {
157
    foreach my $type (keys %{$mappings}) {
113
        my $response = $elasticsearch->indices->put_mapping(
158
        try {
114
            index => $conf->{index_name},
159
            my $response = $elasticsearch->indices->put_mapping(
115
            type => $type,
160
                index => $conf->{index_name},
116
            body => {
161
                type => $type,
117
                $type => $mappings->{$type}
162
                body => {
118
            }
163
                    $type => $mappings->{$type}
119
        );
164
                }
120
        # TODO: process response, produce errors etc
165
            );
166
        } catch {
167
            $self->index_status_recreate_required(1);
168
            my $reason = $_[0]->{vars}->{body}->{error}->{reason};
169
            Koha::Exceptions::Exception->throw(
170
                error => "Unable to update mappings for index \"$conf->{index_name}\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
171
            );
172
        };
121
    }
173
    }
122
    $self->{_mappings_updated} = 1;
174
    $self->index_status_ok(1);
123
}
175
}
124
176
125
=head2 $indexer->update_index_background($biblionums, $records)
177
=head2 $indexer->update_index_background($biblionums, $records)
Lines 179-186 sub delete_index_background { Link Here
179
231
180
=head2 $indexer->drop_index();
232
=head2 $indexer->drop_index();
181
233
182
Drops the index from the elasticsearch server. Calling C<update_index>
234
Drops the index from the elasticsearch server.
183
after this will recreate it again.
184
235
185
=cut
236
=cut
186
237
Lines 189-196 sub drop_index { Link Here
189
    if ($self->index_exists) {
240
    if ($self->index_exists) {
190
        my $conf = $self->get_elasticsearch_params();
241
        my $conf = $self->get_elasticsearch_params();
191
        my $elasticsearch = $self->get_elasticsearch();
242
        my $elasticsearch = $self->get_elasticsearch();
192
        my $response = $elasticsearch->indices->delete(index => $conf->{index_name});
243
        $elasticsearch->indices->delete(index => $conf->{index_name});
193
        # TODO: Handle response? Convert errors to exceptions/die
194
    }
244
    }
195
}
245
}
196
246
Lines 199-211 sub create_index { Link Here
199
    my $conf = $self->get_elasticsearch_params();
249
    my $conf = $self->get_elasticsearch_params();
200
    my $settings = $self->get_elasticsearch_settings();
250
    my $settings = $self->get_elasticsearch_settings();
201
    my $elasticsearch = $self->get_elasticsearch();
251
    my $elasticsearch = $self->get_elasticsearch();
202
    my $response = $elasticsearch->indices->create(
252
    $elasticsearch->indices->create(
203
        index => $conf->{index_name},
253
        index => $conf->{index_name},
204
        body => {
254
        body => {
205
            settings => $settings
255
            settings => $settings
206
        }
256
        }
207
    );
257
    );
208
    # TODO: Handle response? Convert errors to exceptions/die
258
    $self->update_mappings();
209
}
259
}
210
260
211
sub index_exists {
261
sub index_exists {
(-)a/admin/searchengine/elasticsearch/mappings.pl (-1 / +46 lines)
Lines 22-30 use C4::Output; Link Here
22
use C4::Auth;
22
use C4::Auth;
23
23
24
use Koha::SearchEngine::Elasticsearch;
24
use Koha::SearchEngine::Elasticsearch;
25
use Koha::SearchEngine::Elasticsearch::Indexer;
25
use Koha::SearchMarcMaps;
26
use Koha::SearchMarcMaps;
26
use Koha::SearchFields;
27
use Koha::SearchFields;
27
28
29
use Try::Tiny;
30
28
my $input = new CGI;
31
my $input = new CGI;
29
my ( $template, $borrowernumber, $cookie ) = get_template_and_user(
32
my ( $template, $borrowernumber, $cookie ) = get_template_and_user(
30
    {   template_name   => 'admin/searchengine/elasticsearch/mappings.tt',
33
    {   template_name   => 'admin/searchengine/elasticsearch/mappings.tt',
Lines 44-49 my $schema = $database->schema; Link Here
44
47
45
my $marc_type = lc C4::Context->preference('marcflavour');
48
my $marc_type = lc C4::Context->preference('marcflavour');
46
49
50
my @index_names = ('biblios', 'authorities');
51
52
my $update_mappings = sub {
53
    for my $index_name (@index_names) {
54
        my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
55
        try {
56
            $indexer->update_mappings();
57
        } catch {
58
            my $conf = $indexer->get_elasticsearch_params();
59
            push @messages, {
60
                type => 'error',
61
                code => 'error_on_update_es_mappings',
62
                message => $_[0],
63
                index => $conf->{index_name},
64
            };
65
        };
66
    }
67
};
68
47
if ( $op eq 'edit' ) {
69
if ( $op eq 'edit' ) {
48
70
49
    $schema->storage->txn_begin;
71
    $schema->storage->txn_begin;
Lines 95-100 if ( $op eq 'edit' ) { Link Here
95
    } else {
117
    } else {
96
        push @messages, { type => 'message', code => 'success_on_update' };
118
        push @messages, { type => 'message', code => 'success_on_update' };
97
        $schema->storage->txn_commit;
119
        $schema->storage->txn_commit;
120
        $update_mappings->();
98
    }
121
    }
99
}
122
}
100
elsif( $op eq 'reset' ) {
123
elsif( $op eq 'reset' ) {
Lines 103-114 elsif( $op eq 'reset' ) { Link Here
103
    if ( $sure ) {
126
    if ( $sure ) {
104
        Koha::SearchMarcMaps->search->delete;
127
        Koha::SearchMarcMaps->search->delete;
105
        Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;
128
        Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;
129
        $update_mappings->();
106
    }
130
    }
107
}
131
}
108
132
109
my @indexes;
133
my @indexes;
110
134
111
for my $index_name (qw| biblios authorities |) {
135
for my $index_name (@index_names) {
136
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
137
    if (!$indexer->index_status_ok) {
138
        my $conf = $indexer->get_elasticsearch_params();
139
        if ($indexer->index_status_reindex_required) {
140
            push @messages, {
141
                type => 'error',
142
                code => 'reindex_required',
143
                index => $conf->{index_name},
144
            };
145
        }
146
        elsif($indexer->index_status_recreate_required) {
147
            push @messages, {
148
                type => 'error',
149
                code => 'recreate_required',
150
                index => $conf->{index_name},
151
            };
152
        }
153
    }
154
}
155
156
for my $index_name (@index_names) {
112
    my $search_fields = Koha::SearchFields->search(
157
    my $search_fields = Koha::SearchFields->search(
113
        { 'search_marc_map.index_name' => $index_name, 'search_marc_map.marc_type' => $marc_type, },
158
        { 'search_marc_map.index_name' => $index_name, 'search_marc_map.marc_type' => $marc_type, },
114
        {   join => { search_marc_to_fields => 'search_marc_map' },
159
        {   join => { search_marc_to_fields => 'search_marc_map' },
(-)a/installer/data/mysql/atomicupdate/bug_19893_elasticsearch_index_status_sysprefs.sql (+2 lines)
Line 0 Link Here
1
INSERT IGNORE INTO systempreferences (variable, value, explanation, options, type) VALUES ('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL);
2
INSERT IGNORE INTO systempreferences (variable, value, explanation, options, type) VALUES ('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL);
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt (+6 lines)
Lines 79-84 a.add, a.delete { Link Here
79
        [% CASE 'error_on_delete' %]
79
        [% CASE 'error_on_delete' %]
80
          An error occurred when deleting the existing mappings. Nothing has been changed!
80
          An error occurred when deleting the existing mappings. Nothing has been changed!
81
          (search field [% m.values.field_name %] with mapping [% m.values.marc_field %].)
81
          (search field [% m.values.field_name %] with mapping [% m.values.marc_field %].)
82
        [% CASE 'error_on_update_es_mappings' %]
83
          An error occurred when updating Elasticsearch index mappings: [% m.message %].
84
        [% CASE 'reindex_required' %]
85
          Index "[% m.index %]" needs to be reindexed
86
        [% CASE 'recreate_required' %]
87
          Index "[% m.index %]" needs to be recreated
82
        [% CASE 'success_on_update' %]
88
        [% CASE 'success_on_update' %]
83
          Mapping updated successfully.
89
          Mapping updated successfully.
84
        [% CASE %]
90
        [% CASE %]
(-)a/misc/search_tools/rebuild_elastic_search.pl (-3 / +5 lines)
Lines 92-99 use MARC::Record; Link Here
92
use Modern::Perl;
92
use Modern::Perl;
93
use Pod::Usage;
93
use Pod::Usage;
94
94
95
use Data::Dumper; # TODO remove
96
97
my $verbose = 0;
95
my $verbose = 0;
98
my $commit = 5000;
96
my $commit = 5000;
99
my ($delete, $help, $man);
97
my ($delete, $help, $man);
Lines 168-173 sub do_reindex { Link Here
168
    elsif (!$indexer->index_exists()) {
166
    elsif (!$indexer->index_exists()) {
169
        # Create index if does not exist
167
        # Create index if does not exist
170
        $indexer->create_index();
168
        $indexer->create_index();
169
    } elsif ($indexer->index_status_ok) {
170
        # Update mapping unless index is some kind of problematic state
171
        $indexer->update_mappings();
172
    } elsif ($indexer->index_status_recreate_required) {
173
        warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;
171
    }
174
    }
172
175
173
    my $count        = 0;
176
    my $count        = 0;
174
- 

Return to bug 19893