|
Lines 19-24
package Koha::SearchEngine::Elasticsearch::Indexer;
Link Here
|
| 19 |
|
19 |
|
| 20 |
use Carp; |
20 |
use Carp; |
| 21 |
use Modern::Perl; |
21 |
use Modern::Perl; |
|
|
22 |
use Try::Tiny; |
| 23 |
use List::Util qw(any); |
| 22 |
use base qw(Koha::SearchEngine::Elasticsearch); |
24 |
use base qw(Koha::SearchEngine::Elasticsearch); |
| 23 |
use Data::Dumper; |
25 |
use Data::Dumper; |
| 24 |
|
26 |
|
|
Lines 26-31
use Data::Dumper;
Link Here
|
| 26 |
use Catmandu::Importer::MARC; |
28 |
use Catmandu::Importer::MARC; |
| 27 |
use Catmandu::Store::ElasticSearch; |
29 |
use Catmandu::Store::ElasticSearch; |
| 28 |
|
30 |
|
|
|
31 |
use Koha::Exceptions; |
| 32 |
use C4::Context; |
| 33 |
|
| 29 |
Koha::SearchEngine::Elasticsearch::Indexer->mk_accessors(qw( store )); |
34 |
Koha::SearchEngine::Elasticsearch::Indexer->mk_accessors(qw( store )); |
| 30 |
|
35 |
|
| 31 |
=head1 NAME |
36 |
=head1 NAME |
|
Lines 57-62
If that's a problem, clone them first.
Link Here
|
| 57 |
|
62 |
|
| 58 |
=cut |
63 |
=cut |
| 59 |
|
64 |
|
|
|
65 |
use constant { |
| 66 |
INDEX_STATUS_OK => 0, |
| 67 |
INDEX_STATUS_REINDEX_REQUIRED => 1, # Not currently used, but could be useful later, for example if can detect when new field or mapping added |
| 68 |
INDEX_STATUS_RECREATE_REQUIRED => 2, |
| 69 |
}; |
| 70 |
|
| 60 |
sub update_index { |
71 |
sub update_index { |
| 61 |
my ($self, $biblionums, $records) = @_; |
72 |
my ($self, $biblionums, $records) = @_; |
| 62 |
|
73 |
|
|
Lines 66-72
sub update_index {
Link Here
|
| 66 |
$self->_sanitise_records($biblionums, $records); |
77 |
$self->_sanitise_records($biblionums, $records); |
| 67 |
} |
78 |
} |
| 68 |
|
79 |
|
| 69 |
$self->ensure_mappings_updated(); |
|
|
| 70 |
$self->bulk_index($records); |
80 |
$self->bulk_index($records); |
| 71 |
return 1; |
81 |
return 1; |
| 72 |
} |
82 |
} |
|
Lines 96-105
sub bulk_index {
Link Here
|
| 96 |
return 1; |
106 |
return 1; |
| 97 |
} |
107 |
} |
| 98 |
|
108 |
|
| 99 |
sub ensure_mappings_updated { |
109 |
sub index_status_ok { |
| 100 |
my ($self) = @_; |
110 |
my ($self, $set) = @_; |
| 101 |
unless ($self->{_mappings_updated}) { |
111 |
return defined $set ? |
| 102 |
$self->update_mappings(); |
112 |
$self->index_status(INDEX_STATUS_OK) : |
|
|
113 |
$self->index_status == INDEX_STATUS_OK; |
| 114 |
} |
| 115 |
|
| 116 |
sub index_status_reindex_required { |
| 117 |
my ($self, $set) = @_; |
| 118 |
return defined $set ? |
| 119 |
$self->index_status(INDEX_STATUS_REINDEX_REQUIRED) : |
| 120 |
$self->index_status == INDEX_STATUS_REINDEX_REQUIRED; |
| 121 |
} |
| 122 |
|
| 123 |
sub index_status_recreate_required { |
| 124 |
my ($self, $set) = @_; |
| 125 |
return defined $set ? |
| 126 |
$self->index_status(INDEX_STATUS_RECREATE_REQUIRED) : |
| 127 |
$self->index_status == INDEX_STATUS_RECREATE_REQUIRED; |
| 128 |
} |
| 129 |
|
| 130 |
sub index_status { |
| 131 |
my ($self, $status) = @_; |
| 132 |
my $key = 'ElasticsearchIndexStatus_' . $self->index; |
| 133 |
|
| 134 |
if (defined $status) { |
| 135 |
unless (any { $status == $_ } ( |
| 136 |
INDEX_STATUS_OK, |
| 137 |
INDEX_STATUS_REINDEX_REQUIRED, |
| 138 |
INDEX_STATUS_RECREATE_REQUIRED, |
| 139 |
) |
| 140 |
) { |
| 141 |
Koha::Exceptions::Exception->throw("Invalid index status: $status"); |
| 142 |
} |
| 143 |
C4::Context->set_preference($key, $status); |
| 144 |
return $status; |
| 145 |
} |
| 146 |
else { |
| 147 |
return C4::Context->preference($key); |
| 103 |
} |
148 |
} |
| 104 |
} |
149 |
} |
| 105 |
|
150 |
|
|
Lines 110-125
sub update_mappings {
Link Here
|
| 110 |
my $mappings = $self->get_elasticsearch_mappings(); |
155 |
my $mappings = $self->get_elasticsearch_mappings(); |
| 111 |
|
156 |
|
| 112 |
foreach my $type (keys %{$mappings}) { |
157 |
foreach my $type (keys %{$mappings}) { |
| 113 |
my $response = $elasticsearch->indices->put_mapping( |
158 |
try { |
| 114 |
index => $conf->{index_name}, |
159 |
my $response = $elasticsearch->indices->put_mapping( |
| 115 |
type => $type, |
160 |
index => $conf->{index_name}, |
| 116 |
body => { |
161 |
type => $type, |
| 117 |
$type => $mappings->{$type} |
162 |
body => { |
| 118 |
} |
163 |
$type => $mappings->{$type} |
| 119 |
); |
164 |
} |
| 120 |
# TODO: process response, produce errors etc |
165 |
); |
|
|
166 |
} catch { |
| 167 |
$self->index_status_recreate_required(1); |
| 168 |
my $reason = $_[0]->{vars}->{body}->{error}->{reason}; |
| 169 |
Koha::Exceptions::Exception->throw( |
| 170 |
error => "Unable to update mappings for index \"$conf->{index_name}\". Reason was: \"$reason\". Index needs to be recreated and reindexed", |
| 171 |
); |
| 172 |
}; |
| 121 |
} |
173 |
} |
| 122 |
$self->{_mappings_updated} = 1; |
174 |
$self->index_status_ok(1); |
| 123 |
} |
175 |
} |
| 124 |
|
176 |
|
| 125 |
=head2 $indexer->update_index_background($biblionums, $records) |
177 |
=head2 $indexer->update_index_background($biblionums, $records) |
|
Lines 179-186
sub delete_index_background {
Link Here
|
| 179 |
|
231 |
|
| 180 |
=head2 $indexer->drop_index(); |
232 |
=head2 $indexer->drop_index(); |
| 181 |
|
233 |
|
| 182 |
Drops the index from the elasticsearch server. Calling C<update_index> |
234 |
Drops the index from the elasticsearch server. |
| 183 |
after this will recreate it again. |
|
|
| 184 |
|
235 |
|
| 185 |
=cut |
236 |
=cut |
| 186 |
|
237 |
|
|
Lines 189-196
sub drop_index {
Link Here
|
| 189 |
if ($self->index_exists) { |
240 |
if ($self->index_exists) { |
| 190 |
my $conf = $self->get_elasticsearch_params(); |
241 |
my $conf = $self->get_elasticsearch_params(); |
| 191 |
my $elasticsearch = $self->get_elasticsearch(); |
242 |
my $elasticsearch = $self->get_elasticsearch(); |
| 192 |
my $response = $elasticsearch->indices->delete(index => $conf->{index_name}); |
243 |
$elasticsearch->indices->delete(index => $conf->{index_name}); |
| 193 |
# TODO: Handle response? Convert errors to exceptions/die |
|
|
| 194 |
} |
244 |
} |
| 195 |
} |
245 |
} |
| 196 |
|
246 |
|
|
Lines 199-211
sub create_index {
Link Here
|
| 199 |
my $conf = $self->get_elasticsearch_params(); |
249 |
my $conf = $self->get_elasticsearch_params(); |
| 200 |
my $settings = $self->get_elasticsearch_settings(); |
250 |
my $settings = $self->get_elasticsearch_settings(); |
| 201 |
my $elasticsearch = $self->get_elasticsearch(); |
251 |
my $elasticsearch = $self->get_elasticsearch(); |
| 202 |
my $response = $elasticsearch->indices->create( |
252 |
$elasticsearch->indices->create( |
| 203 |
index => $conf->{index_name}, |
253 |
index => $conf->{index_name}, |
| 204 |
body => { |
254 |
body => { |
| 205 |
settings => $settings |
255 |
settings => $settings |
| 206 |
} |
256 |
} |
| 207 |
); |
257 |
); |
| 208 |
# TODO: Handle response? Convert errors to exceptions/die |
258 |
$self->update_mappings(); |
| 209 |
} |
259 |
} |
| 210 |
|
260 |
|
| 211 |
sub index_exists { |
261 |
sub index_exists { |