View | Details | Raw Unified | Return to bug 24823
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-37 / +38 lines)
Lines 45-51 use Encode qw(encode); Link Here
45
use Business::ISBN;
45
use Business::ISBN;
46
use Scalar::Util qw(looks_like_number);
46
use Scalar::Util qw(looks_like_number);
47
47
48
__PACKAGE__->mk_ro_accessors(qw( index ));
48
__PACKAGE__->mk_ro_accessors(qw( index index_name ));
49
__PACKAGE__->mk_accessors(qw( sort_fields ));
49
__PACKAGE__->mk_accessors(qw( sort_fields ));
50
50
51
# Constants to refer to the standard index names
51
# Constants to refer to the standard index names
Lines 66-80 The name of the index to use, generally 'biblios' or 'authorities'. Link Here
66
66
67
=back
67
=back
68
68
69
=item index_name
70
71
The Elasticsearch index name with Koha instance prefix.
72
73
=back
74
75
69
=head1 FUNCTIONS
76
=head1 FUNCTIONS
70
77
71
=cut
78
=cut
72
79
73
sub new {
80
sub new {
74
    my $class = shift @_;
81
    my $class = shift @_;
75
    my $self = $class->SUPER::new(@_);
82
    my ($params) = @_;
83
76
    # Check for a valid index
84
    # Check for a valid index
77
    Koha::Exceptions::MissingParameter->throw('No index name provided') unless $self->index;
85
    Koha::Exceptions::MissingParameter->throw('No index name provided') unless $params->{index};
86
    my $config = _read_configuration();
87
    $params->{index_name} = $config->{index_name} . '_' . $params->{index};
88
89
    my $self = $class->SUPER::new(@_);
78
    return $self;
90
    return $self;
79
}
91
}
80
92
Lines 122-157 This is configured by the following in the C<config> block in koha-conf.xml: Link Here
122
sub get_elasticsearch_params {
134
sub get_elasticsearch_params {
123
    my ($self) = @_;
135
    my ($self) = @_;
124
136
125
    # Copy the hash so that we're not modifying the original
137
    my $conf;
126
    my $conf = C4::Context->config('elasticsearch');
138
    try {
127
    die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
139
        $conf = _read_configuration();
128
    my $es = { %{ $conf } };
140
    } catch {
129
141
        if ( ref($_) eq 'Koha::Exceptions::Config::MissingEntry' ) {
130
    # Helpfully, the multiple server lines end up in an array for us anyway
142
            croak($_->message);
131
    # if there are multiple ones, but not if there's only one.
143
        }
132
    my $server = $es->{server};
144
    };
133
    delete $es->{server};
145
    # Extract relevant parts of configuration
134
    if ( ref($server) eq 'ARRAY' ) {
146
    my $params = {
135
147
        nodes => $conf->{nodes}
136
        # store it called 'nodes' (which is used by newer Search::Elasticsearch)
148
    };
137
        $es->{nodes} = $server;
149
    $params->{cxn_pool} //= 'Static';
138
    }
139
    elsif ($server) {
140
        $es->{nodes} = [$server];
141
    }
142
    else {
143
        die "No elasticsearch servers were specified in koha-conf.xml.\n";
144
    }
145
    die "No elasticsearch index_name was specified in koha-conf.xml.\n"
146
      if ( !$es->{index_name} );
147
    # Append the name of this particular index to our namespace
148
    $es->{index_name} .= '_' . $self->index;
149
150
    $es->{key_prefix} = 'es_';
151
    $es->{cxn_pool} //= 'Static';
152
    $es->{request_timeout} //= 60;
153
150
154
    return $es;
151
    return $params;
155
}
152
}
156
153
157
=head2 get_elasticsearch_settings
154
=head2 get_elasticsearch_settings
Lines 1231-1239 sub _read_configuration { Link Here
1231
    my $configuration;
1228
    my $configuration;
1232
1229
1233
    my $conf = C4::Context->config('elasticsearch');
1230
    my $conf = C4::Context->config('elasticsearch');
1234
    Koha::Exceptions::Config::MissingEntry->throw(
1231
    unless ( defined $conf ) {
1235
        "Missing 'elasticsearch' block in config file")
1232
        Koha::Exceptions::Config::MissingEntry->throw(
1236
      unless defined $conf;
1233
            "Missing <elasticsearch> entry in koha-conf.xml"
1234
        );
1235
    }
1237
1236
1238
    if ( $conf && $conf->{server} ) {
1237
    if ( $conf && $conf->{server} ) {
1239
        my $nodes = $conf->{server};
1238
        my $nodes = $conf->{server};
Lines 1246-1252 sub _read_configuration { Link Here
1246
    }
1245
    }
1247
    else {
1246
    else {
1248
        Koha::Exceptions::Config::MissingEntry->throw(
1247
        Koha::Exceptions::Config::MissingEntry->throw(
1249
            "Missing 'server' entry in config file for elasticsearch");
1248
            "Missing <elasticsearch>/<server> entry in koha-conf.xml"
1249
        );
1250
    }
1250
    }
1251
1251
1252
    if ( defined $conf->{index_name} ) {
1252
    if ( defined $conf->{index_name} ) {
Lines 1254-1260 sub _read_configuration { Link Here
1254
    }
1254
    }
1255
    else {
1255
    else {
1256
        Koha::Exceptions::Config::MissingEntry->throw(
1256
        Koha::Exceptions::Config::MissingEntry->throw(
1257
            "Missing 'index_name' entry in config file for elasticsearch");
1257
            "Missing <elasticsearch>/<index_name> entry in koha-conf.xml",
1258
        );
1258
    }
1259
    }
1259
1260
1260
    return $configuration;
1261
    return $configuration;
(-)a/Koha/SearchEngine/Elasticsearch/Browse.pm (-2 / +1 lines)
Lines 106-114 sub browse { Link Here
106
106
107
    my $query = $self->_build_query($prefix, $field, $options);
107
    my $query = $self->_build_query($prefix, $field, $options);
108
    my $elasticsearch = $self->get_elasticsearch();
108
    my $elasticsearch = $self->get_elasticsearch();
109
    my $conf = $self->get_elasticsearch_params();
110
    my $results = $elasticsearch->search(
109
    my $results = $elasticsearch->search(
111
        index => $conf->{index_name},
110
        index => $self->index_name,
112
        body => $query
111
        body => $query
113
    );
112
    );
114
113
(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-15 / +9 lines)
Lines 96-103 Arrayref of C<MARC::Record>s. Link Here
96
sub update_index {
96
sub update_index {
97
    my ($self, $biblionums, $records) = @_;
97
    my ($self, $biblionums, $records) = @_;
98
98
99
    my $conf = $self->get_elasticsearch_params();
100
    my $elasticsearch = $self->get_elasticsearch();
101
    my $documents = $self->marc_records_to_documents($records);
99
    my $documents = $self->marc_records_to_documents($records);
102
    my @body;
100
    my @body;
103
101
Lines 113-120 sub update_index { Link Here
113
    }
111
    }
114
    my $response;
112
    my $response;
115
    if (@body) {
113
    if (@body) {
114
        my $elasticsearch = $self->get_elasticsearch();
116
        $response = $elasticsearch->bulk(
115
        $response = $elasticsearch->bulk(
117
            index => $conf->{index_name},
116
            index => $self->index_name,
118
            type => 'data', # is just hard coded in Indexer.pm?
117
            type => 'data', # is just hard coded in Indexer.pm?
119
            body => \@body
118
            body => \@body
120
        );
119
        );
Lines 236-249 failes. Link Here
236
235
237
sub update_mappings {
236
sub update_mappings {
238
    my ($self) = @_;
237
    my ($self) = @_;
239
    my $conf = $self->get_elasticsearch_params();
240
    my $elasticsearch = $self->get_elasticsearch();
238
    my $elasticsearch = $self->get_elasticsearch();
241
    my $mappings = $self->get_elasticsearch_mappings();
239
    my $mappings = $self->get_elasticsearch_mappings();
242
240
243
    foreach my $type (keys %{$mappings}) {
241
    foreach my $type (keys %{$mappings}) {
244
        try {
242
        try {
245
            my $response = $elasticsearch->indices->put_mapping(
243
            my $response = $elasticsearch->indices->put_mapping(
246
                index => $conf->{index_name},
244
                index => $self->index_name,
247
                type => $type,
245
                type => $type,
248
                body => {
246
                body => {
249
                    $type => $mappings->{$type}
247
                    $type => $mappings->{$type}
Lines 252-259 sub update_mappings { Link Here
252
        } catch {
250
        } catch {
253
            $self->set_index_status_recreate_required();
251
            $self->set_index_status_recreate_required();
254
            my $reason = $_[0]->{vars}->{body}->{error}->{reason};
252
            my $reason = $_[0]->{vars}->{body}->{error}->{reason};
253
            my $index_name = $self->index_name;
255
            Koha::Exceptions::Exception->throw(
254
            Koha::Exceptions::Exception->throw(
256
                error => "Unable to update mappings for index \"$conf->{index_name}\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
255
                error => "Unable to update mappings for index \"$index_name\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
257
            );
256
            );
258
        };
257
        };
259
    }
258
    }
Lines 288-298 sub delete_index { Link Here
288
    my ($self, $biblionums) = @_;
287
    my ($self, $biblionums) = @_;
289
288
290
    my $elasticsearch = $self->get_elasticsearch();
289
    my $elasticsearch = $self->get_elasticsearch();
291
    my $conf  = $self->get_elasticsearch_params();
292
293
    my @body = map { { delete => { _id => $_ } } } @{$biblionums};
290
    my @body = map { { delete => { _id => $_ } } } @{$biblionums};
294
    my $result = $elasticsearch->bulk(
291
    my $result = $elasticsearch->bulk(
295
        index => $conf->{index_name},
292
        index => $self->index_name,
296
        type => 'data',
293
        type => 'data',
297
        body => \@body,
294
        body => \@body,
298
    );
295
    );
Lines 322-330 Drops the index from the Elasticsearch server. Link Here
322
sub drop_index {
319
sub drop_index {
323
    my ($self) = @_;
320
    my ($self) = @_;
324
    if ($self->index_exists) {
321
    if ($self->index_exists) {
325
        my $conf = $self->get_elasticsearch_params();
326
        my $elasticsearch = $self->get_elasticsearch();
322
        my $elasticsearch = $self->get_elasticsearch();
327
        $elasticsearch->indices->delete(index => $conf->{index_name});
323
        $elasticsearch->indices->delete(index => $self->index_name);
328
        $self->set_index_status_recreate_required();
324
        $self->set_index_status_recreate_required();
329
    }
325
    }
330
}
326
}
Lines 337-347 Creates the index (including mappings) on the Elasticsearch server. Link Here
337
333
338
sub create_index {
334
sub create_index {
339
    my ($self) = @_;
335
    my ($self) = @_;
340
    my $conf = $self->get_elasticsearch_params();
341
    my $settings = $self->get_elasticsearch_settings();
336
    my $settings = $self->get_elasticsearch_settings();
342
    my $elasticsearch = $self->get_elasticsearch();
337
    my $elasticsearch = $self->get_elasticsearch();
343
    $elasticsearch->indices->create(
338
    $elasticsearch->indices->create(
344
        index => $conf->{index_name},
339
        index => $self->index_name,
345
        body => {
340
        body => {
346
            settings => $settings
341
            settings => $settings
347
        }
342
        }
Lines 358-367 empty string to indicate whether index exists or not. Link Here
358
353
359
sub index_exists {
354
sub index_exists {
360
    my ($self) = @_;
355
    my ($self) = @_;
361
    my $conf = $self->get_elasticsearch_params();
362
    my $elasticsearch = $self->get_elasticsearch();
356
    my $elasticsearch = $self->get_elasticsearch();
363
    return $elasticsearch->indices->exists(
357
    return $elasticsearch->indices->exists(
364
        index => $conf->{index_name},
358
        index => $self->index_name,
365
    );
359
    );
366
}
360
}
367
361
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-8 / +5 lines)
Lines 82-88 Returns Link Here
82
sub search {
82
sub search {
83
    my ($self, $query, $page, $count, %options) = @_;
83
    my ($self, $query, $page, $count, %options) = @_;
84
84
85
    my $params = $self->get_elasticsearch_params();
86
    # 20 is the default number of results per page
85
    # 20 is the default number of results per page
87
    $query->{size} = $count // 20;
86
    $query->{size} = $count // 20;
88
    # ES doesn't want pages, it wants a record to start from.
87
    # ES doesn't want pages, it wants a record to start from.
Lines 95-101 sub search { Link Here
95
    my $elasticsearch = $self->get_elasticsearch();
94
    my $elasticsearch = $self->get_elasticsearch();
96
    my $results = eval {
95
    my $results = eval {
97
        $elasticsearch->search(
96
        $elasticsearch->search(
98
            index => $params->{index_name},
97
            index => $self->index_name,
99
            body => $query
98
            body => $query
100
        );
99
        );
101
    };
100
    };
Lines 117-128 faster than pulling all the data in, usually. Link Here
117
sub count {
116
sub count {
118
    my ( $self, $query ) = @_;
117
    my ( $self, $query ) = @_;
119
    my $elasticsearch = $self->get_elasticsearch();
118
    my $elasticsearch = $self->get_elasticsearch();
120
    my $conf = $self->get_elasticsearch_params();
121
119
122
    # TODO: Probably possible to exclude results
120
    # TODO: Probably possible to exclude results
123
    # and just return number of hits
121
    # and just return number of hits
124
    my $result = $elasticsearch->search(
122
    my $result = $elasticsearch->search(
125
        index => $conf->{index_name},
123
        index => $self->index_name,
126
        body => $query
124
        body => $query
127
    );
125
    );
128
126
Lines 407-422 sub max_result_window { Link Here
407
    my ($self) = @_;
405
    my ($self) = @_;
408
406
409
    my $elasticsearch = $self->get_elasticsearch();
407
    my $elasticsearch = $self->get_elasticsearch();
410
    my $conf = $self->get_elasticsearch_params();
411
408
412
    my $response = $elasticsearch->indices->get_settings(
409
    my $response = $elasticsearch->indices->get_settings(
413
        index => $conf->{index_name},
410
        index => $self->index_name,
414
        flat_settings => 'true',
411
        flat_settings => 'true',
415
        include_defaults => 'true'
412
        include_defaults => 'true'
416
    );
413
    );
417
414
418
    my $max_result_window = $response->{$conf->{index_name}}->{settings}->{'index.max_result_window'};
415
    my $max_result_window = $response->{$self->index_name}->{settings}->{'index.max_result_window'};
419
    $max_result_window //= $response->{$conf->{index_name}}->{defaults}->{'index.max_result_window'};
416
    $max_result_window //= $response->{$self->index_name}->{defaults}->{'index.max_result_window'};
420
417
421
    return $max_result_window;
418
    return $max_result_window;
422
}
419
}
(-)a/t/Koha/SearchEngine/Elasticsearch.t (-3 / +3 lines)
Lines 46-52 subtest '_read_configuration() tests' => sub { Link Here
46
      'Configuration problem, exception thrown';
46
      'Configuration problem, exception thrown';
47
    is(
47
    is(
48
        $@->message,
48
        $@->message,
49
        "Missing 'elasticsearch' block in config file",
49
        "Missing <elasticsearch> entry in koha-conf.xml",
50
        'Exception message is correct'
50
        'Exception message is correct'
51
    );
51
    );
52
52
Lines 59-65 subtest '_read_configuration() tests' => sub { Link Here
59
      'Configuration problem, exception thrown';
59
      'Configuration problem, exception thrown';
60
    is(
60
    is(
61
        $@->message,
61
        $@->message,
62
        "Missing 'server' entry in config file for elasticsearch",
62
        "Missing <elasticsearch>/<server> entry in koha-conf.xml",
63
        'Exception message is correct'
63
        'Exception message is correct'
64
    );
64
    );
65
65
Lines 72-78 subtest '_read_configuration() tests' => sub { Link Here
72
      'Configuration problem, exception thrown';
72
      'Configuration problem, exception thrown';
73
    is(
73
    is(
74
        $@->message,
74
        $@->message,
75
        "Missing 'index_name' entry in config file for elasticsearch",
75
        "Missing <elasticsearch>/<index_name> entry in koha-conf.xml",
76
        'Exception message is correct'
76
        'Exception message is correct'
77
    );
77
    );
78
78
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch/Indexer.t (-5 / +5 lines)
Lines 32-43 use_ok('Koha::SearchEngine::Elasticsearch::Indexer'); Link Here
32
subtest 'create_index() tests' => sub {
32
subtest 'create_index() tests' => sub {
33
    plan tests => 5;
33
    plan tests => 5;
34
    my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
34
    my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
35
    $se->mock( 'get_elasticsearch_params', sub {
35
    $se->mock( '_read_configuration', sub {
36
            my ($self, $sub ) = @_;
36
            my ($self, $sub ) = @_;
37
            my $method = $se->original( 'get_elasticsearch_params' );
37
            my $method = $se->original( '_read_configuration' );
38
            my $params = $method->( $self );
38
            my $conf = $method->( $self );
39
            $params->{index_name} .= '__test';
39
            $conf->{index_name} .= '__test';
40
            return $params;
40
            return $conf;
41
        });
41
        });
42
42
43
    my $indexer;
43
    my $indexer;
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch/Search.t (-2 / +1 lines)
Lines 117-123 SKIP: { Link Here
117
    is ($searcher->max_result_window, 10000, 'By default, max_result_window is 10000');
117
    is ($searcher->max_result_window, 10000, 'By default, max_result_window is 10000');
118
118
119
    $searcher->get_elasticsearch()->indices->put_settings(
119
    $searcher->get_elasticsearch()->indices->put_settings(
120
        index => $searcher->get_elasticsearch_params()->{index_name},
120
        index => $searcher->index_name,
121
        body => {
121
        body => {
122
            'index' => {
122
            'index' => {
123
                'max_result_window' => 12000,
123
                'max_result_window' => 12000,
124
- 

Return to bug 24823