View | Details | Raw Unified | Return to bug 24823
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-37 / +38 lines)
Lines 43-49 use MIME::Base64; Link Here
43
use Encode qw(encode);
43
use Encode qw(encode);
44
use Business::ISBN;
44
use Business::ISBN;
45
45
46
__PACKAGE__->mk_ro_accessors(qw( index ));
46
__PACKAGE__->mk_ro_accessors(qw( index index_name ));
47
__PACKAGE__->mk_accessors(qw( sort_fields ));
47
__PACKAGE__->mk_accessors(qw( sort_fields ));
48
48
49
# Constants to refer to the standard index names
49
# Constants to refer to the standard index names
Lines 64-78 The name of the index to use, generally 'biblios' or 'authorities'. Link Here
64
64
65
=back
65
=back
66
66
67
=item index_name
68
69
The Elasticsearch index name with Koha instance prefix.
70
71
=back
72
73
67
=head1 FUNCTIONS
74
=head1 FUNCTIONS
68
75
69
=cut
76
=cut
70
77
71
sub new {
78
sub new {
72
    my $class = shift @_;
79
    my $class = shift @_;
73
    my $self = $class->SUPER::new(@_);
80
    my ($params) = @_;
81
74
    # Check for a valid index
82
    # Check for a valid index
75
    Koha::Exceptions::MissingParameter->throw('No index name provided') unless $self->index;
83
    Koha::Exceptions::MissingParameter->throw('No index name provided') unless $params->{index};
84
    my $config = _read_configuration();
85
    $params->{index_name} = $config->{index_name} . '_' . $params->{index};
86
87
    my $self = $class->SUPER::new(@_);
76
    return $self;
88
    return $self;
77
}
89
}
78
90
Lines 120-155 This is configured by the following in the C<config> block in koha-conf.xml: Link Here
120
sub get_elasticsearch_params {
132
sub get_elasticsearch_params {
121
    my ($self) = @_;
133
    my ($self) = @_;
122
134
123
    # Copy the hash so that we're not modifying the original
135
    my $conf;
124
    my $conf = C4::Context->config('elasticsearch');
136
    try {
125
    die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
137
        $conf = _read_configuration();
126
    my $es = { %{ $conf } };
138
    } catch {
127
139
        if ( ref($_) eq 'Koha::Exceptions::Config::MissingEntry' ) {
128
    # Helpfully, the multiple server lines end up in an array for us anyway
140
            croak($_->message);
129
    # if there are multiple ones, but not if there's only one.
141
        }
130
    my $server = $es->{server};
142
    };
131
    delete $es->{server};
143
    # Extract relevant parts of configuration
132
    if ( ref($server) eq 'ARRAY' ) {
144
    my $params = {
133
145
        nodes => $conf->{nodes}
134
        # store it called 'nodes' (which is used by newer Search::Elasticsearch)
146
    };
135
        $es->{nodes} = $server;
147
    $params->{cxn_pool} //= 'Static';
136
    }
137
    elsif ($server) {
138
        $es->{nodes} = [$server];
139
    }
140
    else {
141
        die "No elasticsearch servers were specified in koha-conf.xml.\n";
142
    }
143
    die "No elasticsearch index_name was specified in koha-conf.xml.\n"
144
      if ( !$es->{index_name} );
145
    # Append the name of this particular index to our namespace
146
    $es->{index_name} .= '_' . $self->index;
147
148
    $es->{key_prefix} = 'es_';
149
    $es->{cxn_pool} //= 'Static';
150
    $es->{request_timeout} //= 60;
151
148
152
    return $es;
149
    return $params;
153
}
150
}
154
151
155
=head2 get_elasticsearch_settings
152
=head2 get_elasticsearch_settings
Lines 1089-1097 sub _read_configuration { Link Here
1089
    my $configuration;
1086
    my $configuration;
1090
1087
1091
    my $conf = C4::Context->config('elasticsearch');
1088
    my $conf = C4::Context->config('elasticsearch');
1092
    Koha::Exceptions::Config::MissingEntry->throw(
1089
    unless ( defined $conf ) {
1093
        "Missing 'elasticsearch' block in config file")
1090
        Koha::Exceptions::Config::MissingEntry->throw(
1094
      unless defined $conf;
1091
            "Missing <elasticsearch> entry in koha-conf.xml"
1092
        );
1093
    }
1095
1094
1096
    if ( $conf && $conf->{server} ) {
1095
    if ( $conf && $conf->{server} ) {
1097
        my $nodes = $conf->{server};
1096
        my $nodes = $conf->{server};
Lines 1104-1110 sub _read_configuration { Link Here
1104
    }
1103
    }
1105
    else {
1104
    else {
1106
        Koha::Exceptions::Config::MissingEntry->throw(
1105
        Koha::Exceptions::Config::MissingEntry->throw(
1107
            "Missing 'server' entry in config file for elasticsearch");
1106
            "Missing <elasticsearch>/<server> entry in koha-conf.xml"
1107
        );
1108
    }
1108
    }
1109
1109
1110
    if ( defined $conf->{index_name} ) {
1110
    if ( defined $conf->{index_name} ) {
Lines 1112-1118 sub _read_configuration { Link Here
1112
    }
1112
    }
1113
    else {
1113
    else {
1114
        Koha::Exceptions::Config::MissingEntry->throw(
1114
        Koha::Exceptions::Config::MissingEntry->throw(
1115
            "Missing 'index_name' entry in config file for elasticsearch");
1115
            "Missing <elasticsearch>/<index_name> entry in koha-conf.xml",
1116
        );
1116
    }
1117
    }
1117
1118
1118
    return $configuration;
1119
    return $configuration;
(-)a/Koha/SearchEngine/Elasticsearch/Browse.pm (-2 / +1 lines)
Lines 106-114 sub browse { Link Here
106
106
107
    my $query = $self->_build_query($prefix, $field, $options);
107
    my $query = $self->_build_query($prefix, $field, $options);
108
    my $elasticsearch = $self->get_elasticsearch();
108
    my $elasticsearch = $self->get_elasticsearch();
109
    my $conf = $self->get_elasticsearch_params();
110
    my $results = $elasticsearch->search(
109
    my $results = $elasticsearch->search(
111
        index => $conf->{index_name},
110
        index => $self->index_name,
112
        body => $query
111
        body => $query
113
    );
112
    );
114
113
(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-15 / +9 lines)
Lines 96-103 Arrayref of C<MARC::Record>s. Link Here
96
sub update_index {
96
sub update_index {
97
    my ($self, $biblionums, $records) = @_;
97
    my ($self, $biblionums, $records) = @_;
98
98
99
    my $conf = $self->get_elasticsearch_params();
100
    my $elasticsearch = $self->get_elasticsearch();
101
    my $documents = $self->marc_records_to_documents($records);
99
    my $documents = $self->marc_records_to_documents($records);
102
    my @body;
100
    my @body;
103
101
Lines 112-119 sub update_index { Link Here
112
        push @body, $document;
110
        push @body, $document;
113
    }
111
    }
114
    if (@body) {
112
    if (@body) {
113
        my $elasticsearch = $self->get_elasticsearch();
115
        my $response = $elasticsearch->bulk(
114
        my $response = $elasticsearch->bulk(
116
            index => $conf->{index_name},
115
            index => $self->index_name,
117
            type => 'data', # is just hard coded in Indexer.pm?
116
            type => 'data', # is just hard coded in Indexer.pm?
118
            body => \@body
117
            body => \@body
119
        );
118
        );
Lines 236-249 failes. Link Here
236
235
237
sub update_mappings {
236
sub update_mappings {
238
    my ($self) = @_;
237
    my ($self) = @_;
239
    my $conf = $self->get_elasticsearch_params();
240
    my $elasticsearch = $self->get_elasticsearch();
238
    my $elasticsearch = $self->get_elasticsearch();
241
    my $mappings = $self->get_elasticsearch_mappings();
239
    my $mappings = $self->get_elasticsearch_mappings();
242
240
243
    foreach my $type (keys %{$mappings}) {
241
    foreach my $type (keys %{$mappings}) {
244
        try {
242
        try {
245
            my $response = $elasticsearch->indices->put_mapping(
243
            my $response = $elasticsearch->indices->put_mapping(
246
                index => $conf->{index_name},
244
                index => $self->index_name,
247
                type => $type,
245
                type => $type,
248
                body => {
246
                body => {
249
                    $type => $mappings->{$type}
247
                    $type => $mappings->{$type}
Lines 252-259 sub update_mappings { Link Here
252
        } catch {
250
        } catch {
253
            $self->set_index_status_recreate_required();
251
            $self->set_index_status_recreate_required();
254
            my $reason = $_[0]->{vars}->{body}->{error}->{reason};
252
            my $reason = $_[0]->{vars}->{body}->{error}->{reason};
253
            my $index_name = $self->index_name;
255
            Koha::Exceptions::Exception->throw(
254
            Koha::Exceptions::Exception->throw(
256
                error => "Unable to update mappings for index \"$conf->{index_name}\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
255
                error => "Unable to update mappings for index \"$index_name\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
257
            );
256
            );
258
        };
257
        };
259
    }
258
    }
Lines 288-298 sub delete_index { Link Here
288
    my ($self, $biblionums) = @_;
287
    my ($self, $biblionums) = @_;
289
288
290
    my $elasticsearch = $self->get_elasticsearch();
289
    my $elasticsearch = $self->get_elasticsearch();
291
    my $conf  = $self->get_elasticsearch_params();
292
293
    my @body = map { { delete => { _id => $_ } } } @{$biblionums};
290
    my @body = map { { delete => { _id => $_ } } } @{$biblionums};
294
    my $result = $elasticsearch->bulk(
291
    my $result = $elasticsearch->bulk(
295
        index => $conf->{index_name},
292
        index => $self->index_name,
296
        type => 'data',
293
        type => 'data',
297
        body => \@body,
294
        body => \@body,
298
    );
295
    );
Lines 322-330 Drops the index from the Elasticsearch server. Link Here
322
sub drop_index {
319
sub drop_index {
323
    my ($self) = @_;
320
    my ($self) = @_;
324
    if ($self->index_exists) {
321
    if ($self->index_exists) {
325
        my $conf = $self->get_elasticsearch_params();
326
        my $elasticsearch = $self->get_elasticsearch();
322
        my $elasticsearch = $self->get_elasticsearch();
327
        $elasticsearch->indices->delete(index => $conf->{index_name});
323
        $elasticsearch->indices->delete(index => $self->index_name);
328
        $self->set_index_status_recreate_required();
324
        $self->set_index_status_recreate_required();
329
    }
325
    }
330
}
326
}
Lines 337-347 Creates the index (including mappings) on the Elasticsearch server. Link Here
337
333
338
sub create_index {
334
sub create_index {
339
    my ($self) = @_;
335
    my ($self) = @_;
340
    my $conf = $self->get_elasticsearch_params();
341
    my $settings = $self->get_elasticsearch_settings();
336
    my $settings = $self->get_elasticsearch_settings();
342
    my $elasticsearch = $self->get_elasticsearch();
337
    my $elasticsearch = $self->get_elasticsearch();
343
    $elasticsearch->indices->create(
338
    $elasticsearch->indices->create(
344
        index => $conf->{index_name},
339
        index => $self->index_name,
345
        body => {
340
        body => {
346
            settings => $settings
341
            settings => $settings
347
        }
342
        }
Lines 358-367 empty string to indicate whether index exists or not. Link Here
358
353
359
sub index_exists {
354
sub index_exists {
360
    my ($self) = @_;
355
    my ($self) = @_;
361
    my $conf = $self->get_elasticsearch_params();
362
    my $elasticsearch = $self->get_elasticsearch();
356
    my $elasticsearch = $self->get_elasticsearch();
363
    return $elasticsearch->indices->exists(
357
    return $elasticsearch->indices->exists(
364
        index => $conf->{index_name},
358
        index => $self->index_name,
365
    );
359
    );
366
}
360
}
367
361
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-8 / +5 lines)
Lines 82-88 Returns Link Here
82
sub search {
82
sub search {
83
    my ($self, $query, $page, $count, %options) = @_;
83
    my ($self, $query, $page, $count, %options) = @_;
84
84
85
    my $params = $self->get_elasticsearch_params();
86
    # 20 is the default number of results per page
85
    # 20 is the default number of results per page
87
    $query->{size} = $count // 20;
86
    $query->{size} = $count // 20;
88
    # ES doesn't want pages, it wants a record to start from.
87
    # ES doesn't want pages, it wants a record to start from.
Lines 95-101 sub search { Link Here
95
    my $elasticsearch = $self->get_elasticsearch();
94
    my $elasticsearch = $self->get_elasticsearch();
96
    my $results = eval {
95
    my $results = eval {
97
        $elasticsearch->search(
96
        $elasticsearch->search(
98
            index => $params->{index_name},
97
            index => $self->index_name,
99
            body => $query
98
            body => $query
100
        );
99
        );
101
    };
100
    };
Lines 117-128 faster than pulling all the data in, usually. Link Here
117
sub count {
116
sub count {
118
    my ( $self, $query ) = @_;
117
    my ( $self, $query ) = @_;
119
    my $elasticsearch = $self->get_elasticsearch();
118
    my $elasticsearch = $self->get_elasticsearch();
120
    my $conf = $self->get_elasticsearch_params();
121
119
122
    # TODO: Probably possible to exclude results
120
    # TODO: Probably possible to exclude results
123
    # and just return number of hits
121
    # and just return number of hits
124
    my $result = $elasticsearch->search(
122
    my $result = $elasticsearch->search(
125
        index => $conf->{index_name},
123
        index => $self->index_name,
126
        body => $query
124
        body => $query
127
    );
125
    );
128
126
Lines 409-424 sub max_result_window { Link Here
409
    my ($self) = @_;
407
    my ($self) = @_;
410
408
411
    my $elasticsearch = $self->get_elasticsearch();
409
    my $elasticsearch = $self->get_elasticsearch();
412
    my $conf = $self->get_elasticsearch_params();
413
410
414
    my $response = $elasticsearch->indices->get_settings(
411
    my $response = $elasticsearch->indices->get_settings(
415
        index => $conf->{index_name},
412
        index => $self->index_name,
416
        flat_settings => 'true',
413
        flat_settings => 'true',
417
        include_defaults => 'true'
414
        include_defaults => 'true'
418
    );
415
    );
419
416
420
    my $max_result_window = $response->{$conf->{index_name}}->{settings}->{'index.max_result_window'};
417
    my $max_result_window = $response->{$self->index_name}->{settings}->{'index.max_result_window'};
421
    $max_result_window //= $response->{$conf->{index_name}}->{defaults}->{'index.max_result_window'};
418
    $max_result_window //= $response->{$self->index_name}->{defaults}->{'index.max_result_window'};
422
419
423
    return $max_result_window;
420
    return $max_result_window;
424
}
421
}
(-)a/t/Koha/SearchEngine/Elasticsearch.t (-3 / +3 lines)
Lines 46-52 subtest '_read_configuration() tests' => sub { Link Here
46
      'Configuration problem, exception thrown';
46
      'Configuration problem, exception thrown';
47
    is(
47
    is(
48
        $@->message,
48
        $@->message,
49
        "Missing 'elasticsearch' block in config file",
49
        "Missing <elasticsearch> entry in koha-conf.xml",
50
        'Exception message is correct'
50
        'Exception message is correct'
51
    );
51
    );
52
52
Lines 59-65 subtest '_read_configuration() tests' => sub { Link Here
59
      'Configuration problem, exception thrown';
59
      'Configuration problem, exception thrown';
60
    is(
60
    is(
61
        $@->message,
61
        $@->message,
62
        "Missing 'server' entry in config file for elasticsearch",
62
        "Missing <elasticsearch>/<server> entry in koha-conf.xml",
63
        'Exception message is correct'
63
        'Exception message is correct'
64
    );
64
    );
65
65
Lines 72-78 subtest '_read_configuration() tests' => sub { Link Here
72
      'Configuration problem, exception thrown';
72
      'Configuration problem, exception thrown';
73
    is(
73
    is(
74
        $@->message,
74
        $@->message,
75
        "Missing 'index_name' entry in config file for elasticsearch",
75
        "Missing <elasticsearch>/<index_name> entry in koha-conf.xml",
76
        'Exception message is correct'
76
        'Exception message is correct'
77
    );
77
    );
78
78
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch/Indexer.t (-5 / +5 lines)
Lines 32-43 use_ok('Koha::SearchEngine::Elasticsearch::Indexer'); Link Here
32
subtest 'create_index() tests' => sub {
32
subtest 'create_index() tests' => sub {
33
    plan tests => 4;
33
    plan tests => 4;
34
    my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
34
    my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
35
    $se->mock( 'get_elasticsearch_params', sub {
35
    $se->mock( '_read_configuration', sub {
36
            my ($self, $sub ) = @_;
36
            my ($self, $sub ) = @_;
37
            my $method = $se->original( 'get_elasticsearch_params' );
37
            my $method = $se->original( '_read_configuration' );
38
            my $params = $method->( $self );
38
            my $conf = $method->( $self );
39
            $params->{index_name} .= '__test';
39
            $conf->{index_name} .= '__test';
40
            return $params;
40
            return $conf;
41
        });
41
        });
42
42
43
    my $indexer;
43
    my $indexer;
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch/Search.t (-2 / +1 lines)
Lines 117-123 SKIP: { Link Here
117
    is ($searcher->max_result_window, 10000, 'By default, max_result_window is 10000');
117
    is ($searcher->max_result_window, 10000, 'By default, max_result_window is 10000');
118
118
119
    $searcher->get_elasticsearch()->indices->put_settings(
119
    $searcher->get_elasticsearch()->indices->put_settings(
120
        index => $searcher->get_elasticsearch_params()->{index_name},
120
        index => $searcher->index_name,
121
        body => {
121
        body => {
122
            'index' => {
122
            'index' => {
123
                'max_result_window' => 12000,
123
                'max_result_window' => 12000,
124
- 

Return to bug 24823