View | Details | Raw Unified | Return to bug 20073
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-120 / +89 lines)
Lines 34-41 use Search::Elasticsearch; Link Here
34
use Try::Tiny;
34
use Try::Tiny;
35
use YAML::Syck;
35
use YAML::Syck;
36
36
37
use Data::Dumper;    # TODO remove
38
39
__PACKAGE__->mk_ro_accessors(qw( index ));
37
__PACKAGE__->mk_ro_accessors(qw( index ));
40
__PACKAGE__->mk_accessors(qw( sort_fields ));
38
__PACKAGE__->mk_accessors(qw( sort_fields ));
41
39
Lines 137-160 A hashref containing the settings is returned. Link Here
137
sub get_elasticsearch_settings {
135
sub get_elasticsearch_settings {
138
    my ($self) = @_;
136
    my ($self) = @_;
139
137
140
    # Ultimately this should come from a file or something, and not be
138
    # Use state to speed up repeated calls
141
    # hardcoded.
139
    state $settings = undef;
142
    my $settings = {
140
    if (!defined $settings) {
143
        index => {
141
        my $config_file = C4::Context->config('elasticsearch_index_config');
144
            analysis => {
142
        $config_file ||= C4::Context->config('intranetdir') . '/etc/searchengine/elasticsearch/index_config.yaml';
145
                analyzer => {
143
        $settings = LoadFile( $config_file );
146
                    analyser_phrase => {
144
    }
147
                        tokenizer => 'icu_tokenizer',
145
148
                        filter    => ['icu_folding'],
149
                    },
150
                    analyser_standard => {
151
                        tokenizer => 'icu_tokenizer',
152
                        filter    => ['icu_folding'],
153
                    },
154
                },
155
            }
156
        }
157
    };
158
    return $settings;
146
    return $settings;
159
}
147
}
160
148
Lines 170-285 created. Link Here
170
sub get_elasticsearch_mappings {
158
sub get_elasticsearch_mappings {
171
    my ($self) = @_;
159
    my ($self) = @_;
172
160
173
    # TODO cache in the object?
161
    # Use state to speed up repeated calls
174
    my $mappings = {
162
    state %all_mappings;
175
        data => {
163
    state %sort_fields;
176
            _all => {type => "string", analyzer => "analyser_standard"},
164
177
            properties => {
165
    if (!defined $all_mappings{$self->index}) {
178
                record => {
166
        $sort_fields{$self->index} = {};
179
                    store          => "true",
167
        my $mappings = {
180
                    include_in_all => JSON::false,
168
            data => _get_elasticsearch_mapping('general', '')
181
                    type           => "text",
169
        };
182
                },
170
        my $marcflavour = lc C4::Context->preference('marcflavour');
183
            }
171
        $self->_foreach_mapping(
184
        }
172
            sub {
185
    };
173
                my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
186
    my %sort_fields;
174
                return if $marc_type ne $marcflavour;
187
    my $marcflavour = lc C4::Context->preference('marcflavour');
175
                # TODO if this gets any sort of complexity to it, it should
188
    $self->_foreach_mapping(
176
                # be broken out into its own function.
189
        sub {
177
190
            my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
178
                # TODO be aware of date formats, but this requires pre-parsing
191
            return if $marc_type ne $marcflavour;
179
                # as ES will simply reject anything with an invalid date.
192
            # TODO if this gets any sort of complexity to it, it should
180
                my $es_type = 'text';
193
            # be broken out into its own function.
181
                if ($type eq 'boolean') {
194
182
                    $es_type = 'boolean';
195
            # TODO be aware of date formats, but this requires pre-parsing
183
                } elsif ($type eq 'number' || $type eq 'sum') {
196
            # as ES will simply reject anything with an invalid date.
184
                    $es_type = 'integer';
197
            my $es_type =
185
                } elsif ($type eq 'isbn' || $type eq 'stdno') {
198
              $type eq 'boolean'
186
                    $es_type = 'stdno';
199
              ? 'boolean'
187
                }
200
              : 'text';
201
202
            if ($es_type eq 'boolean') {
203
                $mappings->{data}{properties}{$name} = _elasticsearch_mapping_for_boolean( $name, $es_type, $facet, $suggestible, $sort, $marc_type );
204
                return; #Boolean cannot have facets nor sorting nor suggestions
205
            } else {
206
                $mappings->{data}{properties}{$name} = _elasticsearch_mapping_for_default( $name, $es_type, $facet, $suggestible, $sort, $marc_type );
207
            }
208
188
209
            if ($facet) {
189
                $mappings->{data}{properties}{$name} = _get_elasticsearch_mapping('search', $es_type);
210
                $mappings->{data}{properties}{ $name . '__facet' } = {
190
211
                    type  => "keyword",
191
                if ($facet) {
212
                };
192
                    $mappings->{data}{properties}{ $name . '__facet' } = _get_elasticsearch_mapping('facet', $es_type);
213
            }
193
                }
214
            if ($suggestible) {
194
                if ($suggestible) {
215
                $mappings->{data}{properties}{ $name . '__suggestion' } = {
195
                    $mappings->{data}{properties}{ $name . '__suggestion' } = _get_elasticsearch_mapping('suggestible', $es_type);
216
                    type => 'completion',
196
                }
217
                    analyzer => 'simple',
197
                # Sort is a bit special as it can be true, false, undef.
218
                    search_analyzer => 'simple',
198
                # We care about "true" or "undef",
219
                };
199
                # "undef" means to do the default thing, which is make it sortable.
220
            }
200
                if (!defined $sort || $sort) {
221
            # Sort is a bit special as it can be true, false, undef.
201
                    $mappings->{data}{properties}{ $name . '__sort' } = _get_elasticsearch_mapping('sort', $es_type);
222
            # We care about "true" or "undef",
202
                    $sort_fields{$self->index}{$name} = 1;
223
            # "undef" means to do the default thing, which is make it sortable.
203
                }
224
            if ($sort || !defined $sort) {
225
                $mappings->{data}{properties}{ $name . '__sort' } = {
226
                    search_analyzer => "analyser_phrase",
227
                    analyzer  => "analyser_phrase",
228
                    type            => "text",
229
                    include_in_all  => JSON::false,
230
                    fields          => {
231
                        phrase => {
232
                            type            => "keyword",
233
                        },
234
                    },
235
                };
236
                $sort_fields{$name} = 1;
237
            }
204
            }
238
        }
205
        );
239
    );
206
        $all_mappings{$self->index} = $mappings;
240
    $self->sort_fields(\%sort_fields);
207
    }
241
    return $mappings;
208
    $self->sort_fields(\%{$sort_fields{$self->index}});
209
210
    return $all_mappings{$self->index};
242
}
211
}
243
212
244
=head2 _elasticsearch_mapping_for_*
213
=head2 _get_elasticsearch_mapping
245
214
246
Get the ES mappings for the given data type or a special mapping case
215
Get the ES mappings for the given purpose and data type
247
216
248
Receives the same parameters from the $self->_foreach_mapping() dispatcher
217
$mapping = _get_elasticsearch_mapping('search', 'text');
249
218
250
=cut
219
=cut
251
220
252
sub _elasticsearch_mapping_for_boolean {
221
sub _get_elasticsearch_mapping {
253
    my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
254
222
255
    return {
223
    my ( $purpose, $type ) = @_;
256
        type            => $type,
257
        null_value      => 0,
258
    };
259
}
260
224
261
sub _elasticsearch_mapping_for_default {
225
    # Use state to speed up repeated calls
262
    my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
226
    state $settings = undef;
263
227
    if (!defined $settings) {
264
    return {
228
        my $config_file = C4::Context->config('elasticsearch_field_config');
265
        search_analyzer => "analyser_standard",
229
        $config_file ||= C4::Context->config('intranetdir') . '/etc/searchengine/elasticsearch/field_config.yaml';
266
        analyzer        => "analyser_standard",
230
        $settings = LoadFile( $config_file );
267
        type            => $type,
231
    }
268
        fields          => {
232
269
            phrase => {
233
    if (!defined $settings->{$purpose}) {
270
                search_analyzer => "analyser_phrase",
234
        die "Field purpose $purpose not defined in field config";
271
                analyzer        => "analyser_phrase",
235
    }
272
                type            => "text",
236
    if ($type eq '') {
273
            },
237
        return $settings->{$purpose};
274
            raw => {
238
    }
275
                type    => "keyword",
239
    if (defined $settings->{$purpose}{$type}) {
276
            }
240
        return $settings->{$purpose}{$type};
277
        },
241
    }
278
    };
242
    if (defined $settings->{$purpose}{'default'}) {
243
        return $settings->{$purpose}{'default'};
244
    }
245
    return undef;
279
}
246
}
280
247
281
sub reset_elasticsearch_mappings {
248
sub reset_elasticsearch_mappings {
282
    my $mappings_yaml = C4::Context->config('intranetdir') . '/admin/searchengine/elasticsearch/mappings.yaml';
249
    my ( $reset_fields ) = @_;
250
    my $mappings_yaml = C4::Context->config('elasticsearch_index_mappings');
251
    $mappings_yaml ||= C4::Context->config('intranetdir') . '/etc/searchengine/elasticsearch/mappings.yaml';
283
    my $indexes = LoadFile( $mappings_yaml );
252
    my $indexes = LoadFile( $mappings_yaml );
284
253
285
    while ( my ( $index_name, $fields ) = each %$indexes ) {
254
    while ( my ( $index_name, $fields ) = each %$indexes ) {
Lines 344-353 sub get_fixer_rules { Link Here
344
            if ($type eq 'sum' ) {
313
            if ($type eq 'sum' ) {
345
                push @rules, "sum('$name')";
314
                push @rules, "sum('$name')";
346
            }
315
            }
347
            if ($self->sort_fields()->{$name}) {
316
            my $sort_fields = $self->sort_fields();
348
                if ($sort || !defined $sort) {
317
            if (defined $sort_fields->{$name} && $sort_fields->{$name}) {
349
                    push @rules, "marc_map('$marc_field','${name}__sort.\$append', $options)";
318
                push @rules, "marc_map('$marc_field','${name}__sort.\$append')";
350
                }
319
                push @rules, "join_field('${name}__sort',' ')";
351
            }
320
            }
352
        }
321
        }
353
    );
322
    );
(-)a/debian/templates/koha-conf-site.xml.in (+11 lines)
Lines 319-328 __END_SRU_PUBLICSERVER__ Link Here
319
 <plack_max_requests>50</plack_max_requests>
319
 <plack_max_requests>50</plack_max_requests>
320
 <plack_workers>2</plack_workers>
320
 <plack_workers>2</plack_workers>
321
321
322
 <!-- Elasticsearch Configuration -->
322
 <elasticsearch>
323
 <elasticsearch>
323
     <server>localhost:9200</server>
324
     <server>localhost:9200</server>
324
     <index_name>koha___KOHASITE__</index_name>
325
     <index_name>koha___KOHASITE__</index_name>
325
 </elasticsearch>
326
 </elasticsearch>
327
 <!-- Uncomment the following line if you want to override the Elasticsearch default index settings -->
328
 <!-- <elasticsearch_index_config>__KOHA_CONF_DIR__/searchengine/elasticsearch/index_config.yaml</elasticsearch_index_config> -->
329
 <!-- Uncomment the following line if you want to override the Elasticsearch default field settings -->
330
 <!-- <elasticsearch_field_config>__KOHA_CONF_DIR__/searchengine/elasticsearch/field_config.yaml</elasticsearch_field_config> -->
331
 <!-- Uncomment the following line if you want to override the Elasticsearch index default settings.
332
      Note that any changes made to the mappings file only take effect if you reset the mappings in
333
      by visiting /cgi-bin/koha/admin/searchengine/elasticsearch/mappings.pl?op=reset&i_know_what_i_am_doing=1&reset_fields=1.
334
      Resetting mappings will override any changes made in the Search engine configuration UI.
335
 -->
336
 <!-- <elasticsearch_index_mappings>__KOHA_CONF_DIR__/searchengine/elasticsearch/mappings.yaml</elasticsearch_index_mappings> -->
326
337
327
 <interlibrary_loans>
338
 <interlibrary_loans>
328
     <!-- Path to where Illbackends are located on the system
339
     <!-- Path to where Illbackends are located on the system
(-)a/etc/koha-conf.xml (+11 lines)
Lines 150-159 __PAZPAR2_TOGGLE_XML_POST__ Link Here
150
 <plack_max_requests>50</plack_max_requests>
150
 <plack_max_requests>50</plack_max_requests>
151
 <plack_workers>2</plack_workers>
151
 <plack_workers>2</plack_workers>
152
152
153
 <!-- Elasticsearch Configuration -->
153
 <elasticsearch>
154
 <elasticsearch>
154
     <server>localhost:9200</server>
155
     <server>localhost:9200</server>
155
     <index_name>koha___DB_NAME__</index_name>
156
     <index_name>koha___DB_NAME__</index_name>
156
 </elasticsearch>
157
 </elasticsearch>
158
 <!-- Uncomment the following line if you want to override the Elasticsearch default index settings -->
159
 <!-- <elasticsearch_index_config>__KOHA_CONF_DIR__/searchengine/elasticsearch/index_config.yaml</elasticsearch_index_config> -->
160
 <!-- Uncomment the following line if you want to override the Elasticsearch default field settings -->
161
 <!-- <elasticsearch_field_config>__KOHA_CONF_DIR__/searchengine/elasticsearch/field_config.yaml</elasticsearch_field_config> -->
162
 <!-- Uncomment the following line if you want to override the Elasticsearch index default settings.
163
      Note that any changes made to the mappings file only take effect if you reset the mappings in
164
      by visiting /cgi-bin/koha/admin/searchengine/elasticsearch/mappings.pl?op=reset&i_know_what_i_am_doing=1&reset_fields=1.
165
      Resetting mappings will override any changes made in the Search engine configuration UI.
166
 -->
167
 <!-- <elasticsearch_index_mappings>__KOHA_CONF_DIR__/searchengine/elasticsearch/mappings.yaml</elasticsearch_index_mappings> -->
157
168
158
 <interlibrary_loans>
169
 <interlibrary_loans>
159
     <!-- Path to where Illbackends are located on the system
170
     <!-- Path to where Illbackends are located on the system
(-)a/etc/searchengine/elasticsearch/field_config.yaml (+61 lines)
Line 0 Link Here
1
---
2
# General field configuration
3
general:
4
  _all:
5
    type: string
6
    analyzer: analyser_standard
7
  properties:
8
    record:
9
      store: true
10
      type: text
11
# Search fields
12
search:
13
  boolean:
14
    type: boolean
15
    null_value: false
16
  integer:
17
    type: integer
18
    null_value: 0
19
  stdno:
20
    type: text
21
    analyzer: analyser_stdno
22
    search_analyzer: analyser_stdno
23
    fields:
24
      phrase:
25
        type: text
26
        analyzer: analyser_stdno
27
        search_analyzer: analyser_stdno
28
      raw:
29
        type: keyword
30
    copy_to: _all
31
  default:
32
    type: text
33
    analyzer: analyser_standard
34
    search_analyzer: analyser_standard
35
    fields:
36
      phrase:
37
        type: text
38
        analyzer: analyser_phrase
39
        search_analyzer: analyser_phrase
40
      raw:
41
        type: keyword
42
    copy_to: _all
43
# Facets
44
facet:
45
  default:
46
    type: keyword
47
# Suggestible
48
suggestible:
49
  default:
50
    type: completion
51
    analyzer: simple
52
    search_analyzer: simple
53
# Sort
54
sort:
55
  default:
56
    type: text
57
    analyzer: analyser_phrase
58
    search_analyzer: analyser_phrase
59
    fields:
60
      phrase:
61
        type: keyword
(-)a/etc/searchengine/elasticsearch/index_config.yaml (+34 lines)
Line 0 Link Here
1
---
2
# Index configuration that defines how different analyzers work.
3
index:
4
  analysis:
5
    analyzer:
6
      # Phrase analyzer is used for phrases (phrase match, sorting)
7
      analyser_phrase:
8
        tokenizer: keyword
9
        filter:
10
          - icu_folding
11
        char_filter:
12
          - punctuation
13
      analyser_standard:
14
        tokenizer: icu_tokenizer
15
        filter:
16
          - icu_folding
17
      analyser_stdno:
18
        tokenizer: whitespace
19
        filter:
20
          - icu_folding
21
        char_filter:
22
          - punctuation
23
    normalizer:
24
      normalizer_keyword:
25
        type: custom
26
        filter:
27
          - icu_folding
28
    char_filter:
29
      # The punctuation filter is used to remove any punctuation chars in fields that don't use icu_tokenizer.
30
      punctuation:
31
        type: pattern_replace
32
        # The pattern contains all ASCII punctuation characters.
33
        pattern: '([\x00-\x1F,\x21-\x2F,\x3A-\x40,\x5B-\x60,\x7B-\x89,\x8B,\x8D,\x8F,\x90-\x99,\x9B,\x9D,\xA0-\xBF,\xD7,\xF7])'
34
        replacement: ''
(-)a/admin/searchengine/elasticsearch/mappings.yaml (-3 / +4 lines)
Lines 1-4 Link Here
1
---
1
---
2
# Basic mappings from MARC fields to Elasticsearch fields.
2
authorities:
3
authorities:
3
  Corporate-name-see-also-from:
4
  Corporate-name-see-also-from:
4
    label: Corporate-name-see-also-from
5
    label: Corporate-name-see-also-from
Lines 2162-2168 biblios: Link Here
2162
        marc_type: unimarc
2163
        marc_type: unimarc
2163
        sort: ~
2164
        sort: ~
2164
        suggestible: ''
2165
        suggestible: ''
2165
    type: ''
2166
    type: 'stdno'
2166
  isbn:
2167
  isbn:
2167
    label: isbn
2168
    label: isbn
2168
    mappings:
2169
    mappings:
Lines 2181-2187 biblios: Link Here
2181
        marc_type: unimarc
2182
        marc_type: unimarc
2182
        sort: ~
2183
        sort: ~
2183
        suggestible: ''
2184
        suggestible: ''
2184
    type: ''
2185
    type: 'isbn'
2185
  issn:
2186
  issn:
2186
    label: issn
2187
    label: issn
2187
    mappings:
2188
    mappings:
Lines 2200-2206 biblios: Link Here
2200
        marc_type: unimarc
2201
        marc_type: unimarc
2201
        sort: ~
2202
        sort: ~
2202
        suggestible: ''
2203
        suggestible: ''
2203
    type: ''
2204
    type: 'stdno'
2204
  issues:
2205
  issues:
2205
    label: issues
2206
    label: issues
2206
    mappings:
2207
    mappings:
(-)a/installer/data/mysql/atomicupdate/bug_20073.perl (+8 lines)
Line 0 Link Here
1
$DBversion = 'XXX';  # will be replaced by the RM
2
if( CheckVersion( $DBversion ) ) {
3
    $dbh->do( "ALTER TABLE search_field CHANGE COLUMN type type ENUM('', 'string', 'date', 'number', 'boolean', 'sum', 'isbn', 'stdno') NOT NULL COMMENT 'what type of data this holds, relevant when storing it in the search engine'" );
4
5
    # Always end with this (adjust the bug info)
6
    SetVersion( $DBversion );
7
    print "Upgrade to $DBversion done (Bug 20073 - Add new types for Elasticsearch fields)\n";
8
}
(-)a/installer/data/mysql/kohastructure.sql (-1 / +1 lines)
Lines 1459-1465 CREATE TABLE `search_field` ( Link Here
1459
  `id` int(11) NOT NULL AUTO_INCREMENT,
1459
  `id` int(11) NOT NULL AUTO_INCREMENT,
1460
  `name` varchar(255) NOT NULL COMMENT 'the name of the field as it will be stored in the search engine',
1460
  `name` varchar(255) NOT NULL COMMENT 'the name of the field as it will be stored in the search engine',
1461
  `label` varchar(255) NOT NULL COMMENT 'the human readable name of the field, for display',
1461
  `label` varchar(255) NOT NULL COMMENT 'the human readable name of the field, for display',
1462
  `type` ENUM('', 'string', 'date', 'number', 'boolean', 'sum') NOT NULL COMMENT 'what type of data this holds, relevant when storing it in the search engine',
1462
  `type` ENUM('', 'string', 'date', 'number', 'boolean', 'sum', 'isbn', 'stdno') NOT NULL COMMENT 'what type of data this holds, relevant when storing it in the search engine',
1463
  PRIMARY KEY (`id`),
1463
  PRIMARY KEY (`id`),
1464
  UNIQUE KEY (`name` (191))
1464
  UNIQUE KEY (`name` (191))
1465
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
1465
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt (+10 lines)
Lines 164-169 a.add, a.delete { Link Here
164
                          [% ELSE %]
164
                          [% ELSE %]
165
                            <option value="sum">Sum</option>
165
                            <option value="sum">Sum</option>
166
                          [% END %]
166
                          [% END %]
167
                          [% IF search_field.type == "isbn" %]
168
                            <option value="isbn" selected="selected">ISBN</option>
169
                          [% ELSE %]
170
                            <option value="isbn">ISBN</option>
171
                          [% END %]
172
                          [% IF search_field.type == "stdno" %]
173
                            <option value="stdno" selected="selected">Std. Number</option>
174
                          [% ELSE %]
175
                            <option value="stdno">Std. Number</option>
176
                          [% END %]
167
                        </select>
177
                        </select>
168
                      </td>
178
                      </td>
169
                    </tr>
179
                    </tr>
(-)a/t/Koha/SearchEngine/Elasticsearch.t (-2 / +25 lines)
Lines 17-23 Link Here
17
17
18
use Modern::Perl;
18
use Modern::Perl;
19
19
20
use Test::More tests => 1;
20
use Test::More tests => 3;
21
use Test::Exception;
21
use Test::Exception;
22
22
23
use t::lib::Mocks;
23
use t::lib::Mocks;
Lines 84-86 subtest '_read_configuration() tests' => sub { Link Here
84
    is( $configuration->{index_name}, 'index', 'Index configuration parsed correctly' );
84
    is( $configuration->{index_name}, 'index', 'Index configuration parsed correctly' );
85
    is_deeply( $configuration->{nodes}, \@servers , 'Server configuration parsed correctly' );
85
    is_deeply( $configuration->{nodes}, \@servers , 'Server configuration parsed correctly' );
86
};
86
};
87
- 
87
88
subtest 'get_elasticsearch_settings() tests' => sub {
89
90
    plan tests => 1;
91
92
    my $settings;
93
94
    # test reading index settings
95
    my $es = Koha::SearchEngine::Elasticsearch->new( {index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX} );
96
    $settings = $es->get_elasticsearch_settings();
97
    is( $settings->{index}{analysis}{analyzer}{analyser_phrase}{tokenizer}, 'keyword', 'Index settings parsed correctly' );
98
};
99
100
subtest 'get_elasticsearch_mappings() tests' => sub {
101
102
    plan tests => 1;
103
104
    my $mappings;
105
106
    # test reading mappings
107
    my $es = Koha::SearchEngine::Elasticsearch->new( {index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX} );
108
    $mappings = $es->get_elasticsearch_mappings();
109
    is( $mappings->{data}{_all}{type}, 'string', 'Field mappings parsed correctly' );
110
};

Return to bug 20073