View | Details | Raw Unified | Return to bug 20589
Collapse All | Expand All

(-)a/Koha/Schema/Result/SearchField.pm (-1 / +5 lines)
Lines 77-83 __PACKAGE__->add_columns( Link Here
77
    is_nullable => 0,
77
    is_nullable => 0,
78
  },
78
  },
79
  "weight",
79
  "weight",
80
  { data_type => "decimal", is_nullable => 1, size => [5, 2] },
80
  { data_type => "decimal", size => [ 5, 2 ], is_nullable => 1 },
81
  "staff_client",
82
  { data_type => "tinyint", size => 1, is_nullable => 0, default => 1 },
83
  "opac",
84
  { data_type => "tinyint", size => 1, is_nullable => 0, default => 1 },
81
);
85
);
82
86
83
=head1 PRIMARY KEY
87
=head1 PRIMARY KEY
(-)a/Koha/Schema/Result/SearchMarcToField.pm (+2 lines)
Lines 71-76 __PACKAGE__->add_columns( Link Here
71
  { data_type => "tinyint", default_value => 0, is_nullable => 1 },
71
  { data_type => "tinyint", default_value => 0, is_nullable => 1 },
72
  "sort",
72
  "sort",
73
  { data_type => "tinyint", is_nullable => 1 },
73
  { data_type => "tinyint", is_nullable => 1 },
74
  "search",
75
  { data_type => "tinyint", size => 1, is_nullable => 0, default => 1 },
74
);
76
);
75
77
76
=head1 PRIMARY KEY
78
=head1 PRIMARY KEY
(-)a/Koha/SearchEngine/Elasticsearch.pm (-8 / +15 lines)
Lines 170-176 sub get_elasticsearch_mappings { Link Here
170
        my $marcflavour = lc C4::Context->preference('marcflavour');
170
        my $marcflavour = lc C4::Context->preference('marcflavour');
171
        $self->_foreach_mapping(
171
        $self->_foreach_mapping(
172
            sub {
172
            sub {
173
                my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
173
                my ( $name, $type, $facet, $suggestible, $sort, $search, $marc_type ) = @_;
174
                return if $marc_type ne $marcflavour;
174
                return if $marc_type ne $marcflavour;
175
                # TODO if this gets any sort of complexity to it, it should
175
                # TODO if this gets any sort of complexity to it, it should
176
                # be broken out into its own function.
176
                # be broken out into its own function.
Lines 185-193 sub get_elasticsearch_mappings { Link Here
185
                } elsif ($type eq 'isbn' || $type eq 'stdno') {
185
                } elsif ($type eq 'isbn' || $type eq 'stdno') {
186
                    $es_type = 'stdno';
186
                    $es_type = 'stdno';
187
                }
187
                }
188
188
                if ($search) {
189
                $mappings->{data}{properties}{$name} = _get_elasticsearch_mapping('search', $es_type);
189
                    $mappings->{data}{properties}{$name} = _get_elasticsearch_mapping('search', $es_type);
190
190
                }
191
                if ($facet) {
191
                if ($facet) {
192
                    $mappings->{data}{properties}{ $name . '__facet' } = _get_elasticsearch_mapping('facet', $es_type);
192
                    $mappings->{data}{properties}{ $name . '__facet' } = _get_elasticsearch_mapping('facet', $es_type);
193
                }
193
                }
Lines 255-265 sub reset_elasticsearch_mappings { Link Here
255
        while ( my ( $field_name, $data ) = each %$fields ) {
255
        while ( my ( $field_name, $data ) = each %$fields ) {
256
            my $field_type = $data->{type};
256
            my $field_type = $data->{type};
257
            my $field_label = $data->{label};
257
            my $field_label = $data->{label};
258
            my $staff_client = exists $data->{staff_client} ? $data->{staff_client} : 1;
259
            my $opac = exists $data->{opac} ? $data->{opac} : 1;
258
            my $mappings = $data->{mappings};
260
            my $mappings = $data->{mappings};
259
            my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type }, { key => 'name' });
261
            my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type, staff_client => $staff_client, opac => $opac }, { key => 'name' });
260
            for my $mapping ( @$mappings ) {
262
            for my $mapping ( @$mappings ) {
261
                my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} });
263
                my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} });
262
                $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet} || 0, suggestible => $mapping->{suggestible} || 0, sort => $mapping->{sort} } );
264
                $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet} || 0, suggestible => $mapping->{suggestible} || 0, sort => $mapping->{sort}, search => $mapping->{search} || 1 } );
263
            }
265
            }
264
        }
266
        }
265
    }
267
    }
Lines 290-300 sub get_fixer_rules { Link Here
290
292
291
    $self->_foreach_mapping(
293
    $self->_foreach_mapping(
292
        sub {
294
        sub {
293
            my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_;
295
            my ( $name, $type, $facet, $suggestible, $sort, $search, $marc_type, $marc_field ) = @_;
294
            return if $marc_type ne $marcflavour;
296
            return if $marc_type ne $marcflavour;
295
            my $options ='';
297
            my $options ='';
296
298
297
            push @rules, "marc_map('$marc_field','${name}.\$append', $options)";
299
            if ($search) {
300
                push @rules, "marc_map('$marc_field','${name}.\$append', $options)";
301
            }
298
            if ($facet) {
302
            if ($facet) {
299
                push @rules, "marc_map('$marc_field','${name}__facet.\$append', $options)";
303
                push @rules, "marc_map('$marc_field','${name}__facet.\$append', $options)";
300
            }
304
            }
Lines 392-397 sub _foreach_mapping { Link Here
392
                'search_marc_to_fields.facet',
396
                'search_marc_to_fields.facet',
393
                'search_marc_to_fields.suggestible',
397
                'search_marc_to_fields.suggestible',
394
                'search_marc_to_fields.sort',
398
                'search_marc_to_fields.sort',
399
                'search_marc_to_fields.search',
395
                'search_marc_map.marc_type',
400
                'search_marc_map.marc_type',
396
                'search_marc_map.marc_field',
401
                'search_marc_map.marc_field',
397
            ],
402
            ],
Lines 399-404 sub _foreach_mapping { Link Here
399
                'facet',
404
                'facet',
400
                'suggestible',
405
                'suggestible',
401
                'sort',
406
                'sort',
407
                'search',
402
                'marc_type',
408
                'marc_type',
403
                'marc_field',
409
                'marc_field',
404
            ],
410
            ],
Lines 412-417 sub _foreach_mapping { Link Here
412
            $search_field->get_column('facet'),
418
            $search_field->get_column('facet'),
413
            $search_field->get_column('suggestible'),
419
            $search_field->get_column('suggestible'),
414
            $search_field->get_column('sort'),
420
            $search_field->get_column('sort'),
421
            $search_field->get_column('search'),
415
            $search_field->get_column('marc_type'),
422
            $search_field->get_column('marc_type'),
416
            $search_field->get_column('marc_field'),
423
            $search_field->get_column('marc_field'),
417
        );
424
        );
(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-54 / +189 lines)
Lines 48-53 use URI::Escape; Link Here
48
48
49
use C4::Context;
49
use C4::Context;
50
use Koha::Exceptions;
50
use Koha::Exceptions;
51
use Koha::Caches;
51
52
52
=head2 build_query
53
=head2 build_query
53
54
Lines 90-98 sub build_query { Link Here
90
            query            => $query,
91
            query            => $query,
91
            fuzziness        => $fuzzy_enabled ? 'auto' : '0',
92
            fuzziness        => $fuzzy_enabled ? 'auto' : '0',
92
            default_operator => 'AND',
93
            default_operator => 'AND',
93
            default_field    => '_all',
94
            fields          => $self->_search_fields({ is_opac => $options{is_opac}, weighted_fields => $options{weighted_fields} }),
94
            lenient          => JSON::true,
95
            lenient          => JSON::true,
95
            fields           => $options{fields} || [],
96
        }
96
        }
97
    };
97
    };
98
98
Lines 106-112 sub build_query { Link Here
106
            # TODO account for fields that don't have a 'phrase' type
106
            # TODO account for fields that don't have a 'phrase' type
107
107
108
            $f = $self->_sort_field($f);
108
            $f = $self->_sort_field($f);
109
            push @{ $res->{sort} }, { "$f.phrase" => { order => $d } };
109
            push @{ $res->{sort} }, { $f => { order => $d } };
110
        }
110
        }
111
    }
111
    }
112
112
Lines 229-245 sub build_query_compat { Link Here
229
        join( ' ', $self->_create_query_string(@search_params) ) || (),
229
        join( ' ', $self->_create_query_string(@search_params) ) || (),
230
        $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () );
230
        $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () );
231
231
232
    my @fields = '_all';
233
    if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) {
234
        push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields;
235
    }
236
237
    # If there's no query on the left, let's remove the junk left behind
232
    # If there's no query on the left, let's remove the junk left behind
238
    $query_str =~ s/^ AND //;
233
    $query_str =~ s/^ AND //;
239
    my %options;
234
    my %options;
240
    $options{fields} = \@fields;
241
    $options{sort} = \@sort_params;
235
    $options{sort} = \@sort_params;
242
    $options{expanded_facet} = $params->{expanded_facet};
236
    $options{expanded_facet} = $params->{expanded_facet};
237
    $options{is_opac} = $params->{is_opac};
238
    $options{weighted_fields} = $params->{weighted_fields};
243
    my $query = $self->build_query( $query_str, %options );
239
    my $query = $self->build_query( $query_str, %options );
244
240
245
    #die Dumper($query);
241
    #die Dumper($query);
Lines 300-361 sub build_authorities_query { Link Here
300
296
301
    foreach my $s ( @{ $search->{searches} } ) {
297
    foreach my $s ( @{ $search->{searches} } ) {
302
        my ( $wh, $op, $val ) = @{$s}{qw(where operator value)};
298
        my ( $wh, $op, $val ) = @{$s}{qw(where operator value)};
303
        $wh = '_all' if $wh eq '';
299
        if (!$wh) {
304
        if ( $op eq 'is' || $op eq '=' ) {
300
            if ( $op eq 'is' || $op eq '=' || $op eq 'exact') {
305
301
                # Match the whole field for all searchable fields, case insensitive,
306
            # look for something that matches a term completely
302
                # UTF normalized.
307
            # note, '=' is about numerical vals. May need special handling.
303
                # Given that field data is "The quick brown fox"
308
            # Also, we lowercase our search because the ES
304
                # "The quick brown fox" and "the quick brown fox" will match
309
            # index lowercases its values, and term searches don't get the
305
                # but not "quick brown fox".
310
            # search analyzer applied to them.
306
                push @query_parts, {
311
            push @query_parts, { term => {"$wh.phrase" => lc $val} };
307
                    multi_match => {
308
                        query => $val,
309
                        fields => $self->_search_fields({ subfield => 'ci_raw' }),
310
                    }
311
                };
312
            }
313
            elsif ( $op eq 'start') {
314
                # Match the prefix within a field for all searchable fields.
315
                # Given that field data is "The quick brown fox"
316
                # "The quick bro" will match, but not "quick bro"
317
318
                # Does not seems to be a multi prefix query
319
                # so we need to create one
320
                my @prefix_queries;
321
                foreach my $field (@{$self->_search_fields()}) {
322
                    push @prefix_queries, {
323
                        prefix => { "$field.ci_raw" => $val }
324
                    };
325
                }
326
                push @query_parts, {
327
                    'bool' => {
328
                        'should' => \@prefix_queries,
329
                        'minimum_should_match' => 1
330
                    }
331
                };
332
            }
333
            else {
334
                # Query all searchable fields.
335
                # Given that field data is "The quick brown fox"
336
                # a search containing any of the words will match, regardless
337
                # of order.
338
                #
339
                # Since default operator is "AND" each of the words must match
340
                # at least one field
341
                push @query_parts, {
342
                    query_string => {
343
                        query => $val,
344
                        fields => $self->_search_fields(),
345
                        default_operator => 'AND',
346
                    }
347
                };
348
            }
312
        }
349
        }
313
        elsif ( $op eq 'exact' ) {
350
        elsif ( $op eq 'is' || $op eq '=' || $op eq 'exact') {
314
            # left and right truncation, otherwise an exact phrase
351
            # Match the whole field, case insensitive, UTF normalized.
315
            push @query_parts, { match_phrase => {"$wh.phrase" => lc $val} };
352
            push @query_parts, { term => { "$wh.ci_raw" => $val } };
316
        }
353
        }
317
        elsif ( $op eq 'start' ) {
354
        elsif ( $op eq 'start' ) {
318
            # startswith search, uses lowercase untokenized version of heading
355
            # Match prefix of the field.
319
            push @query_parts, { prefix => {"$wh.lc_raw" => lc $val} };
356
            push @query_parts, { prefix => {"$wh.ci_raw" => $val} };
320
        }
357
        }
321
        else {
358
        else {
322
            # regular wordlist stuff
359
            # Regular match query for the specific field.
323
#            push @query_parts, { match => {$wh => { query => $val, operator => 'and' }} };
360
            push @query_parts, {
324
            my @values = split(' ',$val);
361
                match => {
325
            foreach my $v (@values) {
362
                    $wh => {
326
                push @query_parts, { wildcard => { "$wh.phrase" => "*" . lc $v . "*" } };
363
                        query => $val,
327
            }
364
                        operator => 'and'
365
                    }
366
                }
367
            };
328
        }
368
        }
329
    }
369
    }
330
370
331
    # Merge the query parts appropriately
371
    # Merge the query parts appropriately
332
    # 'should' behaves like 'or'
372
    # 'should' behaves like 'or'
333
    # 'must' behaves like 'and'
373
    # 'must' behaves like 'and'
334
    # Zebra results seem to match must so using that here
374
    # Zebra behaviour seem to match must so using that here
335
    my $query = { query=>
375
    my $elastic_query = {};
336
                 { bool =>
376
    $elastic_query->{bool}->{must} = \@query_parts;
337
                     { must => \@query_parts  }
377
338
                 }
378
    # Filter by authtypecode if set
339
             };
379
    if ($search->{authtypecode}) {
340
380
        $elastic_query->{bool}->{filter} = {
341
    # We need to add '.phrase' to all the sort headings otherwise it'll sort
381
            term => {
342
    # based on the tokenised form.
382
                "authtype.raw" => $search->{authtypecode}
343
    my %s;
383
            }
344
    if ( exists $search->{sort} ) {
384
        };
345
        foreach my $k ( keys %{ $search->{sort} } ) {
346
            my $f = $self->_sort_field($k);
347
            $s{"$f.phrase"} = $search->{sort}{$k};
348
        }
349
        $search->{sort} = \%s;
350
    }
385
    }
351
386
352
    # add the sort stuff
387
    my $query = {
353
    $query->{sort} = [ $search->{sort} ]  if exists $search->{sort};
388
        query => $elastic_query
389
    };
390
391
    # Add the sort stuff
392
    $query->{sort} = [ $search->{sort} ] if exists $search->{sort};
354
393
355
    return $query;
394
    return $query;
356
}
395
}
357
396
358
359
=head2 build_authorities_query_compat
397
=head2 build_authorities_query_compat
360
398
361
    my ($query) =
399
    my ($query) =
Lines 449-455 sub build_authorities_query_compat { Link Here
449
    my %sort;
487
    my %sort;
450
    my $sort_field =
488
    my $sort_field =
451
        ( $orderby =~ /^Heading/ ) ? 'Heading__sort'
489
        ( $orderby =~ /^Heading/ ) ? 'Heading__sort'
452
      : ( $orderby =~ /^Auth/ )    ? 'Local-Number'
490
      : ( $orderby =~ /^Auth/ )    ? 'Local-Number__sort'
453
      :                              undef;
491
      :                              undef;
454
    if ($sort_field) {
492
    if ($sort_field) {
455
        my $sort_order = ( $orderby =~ /Asc$/ ) ? 'asc' : 'desc';
493
        my $sort_order = ( $orderby =~ /Asc$/ ) ? 'asc' : 'desc';
Lines 516-522 types. Link Here
516
=cut
554
=cut
517
555
518
our %index_field_convert = (
556
our %index_field_convert = (
519
    'kw'      => '_all',
557
    'kw'      => '',
520
    'ti'      => 'title',
558
    'ti'      => 'title',
521
    'au'      => 'author',
559
    'au'      => 'author',
522
    'su'      => 'subject',
560
    'su'      => 'subject',
Lines 542-548 sub _convert_index_fields { Link Here
542
    # If a field starts with mc- we save it as it's used (and removed) later
580
    # If a field starts with mc- we save it as it's used (and removed) later
543
    # when joining things, to indicate we make it an 'OR' join.
581
    # when joining things, to indicate we make it an 'OR' join.
544
    # (Sorry, this got a bit ugly after special cases were found.)
582
    # (Sorry, this got a bit ugly after special cases were found.)
545
    grep { $_->{field} } map {
583
    map {
546
        my ( $f, $t ) = split /,/;
584
        my ( $f, $t ) = split /,/;
547
        my $mc = '';
585
        my $mc = '';
548
        if ($f =~ /^mc-/) {
586
        if ($f =~ /^mc-/) {
Lines 554-560 sub _convert_index_fields { Link Here
554
            type  => $index_type_convert{ $t // '__default' }
592
            type  => $index_type_convert{ $t // '__default' }
555
        };
593
        };
556
        $r->{field} = ($mc . $r->{field}) if $mc && $r->{field};
594
        $r->{field} = ($mc . $r->{field}) if $mc && $r->{field};
557
        $r;
595
        $r->{field} ? $r : undef;
558
    } @indexes;
596
    } @indexes;
559
}
597
}
560
598
Lines 583-590 sub _convert_index_strings { Link Here
583
            push @res, $s;
621
            push @res, $s;
584
            next;
622
            next;
585
        }
623
        }
586
        push @res, $conv->{field} . ":"
624
        push @res, ($conv->{field} ? $conv->{field} . ':' : '')
587
          . $self->_modify_string_by_type( %$conv, operand => $term );
625
            . $self->_modify_string_by_type( %$conv, operand => $term );
588
    }
626
    }
589
    return @res;
627
    return @res;
590
}
628
}
Lines 816-819 sub _truncate_terms { Link Here
816
    return join ' ', @terms;
854
    return join ' ', @terms;
817
}
855
}
818
856
857
=head2 _search_fields
858
    my $weighted_fields = $self->_search_fields({
859
        is_opac => 0,
860
        weighted_fields => 1,
861
        subfield => 'raw'
862
    });
863
864
Generate a list of searchable fields to be used for Elasticsearch queries
865
applied to multiple fields.
866
867
Returns an arrayref of field names for either OPAC or Staff client, with
868
possible weights and subfield appended to each field name depending on the
869
options provided.
870
871
=over 4
872
873
=item C<$params>
874
875
Hashref with options. The parameter C<is_opac> indicates whether the searchable
876
fields for OPAC or Staff client should be retrieved. If C<weighted_fields> is set
877
fields weights will be applied on returned fields. C<subfield> can be used to
878
provide a subfield that will be appended to fields as "C<field_name>.C<subfield>".
879
880
=back
881
882
=cut
883
884
sub _search_fields {
885
    my ($self, $params) = @_;
886
    $params //= {
887
        is_opac => 0,
888
        weighted_fields => 0,
889
        # This is a hack for authorities build_authorities_query
890
        # can hopefully be removed in the future
891
        subfield => undef,
892
    };
893
894
    my $cache = Koha::Caches->get_instance();
895
    my $cache_key = 'elasticsearch_search_fields' . ($params->{is_opac} ? '_opac' : '_staff_client');
896
    my $search_fields = $cache->get_from_cache($cache_key, { unsafe => 1 });
897
898
    if (!$search_fields) {
899
        # The reason we don't use Koha::SearchFields->search here is we don't
900
        # want or need resultset wrapped as Koha::SearchField object.
901
        # It does not make any sense in this context and would cause
902
        # unnecessary overhead sice we are only querying for data
903
        # Also would not work, or produce strange results, with the "columns"
904
        # option.
905
        my $schema = Koha::Database->schema;
906
        my $result = $schema->resultset('SearchField')->search(
907
            {
908
                $params->{is_opac} ? (
909
                    'opac' => 1,
910
                ) : (
911
                    'staff_client' => 1
912
                ),
913
                'search_marc_map.index_name' => $self->index,
914
                'search_marc_map.marc_type' => C4::Context->preference('marcflavour'),
915
                'search_marc_to_fields.search' => 1,
916
            },
917
            {
918
                columns => [qw/name weight/],
919
                # collapse => 1,
920
                join => {search_marc_to_fields => 'search_marc_map'},
921
            }
922
        );
923
        my @search_fields;
924
        while (my $search_field = $result->next) {
925
            push @search_fields, [
926
                $search_field->name,
927
                $search_field->weight ? $search_field->weight : ()
928
            ];
929
        }
930
        $search_fields = \@search_fields;
931
        $cache->set_in_cache($cache_key, $search_fields);
932
    }
933
    if ($params->{subfield}) {
934
        my $subfield = $params->{subfield};
935
        $search_fields = [
936
            map {
937
                # Copy values to avoid mutating cached
938
                # data (since unsafe is used)
939
                my ($field, $weight) = @{$_};
940
                ["${field}.${subfield}", $weight];
941
            } @{$search_fields}
942
        ];
943
    }
944
945
    if ($params->{weighted_fields}) {
946
        return [map { join('^', @{$_}) } @{$search_fields}];
947
    }
948
    else {
949
        # Exclude weight from field
950
        return [map { $_->[0] } @{$search_fields}];
951
    }
952
}
953
819
1;
954
1;
(-)a/Koha/SearchField.pm (-1 lines)
Lines 20-26 use Modern::Perl; Link Here
20
use Carp;
20
use Carp;
21
21
22
use Koha::Database;
22
use Koha::Database;
23
use Koha::SearchMarcMaps;
24
23
25
use base qw(Koha::Object);
24
use base qw(Koha::Object);
26
25
(-)a/Koha/SearchFields.pm (-15 lines)
Lines 35-55 Koha::SearchFields - Koha SearchField Object set class Link Here
35
35
36
=cut
36
=cut
37
37
38
=head3 weighted_fields
39
40
my (@w_fields, @weight) = Koha::SearchFields->weighted_fields();
41
42
=cut
43
44
sub weighted_fields {
45
    my ($self) = @_;
46
47
    return $self->search(
48
        { weight => { '>' => 0, '!=' => undef } },
49
        { order_by => { -desc => 'weight' } }
50
    );
51
}
52
53
=head3 type
38
=head3 type
54
39
55
=cut
40
=cut
(-)a/admin/searchengine/elasticsearch/field_config.yaml (-21 / +13 lines)
Lines 1-9 Link Here
1
---
1
---
2
# General field configuration
2
# General field configuration
3
general:
3
general:
4
  _all:
5
    type: string
6
    analyzer: analyser_standard
7
  properties:
4
  properties:
8
    record:
5
    record:
9
      store: true
6
      store: true
Lines 18-48 search: Link Here
18
    null_value: 0
15
    null_value: 0
19
  stdno:
16
  stdno:
20
    type: text
17
    type: text
21
    analyzer: analyser_stdno
18
    analyzer: analyzer_stdno
22
    search_analyzer: analyser_stdno
19
    search_analyzer: analyzer_stdno
23
    fields:
20
    fields:
24
      phrase:
21
      phrase:
25
        type: text
22
        type: text
26
        analyzer: analyser_stdno
23
        analyzer: analyzer_phrase
27
        search_analyzer: analyser_stdno
24
        search_analyzer: analyzer_phrase
28
      raw:
25
      raw:
29
        type: keyword
26
        type: keyword
30
    copy_to: _all
31
  default:
27
  default:
32
    type: text
28
    type: text
33
    analyzer: analyser_standard
29
    analyzer: analyzer_standard
34
    search_analyzer: analyser_standard
30
    search_analyzer: analyzer_standard
35
    fields:
31
    fields:
36
      phrase:
32
      phrase:
37
        type: text
33
        type: text
38
        analyzer: analyser_phrase
34
        analyzer: analyzer_phrase
39
        search_analyzer: analyser_phrase
35
        search_analyzer: analyzer_phrase
40
      raw:
36
      raw:
41
        type: keyword
37
        type: keyword
42
      lc_raw:
38
        normalizer: nfkc_cf_normalizer
39
      ci_raw:
43
        type: keyword
40
        type: keyword
44
        normalizer: my_normalizer
41
        normalizer: icu_folding_normalizer
45
    copy_to: _all
46
# Facets
42
# Facets
47
facet:
43
facet:
48
  default:
44
  default:
Lines 56-64 suggestible: Link Here
56
# Sort
52
# Sort
57
sort:
53
sort:
58
  default:
54
  default:
59
    type: text
55
    type: icu_collation_keyword
60
    analyzer: analyser_phrase
56
    index: false
61
    search_analyzer: analyser_phrase
62
    fields:
63
      phrase:
64
        type: keyword
(-)a/admin/searchengine/elasticsearch/index_config.yaml (-6 / +6 lines)
Lines 3-31 Link Here
3
index:
3
index:
4
  analysis:
4
  analysis:
5
    analyzer:
5
    analyzer:
6
      # Phrase analyzer is used for phrases (phrase match, sorting)
6
      # Phrase analyzer is used for phrases (exact phrase match)
7
      analyser_phrase:
7
      analyzer_phrase:
8
        tokenizer: keyword
8
        tokenizer: keyword
9
        filter:
9
        filter:
10
          - icu_folding
10
          - icu_folding
11
        char_filter:
11
        char_filter:
12
          - punctuation
12
          - punctuation
13
      analyser_standard:
13
      analyzer_standard:
14
        tokenizer: icu_tokenizer
14
        tokenizer: icu_tokenizer
15
        filter:
15
        filter:
16
          - icu_folding
16
          - icu_folding
17
      analyser_stdno:
17
      analyzer_stdno:
18
        tokenizer: whitespace
18
        tokenizer: whitespace
19
        filter:
19
        filter:
20
          - icu_folding
20
          - icu_folding
21
        char_filter:
21
        char_filter:
22
          - punctuation
22
          - punctuation
23
    normalizer:
23
    normalizer:
24
      normalizer_keyword:
24
      icu_folding_normalizer:
25
        type: custom
25
        type: custom
26
        filter:
26
        filter:
27
          - icu_folding
27
          - icu_folding
28
      my_normalizer:
28
      nfkc_cf_normalizer:
29
        type: custom
29
        type: custom
30
        char_filter: icu_normalizer
30
        char_filter: icu_normalizer
31
    char_filter:
31
    char_filter:
(-)a/admin/searchengine/elasticsearch/mappings.pl (-22 / +61 lines)
Lines 25-30 use C4::Auth; Link Here
25
use Koha::SearchEngine::Elasticsearch;
25
use Koha::SearchEngine::Elasticsearch;
26
use Koha::SearchMarcMaps;
26
use Koha::SearchMarcMaps;
27
use Koha::SearchFields;
27
use Koha::SearchFields;
28
use Koha::Caches;
28
29
29
my $input = new CGI;
30
my $input = new CGI;
30
my ( $template, $borrowernumber, $cookie ) = get_template_and_user(
31
my ( $template, $borrowernumber, $cookie ) = get_template_and_user(
Lines 45-50 my $schema = $database->schema; Link Here
45
46
46
my $marc_type = lc C4::Context->preference('marcflavour');
47
my $marc_type = lc C4::Context->preference('marcflavour');
47
48
49
my $cache = Koha::Caches->get_instance();
50
my $clear_cache = sub {
51
    $cache->clear_from_cache('elasticsearch_search_fields_staff_client');
52
    $cache->clear_from_cache('elasticsearch_search_fields_opac');
53
};
54
48
if ( $op eq 'edit' ) {
55
if ( $op eq 'edit' ) {
49
56
50
    $schema->storage->txn_begin;
57
    $schema->storage->txn_begin;
Lines 53-64 if ( $op eq 'edit' ) { Link Here
53
    my @field_label = $input->param('search_field_label');
60
    my @field_label = $input->param('search_field_label');
54
    my @field_type = $input->param('search_field_type');
61
    my @field_type = $input->param('search_field_type');
55
    my @field_weight = $input->param('search_field_weight');
62
    my @field_weight = $input->param('search_field_weight');
63
    my @field_staff_client = $input->param('search_field_staff_client');
64
    my @field_opac = $input->param('search_field_opac');
56
65
57
    my @index_name          = $input->param('mapping_index_name');
66
    my @index_name          = $input->param('mapping_index_name');
58
    my @search_field_name  = $input->param('mapping_search_field_name');
67
    my @search_field_name   = $input->param('mapping_search_field_name');
59
    my @mapping_sort        = $input->param('mapping_sort');
68
    my @mapping_sort        = $input->param('mapping_sort');
60
    my @mapping_facet       = $input->param('mapping_facet');
69
    my @mapping_facet       = $input->param('mapping_facet');
61
    my @mapping_suggestible = $input->param('mapping_suggestible');
70
    my @mapping_suggestible = $input->param('mapping_suggestible');
71
    my @mapping_search      = $input->param('mapping_search');
62
    my @mapping_marc_field  = $input->param('mapping_marc_field');
72
    my @mapping_marc_field  = $input->param('mapping_marc_field');
63
73
64
    eval {
74
    eval {
Lines 68-74 if ( $op eq 'edit' ) { Link Here
68
            my $field_label = $field_label[$i];
78
            my $field_label = $field_label[$i];
69
            my $field_type = $field_type[$i];
79
            my $field_type = $field_type[$i];
70
            my $field_weight = $field_weight[$i];
80
            my $field_weight = $field_weight[$i];
71
81
            my $field_staff_client = $field_staff_client[$i];
82
            my $field_opac = $field_opac[$i];
72
            my $search_field = Koha::SearchFields->find( { name => $field_name }, { key => 'name' } );
83
            my $search_field = Koha::SearchFields->find( { name => $field_name }, { key => 'name' } );
73
            $search_field->label($field_label);
84
            $search_field->label($field_label);
74
            $search_field->type($field_type);
85
            $search_field->type($field_type);
Lines 83-88 if ( $op eq 'edit' ) { Link Here
83
                $search_field->weight($field_weight);
94
                $search_field->weight($field_weight);
84
            }
95
            }
85
96
97
            $search_field->staff_client($field_staff_client ? 1 : 0);
98
            $search_field->opac($field_opac ? 1 : 0);
86
            $search_field->store;
99
            $search_field->store;
87
        }
100
        }
88
101
Lines 90-107 if ( $op eq 'edit' ) { Link Here
90
103
91
        for my $i ( 0 .. scalar(@index_name) - 1 ) {
104
        for my $i ( 0 .. scalar(@index_name) - 1 ) {
92
            my $index_name          = $index_name[$i];
105
            my $index_name          = $index_name[$i];
93
            my $search_field_name  = $search_field_name[$i];
106
            my $search_field_name   = $search_field_name[$i];
94
            my $mapping_marc_field  = $mapping_marc_field[$i];
107
            my $mapping_marc_field  = $mapping_marc_field[$i];
95
            my $mapping_facet       = $mapping_facet[$i];
108
            my $mapping_facet       = $mapping_facet[$i];
96
            my $mapping_suggestible = $mapping_suggestible[$i];
109
            my $mapping_suggestible = $mapping_suggestible[$i];
97
            my $mapping_sort        = $mapping_sort[$i];
110
            my $mapping_sort        = $mapping_sort[$i] eq 'undef' ? undef : $mapping_sort[$i];
98
            $mapping_sort = undef if $mapping_sort eq 'undef';
111
            my $mapping_search      = $mapping_search[$i];
99
112
100
            my $search_field = Koha::SearchFields->find({ name => $search_field_name }, { key => 'name' });
113
            my $search_field = Koha::SearchFields->find({ name => $search_field_name }, { key => 'name' });
101
            # TODO Check mapping format
114
            # TODO Check mapping format
102
            my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $marc_type, marc_field => $mapping_marc_field });
115
            my $marc_field = Koha::SearchMarcMaps->find_or_create({
103
            $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping_facet, suggestible => $mapping_suggestible, sort => $mapping_sort } );
116
                index_name => $index_name,
104
117
                marc_type => $marc_type,
118
                marc_field => $mapping_marc_field
119
            });
120
            $search_field->add_to_search_marc_maps($marc_field, {
121
                facet => $mapping_facet,
122
                suggestible => $mapping_suggestible,
123
                sort => $mapping_sort,
124
                search => $mapping_search
125
            });
105
        }
126
        }
106
    };
127
    };
107
    if ($@) {
128
    if ($@) {
Lines 111-153 if ( $op eq 'edit' ) { Link Here
111
        push @messages, { type => 'message', code => 'success_on_update' };
132
        push @messages, { type => 'message', code => 'success_on_update' };
112
        $schema->storage->txn_commit;
133
        $schema->storage->txn_commit;
113
    }
134
    }
135
    $clear_cache->();
114
}
136
}
115
elsif( $op eq 'reset_confirmed' ) {
137
elsif( $op eq 'reset_confirm' ) {
116
    Koha::SearchMarcMaps->delete;
138
    Koha::SearchMarcMaps->delete;
117
    Koha::SearchFields->delete;
118
    Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;
139
    Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;
119
    push @messages, { type => 'message', code => 'success_on_reset' };
140
    push @messages, { type => 'message', code => 'success_on_reset' };
141
    $clear_cache->();
120
}
142
}
121
elsif( $op eq 'reset_confirm' ) {
143
elsif( $op eq 'reset_confirm' ) {
122
    $template->param( reset_confirm => 1 );
144
    $template->param( reset_confirm => 1 );
123
}
145
}
124
146
125
126
my @indexes;
147
my @indexes;
127
148
128
for my $index_name (qw| biblios authorities |) {
149
for my $index_name (
150
    $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX,
151
    $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX
152
) {
129
    my $search_fields = Koha::SearchFields->search(
153
    my $search_fields = Koha::SearchFields->search(
130
        { 'search_marc_map.index_name' => $index_name, 'search_marc_map.marc_type' => $marc_type, },
154
        {
131
        {   join => { search_marc_to_fields => 'search_marc_map' },
155
            'search_marc_map.index_name' => $index_name,
132
            '+select' => [ 'search_marc_to_fields.facet', 'search_marc_to_fields.suggestible', 'search_marc_to_fields.sort', 'search_marc_map.marc_field' ],
156
            'search_marc_map.marc_type' => $marc_type,
133
            '+as'     => [ 'facet',                       'suggestible',                       'sort',                       'marc_field' ],
157
        },
134
            order_by => { -asc => [qw/name marc_field/] }
158
        {
159
            join => { search_marc_to_fields => 'search_marc_map' },
160
            '+select' => [
161
                'search_marc_to_fields.facet',
162
                'search_marc_to_fields.suggestible',
163
                'search_marc_to_fields.sort',
164
                'search_marc_to_fields.search',
165
                'search_marc_map.marc_field'
166
            ],
167
            '+as' => [
168
                'facet',
169
                'suggestible',
170
                'sort',
171
                'search',
172
                'marc_field'
173
            ],
135
        }
174
        }
136
    );
175
    );
137
176
138
    my @mappings;
177
    my @mappings;
139
    while ( my $s = $search_fields->next ) {
178
    while ( my $s = $search_fields->next ) {
140
        push @mappings,
179
        push @mappings, {
141
          { search_field_name  => $s->name,
180
            search_field_name  => $s->name,
142
            search_field_label => $s->label,
181
            search_field_label => $s->label,
143
            search_field_type  => $s->type,
182
            search_field_type  => $s->type,
144
            marc_field         => $s->get_column('marc_field'),
183
            marc_field         => $s->get_column('marc_field'),
145
            sort               => $s->get_column('sort') // 'undef', # To avoid warnings "Use of uninitialized value in lc"
184
            sort               => $s->get_column('sort') // 'undef', # To avoid warnings "Use of uninitialized value in lc"
146
            suggestible        => $s->get_column('suggestible'),
185
            suggestible        => $s->get_column('suggestible'),
147
            facet              => $s->get_column('facet'),
186
            facet              => $s->get_column('facet'),
148
          };
187
            search             => $s->get_column('search'),
188
        };
149
    }
189
    }
150
151
    push @indexes, { index_name => $index_name, mappings => \@mappings };
190
    push @indexes, { index_name => $index_name, mappings => \@mappings };
152
}
191
}
153
192
Lines 155-161 my $search_fields = Koha::SearchFields->search( {}, { order_by => ['name'] } ); Link Here
155
my @all_search_fields;
194
my @all_search_fields;
156
while ( my $search_field = $search_fields->next ) {
195
while ( my $search_field = $search_fields->next ) {
157
    my $search_field_unblessed = $search_field->unblessed;
196
    my $search_field_unblessed = $search_field->unblessed;
158
    $search_field_unblessed->{mapped_biblios} = 1 if $search_field->is_mapped_biblios;
197
    $search_field_unblessed->{is_mapped_biblios} = $search_field->is_mapped_biblios;
159
    push @all_search_fields, $search_field_unblessed;
198
    push @all_search_fields, $search_field_unblessed;
160
}
199
}
161
200
(-)a/catalogue/search.pl (-6 / +1 lines)
Lines 456-466 my $expanded_facet = $params->{'expand'}; Link Here
456
# Define some global variables
456
# Define some global variables
457
my ( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type);
457
my ( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type);
458
458
459
my $build_params;
460
unless ( $cgi->param('advsearch') ) {
461
    $build_params->{weighted_fields} = 1;
462
}
463
464
my $builder = Koha::SearchEngine::QueryBuilder->new(
459
my $builder = Koha::SearchEngine::QueryBuilder->new(
465
    { index => $Koha::SearchEngine::BIBLIOS_INDEX } );
460
    { index => $Koha::SearchEngine::BIBLIOS_INDEX } );
466
my $searcher = Koha::SearchEngine::Search->new(
461
my $searcher = Koha::SearchEngine::Search->new(
Lines 473-479 my $searcher = Koha::SearchEngine::Search->new( Link Here
473
    $query_type
468
    $query_type
474
  )
469
  )
475
  = $builder->build_query_compat( \@operators, \@operands, \@indexes, \@limits,
470
  = $builder->build_query_compat( \@operators, \@operands, \@indexes, \@limits,
476
    \@sort_by, $scan, $lang, $build_params );
471
    \@sort_by, $scan, $lang, { weighted_fields => !$cgi->param('advsearch') });
477
472
478
## parse the query_cgi string and put it into a form suitable for <input>s
473
## parse the query_cgi string and put it into a form suitable for <input>s
479
my @query_inputs;
474
my @query_inputs;
(-)a/installer/data/mysql/atomicupdate/bug_20589_add_columns.perl (+14 lines)
Line 0 Link Here
1
$DBversion = 'XXX';
2
if (CheckVersion($DBversion)) {
3
  if (!column_exists('search_marc_to_field', 'search')) {
4
    $dbh->do("ALTER TABLE `search_marc_to_field` ADD COLUMN `search` tinyint(1) NOT NULL DEFAULT 1");
5
  }
6
  if (!column_exists('search_field', 'staff_client')) {
7
    $dbh->do("ALTER TABLE `search_field` ADD COLUMN `staff_client` tinyint(1) NOT NULL DEFAULT 1");
8
  }
9
  if (!column_exists('search_field', 'opac')) {
10
    $dbh->do("ALTER TABLE `search_field` ADD COLUMN `opac` tinyint(1) NOT NULL DEFAULT 1");
11
  }
12
  SetVersion($DBversion)
13
}
14
print "Upgrade to $DBversion done (Bug 20589 - Add field boosting and use elastic query fields parameter instead of depricated _all)\n";
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt (-6 / +60 lines)
Lines 150-157 a.add, a.delete { Link Here
150
                    <th>Name</th>
150
                    <th>Name</th>
151
                    <th>Label</th>
151
                    <th>Label</th>
152
                    <th>Type</th>
152
                    <th>Type</th>
153
                    <th colspan="2">Searchable</th>
153
                    <th>Weight</th>
154
                    <th>Weight</th>
154
                  </tr>
155
                  </tr>
156
                  <tr>
157
                    <th colspan=3>&nbsp;</th>
158
                    <th>Staff client</th>
159
                    <th>OPAC</th>
160
                    <th>&nbsp;</th>
161
                  </tr>
155
                </thead>
162
                </thead>
156
                <tbody>
163
                <tbody>
157
                  [% FOREACH search_field IN all_search_fields %]
164
                  [% FOREACH search_field IN all_search_fields %]
Lines 159-165 a.add, a.delete { Link Here
159
                      <td>
166
                      <td>
160
                        <input type="text" name="search_field_name" value="[% search_field.name | html %]" />
167
                        <input type="text" name="search_field_name" value="[% search_field.name | html %]" />
161
                      </td>
168
                      </td>
162
                      <td><input type="text" name="search_field_label" value="[% search_field.label | html %]" />
169
                      <td>
170
                        <input type="text" name="search_field_label" value="[% search_field.label | html %]" />
171
                      </td>
163
                      <td>
172
                      <td>
164
                        <select name="search_field_type">
173
                        <select name="search_field_type">
165
                          <option value=""></option>
174
                          <option value=""></option>
Lines 201-211 a.add, a.delete { Link Here
201
                        </select>
210
                        </select>
202
                      </td>
211
                      </td>
203
                      <td>
212
                      <td>
204
                      [% IF search_field.mapped_biblios %]
213
                        <select name="search_field_staff_client">
205
                        <input type="number" step="0.01" min="0.01" max="999.99" name="search_field_weight" value="[% search_field.weight | html %]" />
214
                          [% IF search_field.staff_client %]
206
                      [% ELSE %]
215
                            <option value="1" selected="selected">Yes</option>
207
                        <input type="hidden" name="search_field_weight" value="">
216
                            <option value="0">No</option>
208
                      [% END %]
217
                          [% ELSE %]
218
                            <option value="1">Yes</option>
219
                            <option value="0" selected="selected">No</option>
220
                          [% END %]
221
                        </select>
222
                      </td>
223
                      <td>
224
                        <select name="search_field_opac">
225
                          [% IF search_field.opac %]
226
                            <option value="1" selected="selected">Yes</option>
227
                            <option value="0">No</option>
228
                          [% ELSE %]
229
                            <option value="1">Yes</option>
230
                            <option value="0" selected="selected">No</option>
231
                          [% END %]
232
                        </select>
233
                      </td>
234
                      <td>
235
                        [% IF search_field.is_mapped_biblios %]
236
                            <input type="number" step="0.01" min="0.01" max="999.99" name="search_field_weight" value="[% search_field.weight | html %]" />
237
                        [% ELSE %]
238
                            <input type="hidden" name="search_field_weight" value="">
239
                        [% END %]
209
                      </td>
240
                      </td>
210
                    </tr>
241
                    </tr>
211
                  [% END %]
242
                  [% END %]
Lines 221-226 a.add, a.delete { Link Here
221
                          <th>Sortable</th>
252
                          <th>Sortable</th>
222
                          <th>Facetable</th>
253
                          <th>Facetable</th>
223
                          <th>Suggestible</th>
254
                          <th>Suggestible</th>
255
                          <th>Searchable</th>
224
                          <th>Mapping</th>
256
                          <th>Mapping</th>
225
                          <th></th>
257
                          <th></th>
226
                        </tr>
258
                        </tr>
Lines 275-280 a.add, a.delete { Link Here
275
                              </select>
307
                              </select>
276
                            </td>
308
                            </td>
277
                            <td>
309
                            <td>
310
                              <select name="mapping_search">
311
                                [% IF mapping.search %]
312
                                  <option value="0">No</option>
313
                                  <option value="1" selected="selected">Yes</option>
314
                                [% ELSE %]
315
                                  <option value="0" selected="selected">No</option>
316
                                  <option value="1">Yes</option>
317
                                [% END %]
318
                              </select>
319
                            </td>
320
                            <td>
278
                                <input name="mapping_marc_field" type="text" value="[% mapping.marc_field | html %]" />
321
                                <input name="mapping_marc_field" type="text" value="[% mapping.marc_field | html %]" />
279
                            </td>
322
                            </td>
280
                            <td><a class="btn btn-default btn-xs delete" style="cursor: pointer;"><i class="fa fa-trash"></i> Delete</a></td>
323
                            <td><a class="btn btn-default btn-xs delete" style="cursor: pointer;"><i class="fa fa-trash"></i> Delete</a></td>
Lines 320-325 a.add, a.delete { Link Here
320
                              [% END %]
363
                              [% END %]
321
                            </select>
364
                            </select>
322
                          </td>
365
                          </td>
366
                          <td>
367
                            <select data-id="mapping_search">
368
                              [% IF mapping.search %]
369
                                <option value="0">No</option>
370
                                <option value="1" selected="selected">Yes</option>
371
                              [% ELSE %]
372
                                <option value="0" selected="selected">No</option>
373
                                <option value="1">Yes</option>
374
                              [% END %]
375
                            </select>
376
                          </td>
323
                          <td><input data-id="mapping_marc_field" type="text" /></td>
377
                          <td><input data-id="mapping_marc_field" type="text" /></td>
324
                          <td><a class="btn btn-default btn-xs add"><i class="fa fa-plus"></i> Add</a></td>
378
                          <td><a class="btn btn-default btn-xs add"><i class="fa fa-plus"></i> Add</a></td>
325
                        </tr>
379
                        </tr>
(-)a/opac/opac-search.pl (-12 / +15 lines)
Lines 549-567 if (C4::Context->preference('OpacSuppression')) { Link Here
549
    }
549
    }
550
}
550
}
551
551
552
my $build_params = {
553
    expanded_facet => $expanded_facet,
554
    suppress => $suppress
555
};
556
557
unless ( $cgi->param('advsearch') ) {
558
    $build_params->{weighted_fields} = 1;
559
}
560
561
## I. BUILD THE QUERY
552
## I. BUILD THE QUERY
562
( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type)
553
( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type)
563
  = $builder->build_query_compat( \@operators, \@operands,
554
  = $builder->build_query_compat(
564
    \@indexes, \@limits, \@sort_by, 0, $lang, $build_params);
555
    \@operators,
556
    \@operands,
557
    \@indexes,
558
    \@limits,
559
    \@sort_by,
560
    0,
561
    $lang,
562
    {
563
        expanded_facet => $expanded_facet,
564
        suppress => $suppress,
565
        is_opac => 1,
566
        weighted_fields => !$cgi->param('advsearch')
567
    }
568
);
565
569
566
sub _input_cgi_parse {
570
sub _input_cgi_parse {
567
    my @elements;
571
    my @elements;
568
- 

Return to bug 20589