View | Details | Raw Unified | Return to bug 20589
Collapse All | Expand All

(-)a/Koha/Schema/Result/SearchField.pm (-1 / +5 lines)
Lines 77-83 __PACKAGE__->add_columns( Link Here
77
    is_nullable => 0,
77
    is_nullable => 0,
78
  },
78
  },
79
  "weight",
79
  "weight",
80
  { data_type => "decimal", is_nullable => 1, size => [5, 2] },
80
  { data_type => "decimal", size => [ 5, 2 ], is_nullable => 1 },
81
  "staff_client",
82
  { data_type => "tinyint", size => 1, is_nullable => 0, default => 1 },
83
  "opac",
84
  { data_type => "tinyint", size => 1, is_nullable => 0, default => 1 },
81
);
85
);
82
86
83
=head1 PRIMARY KEY
87
=head1 PRIMARY KEY
(-)a/Koha/Schema/Result/SearchMarcToField.pm (+2 lines)
Lines 71-76 __PACKAGE__->add_columns( Link Here
71
  { data_type => "tinyint", default_value => 0, is_nullable => 1 },
71
  { data_type => "tinyint", default_value => 0, is_nullable => 1 },
72
  "sort",
72
  "sort",
73
  { data_type => "tinyint", is_nullable => 1 },
73
  { data_type => "tinyint", is_nullable => 1 },
74
  "search",
75
  { data_type => "tinyint", size => 1, is_nullable => 0, default => 1 },
74
);
76
);
75
77
76
=head1 PRIMARY KEY
78
=head1 PRIMARY KEY
(-)a/Koha/SearchEngine/Elasticsearch.pm (-8 / +15 lines)
Lines 170-176 sub get_elasticsearch_mappings { Link Here
170
        my $marcflavour = lc C4::Context->preference('marcflavour');
170
        my $marcflavour = lc C4::Context->preference('marcflavour');
171
        $self->_foreach_mapping(
171
        $self->_foreach_mapping(
172
            sub {
172
            sub {
173
                my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
173
                my ( $name, $type, $facet, $suggestible, $sort, $search, $marc_type ) = @_;
174
                return if $marc_type ne $marcflavour;
174
                return if $marc_type ne $marcflavour;
175
                # TODO if this gets any sort of complexity to it, it should
175
                # TODO if this gets any sort of complexity to it, it should
176
                # be broken out into its own function.
176
                # be broken out into its own function.
Lines 185-193 sub get_elasticsearch_mappings { Link Here
185
                } elsif ($type eq 'isbn' || $type eq 'stdno') {
185
                } elsif ($type eq 'isbn' || $type eq 'stdno') {
186
                    $es_type = 'stdno';
186
                    $es_type = 'stdno';
187
                }
187
                }
188
188
                if ($search) {
189
                $mappings->{data}{properties}{$name} = _get_elasticsearch_mapping('search', $es_type);
189
                    $mappings->{data}{properties}{$name} = _get_elasticsearch_mapping('search', $es_type);
190
190
                }
191
                if ($facet) {
191
                if ($facet) {
192
                    $mappings->{data}{properties}{ $name . '__facet' } = _get_elasticsearch_mapping('facet', $es_type);
192
                    $mappings->{data}{properties}{ $name . '__facet' } = _get_elasticsearch_mapping('facet', $es_type);
193
                }
193
                }
Lines 255-265 sub reset_elasticsearch_mappings { Link Here
255
        while ( my ( $field_name, $data ) = each %$fields ) {
255
        while ( my ( $field_name, $data ) = each %$fields ) {
256
            my $field_type = $data->{type};
256
            my $field_type = $data->{type};
257
            my $field_label = $data->{label};
257
            my $field_label = $data->{label};
258
            my $staff_client = exists $data->{staff_client} ? $data->{staff_client} : 1;
259
            my $opac = exists $data->{opac} ? $data->{opac} : 1;
258
            my $mappings = $data->{mappings};
260
            my $mappings = $data->{mappings};
259
            my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type }, { key => 'name' });
261
            my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type, staff_client => $staff_client, opac => $opac }, { key => 'name' });
260
            for my $mapping ( @$mappings ) {
262
            for my $mapping ( @$mappings ) {
261
                my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} });
263
                my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} });
262
                $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet} || 0, suggestible => $mapping->{suggestible} || 0, sort => $mapping->{sort} } );
264
                $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet} || 0, suggestible => $mapping->{suggestible} || 0, sort => $mapping->{sort}, search => $mapping->{search} || 1 } );
263
            }
265
            }
264
        }
266
        }
265
    }
267
    }
Lines 290-300 sub get_fixer_rules { Link Here
290
292
291
    $self->_foreach_mapping(
293
    $self->_foreach_mapping(
292
        sub {
294
        sub {
293
            my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_;
295
            my ( $name, $type, $facet, $suggestible, $sort, $search, $marc_type, $marc_field ) = @_;
294
            return if $marc_type ne $marcflavour;
296
            return if $marc_type ne $marcflavour;
295
            my $options ='';
297
            my $options ='';
296
298
297
            push @rules, "marc_map('$marc_field','${name}.\$append', $options)";
299
            if ($search) {
300
                push @rules, "marc_map('$marc_field','${name}.\$append', $options)";
301
            }
298
            if ($facet) {
302
            if ($facet) {
299
                push @rules, "marc_map('$marc_field','${name}__facet.\$append', $options)";
303
                push @rules, "marc_map('$marc_field','${name}__facet.\$append', $options)";
300
            }
304
            }
Lines 392-397 sub _foreach_mapping { Link Here
392
                'search_marc_to_fields.facet',
396
                'search_marc_to_fields.facet',
393
                'search_marc_to_fields.suggestible',
397
                'search_marc_to_fields.suggestible',
394
                'search_marc_to_fields.sort',
398
                'search_marc_to_fields.sort',
399
                'search_marc_to_fields.search',
395
                'search_marc_map.marc_type',
400
                'search_marc_map.marc_type',
396
                'search_marc_map.marc_field',
401
                'search_marc_map.marc_field',
397
            ],
402
            ],
Lines 399-404 sub _foreach_mapping { Link Here
399
                'facet',
404
                'facet',
400
                'suggestible',
405
                'suggestible',
401
                'sort',
406
                'sort',
407
                'search',
402
                'marc_type',
408
                'marc_type',
403
                'marc_field',
409
                'marc_field',
404
            ],
410
            ],
Lines 412-417 sub _foreach_mapping { Link Here
412
            $search_field->get_column('facet'),
418
            $search_field->get_column('facet'),
413
            $search_field->get_column('suggestible'),
419
            $search_field->get_column('suggestible'),
414
            $search_field->get_column('sort'),
420
            $search_field->get_column('sort'),
421
            $search_field->get_column('search'),
415
            $search_field->get_column('marc_type'),
422
            $search_field->get_column('marc_type'),
416
            $search_field->get_column('marc_field'),
423
            $search_field->get_column('marc_field'),
417
        );
424
        );
(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-54 / +162 lines)
Lines 48-53 use URI::Escape; Link Here
48
48
49
use C4::Context;
49
use C4::Context;
50
use Koha::Exceptions;
50
use Koha::Exceptions;
51
use Koha::Caches;
51
52
52
=head2 build_query
53
=head2 build_query
53
54
Lines 90-98 sub build_query { Link Here
90
            query            => $query,
91
            query            => $query,
91
            fuzziness        => $fuzzy_enabled ? 'auto' : '0',
92
            fuzziness        => $fuzzy_enabled ? 'auto' : '0',
92
            default_operator => 'AND',
93
            default_operator => 'AND',
93
            default_field    => '_all',
94
            fields          => $self->_search_fields({ is_opac => $options{is_opac}, weighted_fields => $options{weighted_fields} }),
94
            lenient          => JSON::true,
95
            lenient          => JSON::true,
95
            fields           => $options{fields} || [],
96
        }
96
        }
97
    };
97
    };
98
98
Lines 106-112 sub build_query { Link Here
106
            # TODO account for fields that don't have a 'phrase' type
106
            # TODO account for fields that don't have a 'phrase' type
107
107
108
            $f = $self->_sort_field($f);
108
            $f = $self->_sort_field($f);
109
            push @{ $res->{sort} }, { "$f.phrase" => { order => $d } };
109
            push @{ $res->{sort} }, { $f => { order => $d } };
110
        }
110
        }
111
    }
111
    }
112
112
Lines 229-245 sub build_query_compat { Link Here
229
        join( ' ', $self->_create_query_string(@search_params) ) || (),
229
        join( ' ', $self->_create_query_string(@search_params) ) || (),
230
        $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () );
230
        $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () );
231
231
232
    my @fields = '_all';
233
    if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) {
234
        push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields;
235
    }
236
237
    # If there's no query on the left, let's remove the junk left behind
232
    # If there's no query on the left, let's remove the junk left behind
238
    $query_str =~ s/^ AND //;
233
    $query_str =~ s/^ AND //;
239
    my %options;
234
    my %options;
240
    $options{fields} = \@fields;
241
    $options{sort} = \@sort_params;
235
    $options{sort} = \@sort_params;
242
    $options{expanded_facet} = $params->{expanded_facet};
236
    $options{expanded_facet} = $params->{expanded_facet};
237
    $options{is_opac} = $params->{is_opac};
238
    $options{weighted_fields} = $params->{weighted_fields};
243
    my $query = $self->build_query( $query_str, %options );
239
    my $query = $self->build_query( $query_str, %options );
244
240
245
    #die Dumper($query);
241
    #die Dumper($query);
Lines 300-361 sub build_authorities_query { Link Here
300
296
301
    foreach my $s ( @{ $search->{searches} } ) {
297
    foreach my $s ( @{ $search->{searches} } ) {
302
        my ( $wh, $op, $val ) = @{$s}{qw(where operator value)};
298
        my ( $wh, $op, $val ) = @{$s}{qw(where operator value)};
303
        $wh = '_all' if $wh eq '';
299
        if (!$wh) {
304
        if ( $op eq 'is' || $op eq '=' ) {
300
            if ( $op eq 'is' || $op eq '=' || $op eq 'exact') {
305
301
                # Match the whole field for all searchable fields, case insensitive,
306
            # look for something that matches a term completely
302
                # UTF normalized.
307
            # note, '=' is about numerical vals. May need special handling.
303
                # Given that field data is "The quick brown fox"
308
            # Also, we lowercase our search because the ES
304
                # "The quick brown fox" and "the quick brown fox" will match
309
            # index lowercases its values, and term searches don't get the
305
                # but not "quick brown fox".
310
            # search analyzer applied to them.
306
                push @query_parts, {
311
            push @query_parts, { term => {"$wh.phrase" => lc $val} };
307
                    multi_match => {
308
                        query => $val,
309
                        fields => $self->_search_fields({ subfield => 'ci_raw' }),
310
                    }
311
                };
312
            }
313
            elsif ( $op eq 'start') {
314
                # Match the prefix within a field for all searchable fields.
315
                # Given that field data is "The quick brown fox"
316
                # "The quick bro" will match, but not "quick bro"
317
318
                # Does not seems to be a multi prefix query
319
                # so we need to create one
320
                my @prefix_queries;
321
                foreach my $field (@{$self->_search_fields()}) {
322
                    push @prefix_queries, {
323
                        prefix => { "$field.ci_raw" => $val }
324
                    };
325
                }
326
                push @query_parts, {
327
                    'bool' => {
328
                        'should' => \@prefix_queries,
329
                        'minimum_should_match' => 1
330
                    }
331
                };
332
            }
333
            else {
334
                # Query all searchable fields.
335
                # Given that field data is "The quick brown fox"
336
                # a search containing any of the words will match, regardless
337
                # of order.
338
                #
339
                # Since default operator is "AND" each of the words must match
340
                # at least one field
341
                push @query_parts, {
342
                    query_string => {
343
                        query => $val,
344
                        fields => $self->_search_fields(),
345
                        default_operator => 'AND',
346
                    }
347
                };
348
            }
312
        }
349
        }
313
        elsif ( $op eq 'exact' ) {
350
        elsif ( $op eq 'is' || $op eq '=' || $op eq 'exact') {
314
            # left and right truncation, otherwise an exact phrase
351
            # Match the whole field, case insensitive, UTF normalized.
315
            push @query_parts, { match_phrase => {"$wh.phrase" => lc $val} };
352
            push @query_parts, { term => { "$wh.ci_raw" => $val } };
316
        }
353
        }
317
        elsif ( $op eq 'start' ) {
354
        elsif ( $op eq 'start' ) {
318
            # startswith search, uses lowercase untokenized version of heading
355
            # Match prefix of the field.
319
            push @query_parts, { prefix => {"$wh.lc_raw" => lc $val} };
356
            push @query_parts, { prefix => {"$wh.ci_raw" => $val} };
320
        }
357
        }
321
        else {
358
        else {
322
            # regular wordlist stuff
359
            # Regular match query for the specific field.
323
#            push @query_parts, { match => {$wh => { query => $val, operator => 'and' }} };
360
            push @query_parts, {
324
            my @values = split(' ',$val);
361
                match => {
325
            foreach my $v (@values) {
362
                    $wh => {
326
                push @query_parts, { wildcard => { "$wh.phrase" => "*" . lc $v . "*" } };
363
                        query => $val,
327
            }
364
                        operator => 'and'
365
                    }
366
                }
367
            };
328
        }
368
        }
329
    }
369
    }
330
370
331
    # Merge the query parts appropriately
371
    # Merge the query parts appropriately
332
    # 'should' behaves like 'or'
372
    # 'should' behaves like 'or'
333
    # 'must' behaves like 'and'
373
    # 'must' behaves like 'and'
334
    # Zebra results seem to match must so using that here
374
    # Zebra behaviour seem to match must so using that here
335
    my $query = { query=>
375
    my $elastic_query = {};
336
                 { bool =>
376
    $elastic_query->{bool}->{must} = \@query_parts;
337
                     { must => \@query_parts  }
377
338
                 }
378
    # Filter by authtypecode if set
339
             };
379
    if ($search->{authtypecode}) {
340
380
        $elastic_query->{bool}->{filter} = {
341
    # We need to add '.phrase' to all the sort headings otherwise it'll sort
381
            term => {
342
    # based on the tokenised form.
382
                "authtype.raw" => $search->{authtypecode}
343
    my %s;
383
            }
344
    if ( exists $search->{sort} ) {
384
        };
345
        foreach my $k ( keys %{ $search->{sort} } ) {
346
            my $f = $self->_sort_field($k);
347
            $s{"$f.phrase"} = $search->{sort}{$k};
348
        }
349
        $search->{sort} = \%s;
350
    }
385
    }
351
386
352
    # add the sort stuff
387
    my $query = {
353
    $query->{sort} = [ $search->{sort} ]  if exists $search->{sort};
388
        query => $elastic_query
389
    };
390
391
    # Add the sort stuff
392
    $query->{sort} = [ $search->{sort} ] if exists $search->{sort};
354
393
355
    return $query;
394
    return $query;
356
}
395
}
357
396
358
359
=head2 build_authorities_query_compat
397
=head2 build_authorities_query_compat
360
398
361
    my ($query) =
399
    my ($query) =
Lines 449-455 sub build_authorities_query_compat { Link Here
449
    my %sort;
487
    my %sort;
450
    my $sort_field =
488
    my $sort_field =
451
        ( $orderby =~ /^Heading/ ) ? 'Heading__sort'
489
        ( $orderby =~ /^Heading/ ) ? 'Heading__sort'
452
      : ( $orderby =~ /^Auth/ )    ? 'Local-Number'
490
      : ( $orderby =~ /^Auth/ )    ? 'Local-Number__sort'
453
      :                              undef;
491
      :                              undef;
454
    if ($sort_field) {
492
    if ($sort_field) {
455
        my $sort_order = ( $orderby =~ /Asc$/ ) ? 'asc' : 'desc';
493
        my $sort_order = ( $orderby =~ /Asc$/ ) ? 'asc' : 'desc';
Lines 516-522 types. Link Here
516
=cut
554
=cut
517
555
518
our %index_field_convert = (
556
our %index_field_convert = (
519
    'kw'      => '_all',
557
    'kw'      => '',
520
    'ti'      => 'title',
558
    'ti'      => 'title',
521
    'au'      => 'author',
559
    'au'      => 'author',
522
    'su'      => 'subject',
560
    'su'      => 'subject',
Lines 542-548 sub _convert_index_fields { Link Here
542
    # If a field starts with mc- we save it as it's used (and removed) later
580
    # If a field starts with mc- we save it as it's used (and removed) later
543
    # when joining things, to indicate we make it an 'OR' join.
581
    # when joining things, to indicate we make it an 'OR' join.
544
    # (Sorry, this got a bit ugly after special cases were found.)
582
    # (Sorry, this got a bit ugly after special cases were found.)
545
    grep { $_->{field} } map {
583
    map {
546
        my ( $f, $t ) = split /,/;
584
        my ( $f, $t ) = split /,/;
547
        my $mc = '';
585
        my $mc = '';
548
        if ($f =~ /^mc-/) {
586
        if ($f =~ /^mc-/) {
Lines 554-560 sub _convert_index_fields { Link Here
554
            type  => $index_type_convert{ $t // '__default' }
592
            type  => $index_type_convert{ $t // '__default' }
555
        };
593
        };
556
        $r->{field} = ($mc . $r->{field}) if $mc && $r->{field};
594
        $r->{field} = ($mc . $r->{field}) if $mc && $r->{field};
557
        $r;
595
        $r->{field} ? $r : undef;
558
    } @indexes;
596
    } @indexes;
559
}
597
}
560
598
Lines 583-590 sub _convert_index_strings { Link Here
583
            push @res, $s;
621
            push @res, $s;
584
            next;
622
            next;
585
        }
623
        }
586
        push @res, $conv->{field} . ":"
624
        push @res, ($conv->{field} ? $conv->{field} . ':' : '')
587
          . $self->_modify_string_by_type( %$conv, operand => $term );
625
            . $self->_modify_string_by_type( %$conv, operand => $term );
588
    }
626
    }
589
    return @res;
627
    return @res;
590
}
628
}
Lines 816-819 sub _truncate_terms { Link Here
816
    return join ' ', @terms;
854
    return join ' ', @terms;
817
}
855
}
818
856
857
sub _search_fields {
858
    my ($self, $params) = @_;
859
    $params //= {
860
        is_opac => 0,
861
		weighted_fields => 0,
862
        # This is a hack for authorities build_authorities_query
863
        # can hopefully be removed in the future
864
        subfield => undef,
865
    };
866
867
    my $cache = Koha::Caches->get_instance();
868
    my $cache_key = 'elasticsearch_search_fields' . ($params->{is_opac} ? '_opac' : '_staff_client');
869
    my $search_fields = $cache->get_from_cache($cache_key, { unsafe => 1 });
870
871
    if (!$search_fields) {
872
        # The reason we don't use Koha::SearchFields->search here is we don't
873
        # want or need resultset wrapped as Koha::SearchField object.
874
        # It does not make any sense in this context and would cause
875
        # unnecessary overhead sice we are only querying for data
876
        # Also would not work, or produce strange results, with the "columns"
877
        # option.
878
        my $schema = Koha::Database->schema;
879
        my $result = $schema->resultset('SearchField')->search(
880
            {
881
                $params->{is_opac} ? (
882
                    'opac' => 1,
883
                ) : (
884
                    'staff_client' => 1
885
                ),
886
                'search_marc_map.index_name' => $self->index,
887
                'search_marc_map.marc_type' => C4::Context->preference('marcflavour'),
888
                'search_marc_to_fields.search' => 1,
889
            },
890
            {
891
                columns => [qw/name weight/],
892
                # collapse => 1,
893
                join => {search_marc_to_fields => 'search_marc_map'},
894
            }
895
        );
896
        my @search_fields;
897
        while (my $search_field = $result->next) {
898
            push @search_fields, [
899
                $search_field->name,
900
                $search_field->weight ? $search_field->weight : ()
901
            ];
902
        }
903
        $search_fields = \@search_fields;
904
        $cache->set_in_cache($cache_key, $search_fields);
905
    }
906
    if ($params->{subfield}) {
907
        my $subfield = $params->{subfield};
908
        $search_fields = [
909
            map {
910
                # Copy values to avoid mutating cached
911
                # data (since unsafe is used)
912
                my ($field, $weight) = @{$_};
913
                ["${field}.${subfield}", $weight];
914
            } @{$search_fields}
915
        ];
916
    }
917
918
	if ($params->{weighted_fields}) {
919
        return [map { join('^', @{$_}) } @{$search_fields}];
920
	}
921
    else {
922
		# Exclude weight from field
923
		return [map { $_->[0] } @{$search_fields}];
924
    }
925
}
926
819
1;
927
1;
(-)a/Koha/SearchField.pm (-48 lines)
Lines 20-26 use Modern::Perl; Link Here
20
use Carp;
20
use Carp;
21
21
22
use Koha::Database;
22
use Koha::Database;
23
use Koha::SearchMarcMaps;
24
23
25
use base qw(Koha::Object);
24
use base qw(Koha::Object);
26
25
Lines 39-91 sub add_to_search_marc_maps { Link Here
39
    return $self->_result()->add_to_search_marc_maps($search_field->_result, $params);
38
    return $self->_result()->add_to_search_marc_maps($search_field->_result, $params);
40
}
39
}
41
40
42
=head3 search_marc_maps
43
44
my $search_marc_maps = $search_field->search_marc_maps;
45
46
=cut
47
48
sub search_marc_maps {
49
    my ( $self ) = @_;
50
51
    my $marc_type = lc C4::Context->preference('marcflavour');
52
53
    my $schema = Koha::Database->new->schema;
54
    my $marc_map_fields = $schema->resultset('SearchMarcToField')->search(
55
        {
56
            'me.search_field_id'        => $self->id,
57
            'search_marc_map.marc_type' => $marc_type
58
        },
59
        {
60
            select => [
61
                'search_marc_map.index_name',
62
                'search_marc_map.marc_type',
63
                'search_marc_map.marc_field'
64
            ],
65
            as => [ 'index_name', 'marc_type', 'marc_field' ],
66
            join => 'search_marc_map'
67
        }
68
    );
69
70
    return $marc_map_fields;
71
}
72
73
=head3 is_mapped_biblios
74
75
my $is_mapped_biblios = $search_field->is_mapped_biblios
76
77
=cut
78
79
sub is_mapped_biblios {
80
    my ( $self ) = @_;
81
82
    return $self->search_marc_maps->search(
83
        {
84
            index_name => 'biblios'
85
        }
86
    )->count ? 1 : 0;
87
}
88
89
=head3 type
41
=head3 type
90
42
91
=cut
43
=cut
(-)a/Koha/SearchFields.pm (-15 lines)
Lines 35-55 Koha::SearchFields - Koha SearchField Object set class Link Here
35
35
36
=cut
36
=cut
37
37
38
=head3 weighted_fields
39
40
my (@w_fields, @weight) = Koha::SearchFields->weighted_fields();
41
42
=cut
43
44
sub weighted_fields {
45
    my ($self) = @_;
46
47
    return $self->search(
48
        { weight => { '>' => 0, '!=' => undef } },
49
        { order_by => { -desc => 'weight' } }
50
    );
51
}
52
53
=head3 type
38
=head3 type
54
39
55
=cut
40
=cut
(-)a/admin/searchengine/elasticsearch/field_config.yaml (-21 / +13 lines)
Lines 1-9 Link Here
1
---
1
---
2
# General field configuration
2
# General field configuration
3
general:
3
general:
4
  _all:
5
    type: string
6
    analyzer: analyser_standard
7
  properties:
4
  properties:
8
    record:
5
    record:
9
      store: true
6
      store: true
Lines 18-48 search: Link Here
18
    null_value: 0
15
    null_value: 0
19
  stdno:
16
  stdno:
20
    type: text
17
    type: text
21
    analyzer: analyser_stdno
18
    analyzer: analyzer_stdno
22
    search_analyzer: analyser_stdno
19
    search_analyzer: analyzer_stdno
23
    fields:
20
    fields:
24
      phrase:
21
      phrase:
25
        type: text
22
        type: text
26
        analyzer: analyser_stdno
23
        analyzer: analyzer_phrase
27
        search_analyzer: analyser_stdno
24
        search_analyzer: analyzer_phrase
28
      raw:
25
      raw:
29
        type: keyword
26
        type: keyword
30
    copy_to: _all
31
  default:
27
  default:
32
    type: text
28
    type: text
33
    analyzer: analyser_standard
29
    analyzer: analyzer_standard
34
    search_analyzer: analyser_standard
30
    search_analyzer: analyzer_standard
35
    fields:
31
    fields:
36
      phrase:
32
      phrase:
37
        type: text
33
        type: text
38
        analyzer: analyser_phrase
34
        analyzer: analyzer_phrase
39
        search_analyzer: analyser_phrase
35
        search_analyzer: analyzer_phrase
40
      raw:
36
      raw:
41
        type: keyword
37
        type: keyword
42
      lc_raw:
38
        normalizer: nfkc_cf_normalizer
39
      ci_raw:
43
        type: keyword
40
        type: keyword
44
        normalizer: my_normalizer
41
        normalizer: icu_folding_normalizer
45
    copy_to: _all
46
# Facets
42
# Facets
47
facet:
43
facet:
48
  default:
44
  default:
Lines 56-64 suggestible: Link Here
56
# Sort
52
# Sort
57
sort:
53
sort:
58
  default:
54
  default:
59
    type: text
55
    type: icu_collation_keyword
60
    analyzer: analyser_phrase
56
    index: false
61
    search_analyzer: analyser_phrase
62
    fields:
63
      phrase:
64
        type: keyword
(-)a/admin/searchengine/elasticsearch/index_config.yaml (-6 / +6 lines)
Lines 3-31 Link Here
3
index:
3
index:
4
  analysis:
4
  analysis:
5
    analyzer:
5
    analyzer:
6
      # Phrase analyzer is used for phrases (phrase match, sorting)
6
      # Phrase analyzer is used for phrases (exact phrase match)
7
      analyser_phrase:
7
      analyzer_phrase:
8
        tokenizer: keyword
8
        tokenizer: keyword
9
        filter:
9
        filter:
10
          - icu_folding
10
          - icu_folding
11
        char_filter:
11
        char_filter:
12
          - punctuation
12
          - punctuation
13
      analyser_standard:
13
      analyzer_standard:
14
        tokenizer: icu_tokenizer
14
        tokenizer: icu_tokenizer
15
        filter:
15
        filter:
16
          - icu_folding
16
          - icu_folding
17
      analyser_stdno:
17
      analyzer_stdno:
18
        tokenizer: whitespace
18
        tokenizer: whitespace
19
        filter:
19
        filter:
20
          - icu_folding
20
          - icu_folding
21
        char_filter:
21
        char_filter:
22
          - punctuation
22
          - punctuation
23
    normalizer:
23
    normalizer:
24
      normalizer_keyword:
24
      icu_folding_normalizer:
25
        type: custom
25
        type: custom
26
        filter:
26
        filter:
27
          - icu_folding
27
          - icu_folding
28
      my_normalizer:
28
      nfkc_cf_normalizer:
29
        type: custom
29
        type: custom
30
        char_filter: icu_normalizer
30
        char_filter: icu_normalizer
31
    char_filter:
31
    char_filter:
(-)a/admin/searchengine/elasticsearch/mappings.pl (-31 / +67 lines)
Lines 25-30 use C4::Auth; Link Here
25
use Koha::SearchEngine::Elasticsearch;
25
use Koha::SearchEngine::Elasticsearch;
26
use Koha::SearchMarcMaps;
26
use Koha::SearchMarcMaps;
27
use Koha::SearchFields;
27
use Koha::SearchFields;
28
use Koha::Caches;
28
29
29
my $input = new CGI;
30
my $input = new CGI;
30
my ( $template, $borrowernumber, $cookie ) = get_template_and_user(
31
my ( $template, $borrowernumber, $cookie ) = get_template_and_user(
Lines 45-50 my $schema = $database->schema; Link Here
45
46
46
my $marc_type = lc C4::Context->preference('marcflavour');
47
my $marc_type = lc C4::Context->preference('marcflavour');
47
48
49
my $cache = Koha::Caches->get_instance();
50
my $clear_cache = sub {
51
    $cache->clear_from_cache('elasticsearch_search_fields_staff_client');
52
    $cache->clear_from_cache('elasticsearch_search_fields_opac');
53
};
54
48
if ( $op eq 'edit' ) {
55
if ( $op eq 'edit' ) {
49
56
50
    $schema->storage->txn_begin;
57
    $schema->storage->txn_begin;
Lines 53-64 if ( $op eq 'edit' ) { Link Here
53
    my @field_label = $input->param('search_field_label');
60
    my @field_label = $input->param('search_field_label');
54
    my @field_type = $input->param('search_field_type');
61
    my @field_type = $input->param('search_field_type');
55
    my @field_weight = $input->param('search_field_weight');
62
    my @field_weight = $input->param('search_field_weight');
63
    my @field_staff_client = $input->param('search_field_staff_client');
64
    my @field_opac = $input->param('search_field_opac');
56
65
57
    my @index_name          = $input->param('mapping_index_name');
66
    my @index_name          = $input->param('mapping_index_name');
58
    my @search_field_name  = $input->param('mapping_search_field_name');
67
    my @search_field_name   = $input->param('mapping_search_field_name');
59
    my @mapping_sort        = $input->param('mapping_sort');
68
    my @mapping_sort        = $input->param('mapping_sort');
60
    my @mapping_facet       = $input->param('mapping_facet');
69
    my @mapping_facet       = $input->param('mapping_facet');
61
    my @mapping_suggestible = $input->param('mapping_suggestible');
70
    my @mapping_suggestible = $input->param('mapping_suggestible');
71
    my @mapping_search      = $input->param('mapping_search');
62
    my @mapping_marc_field  = $input->param('mapping_marc_field');
72
    my @mapping_marc_field  = $input->param('mapping_marc_field');
63
73
64
    eval {
74
    eval {
Lines 68-88 if ( $op eq 'edit' ) { Link Here
68
            my $field_label = $field_label[$i];
78
            my $field_label = $field_label[$i];
69
            my $field_type = $field_type[$i];
79
            my $field_type = $field_type[$i];
70
            my $field_weight = $field_weight[$i];
80
            my $field_weight = $field_weight[$i];
71
81
            my $field_staff_client = $field_staff_client[$i];
82
            my $field_opac = $field_opac[$i];
72
            my $search_field = Koha::SearchFields->find( { name => $field_name }, { key => 'name' } );
83
            my $search_field = Koha::SearchFields->find( { name => $field_name }, { key => 'name' } );
73
            $search_field->label($field_label);
84
            $search_field->label($field_label);
74
            $search_field->type($field_type);
85
            $search_field->type($field_type);
75
86
76
            if (!length($field_weight)) {
87
			if (!length($field_weight)) {
77
                $search_field->weight(undef);
88
				$search_field->weight(undef);
78
            }
89
			}
79
            elsif ($field_weight <= 0 || !looks_like_number($field_weight)) {
90
			elsif ($field_weight <= 0 || !looks_like_number($field_weight)) {
80
                push @messages, { type => 'error', code => 'invalid_field_weight', 'weight' => $field_weight };
91
				push @messages, { type => 'error', code => 'invalid_field_weight', 'weight' => $field_weight };
81
            }
92
			}
82
            else {
93
			else {
83
                $search_field->weight($field_weight);
94
				$search_field->weight($field_weight);
84
            }
95
			}
85
96
97
            $search_field->staff_client($field_staff_client ? 1 : 0);
98
            $search_field->opac($field_opac ? 1 : 0);
86
            $search_field->store;
99
            $search_field->store;
87
        }
100
        }
88
101
Lines 90-107 if ( $op eq 'edit' ) { Link Here
90
103
91
        for my $i ( 0 .. scalar(@index_name) - 1 ) {
104
        for my $i ( 0 .. scalar(@index_name) - 1 ) {
92
            my $index_name          = $index_name[$i];
105
            my $index_name          = $index_name[$i];
93
            my $search_field_name  = $search_field_name[$i];
106
            my $search_field_name   = $search_field_name[$i];
94
            my $mapping_marc_field  = $mapping_marc_field[$i];
107
            my $mapping_marc_field  = $mapping_marc_field[$i];
95
            my $mapping_facet       = $mapping_facet[$i];
108
            my $mapping_facet       = $mapping_facet[$i];
96
            my $mapping_suggestible = $mapping_suggestible[$i];
109
            my $mapping_suggestible = $mapping_suggestible[$i];
97
            my $mapping_sort        = $mapping_sort[$i];
110
            my $mapping_sort        = $mapping_sort[$i] eq 'undef' ? undef : $mapping_sort[$i];
98
            $mapping_sort = undef if $mapping_sort eq 'undef';
111
            my $mapping_search      = $mapping_search[$i];
99
112
100
            my $search_field = Koha::SearchFields->find({ name => $search_field_name }, { key => 'name' });
113
            my $search_field = Koha::SearchFields->find({ name => $search_field_name }, { key => 'name' });
101
            # TODO Check mapping format
114
            # TODO Check mapping format
102
            my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $marc_type, marc_field => $mapping_marc_field });
115
            my $marc_field = Koha::SearchMarcMaps->find_or_create({
103
            $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping_facet, suggestible => $mapping_suggestible, sort => $mapping_sort } );
116
                index_name => $index_name,
104
117
                marc_type => $marc_type,
118
                marc_field => $mapping_marc_field
119
            });
120
            $search_field->add_to_search_marc_maps($marc_field, {
121
                facet => $mapping_facet,
122
                suggestible => $mapping_suggestible,
123
                sort => $mapping_sort,
124
                search => $mapping_search
125
            });
105
        }
126
        }
106
    };
127
    };
107
    if ($@) {
128
    if ($@) {
Lines 111-122 if ( $op eq 'edit' ) { Link Here
111
        push @messages, { type => 'message', code => 'success_on_update' };
132
        push @messages, { type => 'message', code => 'success_on_update' };
112
        $schema->storage->txn_commit;
133
        $schema->storage->txn_commit;
113
    }
134
    }
135
    $clear_cache->();
114
}
136
}
115
elsif( $op eq 'reset_confirmed' ) {
137
elsif( $op eq 'confirm' ) {
138
    Koha::SearchMarcMaps->search->delete;
116
    Koha::SearchMarcMaps->delete;
139
    Koha::SearchMarcMaps->delete;
117
    Koha::SearchFields->delete;
118
    Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;
140
    Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;
119
    push @messages, { type => 'message', code => 'success_on_reset' };
141
    $clear_cache->();
120
}
142
}
121
elsif( $op eq 'reset_confirm' ) {
143
elsif( $op eq 'reset_confirm' ) {
122
    $template->param( reset_confirm => 1 );
144
    $template->param( reset_confirm => 1 );
Lines 127-153 my @indexes; Link Here
127
149
128
for my $index_name (qw| biblios authorities |) {
150
for my $index_name (qw| biblios authorities |) {
129
    my $search_fields = Koha::SearchFields->search(
151
    my $search_fields = Koha::SearchFields->search(
130
        { 'search_marc_map.index_name' => $index_name, 'search_marc_map.marc_type' => $marc_type, },
152
        {
131
        {   join => { search_marc_to_fields => 'search_marc_map' },
153
            'search_marc_map.index_name' => $index_name,
132
            '+select' => [ 'search_marc_to_fields.facet', 'search_marc_to_fields.suggestible', 'search_marc_to_fields.sort', 'search_marc_map.marc_field' ],
154
            'search_marc_map.marc_type' => $marc_type,
133
            '+as'     => [ 'facet',                       'suggestible',                       'sort',                       'marc_field' ],
155
        },
134
            order_by => { -asc => [qw/name marc_field/] }
156
        {
157
            join => { search_marc_to_fields => 'search_marc_map' },
158
            '+select' => [
159
                'search_marc_to_fields.facet',
160
                'search_marc_to_fields.suggestible',
161
                'search_marc_to_fields.sort',
162
                'search_marc_to_fields.search',
163
                'search_marc_map.marc_field',
164
            ],
165
            '+as' => [
166
                'facet',
167
                'suggestible',
168
                'sort',
169
                'search',
170
                'marc_field',
171
            ],
135
        }
172
        }
136
    );
173
    );
137
174
138
    my @mappings;
175
    my @mappings;
139
    while ( my $s = $search_fields->next ) {
176
    while ( my $s = $search_fields->next ) {
140
        push @mappings,
177
        push @mappings, {
141
          { search_field_name  => $s->name,
178
            search_field_name  => $s->name,
142
            search_field_label => $s->label,
179
            search_field_label => $s->label,
143
            search_field_type  => $s->type,
180
            search_field_type  => $s->type,
144
            marc_field         => $s->get_column('marc_field'),
181
            marc_field         => $s->get_column('marc_field'),
145
            sort               => $s->get_column('sort') // 'undef', # To avoid warnings "Use of uninitialized value in lc"
182
            sort               => $s->get_column('sort') // 'undef', # To avoid warnings "Use of uninitialized value in lc"
146
            suggestible        => $s->get_column('suggestible'),
183
            suggestible        => $s->get_column('suggestible'),
147
            facet              => $s->get_column('facet'),
184
            facet              => $s->get_column('facet'),
148
          };
185
            search             => $s->get_column('search'),
186
        };
149
    }
187
    }
150
151
    push @indexes, { index_name => $index_name, mappings => \@mappings };
188
    push @indexes, { index_name => $index_name, mappings => \@mappings };
152
}
189
}
153
190
Lines 155-161 my $search_fields = Koha::SearchFields->search( {}, { order_by => ['name'] } ); Link Here
155
my @all_search_fields;
192
my @all_search_fields;
156
while ( my $search_field = $search_fields->next ) {
193
while ( my $search_field = $search_fields->next ) {
157
    my $search_field_unblessed = $search_field->unblessed;
194
    my $search_field_unblessed = $search_field->unblessed;
158
    $search_field_unblessed->{mapped_biblios} = 1 if $search_field->is_mapped_biblios;
159
    push @all_search_fields, $search_field_unblessed;
195
    push @all_search_fields, $search_field_unblessed;
160
}
196
}
161
197
(-)a/catalogue/search.pl (-6 / +1 lines)
Lines 456-466 my $expanded_facet = $params->{'expand'}; Link Here
456
# Define some global variables
456
# Define some global variables
457
my ( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type);
457
my ( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type);
458
458
459
my $build_params;
460
unless ( $cgi->param('advsearch') ) {
461
    $build_params->{weighted_fields} = 1;
462
}
463
464
my $builder = Koha::SearchEngine::QueryBuilder->new(
459
my $builder = Koha::SearchEngine::QueryBuilder->new(
465
    { index => $Koha::SearchEngine::BIBLIOS_INDEX } );
460
    { index => $Koha::SearchEngine::BIBLIOS_INDEX } );
466
my $searcher = Koha::SearchEngine::Search->new(
461
my $searcher = Koha::SearchEngine::Search->new(
Lines 473-479 my $searcher = Koha::SearchEngine::Search->new( Link Here
473
    $query_type
468
    $query_type
474
  )
469
  )
475
  = $builder->build_query_compat( \@operators, \@operands, \@indexes, \@limits,
470
  = $builder->build_query_compat( \@operators, \@operands, \@indexes, \@limits,
476
    \@sort_by, $scan, $lang, $build_params );
471
    \@sort_by, $scan, $lang, { weighted_fields => !$cgi->param('advsearch') });
477
472
478
## parse the query_cgi string and put it into a form suitable for <input>s
473
## parse the query_cgi string and put it into a form suitable for <input>s
479
my @query_inputs;
474
my @query_inputs;
(-)a/installer/data/mysql/atomicupdate/bug_20589_add_columns.perl (+14 lines)
Line 0 Link Here
1
$DBversion = 'XXX';
2
if (CheckVersion($DBversion)) {
3
  if (!column_exists('search_marc_to_field', 'search')) {
4
    $dbh->do("ALTER TABLE `search_marc_to_field` ADD COLUMN `search` tinyint(1) NOT NULL DEFAULT 1");
5
  }
6
  if (!column_exists('search_field', 'staff_client')) {
7
    $dbh->do("ALTER TABLE `search_field` ADD COLUMN `staff_client` tinyint(1) NOT NULL DEFAULT 1");
8
  }
9
  if (!column_exists('search_field', 'opac')) {
10
    $dbh->do("ALTER TABLE `search_field` ADD COLUMN `opac` tinyint(1) NOT NULL DEFAULT 1");
11
  }
12
  SetVersion($DBversion)
13
}
14
print "Upgrade to $DBversion done (Bug 20589 - Add field boosting and use elastic query fields parameter instead of depricated _all)\n";
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt (-5 / +55 lines)
Lines 150-157 a.add, a.delete { Link Here
150
                    <th>Name</th>
150
                    <th>Name</th>
151
                    <th>Label</th>
151
                    <th>Label</th>
152
                    <th>Type</th>
152
                    <th>Type</th>
153
                    <th colspan="2">Searchable</th>
153
                    <th>Weight</th>
154
                    <th>Weight</th>
154
                  </tr>
155
                  </tr>
156
                  <tr>
157
                    <th colspan=3>&nbsp;</th>
158
                    <th>Staff client</th>
159
                    <th>OPAC</th>
160
                    <th>&nbsp;</th>
161
                  </tr>
155
                </thead>
162
                </thead>
156
                <tbody>
163
                <tbody>
157
                  [% FOREACH search_field IN all_search_fields %]
164
                  [% FOREACH search_field IN all_search_fields %]
Lines 159-165 a.add, a.delete { Link Here
159
                      <td>
166
                      <td>
160
                        <input type="text" name="search_field_name" value="[% search_field.name | html %]" />
167
                        <input type="text" name="search_field_name" value="[% search_field.name | html %]" />
161
                      </td>
168
                      </td>
162
                      <td><input type="text" name="search_field_label" value="[% search_field.label | html %]" />
169
                      <td>
170
                        <input type="text" name="search_field_label" value="[% search_field.label | html %]" />
171
                      </td>
163
                      <td>
172
                      <td>
164
                        <select name="search_field_type">
173
                        <select name="search_field_type">
165
                          <option value=""></option>
174
                          <option value=""></option>
Lines 201-211 a.add, a.delete { Link Here
201
                        </select>
210
                        </select>
202
                      </td>
211
                      </td>
203
                      <td>
212
                      <td>
204
                      [% IF search_field.mapped_biblios %]
213
                        <select name="search_field_staff_client">
214
                          [% IF search_field.staff_client %]
215
                            <option value="1" selected="selected">Yes</option>
216
                            <option value="0">No</option>
217
                          [% ELSE %]
218
                            <option value="1">Yes</option>
219
                            <option value="0" selected="selected">No</option>
220
                          [% END %]
221
                        </select>
222
                      </td>
223
                      <td>
224
                        <select name="search_field_opac">
225
                          [% IF search_field.opac %]
226
                            <option value="1" selected="selected">Yes</option>
227
                            <option value="0">No</option>
228
                          [% ELSE %]
229
                            <option value="1">Yes</option>
230
                            <option value="0" selected="selected">No</option>
231
                          [% END %]
232
                        </select>
233
                      </td>
234
                      <td>
205
                        <input type="number" step="0.01" min="0.01" max="999.99" name="search_field_weight" value="[% search_field.weight | html %]" />
235
                        <input type="number" step="0.01" min="0.01" max="999.99" name="search_field_weight" value="[% search_field.weight | html %]" />
206
                      [% ELSE %]
207
                        <input type="hidden" name="search_field_weight" value="">
208
                      [% END %]
209
                      </td>
236
                      </td>
210
                    </tr>
237
                    </tr>
211
                  [% END %]
238
                  [% END %]
Lines 221-226 a.add, a.delete { Link Here
221
                          <th>Sortable</th>
248
                          <th>Sortable</th>
222
                          <th>Facetable</th>
249
                          <th>Facetable</th>
223
                          <th>Suggestible</th>
250
                          <th>Suggestible</th>
251
                          <th>Searchable</th>
224
                          <th>Mapping</th>
252
                          <th>Mapping</th>
225
                          <th></th>
253
                          <th></th>
226
                        </tr>
254
                        </tr>
Lines 275-280 a.add, a.delete { Link Here
275
                              </select>
303
                              </select>
276
                            </td>
304
                            </td>
277
                            <td>
305
                            <td>
306
                              <select name="mapping_search">
307
                                [% IF mapping.search %]
308
                                  <option value="0">No</option>
309
                                  <option value="1" selected="selected">Yes</option>
310
                                [% ELSE %]
311
                                  <option value="0" selected="selected">No</option>
312
                                  <option value="1">Yes</option>
313
                                [% END %]
314
                              </select>
315
                            </td>
316
                            <td>
278
                                <input name="mapping_marc_field" type="text" value="[% mapping.marc_field | html %]" />
317
                                <input name="mapping_marc_field" type="text" value="[% mapping.marc_field | html %]" />
279
                            </td>
318
                            </td>
280
                            <td><a class="btn btn-default btn-xs delete" style="cursor: pointer;"><i class="fa fa-trash"></i> Delete</a></td>
319
                            <td><a class="btn btn-default btn-xs delete" style="cursor: pointer;"><i class="fa fa-trash"></i> Delete</a></td>
Lines 320-325 a.add, a.delete { Link Here
320
                              [% END %]
359
                              [% END %]
321
                            </select>
360
                            </select>
322
                          </td>
361
                          </td>
362
                          <td>
363
                            <select data-id="mapping_search">
364
                              [% IF mapping.search %]
365
                                <option value="0">No</option>
366
                                <option value="1" selected="selected">Yes</option>
367
                              [% ELSE %]
368
                                <option value="0" selected="selected">No</option>
369
                                <option value="1">Yes</option>
370
                              [% END %]
371
                            </select>
372
                          </td>
323
                          <td><input data-id="mapping_marc_field" type="text" /></td>
373
                          <td><input data-id="mapping_marc_field" type="text" /></td>
324
                          <td><a class="btn btn-default btn-xs add"><i class="fa fa-plus"></i> Add</a></td>
374
                          <td><a class="btn btn-default btn-xs add"><i class="fa fa-plus"></i> Add</a></td>
325
                        </tr>
375
                        </tr>
(-)a/opac/opac-search.pl (-12 / +15 lines)
Lines 549-567 if (C4::Context->preference('OpacSuppression')) { Link Here
549
    }
549
    }
550
}
550
}
551
551
552
my $build_params = {
553
    expanded_facet => $expanded_facet,
554
    suppress => $suppress
555
};
556
557
unless ( $cgi->param('advsearch') ) {
558
    $build_params->{weighted_fields} = 1;
559
}
560
561
## I. BUILD THE QUERY
552
## I. BUILD THE QUERY
562
( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type)
553
( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type)
563
  = $builder->build_query_compat( \@operators, \@operands,
554
  = $builder->build_query_compat(
564
    \@indexes, \@limits, \@sort_by, 0, $lang, $build_params);
555
    \@operators,
556
    \@operands,
557
    \@indexes,
558
    \@limits,
559
    \@sort_by,
560
    0,
561
    $lang,
562
    {
563
        expanded_facet => $expanded_facet,
564
        suppress => $suppress,
565
        is_opac => 1,
566
        weighted_fields => !$cgi->param('advsearch')
567
    }
568
);
565
569
566
sub _input_cgi_parse {
570
sub _input_cgi_parse {
567
    my @elements;
571
    my @elements;
568
- 

Return to bug 20589