View | Details | Raw Unified | Return to bug 22592
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-82 / +116 lines)
Lines 135-179 sub build_query { Link Here
135
    return $res;
135
    return $res;
136
}
136
}
137
137
138
=head2 build_browse_query
139
140
    my $browse_query = $builder->build_browse_query($field, $query);
141
142
This performs a "starts with" style query on a particular field. The field
143
to be searched must have been indexed with an appropriate mapping as a
144
"phrase" subfield, which pretty much everything has.
145
146
=cut
147
148
# XXX this isn't really a browse query like we want in the end
149
sub build_browse_query {
150
    my ( $self, $field, $query ) = @_;
151
152
    my $fuzzy_enabled = C4::Context->preference("QueryFuzzy") || 0;
153
154
    return { query => '*' } if !defined $query;
155
156
    # TODO this should come from Koha::SearchEngine::Elasticsearch
157
    my %field_whitelist = (
158
        title  => 1,
159
        author => 1,
160
    );
161
    $field = 'title' if !exists $field_whitelist{$field};
162
    my $sort = $self->_sort_field($field);
163
    my $res = {
164
        query => {
165
            match_phrase_prefix => {
166
                "$field.phrase" => {
167
                    query     => $query,
168
                    operator  => 'or',
169
                    fuzziness => $fuzzy_enabled ? 'auto' : '0',
170
                }
171
            }
172
        },
173
        sort => [ { $sort => { order => "asc" } } ],
174
    };
175
}
176
177
=head2 build_query_compat
138
=head2 build_query_compat
178
139
179
    my (
140
    my (
Lines 199-252 sub build_query_compat { Link Here
199
        $lang, $params )
160
        $lang, $params )
200
      = @_;
161
      = @_;
201
162
202
#die Dumper ( $self, $operators, $operands, $indexes, $orig_limits, $sort_by, $scan, $lang );
163
    my $query;
203
    my @sort_params  = $self->_convert_sort_fields(@$sort_by);
164
    my $query_str = '';
204
    my @index_params = $self->_convert_index_fields(@$indexes);
165
    my $search_param_query_str = '';
205
    my $limits       = $self->_fix_limit_special_cases($orig_limits);
166
    my $limits = ();
206
    if ( $params->{suppress} ) { push @$limits, "suppress:0"; }
167
    if ( $scan ) {
207
168
        ($query, $query_str) = $self->_build_scan_query( $operands, $indexes );
208
    # Merge the indexes in with the search terms and the operands so that
169
        $search_param_query_str = $query_str;
209
    # each search thing is a handy unit.
170
    } else {
210
    unshift @$operators, undef;    # The first one can't have an op
171
        my @sort_params  = $self->_convert_sort_fields(@$sort_by);
211
    my @search_params;
172
        my @index_params = $self->_convert_index_fields(@$indexes);
212
    my $truncate = C4::Context->preference("QueryAutoTruncate") || 0;
173
        my $limits       = $self->_fix_limit_special_cases($orig_limits);
213
    my $ea = each_array( @$operands, @$operators, @index_params );
174
        if ( $params->{suppress} ) { push @$limits, "suppress:0"; }
214
    while ( my ( $oand, $otor, $index ) = $ea->() ) {
175
215
        next if ( !defined($oand) || $oand eq '' );
176
        # Merge the indexes in with the search terms and the operands so that
216
        $oand = $self->_clean_search_term($oand);
177
        # each search thing is a handy unit.
217
        $oand = $self->_truncate_terms($oand) if ($truncate);
178
        unshift @$operators, undef;    # The first one can't have an op
218
        push @search_params, {
179
        my @search_params;
219
            operand => $oand,      # the search terms
180
        my $truncate = C4::Context->preference("QueryAutoTruncate") || 0;
220
            operator => defined($otor) ? uc $otor : undef,    # AND and so on
181
        my $ea = each_array( @$operands, @$operators, @index_params );
221
            $index ? %$index : (),
182
        while ( my ( $oand, $otor, $index ) = $ea->() ) {
222
        };
183
            next if ( !defined($oand) || $oand eq '' );
223
    }
184
            $oand = $self->_clean_search_term($oand);
185
            $oand = $self->_truncate_terms($oand) if ($truncate);
186
            push @search_params, {
187
                operand => $oand,      # the search terms
188
                operator => defined($otor) ? uc $otor : undef,    # AND and so on
189
                $index ? %$index : (),
190
            };
191
        }
224
192
225
    # We build a string query from limits and the queries. An alternative
193
        # We build a string query from limits and the queries. An alternative
226
    # would be to pass them separately into build_query and let it build
194
        # would be to pass them separately into build_query and let it build
227
    # them into a structured ES query itself. Maybe later, though that'd be
195
        # them into a structured ES query itself. Maybe later, though that'd be
228
    # more robust.
196
        # more robust.
229
    my $search_param_query_str = join( ' ', $self->_create_query_string(@search_params) );
197
        $search_param_query_str = join( ' ', $self->_create_query_string(@search_params) );
230
    my $query_str = join( ' AND ',
198
        my $query_str = join( ' AND ',
231
        $search_param_query_str || (),
199
            $search_param_query_str || (),
232
        $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () );
200
            $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () );
233
201
234
    my @fields = '_all';
202
        my @fields = '_all';
235
    if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) {
203
        if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) {
236
        push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields;
204
            push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields;
237
    }
205
        }
238
206
239
    # If there's no query on the left, let's remove the junk left behind
207
        # If there's no query on the left, let's remove the junk left behind
240
    $query_str =~ s/^ AND //;
208
        $query_str =~ s/^ AND //;
241
    my %options;
209
        my %options;
242
    $options{fields} = \@fields;
210
        $options{fields} = \@fields;
243
    $options{sort} = \@sort_params;
211
        $options{sort} = \@sort_params;
244
    my $query = $self->build_query( $query_str, %options );
212
        $options{expanded_facet} = $params->{expanded_facet};
213
        $query = $self->build_query( $query_str, %options );
214
    }
245
215
246
    # We roughly emulate the CGI parameters of the zebra query builder
216
    # We roughly emulate the CGI parameters of the zebra query builder
247
    my $query_cgi = '';
217
    my $query_cgi = '';
248
    shift @$operators; # Shift out the one we unshifted before
218
    shift @$operators; # Shift out the one we unshifted before
249
    $ea = each_array( @$operands, @$operators, @$indexes );
219
    my $ea = each_array( @$operands, @$operators, @$indexes );
250
    while ( my ( $oand, $otor, $index ) = $ea->() ) {
220
    while ( my ( $oand, $otor, $index ) = $ea->() ) {
251
        $query_cgi .= '&' if $query_cgi;
221
        $query_cgi .= '&' if $query_cgi;
252
        $query_cgi .= 'idx=' . uri_escape_utf8( $index // '') . '&q=' . uri_escape_utf8( $oand );
222
        $query_cgi .= 'idx=' . uri_escape_utf8( $index // '') . '&q=' . uri_escape_utf8( $oand );
Lines 480-485 sub build_authorities_query_compat { Link Here
480
    return $query;
450
    return $query;
481
}
451
}
482
452
453
=head2 _build_scan_query
454
455
    my ($query, $query_str) = $builder->build_scan_query(\@operands, \@indexes)
456
457
This will build an aggregation scan query that can be issued to elasticsearch from
458
the provided string input.
459
460
=cut
461
462
our %scan_field_convert = (
463
    'ti' => 'title',
464
    'au' => 'author',
465
    'su' => 'subject',
466
    'se' => 'title-series',
467
    'pb' => 'publisher',
468
);
469
470
sub _build_scan_query {
471
    my ( $self, $operands, $indexes ) = @_;
472
473
    my $term = scalar( @$operands ) == 0 ? '' : $operands->[0];
474
    my $index = scalar( @$indexes ) == 0 ? 'title' : $indexes->[0];
475
476
    my ( $f, $d ) = split( /,/, $index);
477
    $index = $scan_field_convert{$f} || $f;
478
479
    my $res;
480
    $res->{query} = {
481
        query_string => {
482
            query => '*'
483
        }
484
    };
485
    $res->{aggregations} = {
486
        $index => {
487
            terms => {
488
                field => $index . '__facet',
489
                order => { '_term' => 'asc' },
490
                include => $self->_create_regex_filter($self->_clean_search_term($term)) . '.*'
491
            }
492
        }
493
    };
494
    return ($res, $term);
495
}
496
497
=head2 _create_regex_filter
498
499
    my $filter = $builder->create_regex_filter('term')
500
501
This will create a regex filter that can be used with an aggregation query.
502
503
=cut
504
505
sub _create_regex_filter {
506
    my ($self, $term) = @_;
507
508
    my $result = '';
509
    foreach my $c (split(//, quotemeta($term))) {
510
        my $lc = lc($c);
511
        my $uc = uc($c);
512
        $result .= $lc ne $uc ? '[' . $lc . $uc . ']' : $c;
513
    }
514
    return $result;
515
}
516
483
=head2 _convert_sort_fields
517
=head2 _convert_sort_fields
484
518
485
    my @sort_params = _convert_sort_fields(@sort_by)
519
    my @sort_params = _convert_sort_fields(@sort_by)
Lines 726-732 sub _modify_string_by_type { Link Here
726
    return $str unless $str;    # Empty or undef, we can't use it.
760
    return $str unless $str;    # Empty or undef, we can't use it.
727
761
728
    $str .= '*' if $type eq 'right-truncate';
762
    $str .= '*' if $type eq 'right-truncate';
729
    $str = '"' . $str . '"' if $type eq 'phrase';
763
    $str = '"' . $str . '"' if $type eq 'phrase' && $str !~ /^".*"$/;
730
    if ($type eq 'st-year') {
764
    if ($type eq 'st-year') {
731
        if ($str =~ /^(.*)-(.*)$/) {
765
        if ($str =~ /^(.*)-(.*)$/) {
732
            my $from = $1 || '*';
766
            my $from = $1 || '*';
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-1 / +58 lines)
Lines 85-91 sub search { Link Here
85
85
86
    my $params = $self->get_elasticsearch_params();
86
    my $params = $self->get_elasticsearch_params();
87
    # 20 is the default number of results per page
87
    # 20 is the default number of results per page
88
    $query->{size} = $count || 20;
88
    $query->{size} = $count // 20;
89
    # ES doesn't want pages, it wants a record to start from.
89
    # ES doesn't want pages, it wants a record to start from.
90
    if (exists $options{offset}) {
90
    if (exists $options{offset}) {
91
        $query->{from} = $options{offset};
91
        $query->{from} = $options{offset};
Lines 148-153 sub search_compat { Link Here
148
        $servers,  $results_per_page, $offset,       $branches,
148
        $servers,  $results_per_page, $offset,       $branches,
149
        $query_type,       $scan
149
        $query_type,       $scan
150
    ) = @_;
150
    ) = @_;
151
152
    if ( $scan ) {
153
        return $self->_aggregation_scan( $query, $results_per_page, $offset );
154
    }
155
151
    my %options;
156
    my %options;
152
    if ( !defined $offset or $offset < 0 ) {
157
    if ( !defined $offset or $offset < 0 ) {
153
        $offset = 0;
158
        $offset = 0;
Lines 506-509 sub _convert_facets { Link Here
506
    return \@facets;
511
    return \@facets;
507
}
512
}
508
513
514
=head2 _aggregation_scan
515
516
    my $result = $self->_aggregration_scan($query, 10, 0);
517
518
Perform an aggregation request for scan purposes.
519
520
=cut
521
522
sub _aggregation_scan {
523
    my ($self, $query, $results_per_page, $offset) = @_;
524
525
    if (!scalar(keys %{$query->{aggregations}})) {
526
        my %result = {
527
            biblioserver => {
528
                hits => 0,
529
                RECORDS => undef
530
            }
531
        };
532
        return (undef, \%result, undef);
533
    }
534
    my ($field) = keys %{$query->{aggregations}};
535
    $query->{aggregations}{$field}{terms}{size} = 1000;
536
    my $results = $self->search($query, 1, 0);
537
538
    # Convert each result into a MARC::Record
539
    my (@records, $index);
540
    # opac-search expects results to be put in the
541
    # right place in the array, according to $offset
542
    $index = $offset - 1;
543
544
    my $count = scalar(@{$results->{aggregations}{$field}{buckets}});
545
    for (my $index = $offset; $index - $offset < $results_per_page && $index < $count; $index++) {
546
        my $bucket = $results->{aggregations}{$field}{buckets}->[$index];
547
        # Scan values are expressed as 100 a (count) and 245a (term)
548
        my $marc = MARC::Record->new;
549
        $marc->encoding('UTF-8');
550
        $marc->append_fields(
551
            MARC::Field->new((100, ' ',  ' ', 'a' => $bucket->{doc_count}))
552
        );
553
        $marc->append_fields(
554
            MARC::Field->new((245, ' ',  ' ', 'a' => $bucket->{key}))
555
        );
556
        $records[$index] = $marc->as_usmarc();
557
    };
558
    # consumers of this expect a namespaced result, we provide the default
559
    # configuration.
560
    my %result;
561
    $result{biblioserver}{hits} = $count;
562
    $result{biblioserver}{RECORDS} = \@records;
563
    return (undef, \%result, undef);
564
}
565
509
1;
566
1;
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t (-2 / +21 lines)
Lines 45-51 $se->mock( 'get_elasticsearch_mappings', sub { Link Here
45
                    type => 'text'
45
                    type => 'text'
46
                },
46
                },
47
                subject => {
47
                subject => {
48
                    type => 'text'
48
                    type => 'text',
49
                    facet => 1
49
                },
50
                },
50
                itemnumber => {
51
                itemnumber => {
51
                    type => 'integer'
52
                    type => 'integer'
Lines 416-421 subtest 'build_query tests' => sub { Link Here
416
    );
417
    );
417
    is($query_cgi, 'idx=&q=title%3A%22donald%20duck%22', 'query cgi');
418
    is($query_cgi, 'idx=&q=title%3A%22donald%20duck%22', 'query cgi');
418
    is($query_desc, 'title:"donald duck"', 'query desc ok');
419
    is($query_desc, 'title:"donald duck"', 'query desc ok');
420
421
    # Scan queries
422
    ( undef, $query, $simple_query, $query_cgi, $query_desc ) = $qb->build_query_compat( undef, ['new'], ['su'], undef, undef, 1 );
423
    is(
424
        $query->{query}{query_string}{query},
425
        '*',
426
        "scan query is properly formed"
427
    );
428
    is_deeply(
429
        $query->{aggregations}{'subject'}{'terms'},
430
        {
431
            field => 'subject__facet',
432
            order => { '_term' => 'asc' },
433
            include => '[nN][eE][wW].*'
434
        },
435
        "scan aggregation request is properly formed"
436
    );
437
    is($query_cgi, 'idx=su&q=new&scan=1', 'query cgi');
438
    is($query_desc, 'new', 'query desc ok');
419
};
439
};
420
440
421
441
422
- 

Return to bug 22592