View | Details | Raw Unified | Return to bug 22592
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-83 / +116 lines)
Lines 136-180 sub build_query { Link Here
136
    return $res;
136
    return $res;
137
}
137
}
138
138
139
=head2 build_browse_query
140
141
    my $browse_query = $builder->build_browse_query($field, $query);
142
143
This performs a "starts with" style query on a particular field. The field
144
to be searched must have been indexed with an appropriate mapping as a
145
"phrase" subfield, which pretty much everything has.
146
147
=cut
148
149
# XXX this isn't really a browse query like we want in the end
150
sub build_browse_query {
151
    my ( $self, $field, $query ) = @_;
152
153
    my $fuzzy_enabled = C4::Context->preference("QueryFuzzy") || 0;
154
155
    return { query => '*' } if !defined $query;
156
157
    # TODO this should come from Koha::SearchEngine::Elasticsearch
158
    my %field_whitelist = (
159
        title  => 1,
160
        author => 1,
161
    );
162
    $field = 'title' if !exists $field_whitelist{$field};
163
    my $sort = $self->_sort_field($field);
164
    my $res = {
165
        query => {
166
            match_phrase_prefix => {
167
                "$field.phrase" => {
168
                    query     => $query,
169
                    operator  => 'or',
170
                    fuzziness => $fuzzy_enabled ? 'auto' : '0',
171
                }
172
            }
173
        },
174
        sort => [ { $sort => { order => "asc" } } ],
175
    };
176
}
177
178
=head2 build_query_compat
139
=head2 build_query_compat
179
140
180
    my (
141
    my (
Lines 200-254 sub build_query_compat { Link Here
200
        $lang, $params )
161
        $lang, $params )
201
      = @_;
162
      = @_;
202
163
203
#die Dumper ( $self, $operators, $operands, $indexes, $orig_limits, $sort_by, $scan, $lang );
164
    my $query;
204
    my @sort_params  = $self->_convert_sort_fields(@$sort_by);
165
    my $query_str = '';
205
    my @index_params = $self->_convert_index_fields(@$indexes);
166
    my $search_param_query_str = '';
206
    my $limits       = $self->_fix_limit_special_cases($orig_limits);
167
    my $limits = ();
207
    if ( $params->{suppress} ) { push @$limits, "suppress:0"; }
168
    if ( $scan ) {
208
169
        ($query, $query_str) = $self->_build_scan_query( $operands, $indexes );
209
    # Merge the indexes in with the search terms and the operands so that
170
        $search_param_query_str = $query_str;
210
    # each search thing is a handy unit.
171
    } else {
211
    unshift @$operators, undef;    # The first one can't have an op
172
        my @sort_params  = $self->_convert_sort_fields(@$sort_by);
212
    my @search_params;
173
        my @index_params = $self->_convert_index_fields(@$indexes);
213
    my $truncate = C4::Context->preference("QueryAutoTruncate") || 0;
174
        my $limits       = $self->_fix_limit_special_cases($orig_limits);
214
    my $ea = each_array( @$operands, @$operators, @index_params );
175
        if ( $params->{suppress} ) { push @$limits, "suppress:0"; }
215
    while ( my ( $oand, $otor, $index ) = $ea->() ) {
176
216
        next if ( !defined($oand) || $oand eq '' );
177
        # Merge the indexes in with the search terms and the operands so that
217
        $oand = $self->_clean_search_term($oand);
178
        # each search thing is a handy unit.
218
        $oand = $self->_truncate_terms($oand) if ($truncate);
179
        unshift @$operators, undef;    # The first one can't have an op
219
        push @search_params, {
180
        my @search_params;
220
            operand => $oand,      # the search terms
181
        my $truncate = C4::Context->preference("QueryAutoTruncate") || 0;
221
            operator => defined($otor) ? uc $otor : undef,    # AND and so on
182
        my $ea = each_array( @$operands, @$operators, @index_params );
222
            $index ? %$index : (),
183
        while ( my ( $oand, $otor, $index ) = $ea->() ) {
223
        };
184
            next if ( !defined($oand) || $oand eq '' );
224
    }
185
            $oand = $self->_clean_search_term($oand);
186
            $oand = $self->_truncate_terms($oand) if ($truncate);
187
            push @search_params, {
188
                operand => $oand,      # the search terms
189
                operator => defined($otor) ? uc $otor : undef,    # AND and so on
190
                $index ? %$index : (),
191
            };
192
        }
225
193
226
    # We build a string query from limits and the queries. An alternative
194
        # We build a string query from limits and the queries. An alternative
227
    # would be to pass them separately into build_query and let it build
195
        # would be to pass them separately into build_query and let it build
228
    # them into a structured ES query itself. Maybe later, though that'd be
196
        # them into a structured ES query itself. Maybe later, though that'd be
229
    # more robust.
197
        # more robust.
230
    my $search_param_query_str = join( ' ', $self->_create_query_string(@search_params) );
198
        $search_param_query_str = join( ' ', $self->_create_query_string(@search_params) );
231
    my $query_str = join( ' AND ',
199
        my $query_str = join( ' AND ',
232
        $search_param_query_str || (),
200
            $search_param_query_str || (),
233
        $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () );
201
            $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () );
234
202
235
    my @fields = '_all';
203
        my @fields = '_all';
236
    if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) {
204
        if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) {
237
        push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields;
205
            push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields;
238
    }
206
        }
239
207
240
    # If there's no query on the left, let's remove the junk left behind
208
        # If there's no query on the left, let's remove the junk left behind
241
    $query_str =~ s/^ AND //;
209
        $query_str =~ s/^ AND //;
242
    my %options;
210
        my %options;
243
    $options{fields} = \@fields;
211
        $options{fields} = \@fields;
244
    $options{sort} = \@sort_params;
212
        $options{sort} = \@sort_params;
245
    $options{expanded_facet} = $params->{expanded_facet};
213
        $options{expanded_facet} = $params->{expanded_facet};
246
    my $query = $self->build_query( $query_str, %options );
214
        $query = $self->build_query( $query_str, %options );
215
    }
247
216
248
    # We roughly emulate the CGI parameters of the zebra query builder
217
    # We roughly emulate the CGI parameters of the zebra query builder
249
    my $query_cgi = '';
218
    my $query_cgi = '';
250
    shift @$operators; # Shift out the one we unshifted before
219
    shift @$operators; # Shift out the one we unshifted before
251
    $ea = each_array( @$operands, @$operators, @$indexes );
220
    my $ea = each_array( @$operands, @$operators, @$indexes );
252
    while ( my ( $oand, $otor, $index ) = $ea->() ) {
221
    while ( my ( $oand, $otor, $index ) = $ea->() ) {
253
        $query_cgi .= '&' if $query_cgi;
222
        $query_cgi .= '&' if $query_cgi;
254
        $query_cgi .= 'idx=' . uri_escape_utf8( $index // '') . '&q=' . uri_escape_utf8( $oand );
223
        $query_cgi .= 'idx=' . uri_escape_utf8( $index // '') . '&q=' . uri_escape_utf8( $oand );
Lines 482-487 sub build_authorities_query_compat { Link Here
482
    return $query;
451
    return $query;
483
}
452
}
484
453
454
=head2 _build_scan_query
455
456
    my ($query, $query_str) = $builder->build_scan_query(\@operands, \@indexes)
457
458
This will build an aggregation scan query that can be issued to elasticsearch from
459
the provided string input.
460
461
=cut
462
463
our %scan_field_convert = (
464
    'ti' => 'title',
465
    'au' => 'author',
466
    'su' => 'subject',
467
    'se' => 'title-series',
468
    'pb' => 'publisher',
469
);
470
471
sub _build_scan_query {
472
    my ( $self, $operands, $indexes ) = @_;
473
474
    my $term = scalar( @$operands ) == 0 ? '' : $operands->[0];
475
    my $index = scalar( @$indexes ) == 0 ? 'title' : $indexes->[0];
476
477
    my ( $f, $d ) = split( /,/, $index);
478
    $index = $scan_field_convert{$f} || $f;
479
480
    my $res;
481
    $res->{query} = {
482
        query_string => {
483
            query => '*'
484
        }
485
    };
486
    $res->{aggregations} = {
487
        $index => {
488
            terms => {
489
                field => $index . '__facet',
490
                order => { '_term' => 'asc' },
491
                include => $self->_create_regex_filter($self->_clean_search_term($term)) . '.*'
492
            }
493
        }
494
    };
495
    return ($res, $term);
496
}
497
498
=head2 _create_regex_filter
499
500
    my $filter = $builder->create_regex_filter('term')
501
502
This will create a regex filter that can be used with an aggregation query.
503
504
=cut
505
506
sub _create_regex_filter {
507
    my ($self, $term) = @_;
508
509
    my $result = '';
510
    foreach my $c (split(//, quotemeta($term))) {
511
        my $lc = lc($c);
512
        my $uc = uc($c);
513
        $result .= $lc ne $uc ? '[' . $lc . $uc . ']' : $c;
514
    }
515
    return $result;
516
}
517
485
=head2 _convert_sort_fields
518
=head2 _convert_sort_fields
486
519
487
    my @sort_params = _convert_sort_fields(@sort_by)
520
    my @sort_params = _convert_sort_fields(@sort_by)
Lines 728-734 sub _modify_string_by_type { Link Here
728
    return $str unless $str;    # Empty or undef, we can't use it.
761
    return $str unless $str;    # Empty or undef, we can't use it.
729
762
730
    $str .= '*' if $type eq 'right-truncate';
763
    $str .= '*' if $type eq 'right-truncate';
731
    $str = '"' . $str . '"' if $type eq 'phrase';
764
    $str = '"' . $str . '"' if $type eq 'phrase' && $str !~ /^".*"$/;
732
    return $str;
765
    return $str;
733
}
766
}
734
767
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-2 / +59 lines)
Lines 85-91 sub search { Link Here
85
85
86
    my $params = $self->get_elasticsearch_params();
86
    my $params = $self->get_elasticsearch_params();
87
    # 20 is the default number of results per page
87
    # 20 is the default number of results per page
88
    $query->{size} = $count || 20;
88
    $query->{size} = $count // 20;
89
    # ES doesn't want pages, it wants a record to start from.
89
    # ES doesn't want pages, it wants a record to start from.
90
    if (exists $options{offset}) {
90
    if (exists $options{offset}) {
91
        $query->{from} = $options{offset};
91
        $query->{from} = $options{offset};
Lines 146-153 sub search_compat { Link Here
146
    my (
146
    my (
147
        $self,     $query,            $simple_query, $sort_by,
147
        $self,     $query,            $simple_query, $sort_by,
148
        $servers,  $results_per_page, $offset,       $expanded_facet,
148
        $servers,  $results_per_page, $offset,       $expanded_facet,
149
        $branches, $query_type,       $scan
149
        $branches, $item_types,       $query_type,   $scan
150
    ) = @_;
150
    ) = @_;
151
152
    if ( $scan ) {
153
        return $self->_aggregation_scan( $query, $results_per_page, $offset );
154
    }
155
151
    my %options;
156
    my %options;
152
    if ( !defined $offset or $offset < 0 ) {
157
    if ( !defined $offset or $offset < 0 ) {
153
        $offset = 0;
158
        $offset = 0;
Lines 513-516 sub _convert_facets { Link Here
513
    return \@facets;
518
    return \@facets;
514
}
519
}
515
520
521
=head2 _aggregation_scan
522
523
    my $result = $self->_aggregration_scan($query, 10, 0);
524
525
Perform an aggregation request for scan purposes.
526
527
=cut
528
529
sub _aggregation_scan {
530
    my ($self, $query, $results_per_page, $offset) = @_;
531
532
    if (!scalar(keys %{$query->{aggregations}})) {
533
        my %result = {
534
            biblioserver => {
535
                hits => 0,
536
                RECORDS => undef
537
            }
538
        };
539
        return (undef, \%result, undef);
540
    }
541
    my ($field) = keys %{$query->{aggregations}};
542
    $query->{aggregations}{$field}{terms}{size} = 1000;
543
    my $results = $self->search($query, 1, 0);
544
545
    # Convert each result into a MARC::Record
546
    my (@records, $index);
547
    # opac-search expects results to be put in the
548
    # right place in the array, according to $offset
549
    $index = $offset - 1;
550
551
    my $count = scalar(@{$results->{aggregations}{$field}{buckets}});
552
    for (my $index = $offset; $index - $offset < $results_per_page && $index < $count; $index++) {
553
        my $bucket = $results->{aggregations}{$field}{buckets}->[$index];
554
        # Scan values are expressed as 100 a (count) and 245a (term)
555
        my $marc = MARC::Record->new;
556
        $marc->encoding('UTF-8');
557
        $marc->append_fields(
558
            MARC::Field->new((100, ' ',  ' ', 'a' => $bucket->{doc_count}))
559
        );
560
        $marc->append_fields(
561
            MARC::Field->new((245, ' ',  ' ', 'a' => $bucket->{key}))
562
        );
563
        $records[$index] = $marc->as_usmarc();
564
    };
565
    # consumers of this expect a namespaced result, we provide the default
566
    # configuration.
567
    my %result;
568
    $result{biblioserver}{hits} = $count;
569
    $result{biblioserver}{RECORDS} = \@records;
570
    return (undef, \%result, undef);
571
}
572
516
1;
573
1;
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t (-2 / +21 lines)
Lines 45-51 $se->mock( 'get_elasticsearch_mappings', sub { Link Here
45
                    type => 'text'
45
                    type => 'text'
46
                },
46
                },
47
                subject => {
47
                subject => {
48
                    type => 'text'
48
                    type => 'text',
49
                    facet => 1
49
                },
50
                },
50
                itemnumber => {
51
                itemnumber => {
51
                    type => 'integer'
52
                    type => 'integer'
Lines 375-380 subtest 'build_query tests' => sub { Link Here
375
    );
376
    );
376
    is($query_cgi, 'idx=&q=title%3A%22donald%20duck%22', 'query cgi');
377
    is($query_cgi, 'idx=&q=title%3A%22donald%20duck%22', 'query cgi');
377
    is($query_desc, 'title:"donald duck"', 'query desc ok');
378
    is($query_desc, 'title:"donald duck"', 'query desc ok');
379
380
    # Scan queries
381
    ( undef, $query, $simple_query, $query_cgi, $query_desc ) = $qb->build_query_compat( undef, ['new'], ['su'], undef, undef, 1 );
382
    is(
383
        $query->{query}{query_string}{query},
384
        '*',
385
        "scan query is properly formed"
386
    );
387
    is_deeply(
388
        $query->{aggregations}{'subject'}{'terms'},
389
        {
390
            field => 'subject__facet',
391
            order => { '_term' => 'asc' },
392
            include => '[nN][eE][wW].*'
393
        },
394
        "scan aggregation request is properly formed"
395
    );
396
    is($query_cgi, 'idx=su&q=new&scan=1', 'query cgi');
397
    is($query_desc, 'new', 'query desc ok');
378
};
398
};
379
399
380
400
381
- 

Return to bug 22592