View | Details | Raw Unified | Return to bug 22592
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-82 / +113 lines)
Lines 136-180 sub build_query { Link Here
136
    return $res;
136
    return $res;
137
}
137
}
138
138
139
=head2 build_browse_query
140
141
    my $browse_query = $builder->build_browse_query($field, $query);
142
143
This performs a "starts with" style query on a particular field. The field
144
to be searched must have been indexed with an appropriate mapping as a
145
"phrase" subfield, which pretty much everything has.
146
147
=cut
148
149
# XXX this isn't really a browse query like we want in the end
150
sub build_browse_query {
151
    my ( $self, $field, $query ) = @_;
152
153
    my $fuzzy_enabled = C4::Context->preference("QueryFuzzy") || 0;
154
155
    return { query => '*' } if !defined $query;
156
157
    # TODO this should come from Koha::SearchEngine::Elasticsearch
158
    my %field_whitelist = (
159
        title  => 1,
160
        author => 1,
161
    );
162
    $field = 'title' if !exists $field_whitelist{$field};
163
    my $sort = $self->_sort_field($field);
164
    my $res = {
165
        query => {
166
            match_phrase_prefix => {
167
                "$field.phrase" => {
168
                    query     => $query,
169
                    operator  => 'or',
170
                    fuzziness => $fuzzy_enabled ? 'auto' : '0',
171
                }
172
            }
173
        },
174
        sort => [ { $sort => { order => "asc" } } ],
175
    };
176
}
177
178
=head2 build_query_compat
139
=head2 build_query_compat
179
140
180
    my (
141
    my (
Lines 200-253 sub build_query_compat { Link Here
200
        $lang, $params )
161
        $lang, $params )
201
      = @_;
162
      = @_;
202
163
203
#die Dumper ( $self, $operators, $operands, $indexes, $orig_limits, $sort_by, $scan, $lang );
164
    my $query;
204
    my @sort_params  = $self->_convert_sort_fields(@$sort_by);
165
    my $query_str = '';
205
    my @index_params = $self->_convert_index_fields(@$indexes);
166
    my $limits = ();
206
    my $limits       = $self->_fix_limit_special_cases($orig_limits);
167
    if ( $scan ) {
207
    if ( $params->{suppress} ) { push @$limits, "suppress:0"; }
168
        ($query, $query_str) = $self->_build_scan_query( $operands, $indexes );
208
169
    } else {
209
    # Merge the indexes in with the search terms and the operands so that
170
        my @sort_params  = $self->_convert_sort_fields(@$sort_by);
210
    # each search thing is a handy unit.
171
        my @index_params = $self->_convert_index_fields(@$indexes);
211
    unshift @$operators, undef;    # The first one can't have an op
172
        my $limits       = $self->_fix_limit_special_cases($orig_limits);
212
    my @search_params;
173
        if ( $params->{suppress} ) { push @$limits, "suppress:0"; }
213
    my $truncate = C4::Context->preference("QueryAutoTruncate") || 0;
174
214
    my $ea = each_array( @$operands, @$operators, @index_params );
175
        # Merge the indexes in with the search terms and the operands so that
215
    while ( my ( $oand, $otor, $index ) = $ea->() ) {
176
        # each search thing is a handy unit.
216
        next if ( !defined($oand) || $oand eq '' );
177
        unshift @$operators, undef;    # The first one can't have an op
217
        $oand = $self->_clean_search_term($oand);
178
        my @search_params;
218
        $oand = $self->_truncate_terms($oand) if ($truncate);
179
        my $truncate = C4::Context->preference("QueryAutoTruncate") || 0;
219
        push @search_params, {
180
        my $ea = each_array( @$operands, @$operators, @index_params );
220
            operand => $oand,      # the search terms
181
        while ( my ( $oand, $otor, $index ) = $ea->() ) {
221
            operator => defined($otor) ? uc $otor : undef,    # AND and so on
182
            next if ( !defined($oand) || $oand eq '' );
222
            $index ? %$index : (),
183
            $oand = $self->_clean_search_term($oand);
223
        };
184
            $oand = $self->_truncate_terms($oand) if ($truncate);
224
    }
185
            push @search_params, {
186
                operand => $oand,      # the search terms
187
                operator => defined($otor) ? uc $otor : undef,    # AND and so on
188
                $index ? %$index : (),
189
            };
190
        }
225
191
226
    # We build a string query from limits and the queries. An alternative
192
        # We build a string query from limits and the queries. An alternative
227
    # would be to pass them separately into build_query and let it build
193
        # would be to pass them separately into build_query and let it build
228
    # them into a structured ES query itself. Maybe later, though that'd be
194
        # them into a structured ES query itself. Maybe later, though that'd be
229
    # more robust.
195
        # more robust.
230
    my $query_str = join( ' AND ',
196
        $query_str = join( ' AND ',
231
        join( ' ', $self->_create_query_string(@search_params) ) || (),
197
            join( ' ', $self->_create_query_string(@search_params) ) || (),
232
        $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () );
198
            $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () );
233
199
234
    my @fields = '_all';
200
        my @fields = '_all';
235
    if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) {
201
        if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) {
236
        push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields;
202
            push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields;
237
    }
203
        }
238
204
239
    # If there's no query on the left, let's remove the junk left behind
205
        # If there's no query on the left, let's remove the junk left behind
240
    $query_str =~ s/^ AND //;
206
        $query_str =~ s/^ AND //;
241
    my %options;
207
        my %options;
242
    $options{fields} = \@fields;
208
        $options{fields} = \@fields;
243
    $options{sort} = \@sort_params;
209
        $options{sort} = \@sort_params;
244
    $options{expanded_facet} = $params->{expanded_facet};
210
        $options{expanded_facet} = $params->{expanded_facet};
245
    my $query = $self->build_query( $query_str, %options );
211
        $query = $self->build_query( $query_str, %options );
212
    }
246
213
247
    # We roughly emulate the CGI parameters of the zebra query builder
214
    # We roughly emulate the CGI parameters of the zebra query builder
248
    my $query_cgi = '';
215
    my $query_cgi = '';
249
    shift @$operators; # Shift out the one we unshifted before
216
    shift @$operators; # Shift out the one we unshifted before
250
    $ea = each_array( @$operands, @$operators, @$indexes );
217
    my $ea = each_array( @$operands, @$operators, @$indexes );
251
    while ( my ( $oand, $otor, $index ) = $ea->() ) {
218
    while ( my ( $oand, $otor, $index ) = $ea->() ) {
252
        $query_cgi .= '&' if $query_cgi;
219
        $query_cgi .= '&' if $query_cgi;
253
        $query_cgi .= 'idx=' . uri_escape_utf8( $index // '') . '&q=' . uri_escape_utf8( $oand );
220
        $query_cgi .= 'idx=' . uri_escape_utf8( $index // '') . '&q=' . uri_escape_utf8( $oand );
Lines 481-486 sub build_authorities_query_compat { Link Here
481
    return $query;
448
    return $query;
482
}
449
}
483
450
451
=head2 _build_scan_query
452
453
    my ($query, $query_str) = $builder->build_scan_query(\@operands, \@indexes)
454
455
This will build an aggregation scan query that can be issued to elasticsearch from
456
the provided string input.
457
458
=cut
459
460
our %scan_field_convert = (
461
    'ti' => 'title',
462
    'au' => 'author',
463
    'su' => 'subject',
464
    'se' => 'title-series',
465
    'pb' => 'publisher',
466
);
467
468
sub _build_scan_query {
469
    my ( $self, $operands, $indexes ) = @_;
470
471
    my $term = scalar( @$operands ) == 0 ? '' : $operands->[0];
472
    my $index = scalar( @$indexes ) == 0 ? 'title' : $indexes->[0];
473
474
    my ( $f, $d ) = split( /,/, $index);
475
    $index = $scan_field_convert{$f} || $f;
476
477
    my $res;
478
    $res->{query} = {
479
        query_string => {
480
            query => '*'
481
        }
482
    };
483
    $res->{aggregations} = {
484
        $index => {
485
            terms => {
486
                field => $index . '__facet',
487
                order => { '_term' => 'asc' },
488
                include => $self->_create_regex_filter($self->_clean_search_term($term)) . '.*'
489
            }
490
        }
491
    };
492
    return ($res, $term);
493
}
494
495
=head2 _create_regex_filter
496
497
    my $filter = $builder->create_regex_filter('term')
498
499
This will create a regex filter that can be used with an aggregation query.
500
501
=cut
502
503
sub _create_regex_filter {
504
    my ($self, $term) = @_;
505
506
    my $result = '';
507
    foreach my $c (split(//, quotemeta($term))) {
508
        my $lc = lc($c);
509
        my $uc = uc($c);
510
        $result .= $lc ne $uc ? '[' . $lc . $uc . ']' : $c;
511
    }
512
    return $result;
513
}
514
484
=head2 _convert_sort_fields
515
=head2 _convert_sort_fields
485
516
486
    my @sort_params = _convert_sort_fields(@sort_by)
517
    my @sort_params = _convert_sort_fields(@sort_by)
Lines 727-733 sub _modify_string_by_type { Link Here
727
    return $str unless $str;    # Empty or undef, we can't use it.
758
    return $str unless $str;    # Empty or undef, we can't use it.
728
759
729
    $str .= '*' if $type eq 'right-truncate';
760
    $str .= '*' if $type eq 'right-truncate';
730
    $str = '"' . $str . '"' if $type eq 'phrase';
761
    $str = '"' . $str . '"' if $type eq 'phrase' && $str !~ /^".*"$/;
731
    return $str;
762
    return $str;
732
}
763
}
733
764
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-2 / +59 lines)
Lines 85-91 sub search { Link Here
85
85
86
    my $params = $self->get_elasticsearch_params();
86
    my $params = $self->get_elasticsearch_params();
87
    # 20 is the default number of results per page
87
    # 20 is the default number of results per page
88
    $query->{size} = $count || 20;
88
    $query->{size} = $count // 20;
89
    # ES doesn't want pages, it wants a record to start from.
89
    # ES doesn't want pages, it wants a record to start from.
90
    if (exists $options{offset}) {
90
    if (exists $options{offset}) {
91
        $query->{from} = $options{offset};
91
        $query->{from} = $options{offset};
Lines 146-153 sub search_compat { Link Here
146
    my (
146
    my (
147
        $self,     $query,            $simple_query, $sort_by,
147
        $self,     $query,            $simple_query, $sort_by,
148
        $servers,  $results_per_page, $offset,       $expanded_facet,
148
        $servers,  $results_per_page, $offset,       $expanded_facet,
149
        $branches, $query_type,       $scan
149
        $branches, $item_types,       $query_type,   $scan
150
    ) = @_;
150
    ) = @_;
151
152
    if ( $scan ) {
153
        return $self->_aggregation_scan( $query, $results_per_page, $offset );
154
    }
155
151
    my %options;
156
    my %options;
152
    if ( !defined $offset or $offset < 0 ) {
157
    if ( !defined $offset or $offset < 0 ) {
153
        $offset = 0;
158
        $offset = 0;
Lines 513-516 sub _convert_facets { Link Here
513
    return \@facets;
518
    return \@facets;
514
}
519
}
515
520
521
=head2 _aggregation_scan
522
523
    my $result = $self->_aggregration_scan($query, 10, 0);
524
525
Perform an aggregation request for scan purposes.
526
527
=cut
528
529
sub _aggregation_scan {
530
    my ($self, $query, $results_per_page, $offset) = @_;
531
532
    if (!scalar(keys %{$query->{aggregations}})) {
533
        my %result = {
534
            biblioserver => {
535
                hits => 0,
536
                RECORDS => undef
537
            }
538
        };
539
        return (undef, \%result, undef);
540
    }
541
    my ($field) = keys %{$query->{aggregations}};
542
    $query->{aggregations}{$field}{terms}{size} = 1000;
543
    my $results = $self->search($query, 1, 0);
544
545
    # Convert each result into a MARC::Record
546
    my (@records, $index);
547
    # opac-search expects results to be put in the
548
    # right place in the array, according to $offset
549
    $index = $offset - 1;
550
551
    my $count = scalar(@{$results->{aggregations}{$field}{buckets}});
552
    for (my $index = $offset; $index - $offset < $results_per_page && $index < $count; $index++) {
553
        my $bucket = $results->{aggregations}{$field}{buckets}->[$index];
554
        # Scan values are expressed as 100 a (count) and 245a (term)
555
        my $marc = MARC::Record->new;
556
        $marc->encoding('UTF-8');
557
        $marc->append_fields(
558
            MARC::Field->new((100, ' ',  ' ', 'a' => $bucket->{doc_count}))
559
        );
560
        $marc->append_fields(
561
            MARC::Field->new((245, ' ',  ' ', 'a' => $bucket->{key}))
562
        );
563
        $records[$index] = $marc->as_usmarc();
564
    };
565
    # consumers of this expect a namespaced result, we provide the default
566
    # configuration.
567
    my %result;
568
    $result{biblioserver}{hits} = $count;
569
    $result{biblioserver}{RECORDS} = \@records;
570
    return (undef, \%result, undef);
571
}
572
516
1;
573
1;
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t (-3 / +22 lines)
Lines 45-51 $se->mock( 'get_elasticsearch_mappings', sub { Link Here
45
                    type => 'text'
45
                    type => 'text'
46
                },
46
                },
47
                subject => {
47
                subject => {
48
                    type => 'text'
48
                    type => 'text',
49
                    facet => 1
49
                },
50
                },
50
                itemnumber => {
51
                itemnumber => {
51
                    type => 'integer'
52
                    type => 'integer'
Lines 169-175 subtest 'build_authorities_query_compat() tests' => sub { Link Here
169
};
170
};
170
171
171
subtest 'build_query tests' => sub {
172
subtest 'build_query tests' => sub {
172
    plan tests => 29;
173
    plan tests => 33;
173
174
174
    my $qb;
175
    my $qb;
175
176
Lines 347-352 subtest 'build_query tests' => sub { Link Here
347
    );
348
    );
348
    is($query_cgi, 'idx=&q=title%3A%22donald%20duck%22', 'query cgi');
349
    is($query_cgi, 'idx=&q=title%3A%22donald%20duck%22', 'query cgi');
349
    is($query_desc, 'title:"donald duck"', 'query desc ok');
350
    is($query_desc, 'title:"donald duck"', 'query desc ok');
351
352
    # Scan queries
353
    ( undef, $query, $simple_query, $query_cgi, $query_desc ) = $qb->build_query_compat( undef, ['new'], ['su'], undef, undef, 1 );
354
    is(
355
        $query->{query}{query_string}{query},
356
        '*',
357
        "scan query is properly formed"
358
    );
359
    is_deeply(
360
        $query->{aggregations}{'subject'}{'terms'},
361
        {
362
            field => 'subject__facet',
363
            order => { '_term' => 'asc' },
364
            include => '[nN][eE][wW].*'
365
        },
366
        "scan aggregation request is properly formed"
367
    );
368
    is($query_cgi, 'idx=su&q=new&scan=1', 'query cgi');
369
    is($query_desc, 'new', 'query desc ok');
350
};
370
};
351
371
352
372
353
- 

Return to bug 22592