View | Details | Raw Unified | Return to bug 22592
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-82 / +113 lines)
Lines 135-179 sub build_query { Link Here
135
    return $res;
135
    return $res;
136
}
136
}
137
137
138
=head2 build_browse_query
139
140
    my $browse_query = $builder->build_browse_query($field, $query);
141
142
This performs a "starts with" style query on a particular field. The field
143
to be searched must have been indexed with an appropriate mapping as a
144
"phrase" subfield, which pretty much everything has.
145
146
=cut
147
148
# XXX this isn't really a browse query like we want in the end
149
sub build_browse_query {
150
    my ( $self, $field, $query ) = @_;
151
152
    my $fuzzy_enabled = C4::Context->preference("QueryFuzzy") || 0;
153
154
    return { query => '*' } if !defined $query;
155
156
    # TODO this should come from Koha::SearchEngine::Elasticsearch
157
    my %field_whitelist = (
158
        title  => 1,
159
        author => 1,
160
    );
161
    $field = 'title' if !exists $field_whitelist{$field};
162
    my $sort = $self->_sort_field($field);
163
    my $res = {
164
        query => {
165
            match_phrase_prefix => {
166
                "$field.phrase" => {
167
                    query     => $query,
168
                    operator  => 'or',
169
                    fuzziness => $fuzzy_enabled ? 'auto' : '0',
170
                }
171
            }
172
        },
173
        sort => [ { $sort => { order => "asc" } } ],
174
    };
175
}
176
177
=head2 build_query_compat
138
=head2 build_query_compat
178
139
179
    my (
140
    my (
Lines 199-252 sub build_query_compat { Link Here
199
        $lang, $params )
160
        $lang, $params )
200
      = @_;
161
      = @_;
201
162
202
#die Dumper ( $self, $operators, $operands, $indexes, $orig_limits, $sort_by, $scan, $lang );
163
    my $query;
203
    my @sort_params  = $self->_convert_sort_fields(@$sort_by);
164
    my $query_str = '';
204
    my @index_params = $self->_convert_index_fields(@$indexes);
165
    my $limits = ();
205
    my $limits       = $self->_fix_limit_special_cases($orig_limits);
166
    if ( $scan ) {
206
    if ( $params->{suppress} ) { push @$limits, "suppress:0"; }
167
        ($query, $query_str) = $self->_build_scan_query( $operands, $indexes );
207
168
    } else {
208
    # Merge the indexes in with the search terms and the operands so that
169
        my @sort_params  = $self->_convert_sort_fields(@$sort_by);
209
    # each search thing is a handy unit.
170
        my @index_params = $self->_convert_index_fields(@$indexes);
210
    unshift @$operators, undef;    # The first one can't have an op
171
        my $limits       = $self->_fix_limit_special_cases($orig_limits);
211
    my @search_params;
172
        if ( $params->{suppress} ) { push @$limits, "suppress:0"; }
212
    my $truncate = C4::Context->preference("QueryAutoTruncate") || 0;
173
213
    my $ea = each_array( @$operands, @$operators, @index_params );
174
        # Merge the indexes in with the search terms and the operands so that
214
    while ( my ( $oand, $otor, $index ) = $ea->() ) {
175
        # each search thing is a handy unit.
215
        next if ( !defined($oand) || $oand eq '' );
176
        unshift @$operators, undef;    # The first one can't have an op
216
        $oand = $self->_clean_search_term($oand);
177
        my @search_params;
217
        $oand = $self->_truncate_terms($oand) if ($truncate);
178
        my $truncate = C4::Context->preference("QueryAutoTruncate") || 0;
218
        push @search_params, {
179
        my $ea = each_array( @$operands, @$operators, @index_params );
219
            operand => $oand,      # the search terms
180
        while ( my ( $oand, $otor, $index ) = $ea->() ) {
220
            operator => defined($otor) ? uc $otor : undef,    # AND and so on
181
            next if ( !defined($oand) || $oand eq '' );
221
            $index ? %$index : (),
182
            $oand = $self->_clean_search_term($oand);
222
        };
183
            $oand = $self->_truncate_terms($oand) if ($truncate);
223
    }
184
            push @search_params, {
185
                operand => $oand,      # the search terms
186
                operator => defined($otor) ? uc $otor : undef,    # AND and so on
187
                $index ? %$index : (),
188
            };
189
        }
224
190
225
    # We build a string query from limits and the queries. An alternative
191
        # We build a string query from limits and the queries. An alternative
226
    # would be to pass them separately into build_query and let it build
192
        # would be to pass them separately into build_query and let it build
227
    # them into a structured ES query itself. Maybe later, though that'd be
193
        # them into a structured ES query itself. Maybe later, though that'd be
228
    # more robust.
194
        # more robust.
229
    my $query_str = join( ' AND ',
195
        $query_str = join( ' AND ',
230
        join( ' ', $self->_create_query_string(@search_params) ) || (),
196
            join( ' ', $self->_create_query_string(@search_params) ) || (),
231
        $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () );
197
            $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () );
232
198
233
    my @fields = '_all';
199
        my @fields = '_all';
234
    if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) {
200
        if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) {
235
        push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields;
201
            push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields;
236
    }
202
        }
237
203
238
    # If there's no query on the left, let's remove the junk left behind
204
        # If there's no query on the left, let's remove the junk left behind
239
    $query_str =~ s/^ AND //;
205
        $query_str =~ s/^ AND //;
240
    my %options;
206
        my %options;
241
    $options{fields} = \@fields;
207
        $options{fields} = \@fields;
242
    $options{sort} = \@sort_params;
208
        $options{sort} = \@sort_params;
243
    $options{expanded_facet} = $params->{expanded_facet};
209
        $options{expanded_facet} = $params->{expanded_facet};
244
    my $query = $self->build_query( $query_str, %options );
210
        $query = $self->build_query( $query_str, %options );
211
    }
245
212
246
    # We roughly emulate the CGI parameters of the zebra query builder
213
    # We roughly emulate the CGI parameters of the zebra query builder
247
    my $query_cgi = '';
214
    my $query_cgi = '';
248
    shift @$operators; # Shift out the one we unshifted before
215
    shift @$operators; # Shift out the one we unshifted before
249
    $ea = each_array( @$operands, @$operators, @$indexes );
216
    my $ea = each_array( @$operands, @$operators, @$indexes );
250
    while ( my ( $oand, $otor, $index ) = $ea->() ) {
217
    while ( my ( $oand, $otor, $index ) = $ea->() ) {
251
        $query_cgi .= '&' if $query_cgi;
218
        $query_cgi .= '&' if $query_cgi;
252
        $query_cgi .= 'idx=' . uri_escape_utf8( $index // '') . '&q=' . uri_escape_utf8( $oand );
219
        $query_cgi .= 'idx=' . uri_escape_utf8( $index // '') . '&q=' . uri_escape_utf8( $oand );
Lines 480-485 sub build_authorities_query_compat { Link Here
480
    return $query;
447
    return $query;
481
}
448
}
482
449
450
=head2 _build_scan_query
451
452
    my ($query, $query_str) = $builder->build_scan_query(\@operands, \@indexes)
453
454
This will build an aggregation scan query that can be issued to elasticsearch from
455
the provided string input.
456
457
=cut
458
459
our %scan_field_convert = (
460
    'ti' => 'title',
461
    'au' => 'author',
462
    'su' => 'subject',
463
    'se' => 'title-series',
464
    'pb' => 'publisher',
465
);
466
467
sub _build_scan_query {
468
    my ( $self, $operands, $indexes ) = @_;
469
470
    my $term = scalar( @$operands ) == 0 ? '' : $operands->[0];
471
    my $index = scalar( @$indexes ) == 0 ? 'title' : $indexes->[0];
472
473
    my ( $f, $d ) = split( /,/, $index);
474
    $index = $scan_field_convert{$f} || $f;
475
476
    my $res;
477
    $res->{query} = {
478
        query_string => {
479
            query => '*'
480
        }
481
    };
482
    $res->{aggregations} = {
483
        $index => {
484
            terms => {
485
                field => $index . '__facet',
486
                order => { '_term' => 'asc' },
487
                include => $self->_create_regex_filter($self->_clean_search_term($term)) . '.*'
488
            }
489
        }
490
    };
491
    return ($res, $term);
492
}
493
494
=head2 _create_regex_filter
495
496
    my $filter = $builder->create_regex_filter('term')
497
498
This will create a regex filter that can be used with an aggregation query.
499
500
=cut
501
502
sub _create_regex_filter {
503
    my ($self, $term) = @_;
504
505
    my $result = '';
506
    foreach my $c (split(//, quotemeta($term))) {
507
        my $lc = lc($c);
508
        my $uc = uc($c);
509
        $result .= $lc ne $uc ? '[' . $lc . $uc . ']' : $c;
510
    }
511
    return $result;
512
}
513
483
=head2 _convert_sort_fields
514
=head2 _convert_sort_fields
484
515
485
    my @sort_params = _convert_sort_fields(@sort_by)
516
    my @sort_params = _convert_sort_fields(@sort_by)
Lines 726-732 sub _modify_string_by_type { Link Here
726
    return $str unless $str;    # Empty or undef, we can't use it.
757
    return $str unless $str;    # Empty or undef, we can't use it.
727
758
728
    $str .= '*' if $type eq 'right-truncate';
759
    $str .= '*' if $type eq 'right-truncate';
729
    $str = '"' . $str . '"' if $type eq 'phrase';
760
    $str = '"' . $str . '"' if $type eq 'phrase' && $str !~ /^".*"$/;
730
    return $str;
761
    return $str;
731
}
762
}
732
763
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-2 / +58 lines)
Lines 85-91 sub search { Link Here
85
85
86
    my $params = $self->get_elasticsearch_params();
86
    my $params = $self->get_elasticsearch_params();
87
    # 20 is the default number of results per page
87
    # 20 is the default number of results per page
88
    $query->{size} = $count || 20;
88
    $query->{size} = $count // 20;
89
    # ES doesn't want pages, it wants a record to start from.
89
    # ES doesn't want pages, it wants a record to start from.
90
    if (exists $options{offset}) {
90
    if (exists $options{offset}) {
91
        $query->{from} = $options{offset};
91
        $query->{from} = $options{offset};
Lines 146-153 sub search_compat { Link Here
146
    my (
146
    my (
147
        $self,     $query,            $simple_query, $sort_by,
147
        $self,     $query,            $simple_query, $sort_by,
148
        $servers,  $results_per_page, $offset,       $expanded_facet,
148
        $servers,  $results_per_page, $offset,       $expanded_facet,
149
        $branches, $query_type,       $scan
149
        $branches, $item_types,       $query_type,   $scan
150
    ) = @_;
150
    ) = @_;
151
152
    if ( $scan ) {
153
        return $self->_aggregation_scan( $query, $results_per_page, $offset );
154
    }
155
151
    my %options;
156
    my %options;
152
    if ( !defined $offset or $offset < 0 ) {
157
    if ( !defined $offset or $offset < 0 ) {
153
        $offset = 0;
158
        $offset = 0;
Lines 505-509 sub _convert_facets { Link Here
505
    return \@facets;
510
    return \@facets;
506
}
511
}
507
512
513
=head2 _aggregation_scan
514
515
    my $result = $self->_aggregration_scan($query, 10, 0);
516
517
Perform an aggregation request for scan purposes.
518
519
=cut
520
521
sub _aggregation_scan {
522
    my ($self, $query, $results_per_page, $offset) = @_;
523
524
    if (!scalar(keys %{$query->{aggregations}})) {
525
        my %result = {
526
            biblioserver => {
527
                hits => 0,
528
                RECORDS => undef
529
            }
530
        };
531
        return (undef, \%result, undef);
532
    }
533
    my ($field) = keys %{$query->{aggregations}};
534
    $query->{aggregations}{$field}{terms}{size} = 1000;
535
    my $results = $self->search($query, 1, 0);
536
537
    # Convert each result into a MARC::Record
538
    my (@records, $index);
539
    # opac-search expects results to be put in the
540
    # right place in the array, according to $offset
541
    $index = $offset - 1;
542
543
    my $count = scalar(@{$results->{aggregations}{$field}{buckets}});
544
    for (my $index = $offset; $index - $offset < $results_per_page && $index < $count; $index++) {
545
        my $bucket = $results->{aggregations}{$field}{buckets}->[$index];
546
        # Scan values are expressed as 100 a (count) and 245a (term)
547
        my $marc = MARC::Record->new;
548
        $marc->encoding('UTF-8');
549
        $marc->append_fields(
550
            MARC::Field->new((100, ' ',  ' ', 'a' => $bucket->{doc_count}))
551
        );
552
        $marc->append_fields(
553
            MARC::Field->new((245, ' ',  ' ', 'a' => $bucket->{key}))
554
        );
555
        $records[$index] = $marc->as_usmarc();
556
    };
557
    # consumers of this expect a namespaced result, we provide the default
558
    # configuration.
559
    my %result;
560
    $result{biblioserver}{hits} = $count;
561
    $result{biblioserver}{RECORDS} = \@records;
562
    return (undef, \%result, undef);
563
}
508
564
509
1;
565
1;
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t (-3 / +22 lines)
Lines 45-51 $se->mock( 'get_elasticsearch_mappings', sub { Link Here
45
                    type => 'text'
45
                    type => 'text'
46
                },
46
                },
47
                subject => {
47
                subject => {
48
                    type => 'text'
48
                    type => 'text',
49
                    facet => 1
49
                },
50
                },
50
                itemnumber => {
51
                itemnumber => {
51
                    type => 'integer'
52
                    type => 'integer'
Lines 169-175 subtest 'build_authorities_query_compat() tests' => sub { Link Here
169
};
170
};
170
171
171
subtest 'build_query tests' => sub {
172
subtest 'build_query tests' => sub {
172
    plan tests => 29;
173
    plan tests => 33;
173
174
174
    my $qb;
175
    my $qb;
175
176
Lines 347-352 subtest 'build_query tests' => sub { Link Here
347
    );
348
    );
348
    is($query_cgi, 'idx=&q=title%3A%22donald%20duck%22', 'query cgi');
349
    is($query_cgi, 'idx=&q=title%3A%22donald%20duck%22', 'query cgi');
349
    is($query_desc, 'title:"donald duck"', 'query desc ok');
350
    is($query_desc, 'title:"donald duck"', 'query desc ok');
351
352
    # Scan queries
353
    ( undef, $query, $simple_query, $query_cgi, $query_desc ) = $qb->build_query_compat( undef, ['new'], ['su'], undef, undef, 1 );
354
    is(
355
        $query->{query}{query_string}{query},
356
        '*',
357
        "scan query is properly formed"
358
    );
359
    is_deeply(
360
        $query->{aggregations}{'subject'}{'terms'},
361
        {
362
            field => 'subject__facet',
363
            order => { '_term' => 'asc' },
364
            include => '[nN][eE][wW].*'
365
        },
366
        "scan aggregation request is properly formed"
367
    );
368
    is($query_cgi, 'idx=su&q=new&scan=1', 'query cgi');
369
    is($query_desc, 'new', 'query desc ok');
350
};
370
};
351
371
352
372
353
- 

Return to bug 22592