View | Details | Raw Unified | Return to bug 22592
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-79 / +112 lines)
Lines 141-185 sub build_query { Link Here
141
    return $res;
141
    return $res;
142
}
142
}
143
143
144
=head2 build_browse_query
145
146
    my $browse_query = $builder->build_browse_query($field, $query);
147
148
This performs a "starts with" style query on a particular field. The field
149
to be searched must have been indexed with an appropriate mapping as a
150
"phrase" subfield, which pretty much everything has.
151
152
=cut
153
154
# XXX this isn't really a browse query like we want in the end
155
sub build_browse_query {
156
    my ( $self, $field, $query ) = @_;
157
158
    my $fuzzy_enabled = C4::Context->preference("QueryFuzzy") || 0;
159
160
    return { query => '*' } if !defined $query;
161
162
    # TODO this should come from Koha::SearchEngine::Elasticsearch
163
    my %field_whitelist = (
164
        title  => 1,
165
        author => 1,
166
    );
167
    $field = 'title' if !exists $field_whitelist{$field};
168
    my $sort = $self->_sort_field($field);
169
    my $res = {
170
        query => {
171
            match_phrase_prefix => {
172
                "$field.phrase" => {
173
                    query     => $query,
174
                    operator  => 'or',
175
                    fuzziness => $fuzzy_enabled ? 'auto' : '0',
176
                }
177
            }
178
        },
179
        sort => [ { $sort => { order => "asc" } } ],
180
    };
181
}
182
183
=head2 build_query_compat
144
=head2 build_query_compat
184
145
185
    my (
146
    my (
Lines 205-254 sub build_query_compat { Link Here
205
        $lang, $params )
166
        $lang, $params )
206
      = @_;
167
      = @_;
207
168
208
#die Dumper ( $self, $operators, $operands, $indexes, $orig_limits, $sort_by, $scan, $lang );
169
    my $query;
209
    my @sort_params  = $self->_convert_sort_fields(@$sort_by);
170
    my $query_str = '';
210
    my @index_params = $self->_convert_index_fields(@$indexes);
171
    my $search_param_query_str = '';
211
    my $limits       = $self->_fix_limit_special_cases($orig_limits);
172
    my $limits = ();
212
    if ( $params->{suppress} ) { push @$limits, "suppress:0"; }
173
    if ( $scan ) {
213
    # Merge the indexes in with the search terms and the operands so that
174
        ($query, $query_str) = $self->_build_scan_query( $operands, $indexes );
214
    # each search thing is a handy unit.
175
        $search_param_query_str = $query_str;
215
    unshift @$operators, undef;    # The first one can't have an op
176
    } else {
216
    my @search_params;
177
        my @sort_params  = $self->_convert_sort_fields(@$sort_by);
217
    my $truncate = C4::Context->preference("QueryAutoTruncate") || 0;
178
        my @index_params = $self->_convert_index_fields(@$indexes);
218
    my $ea = each_array( @$operands, @$operators, @index_params );
179
        my $limits       = $self->_fix_limit_special_cases($orig_limits);
219
    while ( my ( $oand, $otor, $index ) = $ea->() ) {
180
        if ( $params->{suppress} ) { push @$limits, "suppress:0"; }
220
        next if ( !defined($oand) || $oand eq '' );
181
        # Merge the indexes in with the search terms and the operands so that
221
        $oand = $self->_clean_search_term($oand);
182
        # each search thing is a handy unit.
222
        $oand = $self->_truncate_terms($oand) if ($truncate);
183
        unshift @$operators, undef;    # The first one can't have an op
223
        push @search_params, {
184
        my @search_params;
224
            operand => $oand,      # the search terms
185
        my $truncate = C4::Context->preference("QueryAutoTruncate") || 0;
225
            operator => defined($otor) ? uc $otor : undef,    # AND and so on
186
        my $ea = each_array( @$operands, @$operators, @index_params );
226
            $index ? %$index : (),
187
        while ( my ( $oand, $otor, $index ) = $ea->() ) {
227
        };
188
            next if ( !defined($oand) || $oand eq '' );
228
    }
189
            $oand = $self->_clean_search_term($oand);
190
            $oand = $self->_truncate_terms($oand) if ($truncate);
191
            push @search_params, {
192
                operand => $oand,      # the search terms
193
                operator => defined($otor) ? uc $otor : undef,    # AND and so on
194
                $index ? %$index : (),
195
            };
196
        }
229
197
230
    # We build a string query from limits and the queries. An alternative
198
        # We build a string query from limits and the queries. An alternative
231
    # would be to pass them separately into build_query and let it build
199
        # would be to pass them separately into build_query and let it build
232
    # them into a structured ES query itself. Maybe later, though that'd be
200
        # them into a structured ES query itself. Maybe later, though that'd be
233
    # more robust.
201
        # more robust.
234
    my $search_param_query_str = join( ' ', $self->_create_query_string(@search_params) );
202
        $search_param_query_str = join( ' ', $self->_create_query_string(@search_params) );
235
    my $query_str = join( ' AND ',
203
        $query_str = join( ' AND ',
236
        $search_param_query_str || (),
204
            $search_param_query_str || (),
237
        $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () );
205
            $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () );
238
206
239
    # If there's no query on the left, let's remove the junk left behind
207
        # If there's no query on the left, let's remove the junk left behind
240
    $query_str =~ s/^ AND //;
208
        $query_str =~ s/^ AND //;
241
    my %options;
209
        my %options;
242
    $options{sort} = \@sort_params;
210
        $options{sort} = \@sort_params;
243
    $options{is_opac} = $params->{is_opac};
211
        $options{is_opac} = $params->{is_opac};
244
    $options{weighted_fields} = $params->{weighted_fields};
212
        $options{weighted_fields} = $params->{weighted_fields};
245
    $options{whole_record} = $params->{whole_record};
213
        $options{whole_record} = $params->{whole_record};
246
    my $query = $self->build_query( $query_str, %options );
214
        $query = $self->build_query( $query_str, %options );
215
    }
247
216
248
    # We roughly emulate the CGI parameters of the zebra query builder
217
    # We roughly emulate the CGI parameters of the zebra query builder
249
    my $query_cgi = '';
218
    my $query_cgi = '';
250
    shift @$operators; # Shift out the one we unshifted before
219
    shift @$operators; # Shift out the one we unshifted before
251
    $ea = each_array( @$operands, @$operators, @$indexes );
220
    my $ea = each_array( @$operands, @$operators, @$indexes );
252
    while ( my ( $oand, $otor, $index ) = $ea->() ) {
221
    while ( my ( $oand, $otor, $index ) = $ea->() ) {
253
        $query_cgi .= '&' if $query_cgi;
222
        $query_cgi .= '&' if $query_cgi;
254
        $query_cgi .= 'idx=' . uri_escape_utf8( $index // '') . '&q=' . uri_escape_utf8( $oand );
223
        $query_cgi .= 'idx=' . uri_escape_utf8( $index // '') . '&q=' . uri_escape_utf8( $oand );
Lines 528-533 sub build_authorities_query_compat { Link Here
528
    return $query;
497
    return $query;
529
}
498
}
530
499
500
=head2 _build_scan_query
501
502
    my ($query, $query_str) = $builder->_build_scan_query(\@operands, \@indexes)
503
504
This will build an aggregation scan query that can be issued to elasticsearch from
505
the provided string input.
506
507
=cut
508
509
our %scan_field_convert = (
510
    'ti' => 'title',
511
    'au' => 'author',
512
    'su' => 'subject',
513
    'se' => 'title-series',
514
    'pb' => 'publisher',
515
);
516
517
sub _build_scan_query {
518
    my ( $self, $operands, $indexes ) = @_;
519
520
    my $term = scalar( @$operands ) == 0 ? '' : $operands->[0];
521
    my $index = scalar( @$indexes ) == 0 ? 'subject' : $indexes->[0];
522
523
    my ( $f, $d ) = split( /,/, $index);
524
    $index = $scan_field_convert{$f} || $f;
525
526
    my $res;
527
    $res->{query} = {
528
        query_string => {
529
            query => '*'
530
        }
531
    };
532
    $res->{aggregations} = {
533
        $index => {
534
            terms => {
535
                field => $index . '__facet',
536
                order => { '_term' => 'asc' },
537
                include => $self->_create_regex_filter($self->_clean_search_term($term)) . '.*'
538
            }
539
        }
540
    };
541
    return ($res, $term);
542
}
543
544
=head2 _create_regex_filter
545
546
    my $filter = $builder->_create_regex_filter('term')
547
548
This will create a regex filter that can be used with an aggregation query.
549
550
=cut
551
552
sub _create_regex_filter {
553
    my ($self, $term) = @_;
554
555
    my $result = '';
556
    foreach my $c (split(//, quotemeta($term))) {
557
        my $lc = lc($c);
558
        my $uc = uc($c);
559
        $result .= $lc ne $uc ? '[' . $lc . $uc . ']' : $c;
560
    }
561
    return $result;
562
}
563
531
=head2 _convert_sort_fields
564
=head2 _convert_sort_fields
532
565
533
    my @sort_params = _convert_sort_fields(@sort_by)
566
    my @sort_params = _convert_sort_fields(@sort_by)
Lines 774-780 sub _modify_string_by_type { Link Here
774
    return $str unless $str;    # Empty or undef, we can't use it.
807
    return $str unless $str;    # Empty or undef, we can't use it.
775
808
776
    $str .= '*' if $type eq 'right-truncate';
809
    $str .= '*' if $type eq 'right-truncate';
777
    $str = '"' . $str . '"' if $type eq 'phrase';
810
    $str = '"' . $str . '"' if $type eq 'phrase' && $str !~ /^".*"$/;
778
    if ($type eq 'st-year') {
811
    if ($type eq 'st-year') {
779
        if ($str =~ /^(.*)-(.*)$/) {
812
        if ($str =~ /^(.*)-(.*)$/) {
780
            my $from = $1 || '*';
813
            my $from = $1 || '*';
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-6 / +74 lines)
Lines 85-91 sub search { Link Here
85
85
86
    my $params = $self->get_elasticsearch_params();
86
    my $params = $self->get_elasticsearch_params();
87
    # 20 is the default number of results per page
87
    # 20 is the default number of results per page
88
    $query->{size} = $count || 20;
88
    $query->{size} = $count // 20;
89
    # ES doesn't want pages, it wants a record to start from.
89
    # ES doesn't want pages, it wants a record to start from.
90
    if (exists $options{offset}) {
90
    if (exists $options{offset}) {
91
        $query->{from} = $options{offset};
91
        $query->{from} = $options{offset};
Lines 132-139 sub count { Link Here
132
132
133
    my ( $error, $results, $facets ) = $search->search_compat(
133
    my ( $error, $results, $facets ) = $search->search_compat(
134
        $query,            $simple_query, \@sort_by,       \@servers,
134
        $query,            $simple_query, \@sort_by,       \@servers,
135
        $results_per_page, $offset,       $branches,       $query_type,
135
        $results_per_page, $offset,       undef,           $item_types,
136
        $scan
136
        $query_type,       $scan
137
      )
137
      )
138
138
139
A search interface somewhat compatible with L<C4::Search->getRecords>. Anything
139
A search interface somewhat compatible with L<C4::Search->getRecords>. Anything
Lines 144-153 get ignored here, along with some other things (like C<@servers>.) Link Here
144
144
145
sub search_compat {
145
sub search_compat {
146
    my (
146
    my (
147
        $self,     $query,            $simple_query, $sort_by,
147
        $self,       $query,            $simple_query, $sort_by,
148
        $servers,  $results_per_page, $offset,       $branches,
148
        $servers,    $results_per_page, $offset,       $branches,
149
        $query_type,       $scan
149
        $item_types, $query_type,       $scan
150
    ) = @_;
150
    ) = @_;
151
152
    if ( $scan ) {
153
        return $self->_aggregation_scan( $query, $results_per_page, $offset );
154
    }
155
151
    my %options;
156
    my %options;
152
    if ( !defined $offset or $offset < 0 ) {
157
    if ( !defined $offset or $offset < 0 ) {
153
        $offset = 0;
158
        $offset = 0;
Lines 509-512 sub _convert_facets { Link Here
509
    return \@facets;
514
    return \@facets;
510
}
515
}
511
516
517
=head2 _aggregation_scan
518
519
    my $result = $self->_aggregration_scan($query, 10, 0);
520
521
Perform an aggregation request for scan purposes.
522
523
=cut
524
525
sub _aggregation_scan {
526
    my ($self, $query, $results_per_page, $offset) = @_;
527
528
    if (!scalar(keys %{$query->{aggregations}})) {
529
        my %result = {
530
            biblioserver => {
531
                hits => 0,
532
                RECORDS => undef
533
            }
534
        };
535
        return (undef, \%result, undef);
536
    }
537
    my ($field) = keys %{$query->{aggregations}};
538
    $query->{aggregations}{$field}{terms}{size} = 1000;
539
    my $results = $self->search($query, 1, 0);
540
541
    # Convert each result into a MARC::Record
542
    my (@records, $index);
543
    # opac-search expects results to be put in the
544
    # right place in the array, according to $offset
545
    $index = $offset - 1;
546
547
    my $count = scalar(@{$results->{aggregations}{$field}{buckets}});
548
    for (my $index = $offset; $index - $offset < $results_per_page && $index < $count; $index++) {
549
        my $bucket = $results->{aggregations}{$field}{buckets}->[$index];
550
        # Scan values are expressed as:
551
        # - MARC21: 100a (count) and 245a (term)
552
        # - UNIMARC: 200f (count) and 200a (term)
553
        my $marc = MARC::Record->new;
554
        $marc->encoding('UTF-8');
555
        if (C4::Context->preference('marcflavour') eq 'UNIMARC') {
556
            $marc->append_fields(
557
                MARC::Field->new((200, ' ',  ' ', 'f' => $bucket->{doc_count}))
558
            );
559
            $marc->append_fields(
560
                MARC::Field->new((200, ' ',  ' ', 'a' => $bucket->{key}))
561
            );
562
        } else {
563
            $marc->append_fields(
564
                MARC::Field->new((100, ' ',  ' ', 'a' => $bucket->{doc_count}))
565
            );
566
            $marc->append_fields(
567
                MARC::Field->new((245, ' ',  ' ', 'a' => $bucket->{key}))
568
            );
569
        }
570
        $records[$index] = $marc->as_usmarc();
571
    };
572
    # consumers of this expect a namespaced result, we provide the default
573
    # configuration.
574
    my %result;
575
    $result{biblioserver}{hits} = $count;
576
    $result{biblioserver}{RECORDS} = \@records;
577
    return (undef, \%result, undef);
578
}
579
512
1;
580
1;
(-)a/catalogue/search.pl (-4 / +5 lines)
Lines 498-506 if ($query_cgi) { Link Here
498
        my $input_value = $2;
498
        my $input_value = $2;
499
        push @query_inputs, { input_name => $input_name, input_value => Encode::decode_utf8( uri_unescape( $input_value ) ) };
499
        push @query_inputs, { input_name => $input_name, input_value => Encode::decode_utf8( uri_unescape( $input_value ) ) };
500
        if ($input_name eq 'idx') {
500
        if ($input_name eq 'idx') {
501
            $scan_index_to_use = $input_value; # unless $scan_index_to_use;
501
            # The form contains multiple fields, so take the first value as the scan index
502
            $scan_index_to_use = $input_value unless $scan_index_to_use;
502
        }
503
        }
503
        if ($input_name eq 'q') {
504
        if (!defined $scan_search_term_to_use && $input_name eq 'q') {
504
            $scan_search_term_to_use = Encode::decode_utf8( uri_unescape( $input_value ));
505
            $scan_search_term_to_use = Encode::decode_utf8( uri_unescape( $input_value ));
505
        }
506
        }
506
    }
507
    }
Lines 584-591 for (my $i=0;$i<@servers;$i++) { Link Here
584
            $template->param( EnableSearchHistory => 1 );
585
            $template->param( EnableSearchHistory => 1 );
585
        }
586
        }
586
587
587
        ## If there's just one result, redirect to the detail page
588
        ## If there's just one result, redirect to the detail page unless doing an index scan
588
        if ($total == 1) {         
589
        if ($total == 1 && !$scan) {
589
            my $biblionumber = $newresults[0]->{biblionumber};
590
            my $biblionumber = $newresults[0]->{biblionumber};
590
            my $defaultview = C4::Context->preference('IntranetBiblioDefaultView');
591
            my $defaultview = C4::Context->preference('IntranetBiblioDefaultView');
591
            my $views = { C4::Search::enabled_staff_search_views };
592
            my $views = { C4::Search::enabled_staff_search_views };
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/results.tt (-1 / +2 lines)
Lines 313-319 Link Here
313
                                            [% ELSE %]<option value="ti">Title</option>[% END %]
313
                                            [% ELSE %]<option value="ti">Title</option>[% END %]
314
                                            [% IF ( ms_ticommaphr ) %]<option selected="selected" value="ti,phr">Title phrase</option>
314
                                            [% IF ( ms_ticommaphr ) %]<option selected="selected" value="ti,phr">Title phrase</option>
315
                                            [% ELSE %]<option value="ti,phr">Title phrase</option>[% END %]
315
                                            [% ELSE %]<option value="ti,phr">Title phrase</option>[% END %]
316
                                            [% IF ( ms_aucommaphr ) %]<option selected="selected" value="au,phr">Author</option>
316
                                            [% IF ( ms_au || ms_aucommaphr ) %]<option selected="selected" value="au,phr">Author</option>
317
                                            [% ELSE %]<option value="au,phr">Author</option>[% END %]
317
                                            [% ELSE %]<option value="au,phr">Author</option>[% END %]
318
                                            [% IF ( ms_su ) %]<option selected="selected" value="su">Subject</option>
318
                                            [% IF ( ms_su ) %]<option selected="selected" value="su">Subject</option>
319
                                            [% ELSE %]<option value="su">Subject</option>[% END %]
319
                                            [% ELSE %]<option value="su">Subject</option>[% END %]
Lines 331-336 Link Here
331
                                            [% ELSE %]<option value="ss">ISSN</option>[% END %]
331
                                            [% ELSE %]<option value="ss">ISSN</option>[% END %]
332
                                        </select>
332
                                        </select>
333
                                        <input type="hidden" name="scan" value="1" />
333
                                        <input type="hidden" name="scan" value="1" />
334
                                        <input class="submit" type="submit" value="Submit" />
334
                                    </td>
335
                                    </td>
335
                                </tr>
336
                                </tr>
336
                            </table>
337
                            </table>
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t (-2 / +40 lines)
Lines 47-53 $se->mock( 'get_elasticsearch_mappings', sub { Link Here
47
                    type => 'text'
47
                    type => 'text'
48
                },
48
                },
49
                subject => {
49
                subject => {
50
                    type => 'text'
50
                    type => 'text',
51
                    facet => 1
51
                },
52
                },
52
                itemnumber => {
53
                itemnumber => {
53
                    type => 'integer'
54
                    type => 'integer'
Lines 191-197 subtest 'build_authorities_query_compat() tests' => sub { Link Here
191
};
192
};
192
193
193
subtest 'build_query tests' => sub {
194
subtest 'build_query tests' => sub {
194
    plan tests => 40;
195
    plan tests => 48;
195
196
196
    my $qb;
197
    my $qb;
197
198
Lines 438-443 subtest 'build_query tests' => sub { Link Here
438
    );
439
    );
439
    is($query_cgi, 'idx=&q=title%3A%22donald%20duck%22', 'query cgi');
440
    is($query_cgi, 'idx=&q=title%3A%22donald%20duck%22', 'query cgi');
440
    is($query_desc, 'title:"donald duck"', 'query desc ok');
441
    is($query_desc, 'title:"donald duck"', 'query desc ok');
442
443
    # Scan queries
444
    ( undef, $query, $simple_query, $query_cgi, $query_desc ) = $qb->build_query_compat( undef, ['new'], ['au'], undef, undef, 1 );
445
    is(
446
        $query->{query}{query_string}{query},
447
        '*',
448
        "scan query is properly formed"
449
    );
450
    is_deeply(
451
        $query->{aggregations}{'author'}{'terms'},
452
        {
453
            field => 'author__facet',
454
            order => { '_term' => 'asc' },
455
            include => '[nN][eE][wW].*'
456
        },
457
        "scan aggregation request is properly formed"
458
    );
459
    is($query_cgi, 'idx=au&q=new&scan=1', 'query cgi');
460
    is($query_desc, 'new', 'query desc ok');
461
462
    ( undef, $query, $simple_query, $query_cgi, $query_desc ) = $qb->build_query_compat( undef, ['new'], [], undef, undef, 1 );
463
    is(
464
        $query->{query}{query_string}{query},
465
        '*',
466
        "scan query is properly formed"
467
    );
468
    is_deeply(
469
        $query->{aggregations}{'subject'}{'terms'},
470
        {
471
            field => 'subject__facet',
472
            order => { '_term' => 'asc' },
473
            include => '[nN][eE][wW].*'
474
        },
475
        "scan aggregation request is properly formed"
476
    );
477
    is($query_cgi, 'idx=&q=new&scan=1', 'query cgi');
478
    is($query_desc, 'new', 'query desc ok');
441
};
479
};
442
480
443
481
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch/Search.t (-2 / +11 lines)
Lines 17-23 Link Here
17
17
18
use Modern::Perl;
18
use Modern::Perl;
19
19
20
use Test::More tests => 11;
20
use Test::More tests => 13;
21
use t::lib::Mocks;
21
use t::lib::Mocks;
22
22
23
use Koha::SearchEngine::Elasticsearch::QueryBuilder;
23
use Koha::SearchEngine::Elasticsearch::QueryBuilder;
Lines 100-105 SKIP: { Link Here
100
100
101
    ok ($results = $searcher->search_compat( $query ), 'Test search_compat' );
101
    ok ($results = $searcher->search_compat( $query ), 'Test search_compat' );
102
102
103
    my ( undef, $scan_query ) = $builder->build_query_compat( undef, ['easy'], [], undef, undef, 1 );
104
    ok ((undef, $results) = $searcher->search_compat( $scan_query, undef, [], [], 20, 0, undef, undef, undef, 1 ), 'Test search_compat scan query' );
105
    my $expected = {
106
        biblioserver => {
107
            hits => 0,
108
            RECORDS => []
109
        }
110
    };
111
    is_deeply($results, $expected, 'Scan query results ok');
112
103
    ok (($results,$count) = $searcher->search_auth_compat ( $query ), 'Test search_auth_compat' );
113
    ok (($results,$count) = $searcher->search_auth_compat ( $query ), 'Test search_auth_compat' );
104
114
105
    is ( $count = $searcher->count_auth_use($searcher,1), 0, 'Testing count_auth_use');
115
    is ( $count = $searcher->count_auth_use($searcher,1), 0, 'Testing count_auth_use');
106
- 

Return to bug 22592