Lines 48-53
use URI::Escape;
Link Here
|
48 |
|
48 |
|
49 |
use C4::Context; |
49 |
use C4::Context; |
50 |
use Koha::Exceptions; |
50 |
use Koha::Exceptions; |
|
|
51 |
use Koha::Caches; |
51 |
|
52 |
|
52 |
=head2 build_query |
53 |
=head2 build_query |
53 |
|
54 |
|
Lines 90-96
sub build_query {
Link Here
|
90 |
query => $query, |
91 |
query => $query, |
91 |
fuzziness => $fuzzy_enabled ? 'auto' : '0', |
92 |
fuzziness => $fuzzy_enabled ? 'auto' : '0', |
92 |
default_operator => 'AND', |
93 |
default_operator => 'AND', |
93 |
default_field => '_all', |
94 |
fields => $self->_search_fields({ is_opac => $options{is_opac}, weighted_fields => $options{weighted_fields} }), |
94 |
lenient => JSON::true, |
95 |
lenient => JSON::true, |
95 |
analyze_wildcard => JSON::true, |
96 |
analyze_wildcard => JSON::true, |
96 |
fields => $options{fields} || [], |
97 |
fields => $options{fields} || [], |
Lines 231-246
sub build_query_compat {
Link Here
|
231 |
$search_param_query_str || (), |
232 |
$search_param_query_str || (), |
232 |
$self->_join_queries( $self->_convert_index_strings(@$limits) ) || () ); |
233 |
$self->_join_queries( $self->_convert_index_strings(@$limits) ) || () ); |
233 |
|
234 |
|
234 |
my @fields = '_all'; |
|
|
235 |
if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) { |
236 |
push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields; |
237 |
} |
238 |
|
239 |
# If there's no query on the left, let's remove the junk left behind |
235 |
# If there's no query on the left, let's remove the junk left behind |
240 |
$query_str =~ s/^ AND //; |
236 |
$query_str =~ s/^ AND //; |
241 |
my %options; |
237 |
my %options; |
242 |
$options{fields} = \@fields; |
|
|
243 |
$options{sort} = \@sort_params; |
238 |
$options{sort} = \@sort_params; |
|
|
239 |
$options{is_opac} = $params->{is_opac}; |
240 |
$options{weighted_fields} = $params->{weighted_fields}; |
244 |
my $query = $self->build_query( $query_str, %options ); |
241 |
my $query = $self->build_query( $query_str, %options ); |
245 |
|
242 |
|
246 |
# We roughly emulate the CGI parameters of the zebra query builder |
243 |
# We roughly emulate the CGI parameters of the zebra query builder |
Lines 314-335
sub build_authorities_query {
Link Here
|
314 |
|
311 |
|
315 |
foreach my $s ( @{ $search->{searches} } ) { |
312 |
foreach my $s ( @{ $search->{searches} } ) { |
316 |
my ( $wh, $op, $val ) = @{$s}{qw(where operator value)}; |
313 |
my ( $wh, $op, $val ) = @{$s}{qw(where operator value)}; |
317 |
$wh = '_all' if $wh eq ''; |
314 |
if ( $op eq 'is' || $op eq '=' || $op eq 'exact') { |
318 |
if ( $op eq 'is' || $op eq '=' || $op eq 'exact' ) { |
315 |
if ($wh) { |
319 |
|
316 |
# Match the whole field, case insensitive, UTF normalized. |
320 |
# look for something that matches a term completely |
317 |
push @query_parts, { term => { "$wh.ci_raw" => $val } }; |
321 |
# note, '=' is about numerical vals. May need special handling. |
318 |
} |
322 |
# Also, we lowercase our search because the ES |
319 |
else { |
323 |
# index lowercases its values, and term searches don't get the |
320 |
# Match the whole field for all searchable fields, case insensitive, |
324 |
# search analyzer applied to them. |
321 |
# UTF normalized. |
325 |
push @query_parts, { match_phrase => {"$wh.phrase" => lc $val} }; |
322 |
# Given that field data is "The quick brown fox" |
|
|
323 |
# "The quick brown fox" and "the quick brown fox" will match |
324 |
# but not "quick brown fox". |
325 |
push @query_parts, { |
326 |
multi_match => { |
327 |
query => $val, |
328 |
fields => $self->_search_fields({ subfield => 'ci_raw' }), |
329 |
} |
330 |
}; |
331 |
} |
326 |
} |
332 |
} |
327 |
elsif ( $op eq 'start' ) { |
333 |
elsif ( $op eq 'start') { |
328 |
# startswith search, uses lowercase untokenized version of heading |
334 |
# Match the prefix within a field for all searchable fields. |
329 |
push @query_parts, { match_phrase_prefix => {"$wh.phrase" => lc $val} }; |
335 |
# Given that field data is "The quick brown fox" |
|
|
336 |
# "The quick bro" will match, but not "quick bro" |
337 |
|
338 |
# Does not seems to be a multi prefix query |
339 |
# so we need to create one |
340 |
if ($wh) { |
341 |
# Match prefix of the field. |
342 |
push @query_parts, { prefix => {"$wh.ci_raw" => $val} }; |
343 |
} |
344 |
else { |
345 |
my @prefix_queries; |
346 |
foreach my $field (@{$self->_search_fields()}) { |
347 |
push @prefix_queries, { |
348 |
prefix => { "$field.ci_raw" => $val } |
349 |
}; |
350 |
} |
351 |
push @query_parts, { |
352 |
'bool' => { |
353 |
'should' => \@prefix_queries, |
354 |
'minimum_should_match' => 1 |
355 |
} |
356 |
}; |
357 |
} |
330 |
} |
358 |
} |
331 |
else { |
359 |
else { |
332 |
# regular wordlist stuff |
360 |
# Query all searchable fields. |
|
|
361 |
# Given that field data is "The quick brown fox" |
362 |
# a search containing any of the words will match, regardless |
363 |
# of order. |
364 |
|
333 |
my @tokens = $self->_split_query( $val ); |
365 |
my @tokens = $self->_split_query( $val ); |
334 |
foreach my $token ( @tokens ) { |
366 |
foreach my $token ( @tokens ) { |
335 |
$token = $self->_truncate_terms( |
367 |
$token = $self->_truncate_terms( |
Lines 337-375
sub build_authorities_query {
Link Here
|
337 |
); |
369 |
); |
338 |
} |
370 |
} |
339 |
my $query = $self->_join_queries( @tokens ); |
371 |
my $query = $self->_join_queries( @tokens ); |
340 |
push @query_parts, { query_string => { default_field => $wh, query => $query } }; |
372 |
|
|
|
373 |
if ($wh) { |
374 |
push @query_parts, { query_string => { default_field => $wh, query => $query } }; |
375 |
} |
376 |
else { |
377 |
push @query_parts, { |
378 |
query_string => { |
379 |
query => $query, |
380 |
fields => $self->_search_fields(), |
381 |
} |
382 |
}; |
383 |
} |
341 |
} |
384 |
} |
342 |
} |
385 |
} |
343 |
|
386 |
|
344 |
# Merge the query parts appropriately |
387 |
# Merge the query parts appropriately |
345 |
# 'should' behaves like 'or' |
388 |
# 'should' behaves like 'or' |
346 |
# 'must' behaves like 'and' |
389 |
# 'must' behaves like 'and' |
347 |
# Zebra results seem to match must so using that here |
390 |
# Zebra behaviour seem to match must so using that here |
348 |
my $query = { query => |
391 |
my $elastic_query = {}; |
349 |
{ bool => |
392 |
$elastic_query->{bool}->{must} = \@query_parts; |
350 |
{ must => \@query_parts } |
393 |
|
351 |
} |
394 |
# Filter by authtypecode if set |
352 |
}; |
395 |
if ($search->{authtypecode}) { |
353 |
if ( $search->{authtypecode} ) { |
396 |
$elastic_query->{bool}->{filter} = { |
354 |
$query->{query}->{bool}->{filter} = { term => { 'authtype' => lc $search->{authtypecode} } }; |
397 |
term => { |
|
|
398 |
"authtype.raw" => $search->{authtypecode} |
399 |
} |
400 |
}; |
355 |
} |
401 |
} |
356 |
|
402 |
|
357 |
my %s; |
403 |
my $query = { |
358 |
if ( exists $search->{sort} ) { |
404 |
query => $elastic_query |
359 |
foreach my $k ( keys %{ $search->{sort} } ) { |
405 |
}; |
360 |
my $f = $self->_sort_field($k); |
|
|
361 |
$s{$f} = $search->{sort}{$k}; |
362 |
} |
363 |
$search->{sort} = \%s; |
364 |
} |
365 |
|
406 |
|
366 |
# add the sort stuff |
407 |
# Add the sort stuff |
367 |
$query->{sort} = [ $search->{sort} ] if exists $search->{sort}; |
408 |
$query->{sort} = [ $search->{sort} ] if exists $search->{sort}; |
368 |
|
409 |
|
369 |
return $query; |
410 |
return $query; |
370 |
} |
411 |
} |
371 |
|
412 |
|
372 |
|
|
|
373 |
=head2 build_authorities_query_compat |
413 |
=head2 build_authorities_query_compat |
374 |
|
414 |
|
375 |
my ($query) = |
415 |
my ($query) = |
Lines 467-474
sub build_authorities_query_compat {
Link Here
|
467 |
|
507 |
|
468 |
my %sort; |
508 |
my %sort; |
469 |
my $sort_field = |
509 |
my $sort_field = |
470 |
( $orderby =~ /^heading/ ) ? 'heading' |
510 |
( $orderby =~ /^heading/ ) ? 'heading__sort' |
471 |
: ( $orderby =~ /^auth/ ) ? 'local-number' |
511 |
: ( $orderby =~ /^auth/ ) ? 'local-number__sort' |
472 |
: undef; |
512 |
: undef; |
473 |
if ($sort_field) { |
513 |
if ($sort_field) { |
474 |
my $sort_order = ( $orderby =~ /asc$/ ) ? 'asc' : 'desc'; |
514 |
my $sort_order = ( $orderby =~ /asc$/ ) ? 'asc' : 'desc'; |
Lines 535-541
types.
Link Here
|
535 |
=cut |
575 |
=cut |
536 |
|
576 |
|
537 |
our %index_field_convert = ( |
577 |
our %index_field_convert = ( |
538 |
'kw' => '_all', |
578 |
'kw' => '', |
539 |
'ab' => 'abstract', |
579 |
'ab' => 'abstract', |
540 |
'au' => 'author', |
580 |
'au' => 'author', |
541 |
'lcn' => 'local-classification', |
581 |
'lcn' => 'local-classification', |
Lines 629-635
sub _convert_index_fields {
Link Here
|
629 |
# If a field starts with mc- we save it as it's used (and removed) later |
669 |
# If a field starts with mc- we save it as it's used (and removed) later |
630 |
# when joining things, to indicate we make it an 'OR' join. |
670 |
# when joining things, to indicate we make it an 'OR' join. |
631 |
# (Sorry, this got a bit ugly after special cases were found.) |
671 |
# (Sorry, this got a bit ugly after special cases were found.) |
632 |
grep { $_->{field} } map { |
672 |
map { |
633 |
# Lower case all field names |
673 |
# Lower case all field names |
634 |
my ( $f, $t ) = map(lc, split /,/); |
674 |
my ( $f, $t ) = map(lc, split /,/); |
635 |
my $mc = ''; |
675 |
my $mc = ''; |
Lines 642-648
sub _convert_index_fields {
Link Here
|
642 |
type => $index_type_convert{ $t // '__default' } |
682 |
type => $index_type_convert{ $t // '__default' } |
643 |
}; |
683 |
}; |
644 |
$r->{field} = ($mc . $r->{field}) if $mc && $r->{field}; |
684 |
$r->{field} = ($mc . $r->{field}) if $mc && $r->{field}; |
645 |
$r; |
685 |
$r->{field} ? $r : undef; |
646 |
} @indexes; |
686 |
} @indexes; |
647 |
} |
687 |
} |
648 |
|
688 |
|
Lines 671-678
sub _convert_index_strings {
Link Here
|
671 |
push @res, $s; |
711 |
push @res, $s; |
672 |
next; |
712 |
next; |
673 |
} |
713 |
} |
674 |
push @res, $conv->{field} . ":" |
714 |
push @res, ($conv->{field} ? $conv->{field} . ':' : '') |
675 |
. $self->_modify_string_by_type( %$conv, operand => $term ); |
715 |
. $self->_modify_string_by_type( %$conv, operand => $term ); |
676 |
} |
716 |
} |
677 |
return @res; |
717 |
return @res; |
678 |
} |
718 |
} |
Lines 969-972
sub _split_query {
Link Here
|
969 |
return @tokens; |
1009 |
return @tokens; |
970 |
} |
1010 |
} |
971 |
|
1011 |
|
|
|
1012 |
=head2 _search_fields |
1013 |
my $weighted_fields = $self->_search_fields({ |
1014 |
is_opac => 0, |
1015 |
weighted_fields => 1, |
1016 |
subfield => 'raw' |
1017 |
}); |
1018 |
|
1019 |
Generate a list of searchable fields to be used for Elasticsearch queries |
1020 |
applied to multiple fields. |
1021 |
|
1022 |
Returns an arrayref of field names for either OPAC or Staff client, with |
1023 |
possible weights and subfield appended to each field name depending on the |
1024 |
options provided. |
1025 |
|
1026 |
=over 4 |
1027 |
|
1028 |
=item C<$params> |
1029 |
|
1030 |
Hashref with options. The parameter C<is_opac> indicates whether the searchable |
1031 |
fields for OPAC or Staff client should be retrieved. If C<weighted_fields> is set |
1032 |
fields weights will be applied on returned fields. C<subfield> can be used to |
1033 |
provide a subfield that will be appended to fields as "C<field_name>.C<subfield>". |
1034 |
|
1035 |
=back |
1036 |
|
1037 |
=cut |
1038 |
|
1039 |
sub _search_fields { |
1040 |
my ($self, $params) = @_; |
1041 |
$params //= { |
1042 |
is_opac => 0, |
1043 |
weighted_fields => 0, |
1044 |
# This is a hack for authorities build_authorities_query |
1045 |
# can hopefully be removed in the future |
1046 |
subfield => undef, |
1047 |
}; |
1048 |
|
1049 |
my $cache = Koha::Caches->get_instance(); |
1050 |
my $cache_key = 'elasticsearch_search_fields' . ($params->{is_opac} ? '_opac' : '_staff_client'); |
1051 |
my $search_fields = $cache->get_from_cache($cache_key, { unsafe => 1 }); |
1052 |
|
1053 |
if (!$search_fields) { |
1054 |
# The reason we don't use Koha::SearchFields->search here is we don't |
1055 |
# want or need resultset wrapped as Koha::SearchField object. |
1056 |
# It does not make any sense in this context and would cause |
1057 |
# unnecessary overhead sice we are only querying for data |
1058 |
# Also would not work, or produce strange results, with the "columns" |
1059 |
# option. |
1060 |
my $schema = Koha::Database->schema; |
1061 |
my $result = $schema->resultset('SearchField')->search( |
1062 |
{ |
1063 |
$params->{is_opac} ? ( |
1064 |
'opac' => 1, |
1065 |
) : ( |
1066 |
'staff_client' => 1 |
1067 |
), |
1068 |
'search_marc_map.index_name' => $self->index, |
1069 |
'search_marc_map.marc_type' => C4::Context->preference('marcflavour'), |
1070 |
'search_marc_to_fields.search' => 1, |
1071 |
}, |
1072 |
{ |
1073 |
columns => [qw/name weight/], |
1074 |
collapse => 1, |
1075 |
join => {search_marc_to_fields => 'search_marc_map'}, |
1076 |
} |
1077 |
); |
1078 |
my @search_fields; |
1079 |
while (my $search_field = $result->next) { |
1080 |
push @search_fields, [ |
1081 |
$search_field->name, |
1082 |
$search_field->weight ? $search_field->weight : () |
1083 |
]; |
1084 |
} |
1085 |
$search_fields = \@search_fields; |
1086 |
$cache->set_in_cache($cache_key, $search_fields); |
1087 |
} |
1088 |
if ($params->{subfield}) { |
1089 |
my $subfield = $params->{subfield}; |
1090 |
$search_fields = [ |
1091 |
map { |
1092 |
# Copy values to avoid mutating cached |
1093 |
# data (since unsafe is used) |
1094 |
my ($field, $weight) = @{$_}; |
1095 |
["${field}.${subfield}", $weight]; |
1096 |
} @{$search_fields} |
1097 |
]; |
1098 |
} |
1099 |
|
1100 |
if ($params->{weighted_fields}) { |
1101 |
return [map { join('^', @{$_}) } @{$search_fields}]; |
1102 |
} |
1103 |
else { |
1104 |
# Exclude weight from field |
1105 |
return [map { $_->[0] } @{$search_fields}]; |
1106 |
} |
1107 |
} |
1108 |
|
972 |
1; |
1109 |
1; |