|
Lines 48-53
use URI::Escape;
Link Here
|
| 48 |
|
48 |
|
| 49 |
use C4::Context; |
49 |
use C4::Context; |
| 50 |
use Koha::Exceptions; |
50 |
use Koha::Exceptions; |
|
|
51 |
use Koha::Caches; |
| 51 |
|
52 |
|
| 52 |
=head2 build_query |
53 |
=head2 build_query |
| 53 |
|
54 |
|
|
Lines 90-98
sub build_query {
Link Here
|
| 90 |
query => $query, |
91 |
query => $query, |
| 91 |
fuzziness => $fuzzy_enabled ? 'auto' : '0', |
92 |
fuzziness => $fuzzy_enabled ? 'auto' : '0', |
| 92 |
default_operator => 'AND', |
93 |
default_operator => 'AND', |
| 93 |
default_field => '_all', |
94 |
fields => $self->_search_fields({ is_opac => $options{is_opac}, weighted_fields => $options{weighted_fields} }), |
| 94 |
lenient => JSON::true, |
95 |
lenient => JSON::true, |
| 95 |
fields => $options{fields} || [], |
|
|
| 96 |
} |
96 |
} |
| 97 |
}; |
97 |
}; |
| 98 |
|
98 |
|
|
Lines 227-243
sub build_query_compat {
Link Here
|
| 227 |
join( ' ', $self->_create_query_string(@search_params) ) || (), |
227 |
join( ' ', $self->_create_query_string(@search_params) ) || (), |
| 228 |
$self->_join_queries( $self->_convert_index_strings(@$limits) ) || () ); |
228 |
$self->_join_queries( $self->_convert_index_strings(@$limits) ) || () ); |
| 229 |
|
229 |
|
| 230 |
my @fields = '_all'; |
|
|
| 231 |
if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) { |
| 232 |
push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields; |
| 233 |
} |
| 234 |
|
| 235 |
# If there's no query on the left, let's remove the junk left behind |
230 |
# If there's no query on the left, let's remove the junk left behind |
| 236 |
$query_str =~ s/^ AND //; |
231 |
$query_str =~ s/^ AND //; |
| 237 |
my %options; |
232 |
my %options; |
| 238 |
$options{fields} = \@fields; |
|
|
| 239 |
$options{sort} = \@sort_params; |
233 |
$options{sort} = \@sort_params; |
| 240 |
$options{expanded_facet} = $params->{expanded_facet}; |
234 |
$options{expanded_facet} = $params->{expanded_facet}; |
|
|
235 |
$options{is_opac} = $params->{is_opac}; |
| 236 |
$options{weighted_fields} = $params->{weighted_fields}; |
| 241 |
my $query = $self->build_query( $query_str, %options ); |
237 |
my $query = $self->build_query( $query_str, %options ); |
| 242 |
|
238 |
|
| 243 |
#die Dumper($query); |
239 |
#die Dumper($query); |
|
Lines 298-353
sub build_authorities_query {
Link Here
|
| 298 |
|
294 |
|
| 299 |
foreach my $s ( @{ $search->{searches} } ) { |
295 |
foreach my $s ( @{ $search->{searches} } ) { |
| 300 |
my ( $wh, $op, $val ) = @{$s}{qw(where operator value)}; |
296 |
my ( $wh, $op, $val ) = @{$s}{qw(where operator value)}; |
| 301 |
$wh = '_all' if $wh eq ''; |
297 |
if (!$wh) { |
| 302 |
if ( $op eq 'is' || $op eq '=' || $op eq 'exact' ) { |
298 |
if ( $op eq 'is' || $op eq '=' || $op eq 'exact') { |
| 303 |
|
299 |
# Match the whole field for all searchable fields, case insensitive, |
| 304 |
# look for something that matches a term completely |
300 |
# UTF normalized. |
| 305 |
# note, '=' is about numerical vals. May need special handling. |
301 |
# Given that field data is "The quick brown fox" |
| 306 |
# Also, we lowercase our search because the ES |
302 |
# "The quick brown fox" and "the quick brown fox" will match |
| 307 |
# index lowercases its values, and term searches don't get the |
303 |
# but not "quick brown fox". |
| 308 |
# search analyzer applied to them. |
304 |
push @query_parts, { |
| 309 |
push @query_parts, { match_phrase => {"$wh.phrase" => lc $val} }; |
305 |
multi_match => { |
|
|
306 |
query => $val, |
| 307 |
fields => $self->_search_fields({ subfield => 'ci_raw' }), |
| 308 |
} |
| 309 |
}; |
| 310 |
} |
| 311 |
elsif ( $op eq 'start') { |
| 312 |
# Match the prefix within a field for all searchable fields. |
| 313 |
# Given that field data is "The quick brown fox" |
| 314 |
# "The quick bro" will match, but not "quick bro" |
| 315 |
|
| 316 |
# Does not seems to be a multi prefix query |
| 317 |
# so we need to create one |
| 318 |
my @prefix_queries; |
| 319 |
foreach my $field (@{$self->_search_fields()}) { |
| 320 |
push @prefix_queries, { |
| 321 |
prefix => { "$field.ci_raw" => $val } |
| 322 |
}; |
| 323 |
} |
| 324 |
push @query_parts, { |
| 325 |
'bool' => { |
| 326 |
'should' => \@prefix_queries, |
| 327 |
'minimum_should_match' => 1 |
| 328 |
} |
| 329 |
}; |
| 330 |
} |
| 331 |
else { |
| 332 |
# Query all searchable fields. |
| 333 |
# Given that field data is "The quick brown fox" |
| 334 |
# a search containing any of the words will match, regardless |
| 335 |
# of order. |
| 336 |
# |
| 337 |
# Since default operator is "AND" each of the words must match |
| 338 |
# at least one field |
| 339 |
push @query_parts, { |
| 340 |
query_string => { |
| 341 |
query => $val, |
| 342 |
fields => $self->_search_fields(), |
| 343 |
default_operator => 'AND', |
| 344 |
} |
| 345 |
}; |
| 346 |
} |
| 347 |
} |
| 348 |
elsif ( $op eq 'is' || $op eq '=' || $op eq 'exact') { |
| 349 |
# Match the whole field, case insensitive, UTF normalized. |
| 350 |
push @query_parts, { term => { "$wh.ci_raw" => $val } }; |
| 310 |
} |
351 |
} |
| 311 |
elsif ( $op eq 'start' ) { |
352 |
elsif ( $op eq 'start' ) { |
| 312 |
# startswith search, uses lowercase untokenized version of heading |
353 |
# Match prefix of the field. |
| 313 |
push @query_parts, { match_phrase_prefix => {"$wh.phrase" => lc $val} }; |
354 |
push @query_parts, { prefix => {"$wh.ci_raw" => $val} }; |
| 314 |
} |
355 |
} |
| 315 |
else { |
356 |
else { |
| 316 |
# regular wordlist stuff |
357 |
# Regular match query for the specific field. |
| 317 |
# push @query_parts, { match => {$wh => { query => $val, operator => 'and' }} }; |
358 |
push @query_parts, { |
| 318 |
my @values = split(' ',$val); |
359 |
match => { |
| 319 |
foreach my $v (@values) { |
360 |
$wh => { |
| 320 |
push @query_parts, { wildcard => { "$wh.phrase" => "*" . lc $v . "*" } }; |
361 |
query => $val, |
| 321 |
} |
362 |
operator => 'and' |
|
|
363 |
} |
| 364 |
} |
| 365 |
}; |
| 322 |
} |
366 |
} |
| 323 |
} |
367 |
} |
| 324 |
|
368 |
|
| 325 |
# Merge the query parts appropriately |
369 |
# Merge the query parts appropriately |
| 326 |
# 'should' behaves like 'or' |
370 |
# 'should' behaves like 'or' |
| 327 |
# 'must' behaves like 'and' |
371 |
# 'must' behaves like 'and' |
| 328 |
# Zebra results seem to match must so using that here |
372 |
# Zebra behaviour seem to match must so using that here |
| 329 |
my $query = { query => |
373 |
my $elastic_query = {}; |
| 330 |
{ bool => |
374 |
$elastic_query->{bool}->{must} = \@query_parts; |
| 331 |
{ must => \@query_parts } |
375 |
|
| 332 |
} |
376 |
# Filter by authtypecode if set |
| 333 |
}; |
377 |
if ($search->{authtypecode}) { |
| 334 |
|
378 |
$elastic_query->{bool}->{filter} = { |
| 335 |
my %s; |
379 |
term => { |
| 336 |
if ( exists $search->{sort} ) { |
380 |
"authtype.raw" => $search->{authtypecode} |
| 337 |
foreach my $k ( keys %{ $search->{sort} } ) { |
381 |
} |
| 338 |
my $f = $self->_sort_field($k); |
382 |
}; |
| 339 |
$s{$f} = $search->{sort}{$k}; |
|
|
| 340 |
} |
| 341 |
$search->{sort} = \%s; |
| 342 |
} |
383 |
} |
| 343 |
|
384 |
|
| 344 |
# add the sort stuff |
385 |
my $query = { |
| 345 |
$query->{sort} = [ $search->{sort} ] if exists $search->{sort}; |
386 |
query => $elastic_query |
|
|
387 |
}; |
| 388 |
|
| 389 |
# Add the sort stuff |
| 390 |
$query->{sort} = [ $search->{sort} ] if exists $search->{sort}; |
| 346 |
|
391 |
|
| 347 |
return $query; |
392 |
return $query; |
| 348 |
} |
393 |
} |
| 349 |
|
394 |
|
| 350 |
|
|
|
| 351 |
=head2 build_authorities_query_compat |
395 |
=head2 build_authorities_query_compat |
| 352 |
|
396 |
|
| 353 |
my ($query) = |
397 |
my ($query) = |
|
Lines 441-448
sub build_authorities_query_compat {
Link Here
|
| 441 |
|
485 |
|
| 442 |
my %sort; |
486 |
my %sort; |
| 443 |
my $sort_field = |
487 |
my $sort_field = |
| 444 |
( $orderby =~ /^Heading/ ) ? 'Heading' |
488 |
( $orderby =~ /^Heading/ ) ? 'Heading__sort' |
| 445 |
: ( $orderby =~ /^Auth/ ) ? 'Local-number' |
489 |
: ( $orderby =~ /^Auth/ ) ? 'Local-Number__sort' |
| 446 |
: undef; |
490 |
: undef; |
| 447 |
if ($sort_field) { |
491 |
if ($sort_field) { |
| 448 |
my $sort_order = ( $orderby =~ /Asc$/ ) ? 'asc' : 'desc'; |
492 |
my $sort_order = ( $orderby =~ /Asc$/ ) ? 'asc' : 'desc'; |
|
Lines 509-515
types.
Link Here
|
| 509 |
=cut |
553 |
=cut |
| 510 |
|
554 |
|
| 511 |
our %index_field_convert = ( |
555 |
our %index_field_convert = ( |
| 512 |
'kw' => '_all', |
556 |
'kw' => '', |
| 513 |
'ti' => 'title', |
557 |
'ti' => 'title', |
| 514 |
'au' => 'author', |
558 |
'au' => 'author', |
| 515 |
'su' => 'subject', |
559 |
'su' => 'subject', |
|
Lines 537-543
sub _convert_index_fields {
Link Here
|
| 537 |
# If a field starts with mc- we save it as it's used (and removed) later |
581 |
# If a field starts with mc- we save it as it's used (and removed) later |
| 538 |
# when joining things, to indicate we make it an 'OR' join. |
582 |
# when joining things, to indicate we make it an 'OR' join. |
| 539 |
# (Sorry, this got a bit ugly after special cases were found.) |
583 |
# (Sorry, this got a bit ugly after special cases were found.) |
| 540 |
grep { $_->{field} } map { |
584 |
map { |
| 541 |
my ( $f, $t ) = split /,/; |
585 |
my ( $f, $t ) = split /,/; |
| 542 |
my $mc = ''; |
586 |
my $mc = ''; |
| 543 |
if ($f =~ /^mc-/) { |
587 |
if ($f =~ /^mc-/) { |
|
Lines 549-555
sub _convert_index_fields {
Link Here
|
| 549 |
type => $index_type_convert{ $t // '__default' } |
593 |
type => $index_type_convert{ $t // '__default' } |
| 550 |
}; |
594 |
}; |
| 551 |
$r->{field} = ($mc . $r->{field}) if $mc && $r->{field}; |
595 |
$r->{field} = ($mc . $r->{field}) if $mc && $r->{field}; |
| 552 |
$r; |
596 |
$r->{field} ? $r : undef; |
| 553 |
} @indexes; |
597 |
} @indexes; |
| 554 |
} |
598 |
} |
| 555 |
|
599 |
|
|
Lines 578-585
sub _convert_index_strings {
Link Here
|
| 578 |
push @res, $s; |
622 |
push @res, $s; |
| 579 |
next; |
623 |
next; |
| 580 |
} |
624 |
} |
| 581 |
push @res, $conv->{field} . ":" |
625 |
push @res, ($conv->{field} ? $conv->{field} . ':' : '') |
| 582 |
. $self->_modify_string_by_type( %$conv, operand => $term ); |
626 |
. $self->_modify_string_by_type( %$conv, operand => $term ); |
| 583 |
} |
627 |
} |
| 584 |
return @res; |
628 |
return @res; |
| 585 |
} |
629 |
} |
|
Lines 822-825
sub _truncate_terms {
Link Here
|
| 822 |
return join ' ', @terms; |
866 |
return join ' ', @terms; |
| 823 |
} |
867 |
} |
| 824 |
|
868 |
|
|
|
869 |
=head2 _search_fields |
| 870 |
my $weighted_fields = $self->_search_fields({ |
| 871 |
is_opac => 0, |
| 872 |
weighted_fields => 1, |
| 873 |
subfield => 'raw' |
| 874 |
}); |
| 875 |
|
| 876 |
Generate a list of searchable fields to be used for Elasticsearch queries |
| 877 |
applied to multiple fields. |
| 878 |
|
| 879 |
Returns an arrayref of field names for either OPAC or Staff client, with |
| 880 |
possible weights and subfield appended to each field name depending on the |
| 881 |
options provided. |
| 882 |
|
| 883 |
=over 4 |
| 884 |
|
| 885 |
=item C<$params> |
| 886 |
|
| 887 |
Hashref with options. The parameter C<is_opac> indicates whether the searchable |
| 888 |
fields for OPAC or Staff client should be retrieved. If C<weighted_fields> is set |
| 889 |
fields weights will be applied on returned fields. C<subfield> can be used to |
| 890 |
provide a subfield that will be appended to fields as "C<field_name>.C<subfield>". |
| 891 |
|
| 892 |
=back |
| 893 |
|
| 894 |
=cut |
| 895 |
|
| 896 |
sub _search_fields { |
| 897 |
my ($self, $params) = @_; |
| 898 |
$params //= { |
| 899 |
is_opac => 0, |
| 900 |
weighted_fields => 0, |
| 901 |
# This is a hack for authorities build_authorities_query |
| 902 |
# can hopefully be removed in the future |
| 903 |
subfield => undef, |
| 904 |
}; |
| 905 |
|
| 906 |
my $cache = Koha::Caches->get_instance(); |
| 907 |
my $cache_key = 'elasticsearch_search_fields' . ($params->{is_opac} ? '_opac' : '_staff_client'); |
| 908 |
my $search_fields = $cache->get_from_cache($cache_key, { unsafe => 1 }); |
| 909 |
|
| 910 |
if (!$search_fields) { |
| 911 |
# The reason we don't use Koha::SearchFields->search here is we don't |
| 912 |
# want or need resultset wrapped as Koha::SearchField object. |
| 913 |
# It does not make any sense in this context and would cause |
| 914 |
# unnecessary overhead sice we are only querying for data |
| 915 |
# Also would not work, or produce strange results, with the "columns" |
| 916 |
# option. |
| 917 |
my $schema = Koha::Database->schema; |
| 918 |
my $result = $schema->resultset('SearchField')->search( |
| 919 |
{ |
| 920 |
$params->{is_opac} ? ( |
| 921 |
'opac' => 1, |
| 922 |
) : ( |
| 923 |
'staff_client' => 1 |
| 924 |
), |
| 925 |
'search_marc_map.index_name' => $self->index, |
| 926 |
'search_marc_map.marc_type' => C4::Context->preference('marcflavour'), |
| 927 |
'search_marc_to_fields.search' => 1, |
| 928 |
}, |
| 929 |
{ |
| 930 |
columns => [qw/name weight/], |
| 931 |
# collapse => 1, |
| 932 |
join => {search_marc_to_fields => 'search_marc_map'}, |
| 933 |
} |
| 934 |
); |
| 935 |
my @search_fields; |
| 936 |
while (my $search_field = $result->next) { |
| 937 |
push @search_fields, [ |
| 938 |
$search_field->name, |
| 939 |
$search_field->weight ? $search_field->weight : () |
| 940 |
]; |
| 941 |
} |
| 942 |
$search_fields = \@search_fields; |
| 943 |
$cache->set_in_cache($cache_key, $search_fields); |
| 944 |
} |
| 945 |
if ($params->{subfield}) { |
| 946 |
my $subfield = $params->{subfield}; |
| 947 |
$search_fields = [ |
| 948 |
map { |
| 949 |
# Copy values to avoid mutating cached |
| 950 |
# data (since unsafe is used) |
| 951 |
my ($field, $weight) = @{$_}; |
| 952 |
["${field}.${subfield}", $weight]; |
| 953 |
} @{$search_fields} |
| 954 |
]; |
| 955 |
} |
| 956 |
|
| 957 |
if ($params->{weighted_fields}) { |
| 958 |
return [map { join('^', @{$_}) } @{$search_fields}]; |
| 959 |
} |
| 960 |
else { |
| 961 |
# Exclude weight from field |
| 962 |
return [map { $_->[0] } @{$search_fields}]; |
| 963 |
} |
| 964 |
} |
| 965 |
|
| 825 |
1; |
966 |
1; |