Lines 48-53
use URI::Escape;
Link Here
|
48 |
|
48 |
|
49 |
use C4::Context; |
49 |
use C4::Context; |
50 |
use Koha::Exceptions; |
50 |
use Koha::Exceptions; |
|
|
51 |
use Koha::Caches; |
51 |
|
52 |
|
52 |
=head2 build_query |
53 |
=head2 build_query |
53 |
|
54 |
|
Lines 90-98
sub build_query {
Link Here
|
90 |
query => $query, |
91 |
query => $query, |
91 |
fuzziness => $fuzzy_enabled ? 'auto' : '0', |
92 |
fuzziness => $fuzzy_enabled ? 'auto' : '0', |
92 |
default_operator => 'AND', |
93 |
default_operator => 'AND', |
93 |
default_field => '_all', |
94 |
fields => $self->_search_fields({ is_opac => $options{is_opac}, weighted_fields => $options{weighted_fields} }), |
94 |
lenient => JSON::true, |
95 |
lenient => JSON::true, |
95 |
fields => $options{fields} || [], |
|
|
96 |
} |
96 |
} |
97 |
}; |
97 |
}; |
98 |
|
98 |
|
Lines 106-112
sub build_query {
Link Here
|
106 |
# TODO account for fields that don't have a 'phrase' type |
106 |
# TODO account for fields that don't have a 'phrase' type |
107 |
|
107 |
|
108 |
$f = $self->_sort_field($f); |
108 |
$f = $self->_sort_field($f); |
109 |
push @{ $res->{sort} }, { "$f.phrase" => { order => $d } }; |
109 |
push @{ $res->{sort} }, { $f => { order => $d } }; |
110 |
} |
110 |
} |
111 |
} |
111 |
} |
112 |
|
112 |
|
Lines 229-245
sub build_query_compat {
Link Here
|
229 |
join( ' ', $self->_create_query_string(@search_params) ) || (), |
229 |
join( ' ', $self->_create_query_string(@search_params) ) || (), |
230 |
$self->_join_queries( $self->_convert_index_strings(@$limits) ) || () ); |
230 |
$self->_join_queries( $self->_convert_index_strings(@$limits) ) || () ); |
231 |
|
231 |
|
232 |
my @fields = '_all'; |
|
|
233 |
if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) { |
234 |
push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields; |
235 |
} |
236 |
|
237 |
# If there's no query on the left, let's remove the junk left behind |
232 |
# If there's no query on the left, let's remove the junk left behind |
238 |
$query_str =~ s/^ AND //; |
233 |
$query_str =~ s/^ AND //; |
239 |
my %options; |
234 |
my %options; |
240 |
$options{fields} = \@fields; |
|
|
241 |
$options{sort} = \@sort_params; |
235 |
$options{sort} = \@sort_params; |
242 |
$options{expanded_facet} = $params->{expanded_facet}; |
236 |
$options{expanded_facet} = $params->{expanded_facet}; |
|
|
237 |
$options{is_opac} = $params->{is_opac}; |
238 |
$options{weighted_fields} = $params->{weighted_fields}; |
243 |
my $query = $self->build_query( $query_str, %options ); |
239 |
my $query = $self->build_query( $query_str, %options ); |
244 |
|
240 |
|
245 |
#die Dumper($query); |
241 |
#die Dumper($query); |
Lines 300-361
sub build_authorities_query {
Link Here
|
300 |
|
296 |
|
301 |
foreach my $s ( @{ $search->{searches} } ) { |
297 |
foreach my $s ( @{ $search->{searches} } ) { |
302 |
my ( $wh, $op, $val ) = @{$s}{qw(where operator value)}; |
298 |
my ( $wh, $op, $val ) = @{$s}{qw(where operator value)}; |
303 |
$wh = '_all' if $wh eq ''; |
299 |
if (!$wh) { |
304 |
if ( $op eq 'is' || $op eq '=' ) { |
300 |
if ( $op eq 'is' || $op eq '=' || $op eq 'exact') { |
305 |
|
301 |
# Match the whole field for all searchable fields, case insensitive, |
306 |
# look for something that matches a term completely |
302 |
# UTF normalized. |
307 |
# note, '=' is about numerical vals. May need special handling. |
303 |
# Given that field data is "The quick brown fox" |
308 |
# Also, we lowercase our search because the ES |
304 |
# "The quick brown fox" and "the quick brown fox" will match |
309 |
# index lowercases its values, and term searches don't get the |
305 |
# but not "quick brown fox". |
310 |
# search analyzer applied to them. |
306 |
push @query_parts, { |
311 |
push @query_parts, { term => {"$wh.phrase" => lc $val} }; |
307 |
multi_match => { |
|
|
308 |
query => $val, |
309 |
fields => $self->_search_fields({ subfield => 'ci_raw' }), |
310 |
} |
311 |
}; |
312 |
} |
313 |
elsif ( $op eq 'start') { |
314 |
# Match the prefix within a field for all searchable fields. |
315 |
# Given that field data is "The quick brown fox" |
316 |
# "The quick bro" will match, but not "quick bro" |
317 |
|
318 |
# Does not seems to be a multi prefix query |
319 |
# so we need to create one |
320 |
my @prefix_queries; |
321 |
foreach my $field (@{$self->_search_fields()}) { |
322 |
push @prefix_queries, { |
323 |
prefix => { "$field.ci_raw" => $val } |
324 |
}; |
325 |
} |
326 |
push @query_parts, { |
327 |
'bool' => { |
328 |
'should' => \@prefix_queries, |
329 |
'minimum_should_match' => 1 |
330 |
} |
331 |
}; |
332 |
} |
333 |
else { |
334 |
# Query all searchable fields. |
335 |
# Given that field data is "The quick brown fox" |
336 |
# a search containing any of the words will match, regardless |
337 |
# of order. |
338 |
# |
339 |
# Since default operator is "AND" each of the words must match |
340 |
# at least one field |
341 |
push @query_parts, { |
342 |
query_string => { |
343 |
query => $val, |
344 |
fields => $self->_search_fields(), |
345 |
default_operator => 'AND', |
346 |
} |
347 |
}; |
348 |
} |
312 |
} |
349 |
} |
313 |
elsif ( $op eq 'exact' ) { |
350 |
elsif ( $op eq 'is' || $op eq '=' || $op eq 'exact') { |
314 |
# left and right truncation, otherwise an exact phrase |
351 |
# Match the whole field, case insensitive, UTF normalized. |
315 |
push @query_parts, { match_phrase => {"$wh.phrase" => lc $val} }; |
352 |
push @query_parts, { term => { "$wh.ci_raw" => $val } }; |
316 |
} |
353 |
} |
317 |
elsif ( $op eq 'start' ) { |
354 |
elsif ( $op eq 'start' ) { |
318 |
# startswith search, uses lowercase untokenized version of heading |
355 |
# Match prefix of the field. |
319 |
push @query_parts, { prefix => {"$wh.lc_raw" => lc $val} }; |
356 |
push @query_parts, { prefix => {"$wh.ci_raw" => $val} }; |
320 |
} |
357 |
} |
321 |
else { |
358 |
else { |
322 |
# regular wordlist stuff |
359 |
# Regular match query for the specific field. |
323 |
# push @query_parts, { match => {$wh => { query => $val, operator => 'and' }} }; |
360 |
push @query_parts, { |
324 |
my @values = split(' ',$val); |
361 |
match => { |
325 |
foreach my $v (@values) { |
362 |
$wh => { |
326 |
push @query_parts, { wildcard => { "$wh.phrase" => "*" . lc $v . "*" } }; |
363 |
query => $val, |
327 |
} |
364 |
operator => 'and' |
|
|
365 |
} |
366 |
} |
367 |
}; |
328 |
} |
368 |
} |
329 |
} |
369 |
} |
330 |
|
370 |
|
331 |
# Merge the query parts appropriately |
371 |
# Merge the query parts appropriately |
332 |
# 'should' behaves like 'or' |
372 |
# 'should' behaves like 'or' |
333 |
# 'must' behaves like 'and' |
373 |
# 'must' behaves like 'and' |
334 |
# Zebra results seem to match must so using that here |
374 |
# Zebra behaviour seem to match must so using that here |
335 |
my $query = { query=> |
375 |
my $elastic_query = {}; |
336 |
{ bool => |
376 |
$elastic_query->{bool}->{must} = \@query_parts; |
337 |
{ must => \@query_parts } |
377 |
|
338 |
} |
378 |
# Filter by authtypecode if set |
339 |
}; |
379 |
if ($search->{authtypecode}) { |
340 |
|
380 |
$elastic_query->{bool}->{filter} = { |
341 |
# We need to add '.phrase' to all the sort headings otherwise it'll sort |
381 |
term => { |
342 |
# based on the tokenised form. |
382 |
"authtype.raw" => $search->{authtypecode} |
343 |
my %s; |
383 |
} |
344 |
if ( exists $search->{sort} ) { |
384 |
}; |
345 |
foreach my $k ( keys %{ $search->{sort} } ) { |
|
|
346 |
my $f = $self->_sort_field($k); |
347 |
$s{"$f.phrase"} = $search->{sort}{$k}; |
348 |
} |
349 |
$search->{sort} = \%s; |
350 |
} |
385 |
} |
351 |
|
386 |
|
352 |
# add the sort stuff |
387 |
my $query = { |
353 |
$query->{sort} = [ $search->{sort} ] if exists $search->{sort}; |
388 |
query => $elastic_query |
|
|
389 |
}; |
390 |
|
391 |
# Add the sort stuff |
392 |
$query->{sort} = [ $search->{sort} ] if exists $search->{sort}; |
354 |
|
393 |
|
355 |
return $query; |
394 |
return $query; |
356 |
} |
395 |
} |
357 |
|
396 |
|
358 |
|
|
|
359 |
=head2 build_authorities_query_compat |
397 |
=head2 build_authorities_query_compat |
360 |
|
398 |
|
361 |
my ($query) = |
399 |
my ($query) = |
Lines 449-455
sub build_authorities_query_compat {
Link Here
|
449 |
my %sort; |
487 |
my %sort; |
450 |
my $sort_field = |
488 |
my $sort_field = |
451 |
( $orderby =~ /^Heading/ ) ? 'Heading__sort' |
489 |
( $orderby =~ /^Heading/ ) ? 'Heading__sort' |
452 |
: ( $orderby =~ /^Auth/ ) ? 'Local-Number' |
490 |
: ( $orderby =~ /^Auth/ ) ? 'Local-Number__sort' |
453 |
: undef; |
491 |
: undef; |
454 |
if ($sort_field) { |
492 |
if ($sort_field) { |
455 |
my $sort_order = ( $orderby =~ /Asc$/ ) ? 'asc' : 'desc'; |
493 |
my $sort_order = ( $orderby =~ /Asc$/ ) ? 'asc' : 'desc'; |
Lines 516-522
types.
Link Here
|
516 |
=cut |
554 |
=cut |
517 |
|
555 |
|
518 |
our %index_field_convert = ( |
556 |
our %index_field_convert = ( |
519 |
'kw' => '_all', |
557 |
'kw' => '', |
520 |
'ti' => 'title', |
558 |
'ti' => 'title', |
521 |
'au' => 'author', |
559 |
'au' => 'author', |
522 |
'su' => 'subject', |
560 |
'su' => 'subject', |
Lines 542-548
sub _convert_index_fields {
Link Here
|
542 |
# If a field starts with mc- we save it as it's used (and removed) later |
580 |
# If a field starts with mc- we save it as it's used (and removed) later |
543 |
# when joining things, to indicate we make it an 'OR' join. |
581 |
# when joining things, to indicate we make it an 'OR' join. |
544 |
# (Sorry, this got a bit ugly after special cases were found.) |
582 |
# (Sorry, this got a bit ugly after special cases were found.) |
545 |
grep { $_->{field} } map { |
583 |
map { |
546 |
my ( $f, $t ) = split /,/; |
584 |
my ( $f, $t ) = split /,/; |
547 |
my $mc = ''; |
585 |
my $mc = ''; |
548 |
if ($f =~ /^mc-/) { |
586 |
if ($f =~ /^mc-/) { |
Lines 554-560
sub _convert_index_fields {
Link Here
|
554 |
type => $index_type_convert{ $t // '__default' } |
592 |
type => $index_type_convert{ $t // '__default' } |
555 |
}; |
593 |
}; |
556 |
$r->{field} = ($mc . $r->{field}) if $mc && $r->{field}; |
594 |
$r->{field} = ($mc . $r->{field}) if $mc && $r->{field}; |
557 |
$r; |
595 |
$r->{field} ? $r : undef; |
558 |
} @indexes; |
596 |
} @indexes; |
559 |
} |
597 |
} |
560 |
|
598 |
|
Lines 583-590
sub _convert_index_strings {
Link Here
|
583 |
push @res, $s; |
621 |
push @res, $s; |
584 |
next; |
622 |
next; |
585 |
} |
623 |
} |
586 |
push @res, $conv->{field} . ":" |
624 |
push @res, ($conv->{field} ? $conv->{field} . ':' : '') |
587 |
. $self->_modify_string_by_type( %$conv, operand => $term ); |
625 |
. $self->_modify_string_by_type( %$conv, operand => $term ); |
588 |
} |
626 |
} |
589 |
return @res; |
627 |
return @res; |
590 |
} |
628 |
} |
Lines 816-819
sub _truncate_terms {
Link Here
|
816 |
return join ' ', @terms; |
854 |
return join ' ', @terms; |
817 |
} |
855 |
} |
818 |
|
856 |
|
|
|
857 |
=head2 _search_fields |
858 |
my $weighted_fields = $self->_search_fields({ |
859 |
is_opac => 0, |
860 |
weighted_fields => 1, |
861 |
subfield => 'raw' |
862 |
}); |
863 |
|
864 |
Generate a list of searchable fields to be used for Elasticsearch queries |
865 |
applied to multiple fields. |
866 |
|
867 |
Returns an arrayref of field names for either OPAC or Staff client, with |
868 |
possible weights and subfield appended to each field name depending on the |
869 |
options provided. |
870 |
|
871 |
=over 4 |
872 |
|
873 |
=item C<$params> |
874 |
|
875 |
Hashref with options. The parameter C<is_opac> indicates whether the searchable |
876 |
fields for OPAC or Staff client should be retrieved. If C<weighted_fields> is set |
877 |
fields weights will be applied on returned fields. C<subfield> can be used to |
878 |
provide a subfield that will be appended to fields as "C<field_name>.C<subfield>". |
879 |
|
880 |
=back |
881 |
|
882 |
=cut |
883 |
|
884 |
sub _search_fields { |
885 |
my ($self, $params) = @_; |
886 |
$params //= { |
887 |
is_opac => 0, |
888 |
weighted_fields => 0, |
889 |
# This is a hack for authorities build_authorities_query |
890 |
# can hopefully be removed in the future |
891 |
subfield => undef, |
892 |
}; |
893 |
|
894 |
my $cache = Koha::Caches->get_instance(); |
895 |
my $cache_key = 'elasticsearch_search_fields' . ($params->{is_opac} ? '_opac' : '_staff_client'); |
896 |
my $search_fields = $cache->get_from_cache($cache_key, { unsafe => 1 }); |
897 |
|
898 |
if (!$search_fields) { |
899 |
# The reason we don't use Koha::SearchFields->search here is we don't |
900 |
# want or need resultset wrapped as Koha::SearchField object. |
901 |
# It does not make any sense in this context and would cause |
902 |
# unnecessary overhead sice we are only querying for data |
903 |
# Also would not work, or produce strange results, with the "columns" |
904 |
# option. |
905 |
my $schema = Koha::Database->schema; |
906 |
my $result = $schema->resultset('SearchField')->search( |
907 |
{ |
908 |
$params->{is_opac} ? ( |
909 |
'opac' => 1, |
910 |
) : ( |
911 |
'staff_client' => 1 |
912 |
), |
913 |
'search_marc_map.index_name' => $self->index, |
914 |
'search_marc_map.marc_type' => C4::Context->preference('marcflavour'), |
915 |
'search_marc_to_fields.search' => 1, |
916 |
}, |
917 |
{ |
918 |
columns => [qw/name weight/], |
919 |
# collapse => 1, |
920 |
join => {search_marc_to_fields => 'search_marc_map'}, |
921 |
} |
922 |
); |
923 |
my @search_fields; |
924 |
while (my $search_field = $result->next) { |
925 |
push @search_fields, [ |
926 |
$search_field->name, |
927 |
$search_field->weight ? $search_field->weight : () |
928 |
]; |
929 |
} |
930 |
$search_fields = \@search_fields; |
931 |
$cache->set_in_cache($cache_key, $search_fields); |
932 |
} |
933 |
if ($params->{subfield}) { |
934 |
my $subfield = $params->{subfield}; |
935 |
$search_fields = [ |
936 |
map { |
937 |
# Copy values to avoid mutating cached |
938 |
# data (since unsafe is used) |
939 |
my ($field, $weight) = @{$_}; |
940 |
["${field}.${subfield}", $weight]; |
941 |
} @{$search_fields} |
942 |
]; |
943 |
} |
944 |
|
945 |
if ($params->{weighted_fields}) { |
946 |
return [map { join('^', @{$_}) } @{$search_fields}]; |
947 |
} |
948 |
else { |
949 |
# Exclude weight from field |
950 |
return [map { $_->[0] } @{$search_fields}]; |
951 |
} |
952 |
} |
953 |
|
819 |
1; |
954 |
1; |