From bd349b44b9ae769287278893c9a407779b4b6028 Mon Sep 17 00:00:00 2001 From: David Gustafsson Date: Fri, 13 Apr 2018 15:46:55 +0200 Subject: [PATCH] Bug 20589: Add field boosting and use query_string fields parameter Generate a list of fields for the query_string query fields parameter, with possible boosts, instead of using "_all"-field. Also add "search" flag in search_marc_to_field table so that certain mappings can be excluded from searches. Add option to include/exclude fields in query_string "fields" parameter depending on searching in OPAC or staff client. Refactor code to remove all other dependencies on "_all"-field. How to test: 1) Reindex authorities and biblios. 2) Search biblios and try to verify that this works as expected. 3) Search authorities and try to verify that this works as expected. 4) Go to "Search engine configuration" 5) Change some "Boost", "Staff client", and "OPAC" settings and save. 6) Verify that those settings where saved accordingly. 7) Click the "Biblios" or "Authorities" tab and change one or more "Searchable" settings 8) Verfiy that those settings where saved accordingly. 9) Try to verify that these settings has taken effect by peforming some biblios and/or authorities searches. Sponsorded-by: Gothenburg Univesity Library --- Koha/Schema/Result/SearchField.pm | 6 +- Koha/Schema/Result/SearchMarcToField.pm | 2 + Koha/SearchEngine/Elasticsearch.pm | 37 ++-- Koha/SearchEngine/Elasticsearch/QueryBuilder.pm | 239 ++++++++++++++++----- Koha/SearchField.pm | 1 - Koha/SearchFields.pm | 15 -- admin/searchengine/elasticsearch/field_config.yaml | 34 ++- admin/searchengine/elasticsearch/index_config.yaml | 12 +- admin/searchengine/elasticsearch/mappings.pl | 72 +++++-- catalogue/search.pl | 7 +- .../mysql/atomicupdate/bug_20589_add_columns.perl | 14 ++ .../admin/searchengine/elasticsearch/mappings.tt | 66 +++++- opac/opac-search.pl | 26 ++- 13 files changed, 388 insertions(+), 143 deletions(-) create mode 100644 installer/data/mysql/atomicupdate/bug_20589_add_columns.perl diff --git a/Koha/Schema/Result/SearchField.pm b/Koha/Schema/Result/SearchField.pm index 0f7d943164..3839927ac9 100644 --- a/Koha/Schema/Result/SearchField.pm +++ b/Koha/Schema/Result/SearchField.pm @@ -77,7 +77,11 @@ __PACKAGE__->add_columns( is_nullable => 0, }, "weight", - { data_type => "decimal", is_nullable => 1, size => [5, 2] }, + { data_type => "decimal", size => [ 5, 2 ], is_nullable => 1 }, + "staff_client", + { data_type => "tinyint", size => 1, is_nullable => 0, default => 1 }, + "opac", + { data_type => "tinyint", size => 1, is_nullable => 0, default => 1 }, ); =head1 PRIMARY KEY diff --git a/Koha/Schema/Result/SearchMarcToField.pm b/Koha/Schema/Result/SearchMarcToField.pm index bfd27eab48..5f3ca053e9 100644 --- a/Koha/Schema/Result/SearchMarcToField.pm +++ b/Koha/Schema/Result/SearchMarcToField.pm @@ -71,6 +71,8 @@ __PACKAGE__->add_columns( { data_type => "tinyint", default_value => 0, is_nullable => 1 }, "sort", { data_type => "tinyint", is_nullable => 1 }, + "search", + { data_type => "tinyint", size => 1, is_nullable => 0, default => 1 }, ); =head1 PRIMARY KEY diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm index d51c11e351..74546ea9f5 100644 --- a/Koha/SearchEngine/Elasticsearch.pm +++ b/Koha/SearchEngine/Elasticsearch.pm @@ -199,7 +199,7 @@ sub get_elasticsearch_mappings { my $marcflavour = lc C4::Context->preference('marcflavour'); $self->_foreach_mapping( sub { - my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_; + my ( $name, $type, $facet, $suggestible, $sort, $search, $marc_type ) = @_; return if $marc_type ne $marcflavour; # TODO if this gets any sort of complexity to it, it should # be broken out into its own function. @@ -215,7 +215,9 @@ sub get_elasticsearch_mappings { $es_type = 'stdno'; } - $mappings->{data}{properties}{$name} = _get_elasticsearch_mapping('search', $es_type); + if ($search) { + $mappings->{data}{properties}{$name} = _get_elasticsearch_mapping('search', $es_type); + } if ($facet) { $mappings->{data}{properties}{ $name . '__facet' } = _get_elasticsearch_mapping('facet', $es_type); @@ -284,11 +286,13 @@ sub reset_elasticsearch_mappings { while ( my ( $field_name, $data ) = each %$fields ) { my $field_type = $data->{type}; my $field_label = $data->{label}; + my $staff_client = exists $data->{staff_client} ? $data->{staff_client} : 1; + my $opac = exists $data->{opac} ? $data->{opac} : 1; my $mappings = $data->{mappings}; - my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type }, { key => 'name' }); + my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type, staff_client => $staff_client, opac => $opac }, { key => 'name' }); for my $mapping ( @$mappings ) { my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} }); - $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet} || 0, suggestible => $mapping->{suggestible} || 0, sort => $mapping->{sort} } ); + $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet} || 0, suggestible => $mapping->{suggestible} || 0, sort => $mapping->{sort}, search => $mapping->{search} || 1 } ); } } } @@ -541,9 +545,9 @@ sub marc_records_to_documents { return \@record_documents; } -=head2 _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range) +=head2 _field_mappings($facet, $suggestible, $sort, $search, $target_name, $target_type, $range) - my @mappings = _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range) + my @mappings = _field_mappings($facet, $suggestible, $sort, $search, $target_name, $target_type, $range) Get mappings, an internal data structure later used by L<_process_mappings($mappings, $data, $record_document, $altscript)> to process MARC target @@ -572,6 +576,10 @@ Boolean indicating whether to create a suggestion field for this mapping. Boolean indicating whether to create a sort field for this mapping. +=item C<$search> + +Boolean indicating whether to create a search field for this mapping. + =item C<$target_name> Elasticsearch document target field name. @@ -597,7 +605,7 @@ be extracted. =cut sub _field_mappings { - my ($_self, $facet, $suggestible, $sort, $target_name, $target_type, $range) = @_; + my ($_self, $facet, $suggestible, $sort, $search, $target_name, $target_type, $range) = @_; my %mapping_defaults = (); my @mappings; @@ -625,8 +633,10 @@ sub _field_mappings { }; } - my $mapping = [$target_name, $default_options]; - push @mappings, $mapping; + if ($search) { + my $mapping = [$target_name, $default_options]; + push @mappings, $mapping; + } my @suffixes = (); push @suffixes, 'facet' if $facet; @@ -683,7 +693,7 @@ sub _get_marc_mapping_rules { }; $self->_foreach_mapping(sub { - my ($name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field) = @_; + my ($name, $type, $facet, $suggestible, $sort, $search, $marc_type, $marc_field) = @_; return if $marc_type ne $marcflavour; if ($type eq 'sum') { @@ -746,7 +756,7 @@ sub _get_marc_mapping_rules { } my $range = defined $3 ? $3 : undef; - my @mappings = $self->_field_mappings($facet, $suggestible, $sort, $name, $type, $range); + my @mappings = $self->_field_mappings($facet, $suggestible, $sort, $search, $name, $type, $range); if ($field_tag < 10) { $rules->{control_fields}->{$field_tag} //= []; @@ -766,7 +776,7 @@ sub _get_marc_mapping_rules { } elsif ($marc_field =~ $leader_regexp) { my $range = defined $1 ? $1 : undef; - my @mappings = $self->_field_mappings($facet, $suggestible, $sort, $name, $type, $range); + my @mappings = $self->_field_mappings($facet, $suggestible, $sort, $search, $name, $type, $range); push @{$rules->{leader}}, @mappings; } else { @@ -845,6 +855,7 @@ sub _foreach_mapping { 'search_marc_to_fields.facet', 'search_marc_to_fields.suggestible', 'search_marc_to_fields.sort', + 'search_marc_to_fields.search', 'search_marc_map.marc_type', 'search_marc_map.marc_field', ], @@ -852,6 +863,7 @@ sub _foreach_mapping { 'facet', 'suggestible', 'sort', + 'search', 'marc_type', 'marc_field', ], @@ -865,6 +877,7 @@ sub _foreach_mapping { $search_field->get_column('facet'), $search_field->get_column('suggestible'), $search_field->get_column('sort'), + $search_field->get_column('search'), $search_field->get_column('marc_type'), $search_field->get_column('marc_field'), ); diff --git a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm index 543728e841..0bed95cd6f 100644 --- a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm +++ b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm @@ -48,6 +48,7 @@ use URI::Escape; use C4::Context; use Koha::Exceptions; +use Koha::Caches; =head2 build_query @@ -90,9 +91,8 @@ sub build_query { query => $query, fuzziness => $fuzzy_enabled ? 'auto' : '0', default_operator => 'AND', - default_field => '_all', + fields => $self->_search_fields({ is_opac => $options{is_opac}, weighted_fields => $options{weighted_fields} }), lenient => JSON::true, - fields => $options{fields} || [], } }; @@ -227,17 +227,13 @@ sub build_query_compat { join( ' ', $self->_create_query_string(@search_params) ) || (), $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () ); - my @fields = '_all'; - if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) { - push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields; - } - # If there's no query on the left, let's remove the junk left behind $query_str =~ s/^ AND //; my %options; - $options{fields} = \@fields; $options{sort} = \@sort_params; $options{expanded_facet} = $params->{expanded_facet}; + $options{is_opac} = $params->{is_opac}; + $options{weighted_fields} = $params->{weighted_fields}; my $query = $self->build_query( $query_str, %options ); #die Dumper($query); @@ -298,56 +294,104 @@ sub build_authorities_query { foreach my $s ( @{ $search->{searches} } ) { my ( $wh, $op, $val ) = @{$s}{qw(where operator value)}; - $wh = '_all' if $wh eq ''; - if ( $op eq 'is' || $op eq '=' || $op eq 'exact' ) { - - # look for something that matches a term completely - # note, '=' is about numerical vals. May need special handling. - # Also, we lowercase our search because the ES - # index lowercases its values, and term searches don't get the - # search analyzer applied to them. - push @query_parts, { match_phrase => {"$wh.phrase" => lc $val} }; + if (!$wh) { + if ( $op eq 'is' || $op eq '=' || $op eq 'exact') { + # Match the whole field for all searchable fields, case insensitive, + # UTF normalized. + # Given that field data is "The quick brown fox" + # "The quick brown fox" and "the quick brown fox" will match + # but not "quick brown fox". + push @query_parts, { + multi_match => { + query => $val, + fields => $self->_search_fields({ subfield => 'ci_raw' }), + } + }; + } + elsif ( $op eq 'start') { + # Match the prefix within a field for all searchable fields. + # Given that field data is "The quick brown fox" + # "The quick bro" will match, but not "quick bro" + + # Does not seems to be a multi prefix query + # so we need to create one + my @prefix_queries; + foreach my $field (@{$self->_search_fields()}) { + push @prefix_queries, { + prefix => { "$field.ci_raw" => $val } + }; + } + push @query_parts, { + 'bool' => { + 'should' => \@prefix_queries, + 'minimum_should_match' => 1 + } + }; + } + else { + # Query all searchable fields. + # Given that field data is "The quick brown fox" + # a search containing any of the words will match, regardless + # of order. + # + # Since default operator is "AND" each of the words must match + # at least one field + push @query_parts, { + query_string => { + query => $val, + fields => $self->_search_fields(), + default_operator => 'AND', + } + }; + } + } + elsif ( $op eq 'is' || $op eq '=' || $op eq 'exact') { + # Match the whole field, case insensitive, UTF normalized. + push @query_parts, { term => { "$wh.ci_raw" => $val } }; } elsif ( $op eq 'start' ) { - # startswith search, uses lowercase untokenized version of heading - push @query_parts, { match_phrase_prefix => {"$wh.phrase" => lc $val} }; + # Match prefix of the field. + push @query_parts, { prefix => {"$wh.ci_raw" => $val} }; } else { - # regular wordlist stuff -# push @query_parts, { match => {$wh => { query => $val, operator => 'and' }} }; - my @values = split(' ',$val); - foreach my $v (@values) { - push @query_parts, { wildcard => { "$wh.phrase" => "*" . lc $v . "*" } }; - } + # Regular match query for the specific field. + push @query_parts, { + match => { + $wh => { + query => $val, + operator => 'and' + } + } + }; } } # Merge the query parts appropriately # 'should' behaves like 'or' # 'must' behaves like 'and' - # Zebra results seem to match must so using that here - my $query = { query => - { bool => - { must => \@query_parts } - } - }; - - my %s; - if ( exists $search->{sort} ) { - foreach my $k ( keys %{ $search->{sort} } ) { - my $f = $self->_sort_field($k); - $s{$f} = $search->{sort}{$k}; - } - $search->{sort} = \%s; + # Zebra behaviour seem to match must so using that here + my $elastic_query = {}; + $elastic_query->{bool}->{must} = \@query_parts; + + # Filter by authtypecode if set + if ($search->{authtypecode}) { + $elastic_query->{bool}->{filter} = { + term => { + "authtype.raw" => $search->{authtypecode} + } + }; } - # add the sort stuff - $query->{sort} = [ $search->{sort} ] if exists $search->{sort}; + my $query = { + query => $elastic_query + }; + + # Add the sort stuff + $query->{sort} = [ $search->{sort} ] if exists $search->{sort}; return $query; } - =head2 build_authorities_query_compat my ($query) = @@ -441,8 +485,8 @@ sub build_authorities_query_compat { my %sort; my $sort_field = - ( $orderby =~ /^Heading/ ) ? 'Heading' - : ( $orderby =~ /^Auth/ ) ? 'Local-number' + ( $orderby =~ /^Heading/ ) ? 'Heading__sort' + : ( $orderby =~ /^Auth/ ) ? 'Local-Number__sort' : undef; if ($sort_field) { my $sort_order = ( $orderby =~ /Asc$/ ) ? 'asc' : 'desc'; @@ -509,7 +553,7 @@ types. =cut our %index_field_convert = ( - 'kw' => '_all', + 'kw' => '', 'ti' => 'title', 'au' => 'author', 'su' => 'subject', @@ -537,7 +581,7 @@ sub _convert_index_fields { # If a field starts with mc- we save it as it's used (and removed) later # when joining things, to indicate we make it an 'OR' join. # (Sorry, this got a bit ugly after special cases were found.) - grep { $_->{field} } map { + map { my ( $f, $t ) = split /,/; my $mc = ''; if ($f =~ /^mc-/) { @@ -549,7 +593,7 @@ sub _convert_index_fields { type => $index_type_convert{ $t // '__default' } }; $r->{field} = ($mc . $r->{field}) if $mc && $r->{field}; - $r; + $r->{field} ? $r : undef; } @indexes; } @@ -578,8 +622,8 @@ sub _convert_index_strings { push @res, $s; next; } - push @res, $conv->{field} . ":" - . $self->_modify_string_by_type( %$conv, operand => $term ); + push @res, ($conv->{field} ? $conv->{field} . ':' : '') + . $self->_modify_string_by_type( %$conv, operand => $term ); } return @res; } @@ -822,4 +866,101 @@ sub _truncate_terms { return join ' ', @terms; } +=head2 _search_fields + my $weighted_fields = $self->_search_fields({ + is_opac => 0, + weighted_fields => 1, + subfield => 'raw' + }); + +Generate a list of searchable fields to be used for Elasticsearch queries +applied to multiple fields. + +Returns an arrayref of field names for either OPAC or Staff client, with +possible weights and subfield appended to each field name depending on the +options provided. + +=over 4 + +=item C<$params> + +Hashref with options. The parameter C indicates whether the searchable +fields for OPAC or Staff client should be retrieved. If C is set +fields weights will be applied on returned fields. C can be used to +provide a subfield that will be appended to fields as "C.C". + +=back + +=cut + +sub _search_fields { + my ($self, $params) = @_; + $params //= { + is_opac => 0, + weighted_fields => 0, + # This is a hack for authorities build_authorities_query + # can hopefully be removed in the future + subfield => undef, + }; + + my $cache = Koha::Caches->get_instance(); + my $cache_key = 'elasticsearch_search_fields' . ($params->{is_opac} ? '_opac' : '_staff_client'); + my $search_fields = $cache->get_from_cache($cache_key, { unsafe => 1 }); + + if (!$search_fields) { + # The reason we don't use Koha::SearchFields->search here is we don't + # want or need resultset wrapped as Koha::SearchField object. + # It does not make any sense in this context and would cause + # unnecessary overhead sice we are only querying for data + # Also would not work, or produce strange results, with the "columns" + # option. + my $schema = Koha::Database->schema; + my $result = $schema->resultset('SearchField')->search( + { + $params->{is_opac} ? ( + 'opac' => 1, + ) : ( + 'staff_client' => 1 + ), + 'search_marc_map.index_name' => $self->index, + 'search_marc_map.marc_type' => C4::Context->preference('marcflavour'), + 'search_marc_to_fields.search' => 1, + }, + { + columns => [qw/name weight/], + # collapse => 1, + join => {search_marc_to_fields => 'search_marc_map'}, + } + ); + my @search_fields; + while (my $search_field = $result->next) { + push @search_fields, [ + $search_field->name, + $search_field->weight ? $search_field->weight : () + ]; + } + $search_fields = \@search_fields; + $cache->set_in_cache($cache_key, $search_fields); + } + if ($params->{subfield}) { + my $subfield = $params->{subfield}; + $search_fields = [ + map { + # Copy values to avoid mutating cached + # data (since unsafe is used) + my ($field, $weight) = @{$_}; + ["${field}.${subfield}", $weight]; + } @{$search_fields} + ]; + } + + if ($params->{weighted_fields}) { + return [map { join('^', @{$_}) } @{$search_fields}]; + } + else { + # Exclude weight from field + return [map { $_->[0] } @{$search_fields}]; + } +} + 1; diff --git a/Koha/SearchField.pm b/Koha/SearchField.pm index efabb8db7d..bd19717b71 100644 --- a/Koha/SearchField.pm +++ b/Koha/SearchField.pm @@ -20,7 +20,6 @@ use Modern::Perl; use Carp; use Koha::Database; -use Koha::SearchMarcMaps; use base qw(Koha::Object); diff --git a/Koha/SearchFields.pm b/Koha/SearchFields.pm index b2c985f526..7af3c0b162 100644 --- a/Koha/SearchFields.pm +++ b/Koha/SearchFields.pm @@ -35,21 +35,6 @@ Koha::SearchFields - Koha SearchField Object set class =cut -=head3 weighted_fields - -my (@w_fields, @weight) = Koha::SearchFields->weighted_fields(); - -=cut - -sub weighted_fields { - my ($self) = @_; - - return $self->search( - { weight => { '>' => 0, '!=' => undef } }, - { order_by => { -desc => 'weight' } } - ); -} - =head3 type =cut diff --git a/admin/searchengine/elasticsearch/field_config.yaml b/admin/searchengine/elasticsearch/field_config.yaml index 0baddf3a02..a014ae2ea0 100644 --- a/admin/searchengine/elasticsearch/field_config.yaml +++ b/admin/searchengine/elasticsearch/field_config.yaml @@ -1,9 +1,6 @@ --- # General field configuration general: - _all: - type: string - analyzer: analyser_standard properties: marc_data: store: true @@ -26,31 +23,30 @@ search: null_value: 0 stdno: type: text - analyzer: analyser_stdno - search_analyzer: analyser_stdno + analyzer: analyzer_stdno + search_analyzer: analyzer_stdno fields: phrase: type: text - analyzer: analyser_stdno - search_analyzer: analyser_stdno + analyzer: analyzer_phrase + search_analyzer: analyzer_phrase raw: type: keyword - copy_to: _all default: type: text - analyzer: analyser_standard - search_analyzer: analyser_standard + analyzer: analyzer_standard + search_analyzer: analyzer_standard fields: phrase: type: text - analyzer: analyser_phrase - search_analyzer: analyser_phrase + analyzer: analyzer_phrase + search_analyzer: analyzer_phrase raw: type: keyword - lc_raw: + normalizer: nfkc_cf_normalizer + ci_raw: type: keyword - normalizer: my_normalizer - copy_to: _all + normalizer: icu_folding_normalizer # Facets facet: default: @@ -64,9 +60,5 @@ suggestible: # Sort sort: default: - type: text - analyzer: analyser_phrase - search_analyzer: analyser_phrase - fields: - phrase: - type: keyword + type: icu_collation_keyword + index: false diff --git a/admin/searchengine/elasticsearch/index_config.yaml b/admin/searchengine/elasticsearch/index_config.yaml index dce27bc4fa..3d80732785 100644 --- a/admin/searchengine/elasticsearch/index_config.yaml +++ b/admin/searchengine/elasticsearch/index_config.yaml @@ -3,29 +3,29 @@ index: analysis: analyzer: - # Phrase analyzer is used for phrases (phrase match, sorting) - analyser_phrase: + # Phrase analyzer is used for phrases (exact phrase match) + analyzer_phrase: tokenizer: keyword filter: - icu_folding char_filter: - punctuation - analyser_standard: + analyzer_standard: tokenizer: icu_tokenizer filter: - icu_folding - analyser_stdno: + analyzer_stdno: tokenizer: whitespace filter: - icu_folding char_filter: - punctuation normalizer: - normalizer_keyword: + icu_folding_normalizer: type: custom filter: - icu_folding - my_normalizer: + nfkc_cf_normalizer: type: custom char_filter: icu_normalizer char_filter: diff --git a/admin/searchengine/elasticsearch/mappings.pl b/admin/searchengine/elasticsearch/mappings.pl index 2c64db022e..e7aa6479fa 100755 --- a/admin/searchengine/elasticsearch/mappings.pl +++ b/admin/searchengine/elasticsearch/mappings.pl @@ -26,6 +26,7 @@ use Koha::SearchEngine::Elasticsearch; use Koha::SearchEngine::Elasticsearch::Indexer; use Koha::SearchMarcMaps; use Koha::SearchFields; +use Koha::Caches; use Try::Tiny; @@ -67,6 +68,12 @@ my $update_mappings = sub { } }; +my $cache = Koha::Caches->get_instance(); +my $clear_cache = sub { + $cache->clear_from_cache('elasticsearch_search_fields_staff_client'); + $cache->clear_from_cache('elasticsearch_search_fields_opac'); +}; + if ( $op eq 'edit' ) { $schema->storage->txn_begin; @@ -75,12 +82,15 @@ if ( $op eq 'edit' ) { my @field_label = $input->param('search_field_label'); my @field_type = $input->param('search_field_type'); my @field_weight = $input->param('search_field_weight'); + my @field_staff_client = $input->param('search_field_staff_client'); + my @field_opac = $input->param('search_field_opac'); my @index_name = $input->param('mapping_index_name'); - my @search_field_name = $input->param('mapping_search_field_name'); + my @search_field_name = $input->param('mapping_search_field_name'); my @mapping_sort = $input->param('mapping_sort'); my @mapping_facet = $input->param('mapping_facet'); my @mapping_suggestible = $input->param('mapping_suggestible'); + my @mapping_search = $input->param('mapping_search'); my @mapping_marc_field = $input->param('mapping_marc_field'); eval { @@ -90,6 +100,8 @@ if ( $op eq 'edit' ) { my $field_label = $field_label[$i]; my $field_type = $field_type[$i]; my $field_weight = $field_weight[$i]; + my $field_staff_client = $field_staff_client[$i]; + my $field_opac = $field_opac[$i]; my $search_field = Koha::SearchFields->find( { name => $field_name }, { key => 'name' } ); $search_field->label($field_label); @@ -104,6 +116,8 @@ if ( $op eq 'edit' ) { else { $search_field->weight($field_weight); } + $search_field->staff_client($field_staff_client ? 1 : 0); + $search_field->opac($field_opac ? 1 : 0); $search_field->store; } @@ -112,18 +126,28 @@ if ( $op eq 'edit' ) { for my $i ( 0 .. scalar(@index_name) - 1 ) { my $index_name = $index_name[$i]; - my $search_field_name = $search_field_name[$i]; + my $search_field_name = $search_field_name[$i]; my $mapping_marc_field = $mapping_marc_field[$i]; my $mapping_facet = $mapping_facet[$i]; my $mapping_suggestible = $mapping_suggestible[$i]; my $mapping_sort = $mapping_sort[$i]; $mapping_sort = undef if $mapping_sort eq 'undef'; + my $mapping_sort = $mapping_sort[$i] eq 'undef' ? undef : $mapping_sort[$i]; + my $mapping_search = $mapping_search[$i]; my $search_field = Koha::SearchFields->find({ name => $search_field_name }, { key => 'name' }); # TODO Check mapping format - my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $marc_type, marc_field => $mapping_marc_field }); - $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping_facet, suggestible => $mapping_suggestible, sort => $mapping_sort } ); - + my $marc_field = Koha::SearchMarcMaps->find_or_create({ + index_name => $index_name, + marc_type => $marc_type, + marc_field => $mapping_marc_field + }); + $search_field->add_to_search_marc_maps($marc_field, { + facet => $mapping_facet, + suggestible => $mapping_suggestible, + sort => $mapping_sort, + search => $mapping_search + }); } }; if ($@) { @@ -132,6 +156,7 @@ if ( $op eq 'edit' ) { } else { push @messages, { type => 'message', code => 'success_on_update' }; $schema->storage->txn_commit; + $clear_cache->(); $update_mappings->(); } } @@ -139,13 +164,13 @@ elsif( $op eq 'reset_confirmed' ) { Koha::SearchMarcMaps->delete; Koha::SearchFields->delete; Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings; + $clear_cache->(); push @messages, { type => 'message', code => 'success_on_reset' }; } elsif( $op eq 'reset_confirm' ) { $template->param( reset_confirm => 1 ); } - my @indexes; for my $index_name (@index_names) { @@ -171,25 +196,42 @@ for my $index_name (@index_names) { for my $index_name (@index_names) { my $search_fields = Koha::SearchFields->search( - { 'search_marc_map.index_name' => $index_name, 'search_marc_map.marc_type' => $marc_type, }, - { join => { search_marc_to_fields => 'search_marc_map' }, - '+select' => [ 'search_marc_to_fields.facet', 'search_marc_to_fields.suggestible', 'search_marc_to_fields.sort', 'search_marc_map.marc_field' ], - '+as' => [ 'facet', 'suggestible', 'sort', 'marc_field' ], + { + 'search_marc_map.index_name' => $index_name, + 'search_marc_map.marc_type' => $marc_type, + }, + { + join => { search_marc_to_fields => 'search_marc_map' }, + '+select' => [ + 'search_marc_to_fields.facet', + 'search_marc_to_fields.suggestible', + 'search_marc_to_fields.sort', + 'search_marc_to_fields.search', + 'search_marc_map.marc_field' + ], + '+as' => [ + 'facet', + 'suggestible', + 'sort', + 'search', + 'marc_field' + ], order_by => { -asc => [qw/name marc_field/] } - } - ); + } + ); my @mappings; while ( my $s = $search_fields->next ) { - push @mappings, - { search_field_name => $s->name, + push @mappings, { + search_field_name => $s->name, search_field_label => $s->label, search_field_type => $s->type, marc_field => $s->get_column('marc_field'), sort => $s->get_column('sort') // 'undef', # To avoid warnings "Use of uninitialized value in lc" suggestible => $s->get_column('suggestible'), facet => $s->get_column('facet'), - }; + search => $s->get_column('search') + }; } push @indexes, { index_name => $index_name, mappings => \@mappings }; diff --git a/catalogue/search.pl b/catalogue/search.pl index af70dac698..453d5d05be 100755 --- a/catalogue/search.pl +++ b/catalogue/search.pl @@ -456,11 +456,6 @@ my $expanded_facet = $params->{'expand'}; # Define some global variables my ( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type); -my $build_params; -unless ( $cgi->param('advsearch') ) { - $build_params->{weighted_fields} = 1; -} - my $builder = Koha::SearchEngine::QueryBuilder->new( { index => $Koha::SearchEngine::BIBLIOS_INDEX } ); my $searcher = Koha::SearchEngine::Search->new( @@ -473,7 +468,7 @@ my $searcher = Koha::SearchEngine::Search->new( $query_type ) = $builder->build_query_compat( \@operators, \@operands, \@indexes, \@limits, - \@sort_by, $scan, $lang, $build_params ); + \@sort_by, $scan, $lang, { weighted_fields => !$cgi->param('advsearch') }); ## parse the query_cgi string and put it into a form suitable for s my @query_inputs; diff --git a/installer/data/mysql/atomicupdate/bug_20589_add_columns.perl b/installer/data/mysql/atomicupdate/bug_20589_add_columns.perl new file mode 100644 index 0000000000..73ddd1becb --- /dev/null +++ b/installer/data/mysql/atomicupdate/bug_20589_add_columns.perl @@ -0,0 +1,14 @@ +$DBversion = 'XXX'; +if (CheckVersion($DBversion)) { + if (!column_exists('search_marc_to_field', 'search')) { + $dbh->do("ALTER TABLE `search_marc_to_field` ADD COLUMN `search` tinyint(1) NOT NULL DEFAULT 1"); + } + if (!column_exists('search_field', 'staff_client')) { + $dbh->do("ALTER TABLE `search_field` ADD COLUMN `staff_client` tinyint(1) NOT NULL DEFAULT 1"); + } + if (!column_exists('search_field', 'opac')) { + $dbh->do("ALTER TABLE `search_field` ADD COLUMN `opac` tinyint(1) NOT NULL DEFAULT 1"); + } + SetVersion($DBversion) +} +print "Upgrade to $DBversion done (Bug 20589 - Add field boosting and use elastic query fields parameter instead of depricated _all)\n"; diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt index bb2ade3ea6..9f3820c2af 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt @@ -156,8 +156,15 @@ a.add, a.delete { Name Label Type + Searchable Weight + +   + Staff client + OPAC +   + [% FOREACH search_field IN all_search_fields %] @@ -165,7 +172,9 @@ a.add, a.delete { - + + + - [% IF search_field.mapped_biblios %] - - [% ELSE %] - - [% END %] + + + + + + + [% IF search_field.mapped_biblios %] + + [% ELSE %] + + [% END %] [% END %] @@ -227,6 +258,7 @@ a.add, a.delete { Sortable Facetable Suggestible + Searchable Mapping @@ -281,6 +313,17 @@ a.add, a.delete { + + + Delete @@ -326,6 +369,17 @@ a.add, a.delete { [% END %] + + + Add diff --git a/opac/opac-search.pl b/opac/opac-search.pl index e0d3dd6229..1f00066bdf 100755 --- a/opac/opac-search.pl +++ b/opac/opac-search.pl @@ -549,19 +549,23 @@ if (C4::Context->preference('OpacSuppression')) { } } -my $build_params = { - expanded_facet => $expanded_facet, - suppress => $suppress -}; - -unless ( $cgi->param('advsearch') ) { - $build_params->{weighted_fields} = 1; -} - ## I. BUILD THE QUERY ( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type) - = $builder->build_query_compat( \@operators, \@operands, - \@indexes, \@limits, \@sort_by, 0, $lang, $build_params); + = $builder->build_query_compat( + \@operators, + \@operands, + \@indexes, + \@limits, + \@sort_by, + 0, + $lang, + { + expanded_facet => $expanded_facet, + suppress => $suppress, + is_opac => 1, + weighted_fields => !$cgi->param('advsearch') + } +); sub _input_cgi_parse { my @elements; -- 2.11.0