From c7a432fc6b0c3cf2afa7128e9c300301cda16d94 Mon Sep 17 00:00:00 2001 From: David Gustafsson Date: Fri, 13 Apr 2018 15:46:55 +0200 Subject: [PATCH] Bug 20589: Add field boosting and use query_string fields parameter Generate a list of fields for the query_string query fields parameter, with possible boosts, instead of using "_all"-field. Also add "search" flag in search_marc_to_field table so that certain mappings can be excluded from searches. Add option to include/exclude fields in query_string "fields" parameter depending on searching in OPAC or staff client. Refactor code to remove all other dependencies on "_all"-field. How to test: 1) Reindex authorities and biblios. 2) Search biblios and try to verify that this works as expected. 3) Search authorities and try to verify that this works as expected. 4) Go to "Search engine configuration" 5) Change some "Boost", "Staff client", and "OPAC" settings and save. 6) Verify that those settings where saved accordingly. 7) Click the "Biblios" or "Authorities" tab and change one or more "Searchable" settings 8) Verfiy that those settings where saved accordingly. 9) Try to verify that these settings has taken effect by peforming some biblios and/or authorities searches. Sponsorded-by: Gothenburg Univesity Library --- Koha/Schema/Result/SearchField.pm | 6 +- Koha/Schema/Result/SearchMarcToField.pm | 2 + Koha/SearchEngine/Elasticsearch.pm | 23 +- Koha/SearchEngine/Elasticsearch/QueryBuilder.pm | 243 ++++++++++++++++----- Koha/SearchField.pm | 1 - Koha/SearchFields.pm | 15 -- admin/searchengine/elasticsearch/field_config.yaml | 34 ++- admin/searchengine/elasticsearch/index_config.yaml | 12 +- admin/searchengine/elasticsearch/mappings.pl | 83 +++++-- catalogue/search.pl | 7 +- .../mysql/atomicupdate/bug_20589_add_columns.perl | 14 ++ .../admin/searchengine/elasticsearch/mappings.tt | 66 +++++- opac/opac-search.pl | 26 ++- 13 files changed, 381 insertions(+), 151 deletions(-) create mode 100644 installer/data/mysql/atomicupdate/bug_20589_add_columns.perl diff --git a/Koha/Schema/Result/SearchField.pm b/Koha/Schema/Result/SearchField.pm index 0f7d943164..3839927ac9 100644 --- a/Koha/Schema/Result/SearchField.pm +++ b/Koha/Schema/Result/SearchField.pm @@ -77,7 +77,11 @@ __PACKAGE__->add_columns( is_nullable => 0, }, "weight", - { data_type => "decimal", is_nullable => 1, size => [5, 2] }, + { data_type => "decimal", size => [ 5, 2 ], is_nullable => 1 }, + "staff_client", + { data_type => "tinyint", size => 1, is_nullable => 0, default => 1 }, + "opac", + { data_type => "tinyint", size => 1, is_nullable => 0, default => 1 }, ); =head1 PRIMARY KEY diff --git a/Koha/Schema/Result/SearchMarcToField.pm b/Koha/Schema/Result/SearchMarcToField.pm index bfd27eab48..5f3ca053e9 100644 --- a/Koha/Schema/Result/SearchMarcToField.pm +++ b/Koha/Schema/Result/SearchMarcToField.pm @@ -71,6 +71,8 @@ __PACKAGE__->add_columns( { data_type => "tinyint", default_value => 0, is_nullable => 1 }, "sort", { data_type => "tinyint", is_nullable => 1 }, + "search", + { data_type => "tinyint", size => 1, is_nullable => 0, default => 1 }, ); =head1 PRIMARY KEY diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm index 9c114fb060..9599c0d835 100644 --- a/Koha/SearchEngine/Elasticsearch.pm +++ b/Koha/SearchEngine/Elasticsearch.pm @@ -170,7 +170,7 @@ sub get_elasticsearch_mappings { my $marcflavour = lc C4::Context->preference('marcflavour'); $self->_foreach_mapping( sub { - my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_; + my ( $name, $type, $facet, $suggestible, $sort, $search, $marc_type ) = @_; return if $marc_type ne $marcflavour; # TODO if this gets any sort of complexity to it, it should # be broken out into its own function. @@ -185,9 +185,9 @@ sub get_elasticsearch_mappings { } elsif ($type eq 'isbn' || $type eq 'stdno') { $es_type = 'stdno'; } - - $mappings->{data}{properties}{$name} = _get_elasticsearch_mapping('search', $es_type); - + if ($search) { + $mappings->{data}{properties}{$name} = _get_elasticsearch_mapping('search', $es_type); + } if ($facet) { $mappings->{data}{properties}{ $name . '__facet' } = _get_elasticsearch_mapping('facet', $es_type); } @@ -255,11 +255,13 @@ sub reset_elasticsearch_mappings { while ( my ( $field_name, $data ) = each %$fields ) { my $field_type = $data->{type}; my $field_label = $data->{label}; + my $staff_client = exists $data->{staff_client} ? $data->{staff_client} : 1; + my $opac = exists $data->{opac} ? $data->{opac} : 1; my $mappings = $data->{mappings}; - my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type }, { key => 'name' }); + my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type, staff_client => $staff_client, opac => $opac }, { key => 'name' }); for my $mapping ( @$mappings ) { my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} }); - $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet} || 0, suggestible => $mapping->{suggestible} || 0, sort => $mapping->{sort} } ); + $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet} || 0, suggestible => $mapping->{suggestible} || 0, sort => $mapping->{sort}, search => $mapping->{search} || 1 } ); } } } @@ -290,11 +292,13 @@ sub get_fixer_rules { $self->_foreach_mapping( sub { - my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_; + my ( $name, $type, $facet, $suggestible, $sort, $search, $marc_type, $marc_field ) = @_; return if $marc_type ne $marcflavour; my $options =''; - push @rules, "marc_map('$marc_field','${name}.\$append', $options)"; + if ($search) { + push @rules, "marc_map('$marc_field','${name}.\$append', $options)"; + } if ($facet) { push @rules, "marc_map('$marc_field','${name}__facet.\$append', $options)"; } @@ -392,6 +396,7 @@ sub _foreach_mapping { 'search_marc_to_fields.facet', 'search_marc_to_fields.suggestible', 'search_marc_to_fields.sort', + 'search_marc_to_fields.search', 'search_marc_map.marc_type', 'search_marc_map.marc_field', ], @@ -399,6 +404,7 @@ sub _foreach_mapping { 'facet', 'suggestible', 'sort', + 'search', 'marc_type', 'marc_field', ], @@ -412,6 +418,7 @@ sub _foreach_mapping { $search_field->get_column('facet'), $search_field->get_column('suggestible'), $search_field->get_column('sort'), + $search_field->get_column('search'), $search_field->get_column('marc_type'), $search_field->get_column('marc_field'), ); diff --git a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm index 57836e9d8c..242ea4f211 100644 --- a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm +++ b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm @@ -48,6 +48,7 @@ use URI::Escape; use C4::Context; use Koha::Exceptions; +use Koha::Caches; =head2 build_query @@ -90,9 +91,8 @@ sub build_query { query => $query, fuzziness => $fuzzy_enabled ? 'auto' : '0', default_operator => 'AND', - default_field => '_all', + fields => $self->_search_fields({ is_opac => $options{is_opac}, weighted_fields => $options{weighted_fields} }), lenient => JSON::true, - fields => $options{fields} || [], } }; @@ -106,7 +106,7 @@ sub build_query { # TODO account for fields that don't have a 'phrase' type $f = $self->_sort_field($f); - push @{ $res->{sort} }, { "$f.phrase" => { order => $d } }; + push @{ $res->{sort} }, { $f => { order => $d } }; } } @@ -229,17 +229,13 @@ sub build_query_compat { join( ' ', $self->_create_query_string(@search_params) ) || (), $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () ); - my @fields = '_all'; - if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) { - push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields; - } - # If there's no query on the left, let's remove the junk left behind $query_str =~ s/^ AND //; my %options; - $options{fields} = \@fields; $options{sort} = \@sort_params; $options{expanded_facet} = $params->{expanded_facet}; + $options{is_opac} = $params->{is_opac}; + $options{weighted_fields} = $params->{weighted_fields}; my $query = $self->build_query( $query_str, %options ); #die Dumper($query); @@ -300,62 +296,104 @@ sub build_authorities_query { foreach my $s ( @{ $search->{searches} } ) { my ( $wh, $op, $val ) = @{$s}{qw(where operator value)}; - $wh = '_all' if $wh eq ''; - if ( $op eq 'is' || $op eq '=' ) { - - # look for something that matches a term completely - # note, '=' is about numerical vals. May need special handling. - # Also, we lowercase our search because the ES - # index lowercases its values, and term searches don't get the - # search analyzer applied to them. - push @query_parts, { term => {"$wh.phrase" => lc $val} }; + if (!$wh) { + if ( $op eq 'is' || $op eq '=' || $op eq 'exact') { + # Match the whole field for all searchable fields, case insensitive, + # UTF normalized. + # Given that field data is "The quick brown fox" + # "The quick brown fox" and "the quick brown fox" will match + # but not "quick brown fox". + push @query_parts, { + multi_match => { + query => $val, + fields => $self->_search_fields({ subfield => 'ci_raw' }), + } + }; + } + elsif ( $op eq 'start') { + # Match the prefix within a field for all searchable fields. + # Given that field data is "The quick brown fox" + # "The quick bro" will match, but not "quick bro" + + # Does not seems to be a multi prefix query + # so we need to create one + my @prefix_queries; + foreach my $field (@{$self->_search_fields()}) { + push @prefix_queries, { + prefix => { "$field.ci_raw" => $val } + }; + } + push @query_parts, { + 'bool' => { + 'should' => \@prefix_queries, + 'minimum_should_match' => 1 + } + }; + } + else { + # Query all searchable fields. + # Given that field data is "The quick brown fox" + # a search containing any of the words will match, regardless + # of order. + # + # Since default operator is "AND" each of the words must match + # at least one field + push @query_parts, { + query_string => { + query => $val, + fields => $self->_search_fields(), + default_operator => 'AND', + } + }; + } } - elsif ( $op eq 'exact' ) { - # left and right truncation, otherwise an exact phrase - push @query_parts, { match_phrase => {"$wh.phrase" => lc $val} }; + elsif ( $op eq 'is' || $op eq '=' || $op eq 'exact') { + # Match the whole field, case insensitive, UTF normalized. + push @query_parts, { term => { "$wh.ci_raw" => $val } }; } elsif ( $op eq 'start' ) { - # startswith search, uses lowercase untokenized version of heading - push @query_parts, { prefix => {"$wh.lc_raw" => lc $val} }; + # Match prefix of the field. + push @query_parts, { prefix => {"$wh.ci_raw" => $val} }; } else { - # regular wordlist stuff -# push @query_parts, { match => {$wh => { query => $val, operator => 'and' }} }; - my @values = split(' ',$val); - foreach my $v (@values) { - push @query_parts, { wildcard => { "$wh.phrase" => "*" . lc $v . "*" } }; - } + # Regular match query for the specific field. + push @query_parts, { + match => { + $wh => { + query => $val, + operator => 'and' + } + } + }; } } # Merge the query parts appropriately # 'should' behaves like 'or' # 'must' behaves like 'and' - # Zebra results seem to match must so using that here - my $query = { query=> - { bool => - { must => \@query_parts } - } - }; - - # We need to add '.phrase' to all the sort headings otherwise it'll sort - # based on the tokenised form. - my %s; - if ( exists $search->{sort} ) { - foreach my $k ( keys %{ $search->{sort} } ) { - my $f = $self->_sort_field($k); - $s{"$f.phrase"} = $search->{sort}{$k}; - } - $search->{sort} = \%s; + # Zebra behaviour seem to match must so using that here + my $elastic_query = {}; + $elastic_query->{bool}->{must} = \@query_parts; + + # Filter by authtypecode if set + if ($search->{authtypecode}) { + $elastic_query->{bool}->{filter} = { + term => { + "authtype.raw" => $search->{authtypecode} + } + }; } - # add the sort stuff - $query->{sort} = [ $search->{sort} ] if exists $search->{sort}; + my $query = { + query => $elastic_query + }; + + # Add the sort stuff + $query->{sort} = [ $search->{sort} ] if exists $search->{sort}; return $query; } - =head2 build_authorities_query_compat my ($query) = @@ -449,7 +487,7 @@ sub build_authorities_query_compat { my %sort; my $sort_field = ( $orderby =~ /^Heading/ ) ? 'Heading__sort' - : ( $orderby =~ /^Auth/ ) ? 'Local-Number' + : ( $orderby =~ /^Auth/ ) ? 'Local-Number__sort' : undef; if ($sort_field) { my $sort_order = ( $orderby =~ /Asc$/ ) ? 'asc' : 'desc'; @@ -516,7 +554,7 @@ types. =cut our %index_field_convert = ( - 'kw' => '_all', + 'kw' => '', 'ti' => 'title', 'au' => 'author', 'su' => 'subject', @@ -542,7 +580,7 @@ sub _convert_index_fields { # If a field starts with mc- we save it as it's used (and removed) later # when joining things, to indicate we make it an 'OR' join. # (Sorry, this got a bit ugly after special cases were found.) - grep { $_->{field} } map { + map { my ( $f, $t ) = split /,/; my $mc = ''; if ($f =~ /^mc-/) { @@ -554,7 +592,7 @@ sub _convert_index_fields { type => $index_type_convert{ $t // '__default' } }; $r->{field} = ($mc . $r->{field}) if $mc && $r->{field}; - $r; + $r->{field} ? $r : undef; } @indexes; } @@ -583,8 +621,8 @@ sub _convert_index_strings { push @res, $s; next; } - push @res, $conv->{field} . ":" - . $self->_modify_string_by_type( %$conv, operand => $term ); + push @res, ($conv->{field} ? $conv->{field} . ':' : '') + . $self->_modify_string_by_type( %$conv, operand => $term ); } return @res; } @@ -816,4 +854,101 @@ sub _truncate_terms { return join ' ', @terms; } +=head2 _search_fields + my $weighted_fields = $self->_search_fields({ + is_opac => 0, + weighted_fields => 1, + subfield => 'raw' + }); + +Generate a list of searchable fields to be used for Elasticsearch queries +applied to multiple fields. + +Returns an arrayref of field names for either OPAC or Staff client, with +possible weights and subfield appended to each field name depending on the +options provided. + +=over 4 + +=item C<$params> + +Hashref with options. The parameter C indicates whether the searchable +fields for OPAC or Staff client should be retrieved. If C is set +fields weights will be applied on returned fields. C can be used to +provide a subfield that will be appended to fields as "C.C". + +=back + +=cut + +sub _search_fields { + my ($self, $params) = @_; + $params //= { + is_opac => 0, + weighted_fields => 0, + # This is a hack for authorities build_authorities_query + # can hopefully be removed in the future + subfield => undef, + }; + + my $cache = Koha::Caches->get_instance(); + my $cache_key = 'elasticsearch_search_fields' . ($params->{is_opac} ? '_opac' : '_staff_client'); + my $search_fields = $cache->get_from_cache($cache_key, { unsafe => 1 }); + + if (!$search_fields) { + # The reason we don't use Koha::SearchFields->search here is we don't + # want or need resultset wrapped as Koha::SearchField object. + # It does not make any sense in this context and would cause + # unnecessary overhead sice we are only querying for data + # Also would not work, or produce strange results, with the "columns" + # option. + my $schema = Koha::Database->schema; + my $result = $schema->resultset('SearchField')->search( + { + $params->{is_opac} ? ( + 'opac' => 1, + ) : ( + 'staff_client' => 1 + ), + 'search_marc_map.index_name' => $self->index, + 'search_marc_map.marc_type' => C4::Context->preference('marcflavour'), + 'search_marc_to_fields.search' => 1, + }, + { + columns => [qw/name weight/], + # collapse => 1, + join => {search_marc_to_fields => 'search_marc_map'}, + } + ); + my @search_fields; + while (my $search_field = $result->next) { + push @search_fields, [ + $search_field->name, + $search_field->weight ? $search_field->weight : () + ]; + } + $search_fields = \@search_fields; + $cache->set_in_cache($cache_key, $search_fields); + } + if ($params->{subfield}) { + my $subfield = $params->{subfield}; + $search_fields = [ + map { + # Copy values to avoid mutating cached + # data (since unsafe is used) + my ($field, $weight) = @{$_}; + ["${field}.${subfield}", $weight]; + } @{$search_fields} + ]; + } + + if ($params->{weighted_fields}) { + return [map { join('^', @{$_}) } @{$search_fields}]; + } + else { + # Exclude weight from field + return [map { $_->[0] } @{$search_fields}]; + } +} + 1; diff --git a/Koha/SearchField.pm b/Koha/SearchField.pm index efabb8db7d..bd19717b71 100644 --- a/Koha/SearchField.pm +++ b/Koha/SearchField.pm @@ -20,7 +20,6 @@ use Modern::Perl; use Carp; use Koha::Database; -use Koha::SearchMarcMaps; use base qw(Koha::Object); diff --git a/Koha/SearchFields.pm b/Koha/SearchFields.pm index b2c985f526..7af3c0b162 100644 --- a/Koha/SearchFields.pm +++ b/Koha/SearchFields.pm @@ -35,21 +35,6 @@ Koha::SearchFields - Koha SearchField Object set class =cut -=head3 weighted_fields - -my (@w_fields, @weight) = Koha::SearchFields->weighted_fields(); - -=cut - -sub weighted_fields { - my ($self) = @_; - - return $self->search( - { weight => { '>' => 0, '!=' => undef } }, - { order_by => { -desc => 'weight' } } - ); -} - =head3 type =cut diff --git a/admin/searchengine/elasticsearch/field_config.yaml b/admin/searchengine/elasticsearch/field_config.yaml index dba47f5451..11f48cbeb1 100644 --- a/admin/searchengine/elasticsearch/field_config.yaml +++ b/admin/searchengine/elasticsearch/field_config.yaml @@ -1,9 +1,6 @@ --- # General field configuration general: - _all: - type: string - analyzer: analyser_standard properties: record: store: true @@ -18,31 +15,30 @@ search: null_value: 0 stdno: type: text - analyzer: analyser_stdno - search_analyzer: analyser_stdno + analyzer: analyzer_stdno + search_analyzer: analyzer_stdno fields: phrase: type: text - analyzer: analyser_stdno - search_analyzer: analyser_stdno + analyzer: analyzer_phrase + search_analyzer: analyzer_phrase raw: type: keyword - copy_to: _all default: type: text - analyzer: analyser_standard - search_analyzer: analyser_standard + analyzer: analyzer_standard + search_analyzer: analyzer_standard fields: phrase: type: text - analyzer: analyser_phrase - search_analyzer: analyser_phrase + analyzer: analyzer_phrase + search_analyzer: analyzer_phrase raw: type: keyword - lc_raw: + normalizer: nfkc_cf_normalizer + ci_raw: type: keyword - normalizer: my_normalizer - copy_to: _all + normalizer: icu_folding_normalizer # Facets facet: default: @@ -56,9 +52,5 @@ suggestible: # Sort sort: default: - type: text - analyzer: analyser_phrase - search_analyzer: analyser_phrase - fields: - phrase: - type: keyword + type: icu_collation_keyword + index: false diff --git a/admin/searchengine/elasticsearch/index_config.yaml b/admin/searchengine/elasticsearch/index_config.yaml index dce27bc4fa..3d80732785 100644 --- a/admin/searchengine/elasticsearch/index_config.yaml +++ b/admin/searchengine/elasticsearch/index_config.yaml @@ -3,29 +3,29 @@ index: analysis: analyzer: - # Phrase analyzer is used for phrases (phrase match, sorting) - analyser_phrase: + # Phrase analyzer is used for phrases (exact phrase match) + analyzer_phrase: tokenizer: keyword filter: - icu_folding char_filter: - punctuation - analyser_standard: + analyzer_standard: tokenizer: icu_tokenizer filter: - icu_folding - analyser_stdno: + analyzer_stdno: tokenizer: whitespace filter: - icu_folding char_filter: - punctuation normalizer: - normalizer_keyword: + icu_folding_normalizer: type: custom filter: - icu_folding - my_normalizer: + nfkc_cf_normalizer: type: custom char_filter: icu_normalizer char_filter: diff --git a/admin/searchengine/elasticsearch/mappings.pl b/admin/searchengine/elasticsearch/mappings.pl index e42cc38e29..3351202ee9 100755 --- a/admin/searchengine/elasticsearch/mappings.pl +++ b/admin/searchengine/elasticsearch/mappings.pl @@ -25,6 +25,7 @@ use C4::Auth; use Koha::SearchEngine::Elasticsearch; use Koha::SearchMarcMaps; use Koha::SearchFields; +use Koha::Caches; my $input = new CGI; my ( $template, $borrowernumber, $cookie ) = get_template_and_user( @@ -45,6 +46,12 @@ my $schema = $database->schema; my $marc_type = lc C4::Context->preference('marcflavour'); +my $cache = Koha::Caches->get_instance(); +my $clear_cache = sub { + $cache->clear_from_cache('elasticsearch_search_fields_staff_client'); + $cache->clear_from_cache('elasticsearch_search_fields_opac'); +}; + if ( $op eq 'edit' ) { $schema->storage->txn_begin; @@ -53,12 +60,15 @@ if ( $op eq 'edit' ) { my @field_label = $input->param('search_field_label'); my @field_type = $input->param('search_field_type'); my @field_weight = $input->param('search_field_weight'); + my @field_staff_client = $input->param('search_field_staff_client'); + my @field_opac = $input->param('search_field_opac'); my @index_name = $input->param('mapping_index_name'); - my @search_field_name = $input->param('mapping_search_field_name'); + my @search_field_name = $input->param('mapping_search_field_name'); my @mapping_sort = $input->param('mapping_sort'); my @mapping_facet = $input->param('mapping_facet'); my @mapping_suggestible = $input->param('mapping_suggestible'); + my @mapping_search = $input->param('mapping_search'); my @mapping_marc_field = $input->param('mapping_marc_field'); eval { @@ -68,7 +78,8 @@ if ( $op eq 'edit' ) { my $field_label = $field_label[$i]; my $field_type = $field_type[$i]; my $field_weight = $field_weight[$i]; - + my $field_staff_client = $field_staff_client[$i]; + my $field_opac = $field_opac[$i]; my $search_field = Koha::SearchFields->find( { name => $field_name }, { key => 'name' } ); $search_field->label($field_label); $search_field->type($field_type); @@ -83,6 +94,8 @@ if ( $op eq 'edit' ) { $search_field->weight($field_weight); } + $search_field->staff_client($field_staff_client ? 1 : 0); + $search_field->opac($field_opac ? 1 : 0); $search_field->store; } @@ -90,18 +103,26 @@ if ( $op eq 'edit' ) { for my $i ( 0 .. scalar(@index_name) - 1 ) { my $index_name = $index_name[$i]; - my $search_field_name = $search_field_name[$i]; + my $search_field_name = $search_field_name[$i]; my $mapping_marc_field = $mapping_marc_field[$i]; my $mapping_facet = $mapping_facet[$i]; my $mapping_suggestible = $mapping_suggestible[$i]; - my $mapping_sort = $mapping_sort[$i]; - $mapping_sort = undef if $mapping_sort eq 'undef'; + my $mapping_sort = $mapping_sort[$i] eq 'undef' ? undef : $mapping_sort[$i]; + my $mapping_search = $mapping_search[$i]; my $search_field = Koha::SearchFields->find({ name => $search_field_name }, { key => 'name' }); # TODO Check mapping format - my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $marc_type, marc_field => $mapping_marc_field }); - $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping_facet, suggestible => $mapping_suggestible, sort => $mapping_sort } ); - + my $marc_field = Koha::SearchMarcMaps->find_or_create({ + index_name => $index_name, + marc_type => $marc_type, + marc_field => $mapping_marc_field + }); + $search_field->add_to_search_marc_maps($marc_field, { + facet => $mapping_facet, + suggestible => $mapping_suggestible, + sort => $mapping_sort, + search => $mapping_search + }); } }; if ($@) { @@ -111,43 +132,61 @@ if ( $op eq 'edit' ) { push @messages, { type => 'message', code => 'success_on_update' }; $schema->storage->txn_commit; } + $clear_cache->(); } -elsif( $op eq 'reset_confirmed' ) { +elsif( $op eq 'reset_confirm' ) { Koha::SearchMarcMaps->delete; - Koha::SearchFields->delete; Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings; push @messages, { type => 'message', code => 'success_on_reset' }; + $clear_cache->(); } elsif( $op eq 'reset_confirm' ) { $template->param( reset_confirm => 1 ); } - my @indexes; -for my $index_name (qw| biblios authorities |) { +for my $index_name ( + $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, + $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX +) { my $search_fields = Koha::SearchFields->search( - { 'search_marc_map.index_name' => $index_name, 'search_marc_map.marc_type' => $marc_type, }, - { join => { search_marc_to_fields => 'search_marc_map' }, - '+select' => [ 'search_marc_to_fields.facet', 'search_marc_to_fields.suggestible', 'search_marc_to_fields.sort', 'search_marc_map.marc_field' ], - '+as' => [ 'facet', 'suggestible', 'sort', 'marc_field' ], - order_by => { -asc => [qw/name marc_field/] } + { + 'search_marc_map.index_name' => $index_name, + 'search_marc_map.marc_type' => $marc_type, + }, + { + join => { search_marc_to_fields => 'search_marc_map' }, + '+select' => [ + 'search_marc_to_fields.facet', + 'search_marc_to_fields.suggestible', + 'search_marc_to_fields.sort', + 'search_marc_to_fields.search', + 'search_marc_map.marc_field' + ], + '+as' => [ + 'facet', + 'suggestible', + 'sort', + 'search', + 'marc_field' + ], } ); my @mappings; while ( my $s = $search_fields->next ) { - push @mappings, - { search_field_name => $s->name, + push @mappings, { + search_field_name => $s->name, search_field_label => $s->label, search_field_type => $s->type, marc_field => $s->get_column('marc_field'), sort => $s->get_column('sort') // 'undef', # To avoid warnings "Use of uninitialized value in lc" suggestible => $s->get_column('suggestible'), facet => $s->get_column('facet'), - }; + search => $s->get_column('search'), + }; } - push @indexes, { index_name => $index_name, mappings => \@mappings }; } @@ -155,7 +194,7 @@ my $search_fields = Koha::SearchFields->search( {}, { order_by => ['name'] } ); my @all_search_fields; while ( my $search_field = $search_fields->next ) { my $search_field_unblessed = $search_field->unblessed; - $search_field_unblessed->{mapped_biblios} = 1 if $search_field->is_mapped_biblios; + $search_field_unblessed->{is_mapped_biblios} = $search_field->is_mapped_biblios; push @all_search_fields, $search_field_unblessed; } diff --git a/catalogue/search.pl b/catalogue/search.pl index af70dac698..453d5d05be 100755 --- a/catalogue/search.pl +++ b/catalogue/search.pl @@ -456,11 +456,6 @@ my $expanded_facet = $params->{'expand'}; # Define some global variables my ( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type); -my $build_params; -unless ( $cgi->param('advsearch') ) { - $build_params->{weighted_fields} = 1; -} - my $builder = Koha::SearchEngine::QueryBuilder->new( { index => $Koha::SearchEngine::BIBLIOS_INDEX } ); my $searcher = Koha::SearchEngine::Search->new( @@ -473,7 +468,7 @@ my $searcher = Koha::SearchEngine::Search->new( $query_type ) = $builder->build_query_compat( \@operators, \@operands, \@indexes, \@limits, - \@sort_by, $scan, $lang, $build_params ); + \@sort_by, $scan, $lang, { weighted_fields => !$cgi->param('advsearch') }); ## parse the query_cgi string and put it into a form suitable for s my @query_inputs; diff --git a/installer/data/mysql/atomicupdate/bug_20589_add_columns.perl b/installer/data/mysql/atomicupdate/bug_20589_add_columns.perl new file mode 100644 index 0000000000..73ddd1becb --- /dev/null +++ b/installer/data/mysql/atomicupdate/bug_20589_add_columns.perl @@ -0,0 +1,14 @@ +$DBversion = 'XXX'; +if (CheckVersion($DBversion)) { + if (!column_exists('search_marc_to_field', 'search')) { + $dbh->do("ALTER TABLE `search_marc_to_field` ADD COLUMN `search` tinyint(1) NOT NULL DEFAULT 1"); + } + if (!column_exists('search_field', 'staff_client')) { + $dbh->do("ALTER TABLE `search_field` ADD COLUMN `staff_client` tinyint(1) NOT NULL DEFAULT 1"); + } + if (!column_exists('search_field', 'opac')) { + $dbh->do("ALTER TABLE `search_field` ADD COLUMN `opac` tinyint(1) NOT NULL DEFAULT 1"); + } + SetVersion($DBversion) +} +print "Upgrade to $DBversion done (Bug 20589 - Add field boosting and use elastic query fields parameter instead of depricated _all)\n"; diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt index 490916a4b0..5ce3b15f1d 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt @@ -150,8 +150,15 @@ a.add, a.delete { Name Label Type + Searchable Weight + +   + Staff client + OPAC +   + [% FOREACH search_field IN all_search_fields %] @@ -159,7 +166,9 @@ a.add, a.delete { - + + + - [% IF search_field.mapped_biblios %] - - [% ELSE %] - - [% END %] + + + + + + + [% IF search_field.is_mapped_biblios %] + + [% ELSE %] + + [% END %] [% END %] @@ -221,6 +252,7 @@ a.add, a.delete { Sortable Facetable Suggestible + Searchable Mapping @@ -275,6 +307,17 @@ a.add, a.delete { + + + Delete @@ -320,6 +363,17 @@ a.add, a.delete { [% END %] + + + Add diff --git a/opac/opac-search.pl b/opac/opac-search.pl index e0d3dd6229..1f00066bdf 100755 --- a/opac/opac-search.pl +++ b/opac/opac-search.pl @@ -549,19 +549,23 @@ if (C4::Context->preference('OpacSuppression')) { } } -my $build_params = { - expanded_facet => $expanded_facet, - suppress => $suppress -}; - -unless ( $cgi->param('advsearch') ) { - $build_params->{weighted_fields} = 1; -} - ## I. BUILD THE QUERY ( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type) - = $builder->build_query_compat( \@operators, \@operands, - \@indexes, \@limits, \@sort_by, 0, $lang, $build_params); + = $builder->build_query_compat( + \@operators, + \@operands, + \@indexes, + \@limits, + \@sort_by, + 0, + $lang, + { + expanded_facet => $expanded_facet, + suppress => $suppress, + is_opac => 1, + weighted_fields => !$cgi->param('advsearch') + } +); sub _input_cgi_parse { my @elements; -- 2.11.0