Attachment #83019 for bug #20589

View | Details | Raw Unified | Return to bug 20589
Collapse All | Expand All

Lines 77-83 __PACKAGE__->add_columns( Link Here

(-)a/Koha/Schema/Result/SearchField.pm (-1 / +5 lines)
77	is_nullable => 0,	77	is_nullable => 0,
78	},	78	},
79	"weight",	79	"weight",
80	{ data_type => "decimal", is_nullable => 1, size => [5, 2] },	80	{ data_type => "decimal", size => [ 5, 2 ], is_nullable => 1 },
		81	"staff_client",
		82	{ data_type => "tinyint", size => 1, is_nullable => 0, default => 1 },
		83	"opac",
		84	{ data_type => "tinyint", size => 1, is_nullable => 0, default => 1 },
81	);	85	);
82		86
83	=head1 PRIMARY KEY	87	=head1 PRIMARY KEY

Lines 71-76 __PACKAGE__->add_columns( Link Here

(-)a/Koha/Schema/Result/SearchMarcToField.pm (+2 lines)
71	{ data_type => "tinyint", default_value => 0, is_nullable => 1 },	71	{ data_type => "tinyint", default_value => 0, is_nullable => 1 },
72	"sort",	72	"sort",
73	{ data_type => "tinyint", is_nullable => 1 },	73	{ data_type => "tinyint", is_nullable => 1 },
		74	"search",
		75	{ data_type => "tinyint", size => 1, is_nullable => 0, default => 1 },
74	);	76	);
75		77
76	=head1 PRIMARY KEY	78	=head1 PRIMARY KEY

Lines 199-205 sub get_elasticsearch_mappings { Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (-12 / +25 lines)
199	my $marcflavour = lc C4::Context->preference('marcflavour');	199	my $marcflavour = lc C4::Context->preference('marcflavour');
200	$self->_foreach_mapping(	200	$self->_foreach_mapping(
201	sub {	201	sub {
202	my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;	202	my ( $name, $type, $facet, $suggestible, $sort, $search, $marc_type ) = @_;
203	return if $marc_type ne $marcflavour;	203	return if $marc_type ne $marcflavour;
204	# TODO if this gets any sort of complexity to it, it should	204	# TODO if this gets any sort of complexity to it, it should
205	# be broken out into its own function.	205	# be broken out into its own function.
Lines 215-221 sub get_elasticsearch_mappings { Link Here
215	$es_type = 'stdno';	215	$es_type = 'stdno';
216	}	216	}
217		217
218	$mappings->{data}{properties}{$name} = _get_elasticsearch_mapping('search', $es_type);	218	if ($search) {
		219	$mappings->{data}{properties}{$name} = _get_elasticsearch_mapping('search', $es_type);
		220	}
219		221
220	if ($facet) {	222	if ($facet) {
221	$mappings->{data}{properties}{ $name . '__facet' } = _get_elasticsearch_mapping('facet', $es_type);	223	$mappings->{data}{properties}{ $name . '__facet' } = _get_elasticsearch_mapping('facet', $es_type);
Lines 284-294 sub reset_elasticsearch_mappings { Link Here
284	while ( my ( $field_name, $data ) = each %$fields ) {	286	while ( my ( $field_name, $data ) = each %$fields ) {
285	my $field_type = $data->{type};	287	my $field_type = $data->{type};
286	my $field_label = $data->{label};	288	my $field_label = $data->{label};
		289	my $staff_client = exists $data->{staff_client} ? $data->{staff_client} : 1;
		290	my $opac = exists $data->{opac} ? $data->{opac} : 1;
287	my $mappings = $data->{mappings};	291	my $mappings = $data->{mappings};
288	my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type }, { key => 'name' });	292	my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type, staff_client => $staff_client, opac => $opac }, { key => 'name' });
289	for my $mapping ( @$mappings ) {	293	for my $mapping ( @$mappings ) {
290	my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} });	294	my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} });
291	$search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet} \|\| 0, suggestible => $mapping->{suggestible} \|\| 0, sort => $mapping->{sort} } );	295	$search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet} \|\| 0, suggestible => $mapping->{suggestible} \|\| 0, sort => $mapping->{sort}, search => $mapping->{search} \|\| 1 } );
292	}	296	}
293	}	297	}
294	}	298	}
Lines 541-549 sub marc_records_to_documents { Link Here
541	return \@record_documents;	545	return \@record_documents;
542	}	546	}
543		547
544	=head2 _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)	548	=head2 _field_mappings($facet, $suggestible, $sort, $search, $target_name, $target_type, $range)
545		549
546	my @mappings = _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)	550	my @mappings = _field_mappings($facet, $suggestible, $sort, $search, $target_name, $target_type, $range)
547		551
548	Get mappings, an internal data structure later used by	552	Get mappings, an internal data structure later used by
549	L<_process_mappings($mappings, $data, $record_document, $altscript)> to process MARC target	553	L<_process_mappings($mappings, $data, $record_document, $altscript)> to process MARC target
Lines 572-577 Boolean indicating whether to create a suggestion field for this mapping. Link Here
572		576
573	Boolean indicating whether to create a sort field for this mapping.	577	Boolean indicating whether to create a sort field for this mapping.
574		578
		579	=item C<$search>
		580
		581	Boolean indicating whether to create a search field for this mapping.
		582
575	=item C<$target_name>	583	=item C<$target_name>
576		584
577	Elasticsearch document target field name.	585	Elasticsearch document target field name.
Lines 597-603 be extracted. Link Here
597	=cut	605	=cut
598		606
599	sub _field_mappings {	607	sub _field_mappings {
600	my ($_self, $facet, $suggestible, $sort, $target_name, $target_type, $range) = @_;	608	my ($_self, $facet, $suggestible, $sort, $search, $target_name, $target_type, $range) = @_;
601	my %mapping_defaults = ();	609	my %mapping_defaults = ();
602	my @mappings;	610	my @mappings;
603		611
Lines 625-632 sub _field_mappings { Link Here
625	};	633	};
626	}	634	}
627		635
628	my $mapping = [$target_name, $default_options];	636	if ($search) {
629	push @mappings, $mapping;	637	my $mapping = [$target_name, $default_options];
		638	push @mappings, $mapping;
		639	}
630		640
631	my @suffixes = ();	641	my @suffixes = ();
632	push @suffixes, 'facet' if $facet;	642	push @suffixes, 'facet' if $facet;
Lines 683-689 sub _get_marc_mapping_rules { Link Here
683	};	693	};
684		694
685	$self->_foreach_mapping(sub {	695	$self->_foreach_mapping(sub {
686	my ($name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field) = @_;	696	my ($name, $type, $facet, $suggestible, $sort, $search, $marc_type, $marc_field) = @_;
687	return if $marc_type ne $marcflavour;	697	return if $marc_type ne $marcflavour;
688		698
689	if ($type eq 'sum') {	699	if ($type eq 'sum') {
Lines 746-752 sub _get_marc_mapping_rules { Link Here
746	}	756	}
747		757
748	my $range = defined $3 ? $3 : undef;	758	my $range = defined $3 ? $3 : undef;
749	my @mappings = $self->_field_mappings($facet, $suggestible, $sort, $name, $type, $range);	759	my @mappings = $self->_field_mappings($facet, $suggestible, $sort, $search, $name, $type, $range);
750		760
751	if ($field_tag < 10) {	761	if ($field_tag < 10) {
752	$rules->{control_fields}->{$field_tag} //= [];	762	$rules->{control_fields}->{$field_tag} //= [];
Lines 766-772 sub _get_marc_mapping_rules { Link Here
766	}	776	}
767	elsif ($marc_field =~ $leader_regexp) {	777	elsif ($marc_field =~ $leader_regexp) {
768	my $range = defined $1 ? $1 : undef;	778	my $range = defined $1 ? $1 : undef;
769	my @mappings = $self->_field_mappings($facet, $suggestible, $sort, $name, $type, $range);	779	my @mappings = $self->_field_mappings($facet, $suggestible, $sort, $search, $name, $type, $range);
770	push @{$rules->{leader}}, @mappings;	780	push @{$rules->{leader}}, @mappings;
771	}	781	}
772	else {	782	else {
Lines 845-850 sub _foreach_mapping { Link Here
845	'search_marc_to_fields.facet',	855	'search_marc_to_fields.facet',
846	'search_marc_to_fields.suggestible',	856	'search_marc_to_fields.suggestible',
847	'search_marc_to_fields.sort',	857	'search_marc_to_fields.sort',
		858	'search_marc_to_fields.search',
848	'search_marc_map.marc_type',	859	'search_marc_map.marc_type',
849	'search_marc_map.marc_field',	860	'search_marc_map.marc_field',
850	],	861	],
Lines 852-857 sub _foreach_mapping { Link Here
852	'facet',	863	'facet',
853	'suggestible',	864	'suggestible',
854	'sort',	865	'sort',
		866	'search',
855	'marc_type',	867	'marc_type',
856	'marc_field',	868	'marc_field',
857	],	869	],
Lines 865-870 sub _foreach_mapping { Link Here
865	$search_field->get_column('facet'),	877	$search_field->get_column('facet'),
866	$search_field->get_column('suggestible'),	878	$search_field->get_column('suggestible'),
867	$search_field->get_column('sort'),	879	$search_field->get_column('sort'),
		880	$search_field->get_column('search'),
868	$search_field->get_column('marc_type'),	881	$search_field->get_column('marc_type'),
869	$search_field->get_column('marc_field'),	882	$search_field->get_column('marc_field'),
870	);	883	);





use C4::Context;
use Koha::Exceptions;
use Koha::Caches;

=head2 build_query


            query            => $query,
            fuzziness        => $fuzzy_enabled ? 'auto' : '0',
            default_operator => 'AND',
            fields          => $self->_search_fields({ is_opac => $options{is_opac}, weighted_fields => $options{weighted_fields} }),
            lenient          => JSON::true,
            fields           => $options{fields} || [],
        }
    };


        join( ' ', $self->_create_query_string(@search_params) ) || (),
        $self->_join_queries( $self->_convert_index_strings(@$limits) ) || () );

    my @fields = '_all';
    if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) {
        push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields;
    }

    # If there's no query on the left, let's remove the junk left behind
    $query_str =~ s/^ AND //;
    my %options;
    $options{fields} = \@fields;
    $options{sort} = \@sort_params;
    $options{expanded_facet} = $params->{expanded_facet};
    $options{is_opac} = $params->{is_opac};
    $options{weighted_fields} = $params->{weighted_fields};
    my $query = $self->build_query( $query_str, %options );

    #die Dumper($query);


    foreach my $s ( @{ $search->{searches} } ) {
        my ( $wh, $op, $val ) = @{$s}{qw(where operator value)};
        if (!$wh) {
            if ( $op eq 'is' || $op eq '=' || $op eq 'exact') {
                # Match the whole field for all searchable fields, case insensitive,
                # UTF normalized.
                # Given that field data is "The quick brown fox"
                # "The quick brown fox" and "the quick brown fox" will match
                # but not "quick brown fox".
                push @query_parts, {
                    multi_match => {
                        query => $val,
                        fields => $self->_search_fields({ subfield => 'ci_raw' }),
                    }
                };
            }
            elsif ( $op eq 'start') {
                # Match the prefix within a field for all searchable fields.
                # Given that field data is "The quick brown fox"
                # "The quick bro" will match, but not "quick bro"

                # Does not seems to be a multi prefix query
                # so we need to create one
                my @prefix_queries;
                foreach my $field (@{$self->_search_fields()}) {
                    push @prefix_queries, {
                        prefix => { "$field.ci_raw" => $val }
                    };
                }
                push @query_parts, {
                    'bool' => {
                        'should' => \@prefix_queries,
                        'minimum_should_match' => 1
                    }
                };
            }
            else {
                # Query all searchable fields.
                # Given that field data is "The quick brown fox"
                # a search containing any of the words will match, regardless
                # of order.
                #
                # Since default operator is "AND" each of the words must match
                # at least one field
                push @query_parts, {
                    query_string => {
                        query => $val,
                        fields => $self->_search_fields(),
                        default_operator => 'AND',
                    }
                };
            }
        }
        elsif ( $op eq 'is' || $op eq '=' || $op eq 'exact') {
            # Match the whole field, case insensitive, UTF normalized.
            push @query_parts, { term => { "$wh.ci_raw" => $val } };
        }
        elsif ( $op eq 'start' ) {
            # Match prefix of the field.
            push @query_parts, { prefix => {"$wh.ci_raw" => $val} };
        }
        else {
            # Regular match query for the specific field.
            push @query_parts, {
                match => {
                    $wh => {
                        query => $val,
                        operator => 'and'
                    }
                }
            };
        }
    }

    # Merge the query parts appropriately
    # 'should' behaves like 'or'
    # 'must' behaves like 'and'
    # Zebra behaviour seem to match must so using that here
    my $elastic_query = {};
    $elastic_query->{bool}->{must} = \@query_parts;

    # Filter by authtypecode if set
    if ($search->{authtypecode}) {
        $elastic_query->{bool}->{filter} = {
            term => {
                "authtype.raw" => $search->{authtypecode}
            }
        };
            $s{$f} = $search->{sort}{$k};
        }
        $search->{sort} = \%s;
    }

    my $query = {
        query => $elastic_query
    };

    # Add the sort stuff
    $query->{sort} = [ $search->{sort} ] if exists $search->{sort};

    return $query;
}


=head2 build_authorities_query_compat

    my ($query) =


    my %sort;
    my $sort_field =
        ( $orderby =~ /^Heading/ ) ? 'Heading__sort'
      : ( $orderby =~ /^Auth/ )    ? 'Local-Number__sort'
      :                              undef;
    if ($sort_field) {
        my $sort_order = ( $orderby =~ /Asc$/ ) ? 'asc' : 'desc';

=cut

our %index_field_convert = (
    'kw'      => '',
    'ti'      => 'title',
    'au'      => 'author',
    'su'      => 'subject',

    # If a field starts with mc- we save it as it's used (and removed) later
    # when joining things, to indicate we make it an 'OR' join.
    # (Sorry, this got a bit ugly after special cases were found.)
    map {
        my ( $f, $t ) = split /,/;
        my $mc = '';
        if ($f =~ /^mc-/) {

            type  => $index_type_convert{ $t // '__default' }
        };
        $r->{field} = ($mc . $r->{field}) if $mc && $r->{field};
        $r->{field} ? $r : undef;
    } @indexes;
}


            push @res, $s;
            next;
        }
        push @res, ($conv->{field} ? $conv->{field} . ':' : '')
            . $self->_modify_string_by_type( %$conv, operand => $term );
    }
    return @res;
}

    return join ' ', @terms;
}

=head2 _search_fields
    my $weighted_fields = $self->_search_fields({
        is_opac => 0,
        weighted_fields => 1,
        subfield => 'raw'
    });

Generate a list of searchable fields to be used for Elasticsearch queries
applied to multiple fields.

Returns an arrayref of field names for either OPAC or Staff client, with
possible weights and subfield appended to each field name depending on the
options provided.

=over 4

=item C<$params>

Hashref with options. The parameter C<is_opac> indicates whether the searchable
fields for OPAC or Staff client should be retrieved. If C<weighted_fields> is set
fields weights will be applied on returned fields. C<subfield> can be used to
provide a subfield that will be appended to fields as "C<field_name>.C<subfield>".

=back

=cut

sub _search_fields {
    my ($self, $params) = @_;
    $params //= {
        is_opac => 0,
        weighted_fields => 0,
        # This is a hack for authorities build_authorities_query
        # can hopefully be removed in the future
        subfield => undef,
    };

    my $cache = Koha::Caches->get_instance();
    my $cache_key = 'elasticsearch_search_fields' . ($params->{is_opac} ? '_opac' : '_staff_client');
    my $search_fields = $cache->get_from_cache($cache_key, { unsafe => 1 });

    if (!$search_fields) {
        # The reason we don't use Koha::SearchFields->search here is we don't
        # want or need resultset wrapped as Koha::SearchField object.
        # It does not make any sense in this context and would cause
        # unnecessary overhead sice we are only querying for data
        # Also would not work, or produce strange results, with the "columns"
        # option.
        my $schema = Koha::Database->schema;
        my $result = $schema->resultset('SearchField')->search(
            {
                $params->{is_opac} ? (
                    'opac' => 1,
                ) : (
                    'staff_client' => 1
                ),
                'search_marc_map.index_name' => $self->index,
                'search_marc_map.marc_type' => C4::Context->preference('marcflavour'),
                'search_marc_to_fields.search' => 1,
            },
            {
                columns => [qw/name weight/],
                # collapse => 1,
                join => {search_marc_to_fields => 'search_marc_map'},
            }
        );
        my @search_fields;
        while (my $search_field = $result->next) {
            push @search_fields, [
                $search_field->name,
                $search_field->weight ? $search_field->weight : ()
            ];
        }
        $search_fields = \@search_fields;
        $cache->set_in_cache($cache_key, $search_fields);
    }
    if ($params->{subfield}) {
        my $subfield = $params->{subfield};
        $search_fields = [
            map {
                # Copy values to avoid mutating cached
                # data (since unsafe is used)
                my ($field, $weight) = @{$_};
                ["${field}.${subfield}", $weight];
            } @{$search_fields}
        ];
    }

    if ($params->{weighted_fields}) {
        return [map { join('^', @{$_}) } @{$search_fields}];
    }
    else {
        # Exclude weight from field
        return [map { $_->[0] } @{$search_fields}];
    }
}

1;

Lines 20-26 use Modern::Perl; Link Here

(-)a/Koha/SearchField.pm (-1 lines)
20	use Carp;	20	use Carp;
21		21
22	use Koha::Database;	22	use Koha::Database;
23	use Koha::SearchMarcMaps;
24		23
25	use base qw(Koha::Object);	24	use base qw(Koha::Object);
26		25





=cut

=head3 weighted_fields

my (@w_fields, @weight) = Koha::SearchFields->weighted_fields();

=cut

sub weighted_fields {
    my ($self) = @_;

    return $self->search(
        { weight => { '>' => 0, '!=' => undef } },
        { order_by => { -desc => 'weight' } }
    );
}

=head3 type

=cut




---
# General field configuration
general:
  _all:
    type: string
    analyzer: analyser_standard
  properties:
    marc_data:
      store: true

    null_value: 0
  stdno:
    type: text
    analyzer: analyzer_stdno
    search_analyzer: analyzer_stdno
    fields:
      phrase:
        type: text
        analyzer: analyzer_phrase
        search_analyzer: analyzer_phrase
      raw:
        type: keyword
    copy_to: _all
  default:
    type: text
    analyzer: analyzer_standard
    search_analyzer: analyzer_standard
    fields:
      phrase:
        type: text
        analyzer: analyzer_phrase
        search_analyzer: analyzer_phrase
      raw:
        type: keyword
        normalizer: nfkc_cf_normalizer
      ci_raw:
        type: keyword
        normalizer: icu_folding_normalizer
    copy_to: _all
# Facets
facet:
  default:

# Sort
sort:
  default:
    type: icu_collation_keyword
    index: false
    search_analyzer: analyser_phrase
    fields:
      phrase:
        type: keyword




index:
  analysis:
    analyzer:
      # Phrase analyzer is used for phrases (exact phrase match)
      analyzer_phrase:
        tokenizer: keyword
        filter:
          - icu_folding
        char_filter:
          - punctuation
      analyzer_standard:
        tokenizer: icu_tokenizer
        filter:
          - icu_folding
      analyzer_stdno:
        tokenizer: whitespace
        filter:
          - icu_folding
        char_filter:
          - punctuation
    normalizer:
      icu_folding_normalizer:
        type: custom
        filter:
          - icu_folding
      nfkc_cf_normalizer:
        type: custom
        char_filter: icu_normalizer
    char_filter:

Lines 26-31 use Koha::SearchEngine::Elasticsearch; Link Here

(-)a/admin/searchengine/elasticsearch/mappings.pl (-15 / +57 lines)
26	use Koha::SearchEngine::Elasticsearch::Indexer;	26	use Koha::SearchEngine::Elasticsearch::Indexer;
27	use Koha::SearchMarcMaps;	27	use Koha::SearchMarcMaps;
28	use Koha::SearchFields;	28	use Koha::SearchFields;
		29	use Koha::Caches;
29		30
30	use Try::Tiny;	31	use Try::Tiny;
31		32
Lines 67-72 my $update_mappings = sub { Link Here
67	}	68	}
68	};	69	};
69		70
		71	my $cache = Koha::Caches->get_instance();
		72	my $clear_cache = sub {
		73	$cache->clear_from_cache('elasticsearch_search_fields_staff_client');
		74	$cache->clear_from_cache('elasticsearch_search_fields_opac');
		75	};
		76
70	if ( $op eq 'edit' ) {	77	if ( $op eq 'edit' ) {
71		78
72	$schema->storage->txn_begin;	79	$schema->storage->txn_begin;
Lines 75-86 if ( $op eq 'edit' ) { Link Here
75	my @field_label = $input->param('search_field_label');	82	my @field_label = $input->param('search_field_label');
76	my @field_type = $input->param('search_field_type');	83	my @field_type = $input->param('search_field_type');
77	my @field_weight = $input->param('search_field_weight');	84	my @field_weight = $input->param('search_field_weight');
		85	my @field_staff_client = $input->param('search_field_staff_client');
		86	my @field_opac = $input->param('search_field_opac');
78		87
79	my @index_name = $input->param('mapping_index_name');	88	my @index_name = $input->param('mapping_index_name');
80	my @search_field_name = $input->param('mapping_search_field_name');	89	my @search_field_name = $input->param('mapping_search_field_name');
81	my @mapping_sort = $input->param('mapping_sort');	90	my @mapping_sort = $input->param('mapping_sort');
82	my @mapping_facet = $input->param('mapping_facet');	91	my @mapping_facet = $input->param('mapping_facet');
83	my @mapping_suggestible = $input->param('mapping_suggestible');	92	my @mapping_suggestible = $input->param('mapping_suggestible');
		93	my @mapping_search = $input->param('mapping_search');
84	my @mapping_marc_field = $input->param('mapping_marc_field');	94	my @mapping_marc_field = $input->param('mapping_marc_field');
85		95
86	eval {	96	eval {
Lines 90-95 if ( $op eq 'edit' ) { Link Here
90	my $field_label = $field_label[$i];	100	my $field_label = $field_label[$i];
91	my $field_type = $field_type[$i];	101	my $field_type = $field_type[$i];
92	my $field_weight = $field_weight[$i];	102	my $field_weight = $field_weight[$i];
		103	my $field_staff_client = $field_staff_client[$i];
		104	my $field_opac = $field_opac[$i];
93		105
94	my $search_field = Koha::SearchFields->find( { name => $field_name }, { key => 'name' } );	106	my $search_field = Koha::SearchFields->find( { name => $field_name }, { key => 'name' } );
95	$search_field->label($field_label);	107	$search_field->label($field_label);
Lines 104-109 if ( $op eq 'edit' ) { Link Here
104	else {	116	else {
105	$search_field->weight($field_weight);	117	$search_field->weight($field_weight);
106	}	118	}
		119	$search_field->staff_client($field_staff_client ? 1 : 0);
		120	$search_field->opac($field_opac ? 1 : 0);
107		121
108	$search_field->store;	122	$search_field->store;
109	}	123	}
Lines 112-129 if ( $op eq 'edit' ) { Link Here
112		126
113	for my $i ( 0 .. scalar(@index_name) - 1 ) {	127	for my $i ( 0 .. scalar(@index_name) - 1 ) {
114	my $index_name = $index_name[$i];	128	my $index_name = $index_name[$i];
115	my $search_field_name = $search_field_name[$i];	129	my $search_field_name = $search_field_name[$i];
116	my $mapping_marc_field = $mapping_marc_field[$i];	130	my $mapping_marc_field = $mapping_marc_field[$i];
117	my $mapping_facet = $mapping_facet[$i];	131	my $mapping_facet = $mapping_facet[$i];
118	my $mapping_suggestible = $mapping_suggestible[$i];	132	my $mapping_suggestible = $mapping_suggestible[$i];
119	my $mapping_sort = $mapping_sort[$i];	133	my $mapping_sort = $mapping_sort[$i];
120	$mapping_sort = undef if $mapping_sort eq 'undef';	134	$mapping_sort = undef if $mapping_sort eq 'undef';
		135	my $mapping_sort = $mapping_sort[$i] eq 'undef' ? undef : $mapping_sort[$i];
		136	my $mapping_search = $mapping_search[$i];
121		137
122	my $search_field = Koha::SearchFields->find({ name => $search_field_name }, { key => 'name' });	138	my $search_field = Koha::SearchFields->find({ name => $search_field_name }, { key => 'name' });
123	# TODO Check mapping format	139	# TODO Check mapping format
124	my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $marc_type, marc_field => $mapping_marc_field });	140	my $marc_field = Koha::SearchMarcMaps->find_or_create({
125	$search_field->add_to_search_marc_maps($marc_field, { facet => $mapping_facet, suggestible => $mapping_suggestible, sort => $mapping_sort } );	141	index_name => $index_name,
126		142	marc_type => $marc_type,
		143	marc_field => $mapping_marc_field
		144	});
		145	$search_field->add_to_search_marc_maps($marc_field, {
		146	facet => $mapping_facet,
		147	suggestible => $mapping_suggestible,
		148	sort => $mapping_sort,
		149	search => $mapping_search
		150	});
127	}	151	}
128	};	152	};
129	if ($@) {	153	if ($@) {
Lines 132-137 if ( $op eq 'edit' ) { Link Here
132	} else {	156	} else {
133	push @messages, { type => 'message', code => 'success_on_update' };	157	push @messages, { type => 'message', code => 'success_on_update' };
134	$schema->storage->txn_commit;	158	$schema->storage->txn_commit;
		159	$clear_cache->();
135	$update_mappings->();	160	$update_mappings->();
136	}	161	}
137	}	162	}
Lines 139-151 elsif( $op eq 'reset_confirmed' ) { Link Here
139	Koha::SearchMarcMaps->delete;	164	Koha::SearchMarcMaps->delete;
140	Koha::SearchFields->delete;	165	Koha::SearchFields->delete;
141	Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;	166	Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;
		167	$clear_cache->();
142	push @messages, { type => 'message', code => 'success_on_reset' };	168	push @messages, { type => 'message', code => 'success_on_reset' };
143	}	169	}
144	elsif( $op eq 'reset_confirm' ) {	170	elsif( $op eq 'reset_confirm' ) {
145	$template->param( reset_confirm => 1 );	171	$template->param( reset_confirm => 1 );
146	}	172	}
147		173
148
149	my @indexes;	174	my @indexes;
150		175
151	for my $index_name (@index_names) {	176	for my $index_name (@index_names) {
Lines 171-195 for my $index_name (@index_names) { Link Here
171		196
172	for my $index_name (@index_names) {	197	for my $index_name (@index_names) {
173	my $search_fields = Koha::SearchFields->search(	198	my $search_fields = Koha::SearchFields->search(
174	{ 'search_marc_map.index_name' => $index_name, 'search_marc_map.marc_type' => $marc_type, },	199	{
175	{ join => { search_marc_to_fields => 'search_marc_map' },	200	'search_marc_map.index_name' => $index_name,
176	'+select' => [ 'search_marc_to_fields.facet', 'search_marc_to_fields.suggestible', 'search_marc_to_fields.sort', 'search_marc_map.marc_field' ],	201	'search_marc_map.marc_type' => $marc_type,
177	'+as' => [ 'facet', 'suggestible', 'sort', 'marc_field' ],	202	},
		203	{
		204	join => { search_marc_to_fields => 'search_marc_map' },
		205	'+select' => [
		206	'search_marc_to_fields.facet',
		207	'search_marc_to_fields.suggestible',
		208	'search_marc_to_fields.sort',
		209	'search_marc_to_fields.search',
		210	'search_marc_map.marc_field'
		211	],
		212	'+as' => [
		213	'facet',
		214	'suggestible',
		215	'sort',
		216	'search',
		217	'marc_field'
		218	],
178	order_by => { -asc => [qw/name marc_field/] }	219	order_by => { -asc => [qw/name marc_field/] }
179	}	220	}
180	);	221	);
181		222
182	my @mappings;	223	my @mappings;
183	while ( my $s = $search_fields->next ) {	224	while ( my $s = $search_fields->next ) {
184	push @mappings,	225	push @mappings, {
185	{ search_field_name => $s->name,	226	search_field_name => $s->name,
186	search_field_label => $s->label,	227	search_field_label => $s->label,
187	search_field_type => $s->type,	228	search_field_type => $s->type,
188	marc_field => $s->get_column('marc_field'),	229	marc_field => $s->get_column('marc_field'),
189	sort => $s->get_column('sort') // 'undef', # To avoid warnings "Use of uninitialized value in lc"	230	sort => $s->get_column('sort') // 'undef', # To avoid warnings "Use of uninitialized value in lc"
190	suggestible => $s->get_column('suggestible'),	231	suggestible => $s->get_column('suggestible'),
191	facet => $s->get_column('facet'),	232	facet => $s->get_column('facet'),
192	};	233	search => $s->get_column('search')
		234	};
193	}	235	}
194		236
195	push @indexes, { index_name => $index_name, mappings => \@mappings };	237	push @indexes, { index_name => $index_name, mappings => \@mappings };




# Define some global variables
my ( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type);

my $build_params;
unless ( $cgi->param('advsearch') ) {
    $build_params->{weighted_fields} = 1;
}

my $builder = Koha::SearchEngine::QueryBuilder->new(
    { index => $Koha::SearchEngine::BIBLIOS_INDEX } );
my $searcher = Koha::SearchEngine::Search->new(

    $query_type
  )
  = $builder->build_query_compat( \@operators, \@operands, \@indexes, \@limits,
    \@sort_by, $scan, $lang, { weighted_fields => !$cgi->param('advsearch') });

## parse the query_cgi string and put it into a form suitable for <input>s
my @query_inputs;




$DBversion = 'XXX';
if (CheckVersion($DBversion)) {
  if (!column_exists('search_marc_to_field', 'search')) {
    $dbh->do("ALTER TABLE `search_marc_to_field` ADD COLUMN `search` tinyint(1) NOT NULL DEFAULT 1");
  }
  if (!column_exists('search_field', 'staff_client')) {
    $dbh->do("ALTER TABLE `search_field` ADD COLUMN `staff_client` tinyint(1) NOT NULL DEFAULT 1");
  }
  if (!column_exists('search_field', 'opac')) {
    $dbh->do("ALTER TABLE `search_field` ADD COLUMN `opac` tinyint(1) NOT NULL DEFAULT 1");
  }
  SetVersion($DBversion)
}
print "Upgrade to $DBversion done (Bug 20589 - Add field boosting and use elastic query fields parameter instead of depricated _all)\n";

Lines 156-163 a.add, a.delete { Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt (-6 / +60 lines)
156	<th>Name</th>	156	<th>Name</th>
157	<th>Label</th>	157	<th>Label</th>
158	<th>Type</th>	158	<th>Type</th>
		159	<th colspan="2">Searchable</th>
159	<th>Weight</th>	160	<th>Weight</th>
160	</tr>	161	</tr>
		162	<tr>
		163	<th colspan=3> </th>
		164	<th>Staff client</th>
		165	<th>OPAC</th>
		166	<th> </th>
		167	</tr>
161	</thead>	168	</thead>
162	<tbody>	169	<tbody>
163	[% FOREACH search_field IN all_search_fields %]	170	[% FOREACH search_field IN all_search_fields %]
Lines 165-171 a.add, a.delete { Link Here
165	<td>	172	<td>
166	<input type="text" name="search_field_name" value="[% search_field.name \| html %]" />	173	<input type="text" name="search_field_name" value="[% search_field.name \| html %]" />
167	</td>	174	</td>
168	<td><input type="text" name="search_field_label" value="[% search_field.label \| html %]" />	175	<td>
		176	<input type="text" name="search_field_label" value="[% search_field.label \| html %]" />
		177	</td>
169	<td>	178	<td>
170	<select name="search_field_type">	179	<select name="search_field_type">
171	<option value=""></option>	180	<option value=""></option>
Lines 207-217 a.add, a.delete { Link Here
207	</select>	216	</select>
208	</td>	217	</td>
209	<td>	218	<td>
210	[% IF search_field.mapped_biblios %]	219	<select name="search_field_staff_client">
211	<input type="number" step="0.01" min="0.01" max="999.99" name="search_field_weight" value="[% search_field.weight \| html %]" />	220	[% IF search_field.staff_client %]
212	[% ELSE %]	221	<option value="1" selected="selected">Yes</option>
213	<input type="hidden" name="search_field_weight" value="">	222	<option value="0">No</option>
214	[% END %]	223	[% ELSE %]
		224	<option value="1">Yes</option>
		225	<option value="0" selected="selected">No</option>
		226	[% END %]
		227	</select>
		228	</td>
		229	<td>
		230	<select name="search_field_opac">
		231	[% IF search_field.opac %]
		232	<option value="1" selected="selected">Yes</option>
		233	<option value="0">No</option>
		234	[% ELSE %]
		235	<option value="1">Yes</option>
		236	<option value="0" selected="selected">No</option>
		237	[% END %]
		238	</select>
		239	</td>
		240	<td>
		241	[% IF search_field.mapped_biblios %]
		242	<input type="number" step="0.01" min="0.01" max="999.99" name="search_field_weight" value="[% search_field.weight \| html %]" />
		243	[% ELSE %]
		244	<input type="hidden" name="search_field_weight" value="">
		245	[% END %]
215	</td>	246	</td>
216	</tr>	247	</tr>
217	[% END %]	248	[% END %]
Lines 227-232 a.add, a.delete { Link Here
227	<th>Sortable</th>	258	<th>Sortable</th>
228	<th>Facetable</th>	259	<th>Facetable</th>
229	<th>Suggestible</th>	260	<th>Suggestible</th>
		261	<th>Searchable</th>
230	<th>Mapping</th>	262	<th>Mapping</th>
231	<th></th>	263	<th></th>
232	</tr>	264	</tr>
Lines 281-286 a.add, a.delete { Link Here
281	</select>	313	</select>
282	</td>	314	</td>
283	<td>	315	<td>
		316	<select name="mapping_search">
		317	[% IF mapping.search %]
		318	<option value="0">No</option>
		319	<option value="1" selected="selected">Yes</option>
		320	[% ELSE %]
		321	<option value="0" selected="selected">No</option>
		322	<option value="1">Yes</option>
		323	[% END %]
		324	</select>
		325	</td>
		326	<td>
284	<input name="mapping_marc_field" type="text" value="[% mapping.marc_field \| html %]" />	327	<input name="mapping_marc_field" type="text" value="[% mapping.marc_field \| html %]" />
285	</td>	328	</td>
286	<td><a class="btn btn-default btn-xs delete" style="cursor: pointer;"><i class="fa fa-trash"></i> Delete</a></td>	329	<td><a class="btn btn-default btn-xs delete" style="cursor: pointer;"><i class="fa fa-trash"></i> Delete</a></td>
Lines 326-331 a.add, a.delete { Link Here
326	[% END %]	369	[% END %]
327	</select>	370	</select>
328	</td>	371	</td>
		372	<td>
		373	<select data-id="mapping_search">
		374	[% IF mapping.search %]
		375	<option value="0">No</option>
		376	<option value="1" selected="selected">Yes</option>
		377	[% ELSE %]
		378	<option value="0" selected="selected">No</option>
		379	<option value="1">Yes</option>
		380	[% END %]
		381	</select>
		382	</td>
329	<td><input data-id="mapping_marc_field" type="text" /></td>	383	<td><input data-id="mapping_marc_field" type="text" /></td>
330	<td><a class="btn btn-default btn-xs add"><i class="fa fa-plus"></i> Add</a></td>	384	<td><a class="btn btn-default btn-xs add"><i class="fa fa-plus"></i> Add</a></td>
331	</tr>	385	</tr>




    }
}

my $build_params = {
    expanded_facet => $expanded_facet,
    suppress => $suppress
};

unless ( $cgi->param('advsearch') ) {
    $build_params->{weighted_fields} = 1;
}

## I. BUILD THE QUERY
( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type)
  = $builder->build_query_compat(
    \@operators,
    \@operands,
    \@indexes,
    \@limits,
    \@sort_by,
    0,
    $lang,
    {
        expanded_facet => $expanded_facet,
        suppress => $suppress,
        is_opac => 1,
        weighted_fields => !$cgi->param('advsearch')
    }
);

sub _input_cgi_parse {
    my @elements;
- 

Return to bug 20589

Lines 456-466 my $expanded_facet = $params->{'expand'}; Link Here

(-)a/catalogue/search.pl (-6 / +1 lines)
456	# Define some global variables	456	# Define some global variables
457	my ( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type);	457	my ( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type);
458		458
459	my $build_params;
460	unless ( $cgi->param('advsearch') ) {
461	$build_params->{weighted_fields} = 1;
462	}
463
464	my $builder = Koha::SearchEngine::QueryBuilder->new(	459	my $builder = Koha::SearchEngine::QueryBuilder->new(
465	{ index => $Koha::SearchEngine::BIBLIOS_INDEX } );	460	{ index => $Koha::SearchEngine::BIBLIOS_INDEX } );
466	my $searcher = Koha::SearchEngine::Search->new(	461	my $searcher = Koha::SearchEngine::Search->new(
Lines 473-479 my $searcher = Koha::SearchEngine::Search->new( Link Here
473	$query_type	468	$query_type
474	)	469	)
475	= $builder->build_query_compat( \@operators, \@operands, \@indexes, \@limits,	470	= $builder->build_query_compat( \@operators, \@operands, \@indexes, \@limits,
476	\@sort_by, $scan, $lang, $build_params );	471	\@sort_by, $scan, $lang, { weighted_fields => !$cgi->param('advsearch') });
477		472
478	## parse the query_cgi string and put it into a form suitable for <input>s	473	## parse the query_cgi string and put it into a form suitable for <input>s
479	my @query_inputs;	474	my @query_inputs;

Lines 48-53 use URI::Escape; Link Here

(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-49 / +190 lines)
48		48
49	use C4::Context;	49	use C4::Context;
50	use Koha::Exceptions;	50	use Koha::Exceptions;
		51	use Koha::Caches;
51		52
52	=head2 build_query	53	=head2 build_query
53		54
Lines 90-98 sub build_query { Link Here
90	query => $query,	91	query => $query,
91	fuzziness => $fuzzy_enabled ? 'auto' : '0',	92	fuzziness => $fuzzy_enabled ? 'auto' : '0',
92	default_operator => 'AND',	93	default_operator => 'AND',
93	default_field => '_all',	94	fields => $self->_search_fields({ is_opac => $options{is_opac}, weighted_fields => $options{weighted_fields} }),
94	lenient => JSON::true,	95	lenient => JSON::true,
95	fields => $options{fields} \|\| [],
96	}	96	}
97	};	97	};
98		98
Lines 227-243 sub build_query_compat { Link Here
227	join( ' ', $self->_create_query_string(@search_params) ) \|\| (),	227	join( ' ', $self->_create_query_string(@search_params) ) \|\| (),
228	$self->_join_queries( $self->_convert_index_strings(@$limits) ) \|\| () );	228	$self->_join_queries( $self->_convert_index_strings(@$limits) ) \|\| () );
229		229
230	my @fields = '_all';
231	if ( defined($params->{weighted_fields}) && $params->{weighted_fields} ) {
232	push @fields, sprintf("%s^%s", $_->name, $_->weight) for Koha::SearchFields->weighted_fields;
233	}
234
235	# If there's no query on the left, let's remove the junk left behind	230	# If there's no query on the left, let's remove the junk left behind
236	$query_str =~ s/^ AND //;	231	$query_str =~ s/^ AND //;
237	my %options;	232	my %options;
238	$options{fields} = \@fields;
239	$options{sort} = \@sort_params;	233	$options{sort} = \@sort_params;
240	$options{expanded_facet} = $params->{expanded_facet};	234	$options{expanded_facet} = $params->{expanded_facet};
		235	$options{is_opac} = $params->{is_opac};
		236	$options{weighted_fields} = $params->{weighted_fields};
241	my $query = $self->build_query( $query_str, %options );	237	my $query = $self->build_query( $query_str, %options );
242		238
243	#die Dumper($query);	239	#die Dumper($query);
Lines 298-353 sub build_authorities_query { Link Here
298		294
299	foreach my $s ( @{ $search->{searches} } ) {	295	foreach my $s ( @{ $search->{searches} } ) {
300	my ( $wh, $op, $val ) = @{$s}{qw(where operator value)};	296	my ( $wh, $op, $val ) = @{$s}{qw(where operator value)};
301	$wh = '_all' if $wh eq '';	297	if (!$wh) {
302	if ( $op eq 'is' \|\| $op eq '=' \|\| $op eq 'exact' ) {	298	if ( $op eq 'is' \|\| $op eq '=' \|\| $op eq 'exact') {
303		299	# Match the whole field for all searchable fields, case insensitive,
304	# look for something that matches a term completely	300	# UTF normalized.
305	# note, '=' is about numerical vals. May need special handling.	301	# Given that field data is "The quick brown fox"
306	# Also, we lowercase our search because the ES	302	# "The quick brown fox" and "the quick brown fox" will match
307	# index lowercases its values, and term searches don't get the	303	# but not "quick brown fox".
308	# search analyzer applied to them.	304	push @query_parts, {
309	push @query_parts, { match_phrase => {"$wh.phrase" => lc $val} };	305	multi_match => {
		306	query => $val,
		307	fields => $self->_search_fields({ subfield => 'ci_raw' }),
		308	}
		309	};
		310	}
		311	elsif ( $op eq 'start') {
		312	# Match the prefix within a field for all searchable fields.
		313	# Given that field data is "The quick brown fox"
		314	# "The quick bro" will match, but not "quick bro"
		315
		316	# Does not seems to be a multi prefix query
		317	# so we need to create one
		318	my @prefix_queries;
		319	foreach my $field (@{$self->_search_fields()}) {
		320	push @prefix_queries, {
		321	prefix => { "$field.ci_raw" => $val }
		322	};
		323	}
		324	push @query_parts, {
		325	'bool' => {
		326	'should' => \@prefix_queries,
		327	'minimum_should_match' => 1
		328	}
		329	};
		330	}
		331	else {
		332	# Query all searchable fields.
		333	# Given that field data is "The quick brown fox"
		334	# a search containing any of the words will match, regardless
		335	# of order.
		336	#
		337	# Since default operator is "AND" each of the words must match
		338	# at least one field
		339	push @query_parts, {
		340	query_string => {
		341	query => $val,
		342	fields => $self->_search_fields(),
		343	default_operator => 'AND',
		344	}
		345	};
		346	}
		347	}
		348	elsif ( $op eq 'is' \|\| $op eq '=' \|\| $op eq 'exact') {
		349	# Match the whole field, case insensitive, UTF normalized.
		350	push @query_parts, { term => { "$wh.ci_raw" => $val } };
310	}	351	}
311	elsif ( $op eq 'start' ) {	352	elsif ( $op eq 'start' ) {
312	# startswith search, uses lowercase untokenized version of heading	353	# Match prefix of the field.
313	push @query_parts, { match_phrase_prefix => {"$wh.phrase" => lc $val} };	354	push @query_parts, { prefix => {"$wh.ci_raw" => $val} };
314	}	355	}
315	else {	356	else {
316	# regular wordlist stuff	357	# Regular match query for the specific field.
317	# push @query_parts, { match => {$wh => { query => $val, operator => 'and' }} };	358	push @query_parts, {
318	my @values = split(' ',$val);	359	match => {
319	foreach my $v (@values) {	360	$wh => {
320	push @query_parts, { wildcard => { "$wh.phrase" => "" . lc $v . "" } };	361	query => $val,
321	}	362	operator => 'and'
		363	}
		364	}
		365	};
322	}	366	}
323	}	367	}
324		368
325	# Merge the query parts appropriately	369	# Merge the query parts appropriately
326	# 'should' behaves like 'or'	370	# 'should' behaves like 'or'
327	# 'must' behaves like 'and'	371	# 'must' behaves like 'and'
328	# Zebra results seem to match must so using that here	372	# Zebra behaviour seem to match must so using that here
329	my $query = { query =>	373	my $elastic_query = {};
330	{ bool =>	374	$elastic_query->{bool}->{must} = \@query_parts;
331	{ must => \@query_parts }	375
332	}	376	# Filter by authtypecode if set
333	};	377	if ($search->{authtypecode}) {
334		378	$elastic_query->{bool}->{filter} = {
335	my %s;	379	term => {
336	if ( exists $search->{sort} ) {	380	"authtype.raw" => $search->{authtypecode}
337	foreach my $k ( keys %{ $search->{sort} } ) {	381	}
338	my $f = $self->_sort_field($k);	382	};
339	$s{$f} = $search->{sort}{$k};
340	}
341	$search->{sort} = \%s;
342	}	383	}
343		384
344	# add the sort stuff	385	my $query = {
345	$query->{sort} = [ $search->{sort} ] if exists $search->{sort};	386	query => $elastic_query
		387	};
		388
		389	# Add the sort stuff
		390	$query->{sort} = [ $search->{sort} ] if exists $search->{sort};
346		391
347	return $query;	392	return $query;
348	}	393	}
349		394
350
351	=head2 build_authorities_query_compat	395	=head2 build_authorities_query_compat
352		396
353	my ($query) =	397	my ($query) =
Lines 441-448 sub build_authorities_query_compat { Link Here
441		485
442	my %sort;	486	my %sort;
443	my $sort_field =	487	my $sort_field =
444	( $orderby =~ /^Heading/ ) ? 'Heading'	488	( $orderby =~ /^Heading/ ) ? 'Heading__sort'
445	: ( $orderby =~ /^Auth/ ) ? 'Local-number'	489	: ( $orderby =~ /^Auth/ ) ? 'Local-Number__sort'
446	: undef;	490	: undef;
447	if ($sort_field) {	491	if ($sort_field) {
448	my $sort_order = ( $orderby =~ /Asc$/ ) ? 'asc' : 'desc';	492	my $sort_order = ( $orderby =~ /Asc$/ ) ? 'asc' : 'desc';
Lines 509-515 types. Link Here
509	=cut	553	=cut
510		554
511	our %index_field_convert = (	555	our %index_field_convert = (
512	'kw' => '_all',	556	'kw' => '',
513	'ti' => 'title',	557	'ti' => 'title',
514	'au' => 'author',	558	'au' => 'author',
515	'su' => 'subject',	559	'su' => 'subject',
Lines 537-543 sub _convert_index_fields { Link Here
537	# If a field starts with mc- we save it as it's used (and removed) later	581	# If a field starts with mc- we save it as it's used (and removed) later
538	# when joining things, to indicate we make it an 'OR' join.	582	# when joining things, to indicate we make it an 'OR' join.
539	# (Sorry, this got a bit ugly after special cases were found.)	583	# (Sorry, this got a bit ugly after special cases were found.)
540	grep { $_->{field} } map {	584	map {
541	my ( $f, $t ) = split /,/;	585	my ( $f, $t ) = split /,/;
542	my $mc = '';	586	my $mc = '';
543	if ($f =~ /^mc-/) {	587	if ($f =~ /^mc-/) {
Lines 549-555 sub _convert_index_fields { Link Here
549	type => $index_type_convert{ $t // '__default' }	593	type => $index_type_convert{ $t // '__default' }
550	};	594	};
551	$r->{field} = ($mc . $r->{field}) if $mc && $r->{field};	595	$r->{field} = ($mc . $r->{field}) if $mc && $r->{field};
552	$r;	596	$r->{field} ? $r : undef;
553	} @indexes;	597	} @indexes;
554	}	598	}
555		599
Lines 578-585 sub _convert_index_strings { Link Here
578	push @res, $s;	622	push @res, $s;
579	next;	623	next;
580	}	624	}
581	push @res, $conv->{field} . ":"	625	push @res, ($conv->{field} ? $conv->{field} . ':' : '')
582	. $self->_modify_string_by_type( %$conv, operand => $term );	626	. $self->_modify_string_by_type( %$conv, operand => $term );
583	}	627	}
584	return @res;	628	return @res;
585	}	629	}
Lines 822-825 sub _truncate_terms { Link Here
822	return join ' ', @terms;	866	return join ' ', @terms;
823	}	867	}
824		868
		869	=head2 _search_fields
		870	my $weighted_fields = $self->_search_fields({
		871	is_opac => 0,
		872	weighted_fields => 1,
		873	subfield => 'raw'
		874	});
		875
		876	Generate a list of searchable fields to be used for Elasticsearch queries
		877	applied to multiple fields.
		878
		879	Returns an arrayref of field names for either OPAC or Staff client, with
		880	possible weights and subfield appended to each field name depending on the
		881	options provided.
		882
		883	=over 4
		884
		885	=item C<$params>
		886
		887	Hashref with options. The parameter C<is_opac> indicates whether the searchable
		888	fields for OPAC or Staff client should be retrieved. If C<weighted_fields> is set
		889	fields weights will be applied on returned fields. C<subfield> can be used to
		890	provide a subfield that will be appended to fields as "C<field_name>.C<subfield>".
		891
		892	=back
		893
		894	=cut
		895
		896	sub _search_fields {
		897	my ($self, $params) = @_;
		898	$params //= {
		899	is_opac => 0,
		900	weighted_fields => 0,
		901	# This is a hack for authorities build_authorities_query
		902	# can hopefully be removed in the future
		903	subfield => undef,
		904	};
		905
		906	my $cache = Koha::Caches->get_instance();
		907	my $cache_key = 'elasticsearch_search_fields' . ($params->{is_opac} ? '_opac' : '_staff_client');
		908	my $search_fields = $cache->get_from_cache($cache_key, { unsafe => 1 });
		909
		910	if (!$search_fields) {
		911	# The reason we don't use Koha::SearchFields->search here is we don't
		912	# want or need resultset wrapped as Koha::SearchField object.
		913	# It does not make any sense in this context and would cause
		914	# unnecessary overhead sice we are only querying for data
		915	# Also would not work, or produce strange results, with the "columns"
		916	# option.
		917	my $schema = Koha::Database->schema;
		918	my $result = $schema->resultset('SearchField')->search(
		919	{
		920	$params->{is_opac} ? (
		921	'opac' => 1,
		922	) : (
		923	'staff_client' => 1
		924	),
		925	'search_marc_map.index_name' => $self->index,
		926	'search_marc_map.marc_type' => C4::Context->preference('marcflavour'),
		927	'search_marc_to_fields.search' => 1,
		928	},
		929	{
		930	columns => [qw/name weight/],
		931	# collapse => 1,
		932	join => {search_marc_to_fields => 'search_marc_map'},
933	}
934	);
935	my @search_fields;
936	while (my $search_field = $result->next) {
937	push @search_fields, [
938	$search_field->name,
939	$search_field->weight ? $search_field->weight : ()
940	];
941	}
942	$search_fields = \@search_fields;
943	$cache->set_in_cache($cache_key, $search_fields);
944	}
945	if ($params->{subfield}) {
946	my $subfield = $params->{subfield};
947	$search_fields = [
948	map {
949	# Copy values to avoid mutating cached
950	# data (since unsafe is used)
951	my ($field, $weight) = @{$_};
952	["${field}.${subfield}", $weight];
953	} @{$search_fields}
954	];
955	}
956
957	if ($params->{weighted_fields}) {
958	return [map { join('^', @{$_}) } @{$search_fields}];
959	}
960	else {
961	# Exclude weight from field
962	return [map { $_->[0] } @{$search_fields}];
963	}
964	}
965
825	1;	966	1;

Lines 35-55 Koha::SearchFields - Koha SearchField Object set class Link Here

(-)a/Koha/SearchFields.pm (-15 lines)
35		35
36	=cut	36	=cut
37		37
38	=head3 weighted_fields
39
40	my (@w_fields, @weight) = Koha::SearchFields->weighted_fields();
41
42	=cut
43
44	sub weighted_fields {
45	my ($self) = @_;
46
47	return $self->search(
48	{ weight => { '>' => 0, '!=' => undef } },
49	{ order_by => { -desc => 'weight' } }
50	);
51	}
52
53	=head3 type	38	=head3 type
54		39
55	=cut	40	=cut

Lines 1-9 Link Here

(-)a/admin/searchengine/elasticsearch/field_config.yaml (-21 / +13 lines)
1	---	1	---
2	# General field configuration	2	# General field configuration
3	general:	3	general:
4	_all:
5	type: string
6	analyzer: analyser_standard
7	properties:	4	properties:
8	marc_data:	5	marc_data:
9	store: true	6	store: true
Lines 26-56 search: Link Here
26	null_value: 0	23	null_value: 0
27	stdno:	24	stdno:
28	type: text	25	type: text
29	analyzer: analyser_stdno	26	analyzer: analyzer_stdno
30	search_analyzer: analyser_stdno	27	search_analyzer: analyzer_stdno
31	fields:	28	fields:
32	phrase:	29	phrase:
33	type: text	30	type: text
34	analyzer: analyser_stdno	31	analyzer: analyzer_phrase
35	search_analyzer: analyser_stdno	32	search_analyzer: analyzer_phrase
36	raw:	33	raw:
37	type: keyword	34	type: keyword
38	copy_to: _all
39	default:	35	default:
40	type: text	36	type: text
41	analyzer: analyser_standard	37	analyzer: analyzer_standard
42	search_analyzer: analyser_standard	38	search_analyzer: analyzer_standard
43	fields:	39	fields:
44	phrase:	40	phrase:
45	type: text	41	type: text
46	analyzer: analyser_phrase	42	analyzer: analyzer_phrase
47	search_analyzer: analyser_phrase	43	search_analyzer: analyzer_phrase
48	raw:	44	raw:
49	type: keyword	45	type: keyword
50	lc_raw:	46	normalizer: nfkc_cf_normalizer
		47	ci_raw:
51	type: keyword	48	type: keyword
52	normalizer: my_normalizer	49	normalizer: icu_folding_normalizer
53	copy_to: _all
54	# Facets	50	# Facets
55	facet:	51	facet:
56	default:	52	default:
Lines 64-72 suggestible: Link Here
64	# Sort	60	# Sort
65	sort:	61	sort:
66	default:	62	default:
67	type: text	63	type: icu_collation_keyword
68	analyzer: analyser_phrase	64	index: false
69	search_analyzer: analyser_phrase
70	fields:
71	phrase:
72	type: keyword

Lines 3-31 Link Here

(-)a/admin/searchengine/elasticsearch/index_config.yaml (-6 / +6 lines)
3	index:	3	index:
4	analysis:	4	analysis:
5	analyzer:	5	analyzer:
6	# Phrase analyzer is used for phrases (phrase match, sorting)	6	# Phrase analyzer is used for phrases (exact phrase match)
7	analyser_phrase:	7	analyzer_phrase:
8	tokenizer: keyword	8	tokenizer: keyword
9	filter:	9	filter:
10	- icu_folding	10	- icu_folding
11	char_filter:	11	char_filter:
12	- punctuation	12	- punctuation
13	analyser_standard:	13	analyzer_standard:
14	tokenizer: icu_tokenizer	14	tokenizer: icu_tokenizer
15	filter:	15	filter:
16	- icu_folding	16	- icu_folding
17	analyser_stdno:	17	analyzer_stdno:
18	tokenizer: whitespace	18	tokenizer: whitespace
19	filter:	19	filter:
20	- icu_folding	20	- icu_folding
21	char_filter:	21	char_filter:
22	- punctuation	22	- punctuation
23	normalizer:	23	normalizer:
24	normalizer_keyword:	24	icu_folding_normalizer:
25	type: custom	25	type: custom
26	filter:	26	filter:
27	- icu_folding	27	- icu_folding
28	my_normalizer:	28	nfkc_cf_normalizer:
29	type: custom	29	type: custom
30	char_filter: icu_normalizer	30	char_filter: icu_normalizer
31	char_filter:	31	char_filter:

Line 0 Link Here

(-)a/installer/data/mysql/atomicupdate/bug_20589_add_columns.perl (+14 lines)
	1	$DBversion = 'XXX';
	2	if (CheckVersion($DBversion)) {
	3	if (!column_exists('search_marc_to_field', 'search')) {
	4	$dbh->do("ALTER TABLE `search_marc_to_field` ADD COLUMN `search` tinyint(1) NOT NULL DEFAULT 1");
	5	}
	6	if (!column_exists('search_field', 'staff_client')) {
	7	$dbh->do("ALTER TABLE `search_field` ADD COLUMN `staff_client` tinyint(1) NOT NULL DEFAULT 1");
	8	}
	9	if (!column_exists('search_field', 'opac')) {
	10	$dbh->do("ALTER TABLE `search_field` ADD COLUMN `opac` tinyint(1) NOT NULL DEFAULT 1");
	11	}
	12	SetVersion($DBversion)
	13	}
	14	print "Upgrade to $DBversion done (Bug 20589 - Add field boosting and use elastic query fields parameter instead of depricated _all)\n";

Lines 549-567 if (C4::Context->preference('OpacSuppression')) { Link Here

(-)a/opac/opac-search.pl (-12 / +15 lines)
549	}	549	}
550	}	550	}
551		551
552	my $build_params = {
553	expanded_facet => $expanded_facet,
554	suppress => $suppress
555	};
556
557	unless ( $cgi->param('advsearch') ) {
558	$build_params->{weighted_fields} = 1;
559	}
560
561	## I. BUILD THE QUERY	552	## I. BUILD THE QUERY
562	( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type)	553	( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$query_type)
563	= $builder->build_query_compat( \@operators, \@operands,	554	= $builder->build_query_compat(
564	\@indexes, \@limits, \@sort_by, 0, $lang, $build_params);	555	\@operators,
		556	\@operands,
		557	\@indexes,
		558	\@limits,
		559	\@sort_by,
		560	0,
		561	$lang,
		562	{
		563	expanded_facet => $expanded_facet,
		564	suppress => $suppress,
		565	is_opac => 1,
		566	weighted_fields => !$cgi->param('advsearch')
		567	}
		568	);
565		569
566	sub _input_cgi_parse {	570	sub _input_cgi_parse {
567	my @elements;	571	my @elements;
568	-