Attachment #74373 for bug #20589

View | Details | Raw Unified | Return to bug 20589
Collapse All | Expand All

Lines 68-73 __PACKAGE__->add_columns( Link Here

(-)a/Koha/Schema/Result/SearchField.pm (+6 lines)
68	extra => { list => ["", "string", "date", "number", "boolean", "sum"] },	68	extra => { list => ["", "string", "date", "number", "boolean", "sum"] },
69	is_nullable => 0,	69	is_nullable => 0,
70	},	70	},
		71	"boost",
		72	{ data_type => "decimal", size => [ 5, 2 ], is_nullable => 1 },
		73	"staff_client",
		74	{ data_type => "tinyint", size => 1, is_nullable => 0, default => 1 },
		75	"opac",
		76	{ data_type => "tinyint", size => 1, is_nullable => 0, default => 1 },
71	);	77	);
72		78
73	=head1 PRIMARY KEY	79	=head1 PRIMARY KEY

Lines 71-76 __PACKAGE__->add_columns( Link Here

(-)a/Koha/Schema/Result/SearchMarcToField.pm (+2 lines)
71	{ data_type => "tinyint", default_value => 0, is_nullable => 1 },	71	{ data_type => "tinyint", default_value => 0, is_nullable => 1 },
72	"sort",	72	"sort",
73	{ data_type => "tinyint", is_nullable => 1 },	73	{ data_type => "tinyint", is_nullable => 1 },
		74	"search",
		75	{ data_type => "tinyint", size => 1, is_nullable => 0, default => 1 },
74	);	76	);
75		77
76	=head1 PRIMARY KEY	78	=head1 PRIMARY KEY




    my ($self) = @_;

    # TODO cache in the object?
    my $mappings = {};
        data => {
            _all => {type => "string", analyzer => "analyser_standard"},
            properties => {
                record => {
                    store          => "true",
                    include_in_all => JSON::false,
                    type           => "text",
                },
            }
        }
    };
    my %sort_fields;
    my $marcflavour = lc C4::Context->preference('marcflavour');
    $self->_foreach_mapping(
        sub {
            my ( $name, $type, $facet, $suggestible, $sort, $search, $marc_type ) = @_;
            return if $marc_type ne $marcflavour;
            # TODO if this gets any sort of complexity to it, it should
            # be broken out into its own function.

              ? 'boolean'
              : 'text';

            if ($search) {
                if ($es_type eq 'boolean') {
                    $mappings->{data}{properties}{$name} = _elasticsearch_mapping_for_boolean( $name, $es_type, $facet, $suggestible, $sort, $search, $marc_type );
                } else {
                    $mappings->{data}{properties}{$name} = _elasticsearch_mapping_for_default( $name, $es_type, $facet, $suggestible, $sort, $search, $marc_type );
                }
            }
            if ($es_type eq 'boolean') {
                $mappings->{data}{properties}{$name} = _elasticsearch_mapping_for_boolean( $name, $es_type, $facet, $suggestible, $sort, $marc_type );
                return; #Boolean cannot have facets nor sorting nor suggestions
            } else {
                $mappings->{data}{properties}{$name} = _elasticsearch_mapping_for_default( $name, $es_type, $facet, $suggestible, $sort, $marc_type );
            }

            if ($facet) {
                $mappings->{data}{properties}{ $name . '__facet' } = {
                    type  => "keyword",

                    search_analyzer => "analyser_phrase",
                    analyzer  => "analyser_phrase",
                    type            => "text",
                    include_in_all  => JSON::false,
                    fields          => {
                        phrase => {
                            type            => "keyword",

=cut

sub _elasticsearch_mapping_for_boolean {
    my ( $name, $type, $facet, $suggestible, $sort, $search, $marc_type ) = @_;

    return {
        type            => $type,

}

sub _elasticsearch_mapping_for_default {
    my ( $name, $type, $facet, $suggestible, $sort, $search, $marc_type ) = @_;

    return {
        search_analyzer => "analyser_standard",

        while ( my ( $field_name, $data ) = each %$fields ) {
            my $field_type = $data->{type};
            my $field_label = $data->{label};
            my $staff_client = exists $data->{staff_client} ? $data->{staff_client} : 1;
            my $opac = exists $data->{opac} ? $data->{opac} : 1;
            my $mappings = $data->{mappings};
            my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type, staff_client => $staff_client, opac => $opac }, { key => 'name' });
            for my $mapping ( @$mappings ) {
                my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} });
                $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet} || 0, suggestible => $mapping->{suggestible} || 0, sort => $mapping->{sort}, search => $mapping->{search} || 1 } );
            }
        }
    }


    $self->_foreach_mapping(
        sub {
            my ( $name, $type, $facet, $suggestible, $sort, $search, $marc_type, $marc_field ) = @_;
            return if $marc_type ne $marcflavour;
            my $options = '';


            # The split makes everything into nested arrays, but that's not
            # really a big deal, ES doesn't mind.
            $options = '-split => 1' unless $marc_field =~ m|_/| || $type eq 'sum';
            if ($search) {
                push @rules, "marc_map('$marc_field','${name}.\$append', $options)";
            }
            if ($facet) {
                push @rules, "marc_map('$marc_field','${name}__facet.\$append', $options)";
            }

            $search_field->get_column('facet'),
            $search_field->get_column('suggestible'),
            $search_field->get_column('sort'),
            $search_field->get_column('search'),
            $search_field->get_column('marc_type'),
            $search_field->get_column('marc_field'),
        );





use C4::Context;
use Koha::Exceptions;
use Koha::Caches;

=head2 build_query


            query            => $query,
            fuzziness        => $fuzzy_enabled ? 'auto' : '0',
            default_operator => 'AND',
            fields          => $self->_query_string_fields({ is_opac => $options{is_opac} }),
            lenient          => JSON::true,
        }
    };

    my %options;
    $options{sort} = \@sort_params;
    $options{expanded_facet} = $params->{expanded_facet};
    $options{is_opac} = exists $params->{is_opac} ? $params->{is_opac} : 0;
    my $query = $self->build_query( $query_str, %options );

    #die Dumper($query);

    my @filter_parts;
    foreach my $s ( @{ $search->{searches} } ) {
        my ( $wh, $op, $val ) = @{$s}{qw(where operator value)};
        if (!$wh) {
            my %query_string_params = (
                default_operator => 'AND',
            );
            if ( $op eq 'is' || $op eq '=' ) {
                push @query_parts, {
                    multi_match => {
                        query => $val,
                        fields => $self->_query_string_fields({ subfield => 'raw' }),
                    }
                };
            }
            elsif ( $op eq 'exact' ) {
                # left and right truncation, otherwise an exact phrase
                push @query_parts, {
                    query_string => {
                        query => qq("$val"),
                        fields => $self->_query_string_fields({ subfield => 'phrase' }),
                        %query_string_params,
                    }
                };
            }
            elsif ( $op eq 'start') {
                push @query_parts, {
                    query_string => {
                        query => "$val*",
                        $self->_query_string_fields({ subfield => 'phrase' }),
                        %query_string_params,
                    }
                };
            }
            else {
                push @query_parts, {
                    query_string => {
                        query => $val,
                        fields => $self->_query_string_fields(),
                        %query_string_params,
                    }
                };
            }
        }
        elsif ( $op eq 'is' || $op eq '=' ) {
            # look for something that matches completely
            # note, '=' is about numerical vals. May need special handling.
            # raw is a special subfield that only groups the exact
            # matches. Also, we lowercase our search because the ES
            # index lowercases its values, and term searches don't get the
            # search analyzer applied to them.
            push @query_parts, { term => { "$wh.raw" => lc $val } };
        }
        elsif ( $op eq 'exact' ) {


        }
    }

    # Merge the query and filter parts appropriately
    # 'should' behaves like 'or', if we want 'and', use 'must'
    my $query_part  = { bool => { should => \@query_parts } };
    my $filter_part = { bool => { should => \@filter_parts } };

    # We need to add '.phrase' to all the sort headings otherwise it'll sort
    # based on the tokenised form.
    # TODO: .phrase currently contains the tokenized form, so this will not work
    # instead .raw should be used?
    my %s;
    if ( exists $search->{sort} ) {
        foreach my $k ( keys %{ $search->{sort} } ) {

    # extract the sort stuff
    my %sort;
    %sort = ( sort => [ $search->{sort} ] ) if exists $search->{sort};

    # Merge the query and filter parts appropriately
    # 'should' behaves like 'or', if we want 'and', use 'must'
    my $elastic_query = {};
    if (@filter_parts) {
        $elastic_query->{bool}->{filter} = {
            bool => {
                must => \@filter_parts,
            }
        };
    else {
        $query = { query => $query_part };
    }
    if (@query_parts) {
        $elastic_query->{bool}->{should} = \@query_parts;
    };
    my $params = { query => $elastic_query };
    return { %$params, %sort };
}


=head2 build_authorities_query_compat

    my ($query) =

=cut

our %index_field_convert = (
    'kw'      => '',
    'ti'      => 'title',
    'au'      => 'author',
    'su'      => 'subject',

    # If a field starts with mc- we save it as it's used (and removed) later
    # when joining things, to indicate we make it an 'OR' join.
    # (Sorry, this got a bit ugly after special cases were found.)
    map {
        my ( $f, $t ) = split /,/;
        my $mc = '';
        if ($f =~ /^mc-/) {

            type  => $index_type_convert{ $t // '__default' }
        };
        $r->{field} = ($mc . $r->{field}) if $mc && $r->{field};
        $r->{field} ? $r : undef;
    } @indexes;
}


            push @res, $s;
            next;
        }
        push @res, ($conv->{field} ? $conv->{field} . ':' : '')
            . $self->_modify_string_by_type( %$conv, operand => $term );
    }
    return @res;
}

    return join ' ', @terms;
}

sub _query_string_fields {
    my ($self, $params) = @_;
    $params //= {
        is_opac => 0,
        # This is a terrible hack for authorities build_authorities_query
        # can hopefully be removed in the future
        subfield => undef,
    };

    my $cache = Koha::Caches->get_instance();
    my $cache_key = 'elasticsearch_search_fields' . ($params->{is_opac} ? '_opac' : '_staff_client');
    my $search_fields = $cache->get_from_cache($cache_key, { unsafe => 1 });

    if (!$search_fields) {
        my $schema = Koha::Database->schema;
        my $result = $schema->resultset('SearchField')->search(
            {
                $params->{is_opac} ? (
                    'opac' => 1,
                ) : (
                    'staff_client' => 1
                ),
                'search_marc_map.index_name' => $self->index,
                'search_marc_map.marc_type' => C4::Context->preference('marcflavour'),
                'search_marc_to_fields.search' => 1,
            },
            {
                collapse => 1,
                join => {search_marc_to_fields => 'search_marc_map'},
            }
        );
        my @search_fields;
        while (my $search_field = $result->next) {
            push @search_fields, $search_field->name .
                ($search_field->boost ? '^' . $search_field->boost : '');
        }
        $search_fields = \@search_fields;
        $cache->set_in_cache($cache_key, $search_fields);
    }
    if ($params->{subfield}) {
        my $subfield = $params->{subfield};
        $search_fields = [
            map {
                my ($field, $boost) = $_ =~ /^([^\^]+)(?:\^(.+))?$/;
                "${field}.${subfield}" . ($boost ? "^$boost" : '');
            } @{$search_fields}
        ];
    }
    return $search_fields;
}

1;

Lines 16-21 Link Here

(-)a/admin/searchengine/elasticsearch/mappings.pl (-15 / +59 lines)
16	# along with Koha; if not, see <http://www.gnu.org/licenses>.	16	# along with Koha; if not, see <http://www.gnu.org/licenses>.
17		17
18	use Modern::Perl;	18	use Modern::Perl;
		19	use Scalar::Util qw(looks_like_number);
19	use CGI;	20	use CGI;
20	use C4::Koha;	21	use C4::Koha;
21	use C4::Output;	22	use C4::Output;
Lines 24-29 use C4::Auth; Link Here
24	use Koha::SearchEngine::Elasticsearch;	25	use Koha::SearchEngine::Elasticsearch;
25	use Koha::SearchMarcMaps;	26	use Koha::SearchMarcMaps;
26	use Koha::SearchFields;	27	use Koha::SearchFields;
		28	use Koha::Caches;
27		29
28	my $input = new CGI;	30	my $input = new CGI;
29	my ( $template, $borrowernumber, $cookie ) = get_template_and_user(	31	my ( $template, $borrowernumber, $cookie ) = get_template_and_user(
Lines 44-49 my $schema = $database->schema; Link Here
44		46
45	my $marc_type = lc C4::Context->preference('marcflavour');	47	my $marc_type = lc C4::Context->preference('marcflavour');
46		48
		49	my $cache = Koha::Caches->get_instance();
		50	my $clear_cache = sub {
		51	$cache->clear_from_cache('elasticsearch_search_fields_staff_client');
		52	$cache->clear_from_cache('elasticsearch_search_fields_opac');
		53	};
		54
47	if ( $op eq 'edit' ) {	55	if ( $op eq 'edit' ) {
48		56
49	$schema->storage->txn_begin;	57	$schema->storage->txn_begin;
Lines 51-62 if ( $op eq 'edit' ) { Link Here
51	my @field_name = $input->param('search_field_name');	59	my @field_name = $input->param('search_field_name');
52	my @field_label = $input->param('search_field_label');	60	my @field_label = $input->param('search_field_label');
53	my @field_type = $input->param('search_field_type');	61	my @field_type = $input->param('search_field_type');
		62	my @field_boost = $input->param('search_field_boost');
		63	my @field_staff_client = $input->param('search_field_staff_client');
		64	my @field_opac = $input->param('search_field_opac');
54		65
55	my @index_name = $input->param('mapping_index_name');	66	my @index_name = $input->param('mapping_index_name');
56	my @search_field_name = $input->param('mapping_search_field_name');	67	my @search_field_name = $input->param('mapping_search_field_name');
57	my @mapping_sort = $input->param('mapping_sort');	68	my @mapping_sort = $input->param('mapping_sort');
58	my @mapping_facet = $input->param('mapping_facet');	69	my @mapping_facet = $input->param('mapping_facet');
59	my @mapping_suggestible = $input->param('mapping_suggestible');	70	my @mapping_suggestible = $input->param('mapping_suggestible');
		71	my @mapping_search = $input->param('mapping_search');
60	my @mapping_marc_field = $input->param('mapping_marc_field');	72	my @mapping_marc_field = $input->param('mapping_marc_field');
61		73
62	eval {	74	eval {
Lines 65-73 if ( $op eq 'edit' ) { Link Here
65	my $field_name = $field_name[$i];	77	my $field_name = $field_name[$i];
66	my $field_label = $field_label[$i];	78	my $field_label = $field_label[$i];
67	my $field_type = $field_type[$i];	79	my $field_type = $field_type[$i];
		80	my $field_boost = $field_boost[$i];
		81	my $field_staff_client = $field_staff_client[$i];
		82	my $field_opac = $field_opac[$i];
68	my $search_field = Koha::SearchFields->find( { name => $field_name }, { key => 'name' } );	83	my $search_field = Koha::SearchFields->find( { name => $field_name }, { key => 'name' } );
69	$search_field->label($field_label);	84	$search_field->label($field_label);
70	$search_field->type($field_type);	85	$search_field->type($field_type);
		86	$search_field->boost($field_boost) if looks_like_number($field_boost);
		87	$search_field->staff_client($field_staff_client ? 1 : 0);
		88	$search_field->opac($field_opac ? 1 : 0);
71	$search_field->store;	89	$search_field->store;
72	}	90	}
73		91
Lines 75-92 if ( $op eq 'edit' ) { Link Here
75		93
76	for my $i ( 0 .. scalar(@index_name) - 1 ) {	94	for my $i ( 0 .. scalar(@index_name) - 1 ) {
77	my $index_name = $index_name[$i];	95	my $index_name = $index_name[$i];
78	my $search_field_name = $search_field_name[$i];	96	my $search_field_name = $search_field_name[$i];
79	my $mapping_marc_field = $mapping_marc_field[$i];	97	my $mapping_marc_field = $mapping_marc_field[$i];
80	my $mapping_facet = $mapping_facet[$i];	98	my $mapping_facet = $mapping_facet[$i];
81	my $mapping_suggestible = $mapping_suggestible[$i];	99	my $mapping_suggestible = $mapping_suggestible[$i];
82	my $mapping_sort = $mapping_sort[$i];	100	my $mapping_sort = $mapping_sort[$i] eq 'undef' ? undef : $mapping_sort[$i];
83	$mapping_sort = undef if $mapping_sort eq 'undef';	101	my $mapping_search = $mapping_search[$i];
84		102
85	my $search_field = Koha::SearchFields->find({ name => $search_field_name }, { key => 'name' });	103	my $search_field = Koha::SearchFields->find({ name => $search_field_name }, { key => 'name' });
86	# TODO Check mapping format	104	# TODO Check mapping format
87	my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $marc_type, marc_field => $mapping_marc_field });	105	my $marc_field = Koha::SearchMarcMaps->find_or_create({
88	$search_field->add_to_search_marc_maps($marc_field, { facet => $mapping_facet, suggestible => $mapping_suggestible, sort => $mapping_sort } );	106	index_name => $index_name,
89		107	marc_type => $marc_type,
		108	marc_field => $mapping_marc_field
		109	});
		110	$search_field->add_to_search_marc_maps($marc_field, {
		111	facet => $mapping_facet,
		112	suggestible => $mapping_suggestible,
		113	sort => $mapping_sort,
		114	search => $mapping_search,
		115	});
90	}	116	}
91	};	117	};
92	if ($@) {	118	if ($@) {
Lines 96-101 if ( $op eq 'edit' ) { Link Here
96	push @messages, { type => 'message', code => 'success_on_update' };	122	push @messages, { type => 'message', code => 'success_on_update' };
97	$schema->storage->txn_commit;	123	$schema->storage->txn_commit;
98	}	124	}
		125	$clear_cache->();
99	}	126	}
100	elsif( $op eq 'reset' ) {	127	elsif( $op eq 'reset' ) {
101	# TODO Move this feature to the interface	128	# TODO Move this feature to the interface
Lines 104-135 elsif( $op eq 'reset' ) { Link Here
104	Koha::SearchMarcMaps->search->delete;	131	Koha::SearchMarcMaps->search->delete;
105	Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;	132	Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;
106	}	133	}
		134	$clear_cache->();
107	}	135	}
108		136
109	my @indexes;	137	my @indexes;
110		138
111	for my $index_name (qw\| biblios authorities \|) {	139	for my $index_name (qw\| biblios authorities \|) {
112	my $search_fields = Koha::SearchFields->search(	140	my $search_fields = Koha::SearchFields->search(
113	{ 'search_marc_map.index_name' => $index_name, 'search_marc_map.marc_type' => $marc_type, },	141	{
114	{ join => { search_marc_to_fields => 'search_marc_map' },	142	'search_marc_map.index_name' => $index_name,
115	'+select' => [ 'search_marc_to_fields.facet', 'search_marc_to_fields.suggestible', 'search_marc_to_fields.sort', 'search_marc_map.marc_field' ],	143	'search_marc_map.marc_type' => $marc_type,
116	'+as' => [ 'facet', 'suggestible', 'sort', 'marc_field' ],	144	},
		145	{
		146	join => { search_marc_to_fields => 'search_marc_map' },
		147	'+select' => [
		148	'search_marc_to_fields.facet',
		149	'search_marc_to_fields.suggestible',
		150	'search_marc_to_fields.sort',
		151	'search_marc_to_fields.search',
		152	'search_marc_map.marc_field',
		153	],
		154	'+as' => [
		155	'facet',
		156	'suggestible',
		157	'sort',
		158	'search',
		159	'marc_field',
		160	],
117	}	161	}
118	);	162	);
119		163
120	my @mappings;	164	my @mappings;
121	while ( my $s = $search_fields->next ) {	165	while ( my $s = $search_fields->next ) {
122	push @mappings,	166	push @mappings, {
123	{ search_field_name => $s->name,	167	search_field_name => $s->name,
124	search_field_label => $s->label,	168	search_field_label => $s->label,
125	search_field_type => $s->type,	169	search_field_type => $s->type,
126	marc_field => $s->get_column('marc_field'),	170	marc_field => $s->get_column('marc_field'),
127	sort => $s->get_column('sort') // 'undef', # To avoid warnings "Use of uninitialized value in lc"	171	sort => $s->get_column('sort') // 'undef', # To avoid warnings "Use of uninitialized value in lc"
128	suggestible => $s->get_column('suggestible'),	172	suggestible => $s->get_column('suggestible'),
129	facet => $s->get_column('facet'),	173	facet => $s->get_column('facet'),
130	};	174	search => $s->get_column('search'),
		175	};
131	}	176	}
132
133	push @indexes, { index_name => $index_name, mappings => \@mappings };	177	push @indexes, { index_name => $index_name, mappings => \@mappings };
134	}	178	}
135		179




$DBversion = 'XXX';
if (CheckVersion($DBversion)) {
  if (!column_exists('search_marc_to_field', 'search')) {
    $dbh->do("ALTER TABLE `search_marc_to_field` ADD COLUMN `search` tinyint(1) NOT NULL DEFAULT 1");
  }
  if (!column_exists('search_field', 'boost')) {
    $dbh->do("ALTER TABLE `search_field` ADD COLUMN `boost` decimal(5,2) DEFAULT NULL");
  }
  if (!column_exists('search_field', 'staff_client')) {
    $dbh->do("ALTER TABLE `search_field` ADD COLUMN `staff_client` tinyint(1) NOT NULL DEFAULT 1");
  }
  if (!column_exists('search_field', 'opac')) {
    $dbh->do("ALTER TABLE `search_field` ADD COLUMN `opac` tinyint(1) NOT NULL DEFAULT 1");
  }
  SetVersion($DBversion)
}
print "Upgrade to $DBversion done (Bug 20589 - Add field boosting and use elastic query fields parameter instead of depricated _all)\n";

Lines 126-131 a.add, a.delete { Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt (-1 / +59 lines)
126	<th>Name</th>	126	<th>Name</th>
127	<th>Label</th>	127	<th>Label</th>
128	<th>Type</th>	128	<th>Type</th>
		129	<th colspan="2">Searchable</th>
		130	<th>Boost</th>
		131	</tr>
		132	<tr>
		133	<th colspan=3> </th>
		134	<th>Staff client</th>
		135	<th>OPAC</th>
		136	<th> </th>
129	</tr>	137	</tr>
130	</thead>	138	</thead>
131	<tbody>	139	<tbody>
Lines 134-140 a.add, a.delete { Link Here
134	<td>	142	<td>
135	<input type="text" name="search_field_name" value="[% search_field.name %]" />	143	<input type="text" name="search_field_name" value="[% search_field.name %]" />
136	</td>	144	</td>
137	<td><input type="text" name="search_field_label" value="[% search_field.label %]" />	145	<td>
		146	<input type="text" name="search_field_label" value="[% search_field.label %]" />
		147	</td>
138	<td>	148	<td>
139	<select name="search_field_type">	149	<select name="search_field_type">
140	<option value=""></option>	150	<option value=""></option>
Lines 165-170 a.add, a.delete { Link Here
165	[% END %]	175	[% END %]
166	</select>	176	</select>
167	</td>	177	</td>
		178	<td>
		179	<select name="search_field_staff_client">
		180	[% IF search_field.staff_client %]
		181	<option value="1" selected="selected">Yes</option>
		182	<option value="0">No</option>
		183	[% ELSE %]
		184	<option value="1">Yes</option>
		185	<option value="0" selected="selected">No</option>
		186	[% END %]
		187	</select>
		188	</td>
		189	<td>
		190	<select name="search_field_opac">
		191	[% IF search_field.opac %]
		192	<option value="1" selected="selected">Yes</option>
		193	<option value="0">No</option>
		194	[% ELSE %]
		195	<option value="1">Yes</option>
		196	<option value="0" selected="selected">No</option>
		197	[% END %]
		198	</select>
		199	</td>
		200	<td>
		201	<input type="text" size="3" name="search_field_boost" value="[% search_field.boost %]" />
		202	</td>
168	</tr>	203	</tr>
169	[% END %]	204	[% END %]
170	</tbody>	205	</tbody>
Lines 179-184 a.add, a.delete { Link Here
179	<th>Sortable</th>	214	<th>Sortable</th>
180	<th>Facetable</th>	215	<th>Facetable</th>
181	<th>Suggestible</th>	216	<th>Suggestible</th>
		217	<th>Searchable</th>
182	<th>Mapping</th>	218	<th>Mapping</th>
183	<th></th>	219	<th></th>
184	</tr>	220	</tr>
Lines 233-238 a.add, a.delete { Link Here
233	</select>	269	</select>
234	</td>	270	</td>
235	<td>	271	<td>
		272	<select name="mapping_search">
		273	[% IF mapping.search %]
		274	<option value="0">No</option>
		275	<option value="1" selected="selected">Yes</option>
		276	[% ELSE %]
		277	<option value="0" selected="selected">No</option>
		278	<option value="1">Yes</option>
		279	[% END %]
		280	</select>
		281	</td>
		282	<td>
236	<input name="mapping_marc_field" type="text" value="[% mapping.marc_field %]" />	283	<input name="mapping_marc_field" type="text" value="[% mapping.marc_field %]" />
237	</td>	284	</td>
238	<td><a class="btn btn-default btn-xs delete" style="cursor: pointer;"><i class="fa fa-trash"></i> Delete</a></td>	285	<td><a class="btn btn-default btn-xs delete" style="cursor: pointer;"><i class="fa fa-trash"></i> Delete</a></td>
Lines 278-283 a.add, a.delete { Link Here
278	[% END %]	325	[% END %]
279	</select>	326	</select>
280	</td>	327	</td>
		328	<td>
		329	<select data-id="mapping_search">
		330	[% IF mapping.search %]
		331	<option value="0">No</option>
		332	<option value="1" selected="selected">Yes</option>
		333	[% ELSE %]
		334	<option value="0" selected="selected">No</option>
		335	<option value="1">Yes</option>
		336	[% END %]
		337	</select>
		338	</td>
281	<td><input data-id="mapping_marc_field" type="text" /></td>	339	<td><input data-id="mapping_marc_field" type="text" /></td>
282	<td><a class="btn btn-default btn-xs add"><i class="fa fa-plus"></i> Add</a></td>	340	<td><a class="btn btn-default btn-xs add"><i class="fa fa-plus"></i> Add</a></td>
283	</tr>	341	</tr>

Lines 555-561 if (C4::Context->preference('OpacSuppression')) { Link Here

(-)a/opac/opac-search.pl (-2 / +1 lines)
555	\@sort_by,	555	\@sort_by,
556	0,	556	0,
557	$lang,	557	$lang,
558	{ expanded_facet => $expanded_facet, suppress => $suppress }	558	{ expanded_facet => $expanded_facet, suppress => $suppress, is_opac => 1 }
559	);	559	);
560		560
561	sub _input_cgi_parse {	561	sub _input_cgi_parse {
562	-

Return to bug 20589

Lines 171-193 sub get_elasticsearch_mappings { Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (-24 / +20 lines)
171	my ($self) = @_;	171	my ($self) = @_;
172		172
173	# TODO cache in the object?	173	# TODO cache in the object?
174	my $mappings = {	174	my $mappings = {};
175	data => {
176	_all => {type => "string", analyzer => "analyser_standard"},
177	properties => {
178	record => {
179	store => "true",
180	include_in_all => JSON::false,
181	type => "text",
182	},
183	}
184	}
185	};
186	my %sort_fields;	175	my %sort_fields;
187	my $marcflavour = lc C4::Context->preference('marcflavour');	176	my $marcflavour = lc C4::Context->preference('marcflavour');
188	$self->_foreach_mapping(	177	$self->_foreach_mapping(
189	sub {	178	sub {
190	my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;	179	my ( $name, $type, $facet, $suggestible, $sort, $search, $marc_type ) = @_;
191	return if $marc_type ne $marcflavour;	180	return if $marc_type ne $marcflavour;
192	# TODO if this gets any sort of complexity to it, it should	181	# TODO if this gets any sort of complexity to it, it should
193	# be broken out into its own function.	182	# be broken out into its own function.
Lines 199-211 sub get_elasticsearch_mappings { Link Here
199	? 'boolean'	188	? 'boolean'
200	: 'text';	189	: 'text';
201		190
		191	if ($search) {
		192	if ($es_type eq 'boolean') {
		193	$mappings->{data}{properties}{$name} = _elasticsearch_mapping_for_boolean( $name, $es_type, $facet, $suggestible, $sort, $search, $marc_type );
		194	} else {
		195	$mappings->{data}{properties}{$name} = _elasticsearch_mapping_for_default( $name, $es_type, $facet, $suggestible, $sort, $search, $marc_type );
		196	}
		197	}
202	if ($es_type eq 'boolean') {	198	if ($es_type eq 'boolean') {
203	$mappings->{data}{properties}{$name} = _elasticsearch_mapping_for_boolean( $name, $es_type, $facet, $suggestible, $sort, $marc_type );
204	return; #Boolean cannot have facets nor sorting nor suggestions	199	return; #Boolean cannot have facets nor sorting nor suggestions
205	} else {
206	$mappings->{data}{properties}{$name} = _elasticsearch_mapping_for_default( $name, $es_type, $facet, $suggestible, $sort, $marc_type );
207	}	200	}
208
209	if ($facet) {	201	if ($facet) {
210	$mappings->{data}{properties}{ $name . '__facet' } = {	202	$mappings->{data}{properties}{ $name . '__facet' } = {
211	type => "keyword",	203	type => "keyword",
Lines 226-232 sub get_elasticsearch_mappings { Link Here
226	search_analyzer => "analyser_phrase",	218	search_analyzer => "analyser_phrase",
227	analyzer => "analyser_phrase",	219	analyzer => "analyser_phrase",
228	type => "text",	220	type => "text",
229	include_in_all => JSON::false,
230	fields => {	221	fields => {
231	phrase => {	222	phrase => {
232	type => "keyword",	223	type => "keyword",
Lines 250-256 Receives the same parameters from the $self->_foreach_mapping() dispatcher Link Here
250	=cut	241	=cut
251		242
252	sub _elasticsearch_mapping_for_boolean {	243	sub _elasticsearch_mapping_for_boolean {
253	my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;	244	my ( $name, $type, $facet, $suggestible, $sort, $search, $marc_type ) = @_;
254		245
255	return {	246	return {
256	type => $type,	247	type => $type,
Lines 259-265 sub _elasticsearch_mapping_for_boolean { Link Here
259	}	250	}
260		251
261	sub _elasticsearch_mapping_for_default {	252	sub _elasticsearch_mapping_for_default {
262	my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;	253	my ( $name, $type, $facet, $suggestible, $sort, $search, $marc_type ) = @_;
263		254
264	return {	255	return {
265	search_analyzer => "analyser_standard",	256	search_analyzer => "analyser_standard",
Lines 286-296 sub reset_elasticsearch_mappings { Link Here
286	while ( my ( $field_name, $data ) = each %$fields ) {	277	while ( my ( $field_name, $data ) = each %$fields ) {
287	my $field_type = $data->{type};	278	my $field_type = $data->{type};
288	my $field_label = $data->{label};	279	my $field_label = $data->{label};
		280	my $staff_client = exists $data->{staff_client} ? $data->{staff_client} : 1;
		281	my $opac = exists $data->{opac} ? $data->{opac} : 1;
289	my $mappings = $data->{mappings};	282	my $mappings = $data->{mappings};
290	my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type }, { key => 'name' });	283	my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type, staff_client => $staff_client, opac => $opac }, { key => 'name' });
291	for my $mapping ( @$mappings ) {	284	for my $mapping ( @$mappings ) {
292	my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} });	285	my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} });
293	$search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet} \|\| 0, suggestible => $mapping->{suggestible} \|\| 0, sort => $mapping->{sort} } );	286	$search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet} \|\| 0, suggestible => $mapping->{suggestible} \|\| 0, sort => $mapping->{sort}, search => $mapping->{search} \|\| 1 } );
294	}	287	}
295	}	288	}
296	}	289	}
Lines 321-327 sub get_fixer_rules { Link Here
321		314
322	$self->_foreach_mapping(	315	$self->_foreach_mapping(
323	sub {	316	sub {
324	my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_;	317	my ( $name, $type, $facet, $suggestible, $sort, $search, $marc_type, $marc_field ) = @_;
325	return if $marc_type ne $marcflavour;	318	return if $marc_type ne $marcflavour;
326	my $options = '';	319	my $options = '';
327		320
Lines 330-336 sub get_fixer_rules { Link Here
330	# The split makes everything into nested arrays, but that's not	323	# The split makes everything into nested arrays, but that's not
331	# really a big deal, ES doesn't mind.	324	# really a big deal, ES doesn't mind.
332	$options = '-split => 1' unless $marc_field =~ m\|_/\| \|\| $type eq 'sum';	325	$options = '-split => 1' unless $marc_field =~ m\|_/\| \|\| $type eq 'sum';
333	push @rules, "marc_map('$marc_field','${name}.\$append', $options)";	326	if ($search) {
		327	push @rules, "marc_map('$marc_field','${name}.\$append', $options)";
		328	}
334	if ($facet) {	329	if ($facet) {
335	push @rules, "marc_map('$marc_field','${name}__facet.\$append', $options)";	330	push @rules, "marc_map('$marc_field','${name}__facet.\$append', $options)";
336	}	331	}
Lines 448-453 sub _foreach_mapping { Link Here
448	$search_field->get_column('facet'),	443	$search_field->get_column('facet'),
449	$search_field->get_column('suggestible'),	444	$search_field->get_column('suggestible'),
450	$search_field->get_column('sort'),	445	$search_field->get_column('sort'),
		446	$search_field->get_column('search'),
451	$search_field->get_column('marc_type'),	447	$search_field->get_column('marc_type'),
452	$search_field->get_column('marc_field'),	448	$search_field->get_column('marc_field'),
453	);	449	);

Lines 48-53 use URI::Escape; Link Here

(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-27 / +119 lines)
48		48
49	use C4::Context;	49	use C4::Context;
50	use Koha::Exceptions;	50	use Koha::Exceptions;
		51	use Koha::Caches;
51		52
52	=head2 build_query	53	=head2 build_query
53		54
Lines 90-96 sub build_query { Link Here
90	query => $query,	91	query => $query,
91	fuzziness => $fuzzy_enabled ? 'auto' : '0',	92	fuzziness => $fuzzy_enabled ? 'auto' : '0',
92	default_operator => 'AND',	93	default_operator => 'AND',
93	default_field => '_all',	94	fields => $self->_query_string_fields({ is_opac => $options{is_opac} }),
94	lenient => JSON::true,	95	lenient => JSON::true,
95	}	96	}
96	};	97	};
Lines 233-238 sub build_query_compat { Link Here
233	my %options;	234	my %options;
234	$options{sort} = \@sort_params;	235	$options{sort} = \@sort_params;
235	$options{expanded_facet} = $params->{expanded_facet};	236	$options{expanded_facet} = $params->{expanded_facet};
		237	$options{is_opac} = exists $params->{is_opac} ? $params->{is_opac} : 0;
236	my $query = $self->build_query( $query_str, %options );	238	my $query = $self->build_query( $query_str, %options );
237		239
238	#die Dumper($query);	240	#die Dumper($query);
Lines 293-308 sub build_authorities_query { Link Here
293	my @filter_parts;	295	my @filter_parts;
294	foreach my $s ( @{ $search->{searches} } ) {	296	foreach my $s ( @{ $search->{searches} } ) {
295	my ( $wh, $op, $val ) = @{$s}{qw(where operator value)};	297	my ( $wh, $op, $val ) = @{$s}{qw(where operator value)};
296	$wh = '_all' if $wh eq '';	298	if (!$wh) {
297	if ( $op eq 'is' \|\| $op eq '=' ) {	299	my %query_string_params = (
298		300	default_operator => 'AND',
		301	);
		302	if ( $op eq 'is' \|\| $op eq '=' ) {
		303	push @query_parts, {
		304	multi_match => {
		305	query => $val,
		306	fields => $self->_query_string_fields({ subfield => 'raw' }),
		307	}
		308	};
		309	}
		310	elsif ( $op eq 'exact' ) {
		311	# left and right truncation, otherwise an exact phrase
		312	push @query_parts, {
		313	query_string => {
		314	query => qq("$val"),
		315	fields => $self->_query_string_fields({ subfield => 'phrase' }),
		316	%query_string_params,
		317	}
		318	};
		319	}
		320	elsif ( $op eq 'start') {
		321	push @query_parts, {
		322	query_string => {
		323	query => "$val*",
		324	$self->_query_string_fields({ subfield => 'phrase' }),
		325	%query_string_params,
		326	}
		327	};
		328	}
		329	else {
		330	push @query_parts, {
		331	query_string => {
		332	query => $val,
		333	fields => $self->_query_string_fields(),
		334	%query_string_params,
		335	}
		336	};
		337	}
		338	}
		339	elsif ( $op eq 'is' \|\| $op eq '=' ) {
299	# look for something that matches completely	340	# look for something that matches completely
300	# note, '=' is about numerical vals. May need special handling.	341	# note, '=' is about numerical vals. May need special handling.
301	# _allphrase is a special field that only groups the exact	342	# raw is a special subfield that only groups the exact
302	# matches. Also, we lowercase our search because the ES	343	# matches. Also, we lowercase our search because the ES
303	# index lowercases its values, and term searches don't get the	344	# index lowercases its values, and term searches don't get the
304	# search analyzer applied to them.	345	# search analyzer applied to them.
305	push @filter_parts, { term => { "$wh.phrase" => lc $val } };	346	push @query_parts, { term => { "$wh.raw" => lc $val } };
306	}	347	}
307	elsif ( $op eq 'exact' ) {	348	elsif ( $op eq 'exact' ) {
308		349
Lines 320-332 sub build_authorities_query { Link Here
320	}	361	}
321	}	362	}
322		363
323	# Merge the query and filter parts appropriately
324	# 'should' behaves like 'or', if we want 'and', use 'must'
325	my $query_part = { bool => { should => \@query_parts } };
326	my $filter_part = { bool => { should => \@filter_parts } };
327
328	# We need to add '.phrase' to all the sort headings otherwise it'll sort	364	# We need to add '.phrase' to all the sort headings otherwise it'll sort
329	# based on the tokenised form.	365	# based on the tokenised form.
		366	# TODO: .phrase currently contains the tokenized form, so this will not work
		367	# instead .raw should be used?
330	my %s;	368	my %s;
331	if ( exists $search->{sort} ) {	369	if ( exists $search->{sort} ) {
332	foreach my $k ( keys %{ $search->{sort} } ) {	370	foreach my $k ( keys %{ $search->{sort} } ) {
Lines 339-359 sub build_authorities_query { Link Here
339	# extract the sort stuff	377	# extract the sort stuff
340	my %sort;	378	my %sort;
341	%sort = ( sort => [ $search->{sort} ] ) if exists $search->{sort};	379	%sort = ( sort => [ $search->{sort} ] ) if exists $search->{sort};
342	my $query;	380
		381	# Merge the query and filter parts appropriately
		382	# 'should' behaves like 'or', if we want 'and', use 'must'
		383	my $elastic_query = {};
343	if (@filter_parts) {	384	if (@filter_parts) {
344	$query =	385	$elastic_query->{bool}->{filter} = {
345	{ query =>	386	bool => {
346	{ filtered => { filter => $filter_part, query => $query_part } }	387	must => \@filter_parts,
347	};	388	}
348	}	389	};
349	else {
350	$query = { query => $query_part };
351	}	390	}
352	$query = { %$query, %sort };	391	if (@query_parts) {
353	return $query;	392	$elastic_query->{bool}->{should} = \@query_parts;
		393	};
		394	my $params = { query => $elastic_query };
		395	return { %$params, %sort };
354	}	396	}
355		397
356
357	=head2 build_authorities_query_compat	398	=head2 build_authorities_query_compat
358		399
359	my ($query) =	400	my ($query) =
Lines 513-519 types. Link Here
513	=cut	554	=cut
514		555
515	our %index_field_convert = (	556	our %index_field_convert = (
516	'kw' => '_all',	557	'kw' => '',
517	'ti' => 'title',	558	'ti' => 'title',
518	'au' => 'author',	559	'au' => 'author',
519	'su' => 'subject',	560	'su' => 'subject',
Lines 539-545 sub _convert_index_fields { Link Here
539	# If a field starts with mc- we save it as it's used (and removed) later	580	# If a field starts with mc- we save it as it's used (and removed) later
540	# when joining things, to indicate we make it an 'OR' join.	581	# when joining things, to indicate we make it an 'OR' join.
541	# (Sorry, this got a bit ugly after special cases were found.)	582	# (Sorry, this got a bit ugly after special cases were found.)
542	grep { $_->{field} } map {	583	map {
543	my ( $f, $t ) = split /,/;	584	my ( $f, $t ) = split /,/;
544	my $mc = '';	585	my $mc = '';
545	if ($f =~ /^mc-/) {	586	if ($f =~ /^mc-/) {
Lines 551-557 sub _convert_index_fields { Link Here
551	type => $index_type_convert{ $t // '__default' }	592	type => $index_type_convert{ $t // '__default' }
552	};	593	};
553	$r->{field} = ($mc . $r->{field}) if $mc && $r->{field};	594	$r->{field} = ($mc . $r->{field}) if $mc && $r->{field};
554	$r;	595	$r->{field} ? $r : undef;
555	} @indexes;	596	} @indexes;
556	}	597	}
557		598
Lines 580-587 sub _convert_index_strings { Link Here
580	push @res, $s;	621	push @res, $s;
581	next;	622	next;
582	}	623	}
583	push @res, $conv->{field} . ":"	624	push @res, ($conv->{field} ? $conv->{field} . ':' : '')
584	. $self->_modify_string_by_type( %$conv, operand => $term );	625	. $self->_modify_string_by_type( %$conv, operand => $term );
585	}	626	}
586	return @res;	627	return @res;
587	}	628	}
Lines 813-816 sub _truncate_terms { Link Here
813	return join ' ', @terms;	854	return join ' ', @terms;
814	}	855	}
815		856
		857	sub _query_string_fields {
		858	my ($self, $params) = @_;
		859	$params //= {
		860	is_opac => 0,
		861	# This is a terrible hack for authorities build_authorities_query
		862	# can hopefully be removed in the future
		863	subfield => undef,
		864	};
		865
		866	my $cache = Koha::Caches->get_instance();
		867	my $cache_key = 'elasticsearch_search_fields' . ($params->{is_opac} ? '_opac' : '_staff_client');
		868	my $search_fields = $cache->get_from_cache($cache_key, { unsafe => 1 });
		869
		870	if (!$search_fields) {
		871	my $schema = Koha::Database->schema;
		872	my $result = $schema->resultset('SearchField')->search(
		873	{
		874	$params->{is_opac} ? (
		875	'opac' => 1,
		876	) : (
		877	'staff_client' => 1
		878	),
		879	'search_marc_map.index_name' => $self->index,
		880	'search_marc_map.marc_type' => C4::Context->preference('marcflavour'),
		881	'search_marc_to_fields.search' => 1,
		882	},
		883	{
		884	collapse => 1,
		885	join => {search_marc_to_fields => 'search_marc_map'},
		886	}
		887	);
		888	my @search_fields;
		889	while (my $search_field = $result->next) {
		890	push @search_fields, $search_field->name .
		891	($search_field->boost ? '^' . $search_field->boost : '');
		892	}
		893	$search_fields = \@search_fields;
		894	$cache->set_in_cache($cache_key, $search_fields);
		895	}
		896	if ($params->{subfield}) {
		897	my $subfield = $params->{subfield};
		898	$search_fields = [
		899	map {
		900	my ($field, $boost) = $_ =~ /^([^\^]+)(?:\^(.+))?$/;
		901	"${field}.${subfield}" . ($boost ? "^$boost" : '');
		902	} @{$search_fields}
		903	];
		904	}
		905	return $search_fields;
		906	}
		907
816	1;	908	1;

Line 0 Link Here

(-)a/installer/data/mysql/atomicupdate/bug_20589_add_columns.perl (+17 lines)
	1	$DBversion = 'XXX';
	2	if (CheckVersion($DBversion)) {
	3	if (!column_exists('search_marc_to_field', 'search')) {
	4	$dbh->do("ALTER TABLE `search_marc_to_field` ADD COLUMN `search` tinyint(1) NOT NULL DEFAULT 1");
	5	}
	6	if (!column_exists('search_field', 'boost')) {
	7	$dbh->do("ALTER TABLE `search_field` ADD COLUMN `boost` decimal(5,2) DEFAULT NULL");
	8	}
	9	if (!column_exists('search_field', 'staff_client')) {
	10	$dbh->do("ALTER TABLE `search_field` ADD COLUMN `staff_client` tinyint(1) NOT NULL DEFAULT 1");
	11	}
	12	if (!column_exists('search_field', 'opac')) {
	13	$dbh->do("ALTER TABLE `search_field` ADD COLUMN `opac` tinyint(1) NOT NULL DEFAULT 1");
	14	}
	15	SetVersion($DBversion)
	16	}
	17	print "Upgrade to $DBversion done (Bug 20589 - Add field boosting and use elastic query fields parameter instead of depricated _all)\n";