Attachment #130811 for bug #25669

View | Details | Raw Unified | Return to bug 25669
Collapse All | Expand All




        $self->{elasticsearch} = Search::Elasticsearch->new(
            $self->get_elasticsearch_params()
        );
        $self->{es_version} = $self->{elasticsearch}->info->{version}{number};
    }
    return $self->{elasticsearch};
}

    if (!defined $all_mappings{$self->index}) {
        $sort_fields{$self->index} = {};
        # Clone the general mapping to break ties with the original hash
        my $mappings = clone(_get_elasticsearch_field_config('general', ''));
            data => clone(_get_elasticsearch_field_config('general', ''))
        };
        my $marcflavour = lc C4::Context->preference('marcflavour');
        $self->_foreach_mapping(
            sub {

                }

                if ($search) {
                    $mappings->{properties}{$name} = _get_elasticsearch_field_config('search', $es_type);
                }

                if ($facet) {
                    $mappings->{properties}{ $name . '__facet' } = _get_elasticsearch_field_config('facet', $es_type);
                }
                if ($suggestible) {
                    $mappings->{properties}{ $name . '__suggestion' } = _get_elasticsearch_field_config('suggestible', $es_type);
                }
                # Sort is a bit special as it can be true, false, undef.
                # We care about "true" or "undef",
                # "undef" means to do the default thing, which is make it sortable.
                if (!defined $sort || $sort) {
                    $mappings->{properties}{ $name . '__sort' } = _get_elasticsearch_field_config('sort', $es_type);
                    $sort_fields{$self->index}{$name} = 1;
                }
            }
        );
        $mappings->{properties}{ 'match-heading' } = _get_elasticsearch_field_config('search', 'text') if $self->index eq 'authorities';
        $all_mappings{$self->index} = $mappings;
    }
    $self->sort_fields(\%{$sort_fields{$self->index}});

Lines 21-26 use Carp qw( carp croak ); Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-22 / +32 lines)
21	use Modern::Perl;	21	use Modern::Perl;
22	use Try::Tiny qw( catch try );	22	use Try::Tiny qw( catch try );
23	use List::Util qw( any );	23	use List::Util qw( any );
		24	use version;
24	use base qw(Koha::SearchEngine::Elasticsearch);	25	use base qw(Koha::SearchEngine::Elasticsearch);
25		26
26	use Koha::Exceptions;	27	use Koha::Exceptions;
Lines 115-125 sub update_index { Link Here
115	if (@body) {	116	if (@body) {
116	try{	117	try{
117	my $elasticsearch = $self->get_elasticsearch();	118	my $elasticsearch = $self->get_elasticsearch();
118	$response = $elasticsearch->bulk(	119	my %es_bulk = (
119	index => $self->index_name,	120	index => $self->index_name,
120	type => 'data', # is just hard coded in Indexer.pm?
121	body => \@body	121	body => \@body
122	);	122	);
		123	if (version->parse($self->{es_version}) < version->parse('7.0.0')) {
		124	$es_bulk{'type'} = 'data';
		125	}
		126	$response = $elasticsearch->bulk(%es_bulk);
123	if ($response->{errors}) {	127	if ($response->{errors}) {
124	carp "One or more ElasticSearch errors occurred when indexing documents";	128	carp "One or more ElasticSearch errors occurred when indexing documents";
125	}	129	}
Lines 250-273 sub update_mappings { Link Here
250	my $elasticsearch = $self->get_elasticsearch();	254	my $elasticsearch = $self->get_elasticsearch();
251	my $mappings = $self->get_elasticsearch_mappings();	255	my $mappings = $self->get_elasticsearch_mappings();
252		256
253	foreach my $type (keys %{$mappings}) {	257	try {
254	try {	258	my %es_mapping = (
255	my $response = $elasticsearch->indices->put_mapping(	259	index => $self->index_name,
256	index => $self->index_name,	260	body => $mappings
257	type => $type,	261	);
258	body => {	262	if (version->parse($self->{es_version}) < version->parse('7.0.0')) {
259	$type => $mappings->{$type}	263	if (version->parse($self->{es_version}) >= version->parse('6.8.0') && version->parse($elasticsearch->VERSION) >= version->parse('6.8.0')) {
260	}	264	$es_mapping{'include_type_name'} = \1;
261	);	265	}
262	} catch {	266	$es_mapping{'type'} = 'data';
263	$self->set_index_status_recreate_required();	267	$es_mapping{'body'} = {'data' => $mappings};
264	my $reason = $_[0]->{vars}->{body}->{error}->{reason};	268	}
265	my $index_name = $self->index_name;	269	my $response = $elasticsearch->indices->put_mapping(%es_mapping);
266	Koha::Exceptions::Exception->throw(	270	} catch {
267	error => "Unable to update mappings for index \"$index_name\". Reason was: \"$reason\". Index needs to be recreated and reindexed",	271	$self->set_index_status_recreate_required();
268	);	272	my $reason = $_[0]->{vars}->{body}->{error}->{reason};
269	};	273	my $index_name = $self->index_name;
270	}	274	Koha::Exceptions::Exception->throw(
		275	error => "Unable to update mappings for index \"$index_name\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
		276	);
		277	};
271	$self->set_index_status_ok();	278	$self->set_index_status_ok();
272	}	279	}
273		280
Lines 346-356 sub delete_index { Link Here
346		353
347	my $elasticsearch = $self->get_elasticsearch();	354	my $elasticsearch = $self->get_elasticsearch();
348	my @body = map { { delete => { _id => "$_" } } } @{$biblionums};	355	my @body = map { { delete => { _id => "$_" } } } @{$biblionums};
349	my $result = $elasticsearch->bulk(	356	my %es_bulk = (
350	index => $self->index_name,	357	index => $self->index_name,
351	type => 'data',
352	body => \@body,	358	body => \@body,
353	);	359	);
		360	if (version->parse($self->{es_version}) < version->parse('7.0.0')) {
		361	$es_bulk{'type'} = 'data';
		362	}
		363	my $result = $elasticsearch->bulk(%es_bulk);
354	if ($result->{errors}) {	364	if ($result->{errors}) {
355	croak "An Elasticsearch error occurred during bulk delete";	365	croak "An Elasticsearch error occurred during bulk delete";
356	}	366	}

Lines 577-583 sub build_authorities_query_compat { Link Here

(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-2 / +2 lines)
577		577
578	$m = exists $koha_to_index_name->{$m} ? $koha_to_index_name->{$m} : $m;	578	$m = exists $koha_to_index_name->{$m} ? $koha_to_index_name->{$m} : $m;
579	push @indexes, $m;	579	push @indexes, $m;
580	warn "Unknown search field $m in marclist" unless (defined $mappings->{data}->{properties}->{$m} \|\| $m eq '' \|\| $m eq 'match-heading');	580	warn "Unknown search field $m in marclist" unless (defined $mappings->{properties}->{$m} \|\| $m eq '' \|\| $m eq 'match-heading');
581	}	581	}
582	for ( my $i = 0 ; $i < @$value ; $i++ ) {	582	for ( my $i = 0 ; $i < @$value ; $i++ ) {
583	next unless $value->[$i]; #clean empty form values, ES doesn't like undefined searches	583	next unless $value->[$i]; #clean empty form values, ES doesn't like undefined searches
Lines 1123-1129 sub _sort_field { Link Here
1123	my ($self, $f) = @_;	1123	my ($self, $f) = @_;
1124		1124
1125	my $mappings = $self->get_elasticsearch_mappings();	1125	my $mappings = $self->get_elasticsearch_mappings();
1126	my $textField = defined $mappings->{data}{properties}{$f}{type} && $mappings->{data}{properties}{$f}{type} eq 'text';	1126	my $textField = defined $mappings->{properties}{$f}{type} && $mappings->{properties}{$f}{type} eq 'text';
1127	if (!defined $self->sort_fields()->{$f} \|\| $self->sort_fields()->{$f}) {	1127	if (!defined $self->sort_fields()->{$f} \|\| $self->sort_fields()->{$f}) {
1128	$f .= '__sort';	1128	$f .= '__sort';
1129	} else {	1129	} else {

Lines 135-141 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here

(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t (-1 / +1 lines)
135	# test reading mappings	135	# test reading mappings
136	my $es = Koha::SearchEngine::Elasticsearch->new( {index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX} );	136	my $es = Koha::SearchEngine::Elasticsearch->new( {index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX} );
137	$mappings = $es->get_elasticsearch_mappings();	137	$mappings = $es->get_elasticsearch_mappings();
138	is( $mappings->{data}{properties}{isbn__sort}{index}, 'false', 'Field mappings parsed correctly' );	138	is( $mappings->{properties}{isbn__sort}{index}, 'false', 'Field mappings parsed correctly' );
139	};	139	};
140		140
141	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {	141	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {




    my %all_mappings;

    my $mappings = {
        properties => {
            title => {
                type => 'text'
            },
            title__sort => {
                type => 'text'
            },
            subject => {
                type => 'text',
                facet => 1
            },
            'subject-heading-thesaurus' => {
                type => 'text',
                facet => 1
            },
            itemnumber => {
                type => 'integer'
            },
            sortablenumber => {
                type => 'integer'
            },
            sortablenumber__sort => {
                type => 'integer'
            },
            heading => {
                type => 'text'
            },
            'heading-main' => {
                type => 'text'
            },
            heading__sort => {
                type => 'text'
            },
            match => {
                type => 'text'
            },
            'match-heading' => {
                type => 'text'
            },
            'match-heading-see-from' => {
                type => 'text'
            },
                },
            }
        }
    };
    $all_mappings{$self->index} = $mappings;




    my %all_mappings;

    my $mappings = {
        properties => {
            title => {
                type => 'text'
            },
            title__sort => {
                type => 'text'
            },
            subject => {
                type => 'text'
            },
            itemnumber => {
                type => 'integer'
            },
            sortablenumber => {
                type => 'integer'
            },
            sortablenumber__sort => {
                type => 'integer'
                    type => 'integer'
                }
            }
        }
    };
- 

Return to bug 25669

Lines 104-109 sub get_elasticsearch { Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (-8 / +7 lines)
104	$self->{elasticsearch} = Search::Elasticsearch->new(	104	$self->{elasticsearch} = Search::Elasticsearch->new(
105	$self->get_elasticsearch_params()	105	$self->get_elasticsearch_params()
106	);	106	);
		107	$self->{es_version} = $self->{elasticsearch}->info->{version}{number};
107	}	108	}
108	return $self->{elasticsearch};	109	return $self->{elasticsearch};
109	}	110	}
Lines 189-197 sub get_elasticsearch_mappings { Link Here
189	if (!defined $all_mappings{$self->index}) {	190	if (!defined $all_mappings{$self->index}) {
190	$sort_fields{$self->index} = {};	191	$sort_fields{$self->index} = {};
191	# Clone the general mapping to break ties with the original hash	192	# Clone the general mapping to break ties with the original hash
192	my $mappings = {	193	my $mappings = clone(_get_elasticsearch_field_config('general', ''));
193	data => clone(_get_elasticsearch_field_config('general', ''))
194	};
195	my $marcflavour = lc C4::Context->preference('marcflavour');	194	my $marcflavour = lc C4::Context->preference('marcflavour');
196	$self->_foreach_mapping(	195	$self->_foreach_mapping(
197	sub {	196	sub {
Lines 214-238 sub get_elasticsearch_mappings { Link Here
214	}	213	}
215		214
216	if ($search) {	215	if ($search) {
217	$mappings->{data}{properties}{$name} = _get_elasticsearch_field_config('search', $es_type);	216	$mappings->{properties}{$name} = _get_elasticsearch_field_config('search', $es_type);
218	}	217	}
219		218
220	if ($facet) {	219	if ($facet) {
221	$mappings->{data}{properties}{ $name . '__facet' } = _get_elasticsearch_field_config('facet', $es_type);	220	$mappings->{properties}{ $name . '__facet' } = _get_elasticsearch_field_config('facet', $es_type);
222	}	221	}
223	if ($suggestible) {	222	if ($suggestible) {
224	$mappings->{data}{properties}{ $name . '__suggestion' } = _get_elasticsearch_field_config('suggestible', $es_type);	223	$mappings->{properties}{ $name . '__suggestion' } = _get_elasticsearch_field_config('suggestible', $es_type);
225	}	224	}
226	# Sort is a bit special as it can be true, false, undef.	225	# Sort is a bit special as it can be true, false, undef.
227	# We care about "true" or "undef",	226	# We care about "true" or "undef",
228	# "undef" means to do the default thing, which is make it sortable.	227	# "undef" means to do the default thing, which is make it sortable.
229	if (!defined $sort \|\| $sort) {	228	if (!defined $sort \|\| $sort) {
230	$mappings->{data}{properties}{ $name . '__sort' } = _get_elasticsearch_field_config('sort', $es_type);	229	$mappings->{properties}{ $name . '__sort' } = _get_elasticsearch_field_config('sort', $es_type);
231	$sort_fields{$self->index}{$name} = 1;	230	$sort_fields{$self->index}{$name} = 1;
232	}	231	}
233	}	232	}
234	);	233	);
235	$mappings->{data}{properties}{ 'match-heading' } = _get_elasticsearch_field_config('search', 'text') if $self->index eq 'authorities';	234	$mappings->{properties}{ 'match-heading' } = _get_elasticsearch_field_config('search', 'text') if $self->index eq 'authorities';
236	$all_mappings{$self->index} = $mappings;	235	$all_mappings{$self->index} = $mappings;
237	}	236	}
238	$self->sort_fields(\%{$sort_fields{$self->index}});	237	$self->sort_fields(\%{$sort_fields{$self->index}});

Lines 39-88 $se->mock( 'get_elasticsearch_mappings', sub { Link Here

(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t (-44 / +42 lines)
39	my %all_mappings;	39	my %all_mappings;
40		40
41	my $mappings = {	41	my $mappings = {
42	data => {	42	properties => {
43	properties => {	43	title => {
44	title => {	44	type => 'text'
45	type => 'text'	45	},
46	},	46	title__sort => {
47	title__sort => {	47	type => 'text'
48	type => 'text'	48	},
49	},	49	subject => {
50	subject => {	50	type => 'text',
51	type => 'text',	51	facet => 1
52	facet => 1	52	},
53	},	53	'subject-heading-thesaurus' => {
54	'subject-heading-thesaurus' => {	54	type => 'text',
55	type => 'text',	55	facet => 1
56	facet => 1	56	},
57	},	57	itemnumber => {
58	itemnumber => {	58	type => 'integer'
59	type => 'integer'	59	},
60	},	60	sortablenumber => {
61	sortablenumber => {	61	type => 'integer'
62	type => 'integer'	62	},
63	},	63	sortablenumber__sort => {
64	sortablenumber__sort => {	64	type => 'integer'
65	type => 'integer'	65	},
66	},	66	heading => {
67	heading => {	67	type => 'text'
68	type => 'text'	68	},
69	},	69	'heading-main' => {
70	'heading-main' => {	70	type => 'text'
71	type => 'text'	71	},
72	},	72	heading__sort => {
73	heading__sort => {	73	type => 'text'
74	type => 'text'	74	},
75	},	75	match => {
76	match => {	76	type => 'text'
77	type => 'text'	77	},
78	},	78	'match-heading' => {
79	'match-heading' => {	79	type => 'text'
80	type => 'text'	80	},
81	},	81	'match-heading-see-from' => {
82	'match-heading-see-from' => {	82	type => 'text'
83	type => 'text'	83	},
84	},
85	}
86	}	84	}
87	};	85	};
88	$all_mappings{$self->index} = $mappings;	86	$all_mappings{$self->index} = $mappings;