Attachment #131009 for bug #25669

View | Details | Raw Unified | Return to bug 25669
Collapse All | Expand All




    if (@body) {
        try{
            my $elasticsearch = $self->get_elasticsearch();
            #TODO Remove check for no_type and use bulk without type when support for Elasticsearch 6 is dropped
            if ($self->{no_type}) {
                $response = $elasticsearch->bulk(
                    index => $self->index_name,
                    body => \@body
                );
            }
            else {
                $response = $elasticsearch->bulk(
                    index => $self->index_name,
                    type => 'data',
                    body => \@body
                );
            }
            if ($response->{errors}) {
                carp "One or more ElasticSearch errors occurred when indexing documents";
            }

    my $elasticsearch = $self->get_elasticsearch();
    my $mappings = $self->get_elasticsearch_mappings();

    try {
        #TODO Remove check for no_type and use put_mapping without type when support for Elasticsearch 6 is dropped
        if ($self->{no_type}) {
            my $response = $elasticsearch->indices->put_mapping(
                index => $self->index_name,
                body => $mappings
            );
        }
        else {
            my $response = $elasticsearch->indices->put_mapping(
                index => $self->index_name,
                type => 'data',
                body => {
                    data => $mappings
                }
            );
        }
    } catch {
        $self->set_index_status_recreate_required();
        my $reason = $_[0]->{vars}->{body}->{error}->{reason};
        my $index_name = $self->index_name;
        Koha::Exceptions::Exception->throw(
            error => "Unable to update mappings for index \"$index_name\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
        );
    };
    $self->set_index_status_ok();
}



    my $elasticsearch = $self->get_elasticsearch();
    my @body = map { { delete => { _id => "$_" } } } @{$biblionums};
    #TODO Remove check for no_type and use bulk without type when support for Elasticsearch 6 is dropped
    my $result = $self->{no_type} ?
        $elasticsearch->bulk(
            index => $self->index_name,
            body => \@body,
        )
        :
        $elasticsearch->bulk(
            index => $self->index_name,
            type => 'data',
            body => \@body,
        );
    if ($result->{errors}) {
        croak "An Elasticsearch error occurred during bulk delete";
    }

Lines 577-583 sub build_authorities_query_compat { Link Here

(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-2 / +2 lines)
577		577
578	$m = exists $koha_to_index_name->{$m} ? $koha_to_index_name->{$m} : $m;	578	$m = exists $koha_to_index_name->{$m} ? $koha_to_index_name->{$m} : $m;
579	push @indexes, $m;	579	push @indexes, $m;
580	warn "Unknown search field $m in marclist" unless (defined $mappings->{data}->{properties}->{$m} \|\| $m eq '' \|\| $m eq 'match-heading');	580	warn "Unknown search field $m in marclist" unless (defined $mappings->{properties}->{$m} \|\| $m eq '' \|\| $m eq 'match-heading');
581	}	581	}
582	for ( my $i = 0 ; $i < @$value ; $i++ ) {	582	for ( my $i = 0 ; $i < @$value ; $i++ ) {
583	next unless $value->[$i]; #clean empty form values, ES doesn't like undefined searches	583	next unless $value->[$i]; #clean empty form values, ES doesn't like undefined searches
Lines 1123-1129 sub _sort_field { Link Here
1123	my ($self, $f) = @_;	1123	my ($self, $f) = @_;
1124		1124
1125	my $mappings = $self->get_elasticsearch_mappings();	1125	my $mappings = $self->get_elasticsearch_mappings();
1126	my $textField = defined $mappings->{data}{properties}{$f}{type} && $mappings->{data}{properties}{$f}{type} eq 'text';	1126	my $textField = defined $mappings->{properties}{$f}{type} && $mappings->{properties}{$f}{type} eq 'text';
1127	if (!defined $self->sort_fields()->{$f} \|\| $self->sort_fields()->{$f}) {	1127	if (!defined $self->sort_fields()->{$f} \|\| $self->sort_fields()->{$f}) {
1128	$f .= '__sort';	1128	$f .= '__sort';
1129	} else {	1129	} else {

Lines 177-182 Link Here

(-)a/etc/koha-conf.xml (+2 lines)
177	Resetting mappings will override any changes made in the Search engine configuration UI.	177	Resetting mappings will override any changes made in the Search engine configuration UI.
178	-->	178	-->
179	<!-- <elasticsearch_index_mappings>__KOHA_CONF_DIR__/searchengine/elasticsearch/mappings.yaml</elasticsearch_index_mappings> -->	179	<!-- <elasticsearch_index_mappings>__KOHA_CONF_DIR__/searchengine/elasticsearch/mappings.yaml</elasticsearch_index_mappings> -->
		180	<!-- Uncomment the following line if you have Elasticsearch 7 or higher. TODO Remove when support for Elasticsearch 6 is dropped. -->
		181	<!-- <elasticsearch_no_type>1</elasticsearch_no_type> -->
180		182
181	<interlibrary_loans>	183	<interlibrary_loans>
182	<!-- Path to where Illbackends are located on the system	184	<!-- Path to where Illbackends are located on the system

Lines 135-141 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here

(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t (-1 / +1 lines)
135	# test reading mappings	135	# test reading mappings
136	my $es = Koha::SearchEngine::Elasticsearch->new( {index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX} );	136	my $es = Koha::SearchEngine::Elasticsearch->new( {index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX} );
137	$mappings = $es->get_elasticsearch_mappings();	137	$mappings = $es->get_elasticsearch_mappings();
138	is( $mappings->{data}{properties}{isbn__sort}{index}, 'false', 'Field mappings parsed correctly' );	138	is( $mappings->{properties}{isbn__sort}{index}, 'false', 'Field mappings parsed correctly' );
139	};	139	};
140		140
141	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {	141	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {




    my %all_mappings;

    my $mappings = {
        properties => {
            title => {
                type => 'text'
            },
            title__sort => {
                type => 'text'
            },
            subject => {
                type => 'text',
                facet => 1
            },
            'subject-heading-thesaurus' => {
                type => 'text',
                facet => 1
            },
            itemnumber => {
                type => 'integer'
            },
            sortablenumber => {
                type => 'integer'
            },
            sortablenumber__sort => {
                type => 'integer'
            },
            heading => {
                type => 'text'
            },
            'heading-main' => {
                type => 'text'
            },
            heading__sort => {
                type => 'text'
            },
            match => {
                type => 'text'
            },
            'match-heading' => {
                type => 'text'
            },
            'match-heading-see-from' => {
                type => 'text'
            },
                },
            }
        }
    };
    $all_mappings{$self->index} = $mappings;




    my %all_mappings;

    my $mappings = {
        properties => {
            title => {
                type => 'text'
            },
            title__sort => {
                type => 'text'
            },
            subject => {
                type => 'text'
            },
            itemnumber => {
                type => 'integer'
            },
            sortablenumber => {
                type => 'integer'
            },
            sortablenumber__sort => {
                type => 'integer'
                    type => 'integer'
                }
            }
        }
    };
- 

Return to bug 25669

Lines 115-125 sub update_index { Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-23 / +46 lines)
115	if (@body) {	115	if (@body) {
116	try{	116	try{
117	my $elasticsearch = $self->get_elasticsearch();	117	my $elasticsearch = $self->get_elasticsearch();
118	$response = $elasticsearch->bulk(	118	#TODO Remove check for no_type and use bulk without type when support for Elasticsearch 6 is dropped
119	index => $self->index_name,	119	if ($self->{no_type}) {
120	type => 'data', # is just hard coded in Indexer.pm?	120	$response = $elasticsearch->bulk(
121	body => \@body	121	index => $self->index_name,
122	);	122	body => \@body
		123	);
		124	}
		125	else {
		126	$response = $elasticsearch->bulk(
		127	index => $self->index_name,
		128	type => 'data',
		129	body => \@body
		130	);
		131	}
123	if ($response->{errors}) {	132	if ($response->{errors}) {
124	carp "One or more ElasticSearch errors occurred when indexing documents";	133	carp "One or more ElasticSearch errors occurred when indexing documents";
125	}	134	}
Lines 250-273 sub update_mappings { Link Here
250	my $elasticsearch = $self->get_elasticsearch();	259	my $elasticsearch = $self->get_elasticsearch();
251	my $mappings = $self->get_elasticsearch_mappings();	260	my $mappings = $self->get_elasticsearch_mappings();
252		261
253	foreach my $type (keys %{$mappings}) {	262	try {
254	try {	263	#TODO Remove check for no_type and use put_mapping without type when support for Elasticsearch 6 is dropped
		264	if ($self->{no_type}) {
		265	my $response = $elasticsearch->indices->put_mapping(
		266	index => $self->index_name,
		267	body => $mappings
		268	);
		269	}
		270	else {
255	my $response = $elasticsearch->indices->put_mapping(	271	my $response = $elasticsearch->indices->put_mapping(
256	index => $self->index_name,	272	index => $self->index_name,
257	type => $type,	273	type => 'data',
258	body => {	274	body => {
259	$type => $mappings->{$type}	275	data => $mappings
260	}	276	}
261	);	277	);
262	} catch {	278	}
263	$self->set_index_status_recreate_required();	279	} catch {
264	my $reason = $_[0]->{vars}->{body}->{error}->{reason};	280	$self->set_index_status_recreate_required();
265	my $index_name = $self->index_name;	281	my $reason = $_[0]->{vars}->{body}->{error}->{reason};
266	Koha::Exceptions::Exception->throw(	282	my $index_name = $self->index_name;
267	error => "Unable to update mappings for index \"$index_name\". Reason was: \"$reason\". Index needs to be recreated and reindexed",	283	Koha::Exceptions::Exception->throw(
268	);	284	error => "Unable to update mappings for index \"$index_name\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
269	};	285	);
270	}	286	};
271	$self->set_index_status_ok();	287	$self->set_index_status_ok();
272	}	288	}
273		289
Lines 346-356 sub delete_index { Link Here
346		362
347	my $elasticsearch = $self->get_elasticsearch();	363	my $elasticsearch = $self->get_elasticsearch();
348	my @body = map { { delete => { _id => "$_" } } } @{$biblionums};	364	my @body = map { { delete => { _id => "$_" } } } @{$biblionums};
349	my $result = $elasticsearch->bulk(	365	#TODO Remove check for no_type and use bulk without type when support for Elasticsearch 6 is dropped
350	index => $self->index_name,	366	my $result = $self->{no_type} ?
351	type => 'data',	367	$elasticsearch->bulk(
352	body => \@body,	368	index => $self->index_name,
353	);	369	body => \@body,
		370	)
		371	:
		372	$elasticsearch->bulk(
		373	index => $self->index_name,
		374	type => 'data',
		375	body => \@body,
		376	);
354	if ($result->{errors}) {	377	if ($result->{errors}) {
355	croak "An Elasticsearch error occurred during bulk delete";	378	croak "An Elasticsearch error occurred during bulk delete";
356	}	379	}

Lines 104-109 sub get_elasticsearch { Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (-8 / +8 lines)
104	$self->{elasticsearch} = Search::Elasticsearch->new(	104	$self->{elasticsearch} = Search::Elasticsearch->new(
105	$self->get_elasticsearch_params()	105	$self->get_elasticsearch_params()
106	);	106	);
		107	# TODO Remove no_type when support for Elasticsearch 6 is dropped
		108	$self->{no_type} = C4::Context->config('elasticsearch_no_type');
107	}	109	}
108	return $self->{elasticsearch};	110	return $self->{elasticsearch};
109	}	111	}
Lines 189-197 sub get_elasticsearch_mappings { Link Here
189	if (!defined $all_mappings{$self->index}) {	191	if (!defined $all_mappings{$self->index}) {
190	$sort_fields{$self->index} = {};	192	$sort_fields{$self->index} = {};
191	# Clone the general mapping to break ties with the original hash	193	# Clone the general mapping to break ties with the original hash
192	my $mappings = {	194	my $mappings = clone(_get_elasticsearch_field_config('general', ''));
193	data => clone(_get_elasticsearch_field_config('general', ''))
194	};
195	my $marcflavour = lc C4::Context->preference('marcflavour');	195	my $marcflavour = lc C4::Context->preference('marcflavour');
196	$self->_foreach_mapping(	196	$self->_foreach_mapping(
197	sub {	197	sub {
Lines 214-238 sub get_elasticsearch_mappings { Link Here
214	}	214	}
215		215
216	if ($search) {	216	if ($search) {
217	$mappings->{data}{properties}{$name} = _get_elasticsearch_field_config('search', $es_type);	217	$mappings->{properties}{$name} = _get_elasticsearch_field_config('search', $es_type);
218	}	218	}
219		219
220	if ($facet) {	220	if ($facet) {
221	$mappings->{data}{properties}{ $name . '__facet' } = _get_elasticsearch_field_config('facet', $es_type);	221	$mappings->{properties}{ $name . '__facet' } = _get_elasticsearch_field_config('facet', $es_type);
222	}	222	}
223	if ($suggestible) {	223	if ($suggestible) {
224	$mappings->{data}{properties}{ $name . '__suggestion' } = _get_elasticsearch_field_config('suggestible', $es_type);	224	$mappings->{properties}{ $name . '__suggestion' } = _get_elasticsearch_field_config('suggestible', $es_type);
225	}	225	}
226	# Sort is a bit special as it can be true, false, undef.	226	# Sort is a bit special as it can be true, false, undef.
227	# We care about "true" or "undef",	227	# We care about "true" or "undef",
228	# "undef" means to do the default thing, which is make it sortable.	228	# "undef" means to do the default thing, which is make it sortable.
229	if (!defined $sort \|\| $sort) {	229	if (!defined $sort \|\| $sort) {
230	$mappings->{data}{properties}{ $name . '__sort' } = _get_elasticsearch_field_config('sort', $es_type);	230	$mappings->{properties}{ $name . '__sort' } = _get_elasticsearch_field_config('sort', $es_type);
231	$sort_fields{$self->index}{$name} = 1;	231	$sort_fields{$self->index}{$name} = 1;
232	}	232	}
233	}	233	}
234	);	234	);
235	$mappings->{data}{properties}{ 'match-heading' } = _get_elasticsearch_field_config('search', 'text') if $self->index eq 'authorities';	235	$mappings->{properties}{ 'match-heading' } = _get_elasticsearch_field_config('search', 'text') if $self->index eq 'authorities';
236	$all_mappings{$self->index} = $mappings;	236	$all_mappings{$self->index} = $mappings;
237	}	237	}
238	$self->sort_fields(\%{$sort_fields{$self->index}});	238	$self->sort_fields(\%{$sort_fields{$self->index}});

Lines 39-88 $se->mock( 'get_elasticsearch_mappings', sub { Link Here

(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t (-44 / +42 lines)
39	my %all_mappings;	39	my %all_mappings;
40		40
41	my $mappings = {	41	my $mappings = {
42	data => {	42	properties => {
43	properties => {	43	title => {
44	title => {	44	type => 'text'
45	type => 'text'	45	},
46	},	46	title__sort => {
47	title__sort => {	47	type => 'text'
48	type => 'text'	48	},
49	},	49	subject => {
50	subject => {	50	type => 'text',
51	type => 'text',	51	facet => 1
52	facet => 1	52	},
53	},	53	'subject-heading-thesaurus' => {
54	'subject-heading-thesaurus' => {	54	type => 'text',
55	type => 'text',	55	facet => 1
56	facet => 1	56	},
57	},	57	itemnumber => {
58	itemnumber => {	58	type => 'integer'
59	type => 'integer'	59	},
60	},	60	sortablenumber => {
61	sortablenumber => {	61	type => 'integer'
62	type => 'integer'	62	},
63	},	63	sortablenumber__sort => {
64	sortablenumber__sort => {	64	type => 'integer'
65	type => 'integer'	65	},
66	},	66	heading => {
67	heading => {	67	type => 'text'
68	type => 'text'	68	},
69	},	69	'heading-main' => {
70	'heading-main' => {	70	type => 'text'
71	type => 'text'	71	},
72	},	72	heading__sort => {
73	heading__sort => {	73	type => 'text'
74	type => 'text'	74	},
75	},	75	match => {
76	match => {	76	type => 'text'
77	type => 'text'	77	},
78	},	78	'match-heading' => {
79	'match-heading' => {	79	type => 'text'
80	type => 'text'	80	},
81	},	81	'match-heading-see-from' => {
82	'match-heading-see-from' => {	82	type => 'text'
83	type => 'text'	83	},
84	},
85	}
86	}	84	}
87	};	85	};
88	$all_mappings{$self->index} = $mappings;	86	$all_mappings{$self->index} = $mappings;