Attachment #82258 for bug #19893

View | Details | Raw Unified | Return to bug 19893
Collapse All | Expand All

Lines 19-24 package Koha::SearchEngine::Elasticsearch::Indexer; Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-20 / +70 lines)
19		19
20	use Carp;	20	use Carp;
21	use Modern::Perl;	21	use Modern::Perl;
		22	use Try::Tiny;
		23	use List::Util qw(any);
22	use base qw(Koha::SearchEngine::Elasticsearch);	24	use base qw(Koha::SearchEngine::Elasticsearch);
23	use Data::Dumper;	25	use Data::Dumper;
24		26
Lines 26-31 use Data::Dumper; Link Here
26	use Catmandu::Importer::MARC;	28	use Catmandu::Importer::MARC;
27	use Catmandu::Store::ElasticSearch;	29	use Catmandu::Store::ElasticSearch;
28		30
		31	use Koha::Exceptions;
		32	use C4::Context;
		33
29	Koha::SearchEngine::Elasticsearch::Indexer->mk_accessors(qw( store ));	34	Koha::SearchEngine::Elasticsearch::Indexer->mk_accessors(qw( store ));
30		35
31	=head1 NAME	36	=head1 NAME
Lines 57-62 If that's a problem, clone them first. Link Here
57		62
58	=cut	63	=cut
59		64
		65	use constant {
		66	INDEX_STATUS_OK => 0,
		67	INDEX_STATUS_REINDEX_REQUIRED => 1, # Not currently used, but could be useful later, for example if can detect when new field or mapping added
		68	INDEX_STATUS_RECREATE_REQUIRED => 2,
		69	};
		70
60	sub update_index {	71	sub update_index {
61	my ($self, $biblionums, $records) = @_;	72	my ($self, $biblionums, $records) = @_;
62		73
Lines 66-72 sub update_index { Link Here
66	$self->_sanitise_records($biblionums, $records);	77	$self->_sanitise_records($biblionums, $records);
67	}	78	}
68		79
69	$self->ensure_mappings_updated();
70	$self->bulk_index($records);	80	$self->bulk_index($records);
71	return 1;	81	return 1;
72	}	82	}
Lines 98-107 sub bulk_index { Link Here
98	return 1;	108	return 1;
99	}	109	}
100		110
101	sub ensure_mappings_updated {	111	sub index_status_ok {
102	my ($self) = @_;	112	my ($self, $set) = @_;
103	unless ($self->{_mappings_updated}) {	113	return defined $set ?
104	$self->update_mappings();	114	$self->index_status(INDEX_STATUS_OK) :
		115	$self->index_status == INDEX_STATUS_OK;
		116	}
		117
		118	sub index_status_reindex_required {
		119	my ($self, $set) = @_;
		120	return defined $set ?
		121	$self->index_status(INDEX_STATUS_REINDEX_REQUIRED) :
		122	$self->index_status == INDEX_STATUS_REINDEX_REQUIRED;
		123	}
		124
		125	sub index_status_recreate_required {
		126	my ($self, $set) = @_;
		127	return defined $set ?
		128	$self->index_status(INDEX_STATUS_RECREATE_REQUIRED) :
		129	$self->index_status == INDEX_STATUS_RECREATE_REQUIRED;
		130	}
		131
		132	sub index_status {
		133	my ($self, $status) = @_;
		134	my $key = 'ElasticsearchIndexStatus_' . $self->index;
		135
		136	if (defined $status) {
		137	unless (any { $status == $_ } (
		138	INDEX_STATUS_OK,
		139	INDEX_STATUS_REINDEX_REQUIRED,
		140	INDEX_STATUS_RECREATE_REQUIRED,
		141	)
		142	) {
		143	Koha::Exceptions::Exception->throw("Invalid index status: $status");
		144	}
		145	C4::Context->set_preference($key, $status);
		146	return $status;
		147	}
		148	else {
		149	return C4::Context->preference($key);
105	}	150	}
106	}	151	}
107		152
Lines 112-127 sub update_mappings { Link Here
112	my $mappings = $self->get_elasticsearch_mappings();	157	my $mappings = $self->get_elasticsearch_mappings();
113		158
114	foreach my $type (keys %{$mappings}) {	159	foreach my $type (keys %{$mappings}) {
115	my $response = $elasticsearch->indices->put_mapping(	160	try {
116	index => $conf->{index_name},	161	my $response = $elasticsearch->indices->put_mapping(
117	type => $type,	162	index => $conf->{index_name},
118	body => {	163	type => $type,
119	$type => $mappings->{$type}	164	body => {
120	}	165	$type => $mappings->{$type}
121	);	166	}
122	# TODO: process response, produce errors etc	167	);
		168	} catch {
		169	$self->index_status_recreate_required(1);
		170	my $reason = $_[0]->{vars}->{body}->{error}->{reason};
		171	Koha::Exceptions::Exception->throw(
		172	error => "Unable to update mappings for index \"$conf->{index_name}\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
		173	);
		174	};
123	}	175	}
124	$self->{_mappings_updated} = 1;	176	$self->index_status_ok(1);
125	}	177	}
126		178
127	=head2 $indexer->update_index_background($biblionums, $records)	179	=head2 $indexer->update_index_background($biblionums, $records)
Lines 181-188 sub delete_index_background { Link Here
181		233
182	=head2 $indexer->drop_index();	234	=head2 $indexer->drop_index();
183		235
184	Drops the index from the elasticsearch server. Calling C<update_index>	236	Drops the index from the elasticsearch server.
185	after this will recreate it again.
186		237
187	=cut	238	=cut
188		239
Lines 191-198 sub drop_index { Link Here
191	if ($self->index_exists) {	242	if ($self->index_exists) {
192	my $conf = $self->get_elasticsearch_params();	243	my $conf = $self->get_elasticsearch_params();
193	my $elasticsearch = $self->get_elasticsearch();	244	my $elasticsearch = $self->get_elasticsearch();
194	my $response = $elasticsearch->indices->delete(index => $conf->{index_name});	245	$elasticsearch->indices->delete(index => $conf->{index_name});
195	# TODO: Handle response? Convert errors to exceptions/die
196	}	246	}
197	}	247	}
198		248
Lines 201-213 sub create_index { Link Here
201	my $conf = $self->get_elasticsearch_params();	251	my $conf = $self->get_elasticsearch_params();
202	my $settings = $self->get_elasticsearch_settings();	252	my $settings = $self->get_elasticsearch_settings();
203	my $elasticsearch = $self->get_elasticsearch();	253	my $elasticsearch = $self->get_elasticsearch();
204	my $response = $elasticsearch->indices->create(	254	$elasticsearch->indices->create(
205	index => $conf->{index_name},	255	index => $conf->{index_name},
206	body => {	256	body => {
207	settings => $settings	257	settings => $settings
208	}	258	}
209	);	259	);
210	# TODO: Handle response? Convert errors to exceptions/die	260	$self->update_mappings();
211	}	261	}
212		262
213	sub index_exists {	263	sub index_exists {

Lines 23-31 use C4::Output; Link Here

(-)a/admin/searchengine/elasticsearch/mappings.pl (-1 / +45 lines)
23	use C4::Auth;	23	use C4::Auth;
24		24
25	use Koha::SearchEngine::Elasticsearch;	25	use Koha::SearchEngine::Elasticsearch;
		26	use Koha::SearchEngine::Elasticsearch::Indexer;
26	use Koha::SearchMarcMaps;	27	use Koha::SearchMarcMaps;
27	use Koha::SearchFields;	28	use Koha::SearchFields;
28		29
		30	use Try::Tiny;
		31
29	my $input = new CGI;	32	my $input = new CGI;
30	my ( $template, $borrowernumber, $cookie ) = get_template_and_user(	33	my ( $template, $borrowernumber, $cookie ) = get_template_and_user(
31	{ template_name => 'admin/searchengine/elasticsearch/mappings.tt',	34	{ template_name => 'admin/searchengine/elasticsearch/mappings.tt',
Lines 45-50 my $schema = $database->schema; Link Here
45		48
46	my $marc_type = lc C4::Context->preference('marcflavour');	49	my $marc_type = lc C4::Context->preference('marcflavour');
47		50
		51	my @index_names = ('biblios', 'authorities');
		52
		53	my $update_mappings = sub {
		54	for my $index_name (@index_names) {
		55	my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
		56	try {
		57	$indexer->update_mappings();
		58	} catch {
		59	my $conf = $indexer->get_elasticsearch_params();
		60	push @messages, {
		61	type => 'error',
		62	code => 'error_on_update_es_mappings',
		63	message => $_[0],
		64	index => $conf->{index_name},
		65	};
		66	};
		67	}
		68	};
		69
48	if ( $op eq 'edit' ) {	70	if ( $op eq 'edit' ) {
49		71
50	$schema->storage->txn_begin;	72	$schema->storage->txn_begin;
Lines 110-115 if ( $op eq 'edit' ) { Link Here
110	} else {	132	} else {
111	push @messages, { type => 'message', code => 'success_on_update' };	133	push @messages, { type => 'message', code => 'success_on_update' };
112	$schema->storage->txn_commit;	134	$schema->storage->txn_commit;
		135	$update_mappings->();
113	}	136	}
114	}	137	}
115	elsif( $op eq 'reset_confirmed' ) {	138	elsif( $op eq 'reset_confirmed' ) {
Lines 125-131 elsif( $op eq 'reset_confirm' ) { Link Here
125		148
126	my @indexes;	149	my @indexes;
127		150
128	for my $index_name (qw\| biblios authorities \|) {	151	for my $index_name (@index_names) {
		152	my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
		153	if (!$indexer->index_status_ok) {
		154	my $conf = $indexer->get_elasticsearch_params();
		155	if ($indexer->index_status_reindex_required) {
		156	push @messages, {
		157	type => 'error',
		158	code => 'reindex_required',
		159	index => $conf->{index_name},
		160	};
		161	}
		162	elsif($indexer->index_status_recreate_required) {
		163	push @messages, {
		164	type => 'error',
		165	code => 'recreate_required',
		166	index => $conf->{index_name},
		167	};
		168	}
		169	}
		170	}
		171
		172	for my $index_name (@index_names) {
129	my $search_fields = Koha::SearchFields->search(	173	my $search_fields = Koha::SearchFields->search(
130	{ 'search_marc_map.index_name' => $index_name, 'search_marc_map.marc_type' => $marc_type, },	174	{ 'search_marc_map.index_name' => $index_name, 'search_marc_map.marc_type' => $marc_type, },
131	{ join => { search_marc_to_fields => 'search_marc_map' },	175	{ join => { search_marc_to_fields => 'search_marc_map' },

Line 0 Link Here

(-)a/installer/data/mysql/atomicupdate/bug_19893_elasticsearch_index_status_sysprefs.sql (+2 lines)
	1	INSERT IGNORE INTO systempreferences (variable, value, explanation, options, type) VALUES ('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL);
	2	INSERT IGNORE INTO systempreferences (variable, value, explanation, options, type) VALUES ('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL);

Lines 81-86 a.add, a.delete { Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt (+6 lines)
81	(search field [% m.values.field_name \| html %] with mapping [% m.values.marc_field \| html %].)	81	(search field [% m.values.field_name \| html %] with mapping [% m.values.marc_field \| html %].)
82	[% CASE 'invalid_field_weight' %]	82	[% CASE 'invalid_field_weight' %]
83	Invalid field weight "[% m.weight \| html %]", must be a positive decimal number.	83	Invalid field weight "[% m.weight \| html %]", must be a positive decimal number.
		84	[% CASE 'error_on_update_es_mappings' %]
		85	An error occurred when updating Elasticsearch index mappings: [% m.message \| html %].
		86	[% CASE 'reindex_required' %]
		87	Index "[% m.index \| html %]" needs to be reindexed
		88	[% CASE 'recreate_required' %]
		89	Index "[% m.index \| html %]" needs to be recreated
84	[% CASE 'success_on_update' %]	90	[% CASE 'success_on_update' %]
85	Mappings updated successfully.	91	Mappings updated successfully.
86	[% CASE 'success_on_reset' %]	92	[% CASE 'success_on_reset' %]

Lines 168-173 sub do_reindex { Link Here

(-)a/misc/search_tools/rebuild_elastic_search.pl (-1 / +5 lines)
168	elsif (!$indexer->index_exists()) {	168	elsif (!$indexer->index_exists()) {
169	# Create index if does not exist	169	# Create index if does not exist
170	$indexer->create_index();	170	$indexer->create_index();
		171	} elsif ($indexer->index_status_ok) {
		172	# Update mapping unless index is some kind of problematic state
		173	$indexer->update_mappings();
		174	} elsif ($indexer->index_status_recreate_required) {
		175	warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;
171	}	176	}
172		177
173	my $count = 0;	178	my $count = 0;
174	-

Return to bug 19893