Attachment #80971 for bug #19893

View | Details | Raw Unified | Return to bug 19893
Collapse All | Expand All

Lines 73-78 sub new { Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (-8 / +55 lines)
73	return $self;	73	return $self;
74	}	74	}
75		75
		76	=head2 get_elasticsearch
		77
		78	my $elasticsearch_client = $self->get_elasticsearch();
		79
		80	Returns a C<Search::Elasticsearch> client. The client is cached on a C<Koha::SearchEngine::ElasticSearch>
		81	instance level and will be reused if method is called multiple times.
		82
		83	=cut
		84
76	sub get_elasticsearch {	85	sub get_elasticsearch {
77	my $self = shift @_;	86	my $self = shift @_;
78	unless (defined $self->{elasticsearch}) {	87	unless (defined $self->{elasticsearch}) {
Lines 145-152 sub get_elasticsearch_params { Link Here
145		154
146	my $settings = $self->get_elasticsearch_settings();	155	my $settings = $self->get_elasticsearch_settings();
147		156
148	This provides the settings provided to elasticsearch when an index is created.	157	This provides the settings provided to Elasticsearch when an index is created.
149	These can do things like define tokenisation methods.	158	These can do things like define tokenization methods.
150		159
151	A hashref containing the settings is returned.	160	A hashref containing the settings is returned.
152		161
Lines 170-176 sub get_elasticsearch_settings { Link Here
170		179
171	my $mappings = $self->get_elasticsearch_mappings();	180	my $mappings = $self->get_elasticsearch_mappings();
172		181
173	This provides the mappings that get passed to elasticsearch when an index is	182	This provides the mappings that get passed to Elasticsearch when an index is
174	created.	183	created.
175		184
176	=cut	185	=cut
Lines 232-238 sub get_elasticsearch_mappings { Link Here
232		241
233	=head2 _get_elasticsearch_mapping	242	=head2 _get_elasticsearch_mapping
234		243
235	Get the ES mappings for the given purpose and data type	244	Get the Elasticsearch mappings for the given purpose and data type.
236		245
237	$mapping = _get_elasticsearch_mapping('search', 'text');	246	$mapping = _get_elasticsearch_mapping('search', 'text');
238		247
Lines 301-313 sub sort_fields { Link Here
301	return $self->_sort_fields_accessor();	310	return $self->_sort_fields_accessor();
302	}	311	}
303		312
		313	=head2 marc_records_to_documents($marc_records)
		314
		315	my @record_documents = $self->marc_records_to_documents($marc_records);
		316
		317	Using mappings stored in databse convert C<$marc_records> to ElasticSearch documents.
		318
		319	Returns array of hash references, representing ElasticSearch documents,
		320	acceptable as body payload in C<Search::Elasticsearch> requests.
		321
		322	=over 4
		323
		324	=item C<$marc_documents>
		325
		326	Reference to array of C<MARC::Record> objects to be converted to ElasticSearch documents.
		327
		328	=back
		329
		330	=cut
		331
304	sub marc_records_to_documents {	332	sub marc_records_to_documents {
305	my ($self, $records) = @_;	333	my ($self, $records) = @_;
306	my $rules = $self->get_marc_mapping_rules();	334	my $rules = $self->_get_marc_mapping_rules();
307	my $control_fields_rules = $rules->{control_fields};	335	my $control_fields_rules = $rules->{control_fields};
308	my $data_fields_rules = $rules->{data_fields};	336	my $data_fields_rules = $rules->{data_fields};
309	my $marcflavour = lc C4::Context->preference('marcflavour');	337	my $marcflavour = lc C4::Context->preference('marcflavour');
310	my $serialization_format = C4::Context->preference('ElasticsearchMARCSerializationFormat');
311		338
312	my @record_documents;	339	my @record_documents;
313		340
Lines 426-433 sub marc_records_to_documents { Link Here
426	return \@record_documents;	453	return \@record_documents;
427	}	454	}
428		455
429	# Provides the rules for marc to Elasticsearch JSON document conversion.	456	=head2 _get_marc_mapping_rules
430	sub get_marc_mapping_rules {	457
		458	my $mapping_rules = $self->_get_marc_mapping_rules()
		459
		460	Generates rules from mappings stored in databse for MARC records to Elasticsearch JSON document conversion.
		461
		462	Since field retrieval is slow in C<MARC::Records> (all fields are itereted through for
		463	each call to C<MARC::Record>->field) we create an optimized structure of mapping
		464	rules keyed by MARC field tags holding all the mapping rules for that particular tag.
		465
		466	We can then iterate through all MARC fields for each record and apply all relevant
		467	rules once per fields instead of retreiving field multiple times for each mapping rule
		468	wich is terribly slow.
		469
		470	=cut
		471
		472	# TODO: This structure can be used for processing multiple MARC::Records so is currently
		473	# rebuilt for each batch. Since it is cacheable it could also be stored in an in
		474	# memory cache which it is currently not. The performance gain of caching
		475	# would probably be marginal, but to do this could be a further improvement.
		476
		477	sub _get_marc_mapping_rules {
431	my ($self) = @_;	478	my ($self) = @_;
432		479
433	my $marcflavour = lc C4::Context->preference('marcflavour');	480	my $marcflavour = lc C4::Context->preference('marcflavour');




    $indexer->drop_index();
    $indexer->update_index(\@biblionumbers, \@records);

=head1 FUNCTIONS

=head1 CONSTANTS

=over 4

=item C<Koha::SearchEngine::Elasticsearch::Indexer::INDEX_STATUS_OK>

Represents an index state where index is created and in a working state.
will be rewritten using the values in C<$biblionums> to ensure they are correct.
If C<$biblionums> is C<undef>, this won't happen, but you should be sure that
999$c is correct on your own then.

=item C<Koha::SearchEngine::Elasticsearch::Indexer::INDEX_STATUS_REINDEX_REQUIRED>

Not currently used, but could be useful later, for example if can detect when new field or mapping added.

=item C<Koha::SearchEngine::Elasticsearch::Indexer::INDEX_STATUS_RECREATE_REQUIRED>

Representings an index state where index needs to be recreated and is not in a working state.

=back

=cut

use constant {
    INDEX_STATUS_OK => 0,
    INDEX_STATUS_REINDEX_REQUIRED => 1,
    INDEX_STATUS_RECREATE_REQUIRED => 2,
};

=head1 FUNCTIONS

=head2 update_index($biblionums, $records)

    try {
        $self->update_index($biblionums, $records);
    } catch {
        die("Something whent wrong trying to update index:" .  $_[0]);
    }

Converts C<MARC::Records> C<$records> to Elasticsearch documents and perform
an update request for these records on the Elasticsearch index.

The values in the arrays must match up, and the 999$c value in the MARC record
will be rewritten using the values in C<$biblionums> to ensure they are correct.
If C<$biblionums> is C<undef>, this won't happen, so in that case you should make
sure that 999$c is correct.

Note that this will modify the original record if C<$biblionums> is supplied.
If that's a problem, clone them first.

=over 4

=item C<$biblionums>

Arrayref of biblio numbers for the C<$records>, the order must be the same
as C<$records>.

=item C<$records>

Arrayref of C<MARC::Record>s.

=back

=cut

sub update_index {
    my ($self, $biblionums, $records) = @_;

    # TODO should have a separate path for dealing with a large number
    # of records at once where we use the bulk update functions in ES.
    if ($biblionums) {
        $self->_sanitise_records($biblionums, $records);
    }

    $self->bulk_index($records);
    return 1;
}

sub bulk_index {
    my ($self, $records) = @_;
    my $conf = $self->get_elasticsearch_params();
    my $elasticsearch = $self->get_elasticsearch();
    my $documents = $self->marc_records_to_documents($records);

    return 1;
}

=head2 set_index_status_ok

Convenience method for setting index status to C<INDEX_STATUS_OK>.

=cut

sub set_index_status_ok {
    my ($self) = @_;
    $self->index_status(INDEX_STATUS_OK);
}

=head2 is_index_status_ok

Convenience method for checking if index status is C<INDEX_STATUS_OK>.

=cut

sub is_index_status_ok {
    my ($self) = @_;
    return $self->index_status == INDEX_STATUS_OK;
}

=head2 set_index_status_reindex_required

Convenience method for setting index status to C<INDEX_REINDEX_REQUIRED>.

=cut

sub set_index_status_reindex_required {
    my ($self) = @_;
    $self->index_status(INDEX_STATUS_REINDEX_REQUIRED);
}

=head2 is_index_status_reindex_required

Convenience method for checking if index status is C<INDEX_STATUS_REINDEX_REQUIRED>.

=cut

sub is_index_status_reindex_required {
    my ($self) = @_;
    return $self->index_status == INDEX_STATUS_REINDEX_REQUIRED;
}

=head2 set_index_status_recreate_required

Convenience method for setting index status to C<INDEX_STATUS_RECREATE_REQUIRED>.

=cut

sub set_index_status_recreate_required {
    my ($self) = @_;
    $self->index_status(INDEX_STATUS_RECREATE_REQUIRED);
}

=head2 is_index_status_recreate_required

Convenience method for checking if index status is C<INDEX_STATUS_RECREATE_REQUIRED>.

=cut

sub is_index_status_recreate_required {
    my ($self) = @_;
    return $self->index_status == INDEX_STATUS_RECREATE_REQUIRED;
}

=head2 index_status($status)

Will either set the current index status to C<$status> and return C<$status>,
or return the current index status if called with no arguments.

=over 4

=item C<$status>

Optional argument. If passed will set current index status to C<$status> if C<$status> is
a valid status. See L</CONSTANTS>.

=back

=cut

sub index_status {
    my ($self, $status) = @_;
    my $key = 'ElasticsearchIndexStatus_' . $self->index;

    }
}

=head2 update_mappings

Generate Elasticsearch mappings from mappings stored in database and
perform a request to update Elasticsearch index mappings. Will throw an
error and set index status to C<INDEX_STATUS_RECREATE_REQUIRED> if update
failes.

=cut

sub update_mappings {
    my ($self) = @_;
    my $conf = $self->get_elasticsearch_params();

                }
            );
        } catch {
            $self->set_index_status_recreate_required();
            my $reason = $_[0]->{vars}->{body}->{error}->{reason};
            Koha::Exceptions::Exception->throw(
                error => "Unable to update mappings for index \"$conf->{index_name}\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
            );
        };
    }
    $self->set_index_status_ok();
}

=head2 update_index_background($biblionums, $records)

This has exactly the same API as C<update_index> however it'll
return immediately. It'll start a background process that does the adding.

If it fails to add to Elasticsearch then it'll add to a queue that will cause
it to be updated by a regular index cron job in the future.

=cut

# TODO implement in the future - I don't know the best way of doing this yet.
# If fork: make sure process group is changed so apache doesn't wait for us.

=cut

sub update_index_background {
    my $self = shift;
    $self->update_index(@_);
}

=head2 delete_index($biblionums)

C<$biblionums> is an arrayref of biblionumbers to delete from the index.


    if ( !$self->store ) {
        my $params  = $self->get_elasticsearch_params();
        $self->store(
            Catmandu::Store::Elasticsearch->new(
                %$params,
                index_settings => $self->get_elasticsearch_settings(),
                index_mappings => $self->get_elasticsearch_mappings(),

    $self->store->bag->commit;
}

=head2 delete_index_background($biblionums)

Identical to L</delete_index($biblionums)>
background process to do the actual deleting.

=cut

# TODO: Should be made async

sub delete_index_background {
    my $self = shift;
    $self->delete_index(@_);
}

=head2 drop_index

Drops the index from the Elasticsearch server.

=cut


        my $conf = $self->get_elasticsearch_params();
        my $elasticsearch = $self->get_elasticsearch();
        $elasticsearch->indices->delete(index => $conf->{index_name});
        $self->set_index_status_recreate_required();
    }
}

=head2 create_index

Creates the index (including mappings) on the Elasticsearch server.

=cut

sub create_index {
    my ($self) = @_;
    my $conf = $self->get_elasticsearch_params();

    $self->update_mappings();
}

=head2 index_exists

Checks if index has been created on the ElasticSearch server. Returns C<1> or the
empty string to indicate whether index exists or not.

=cut

sub index_exists {
    my ($self) = @_;
    my $conf = $self->get_elasticsearch_params();





for my $index_name (@index_names) {
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
    if (!$indexer->is_index_status_ok) {
        my $conf = $indexer->get_elasticsearch_params();
        if ($indexer->is_index_status_reindex_required) {
            push @messages, {
                type => 'error',
                code => 'reindex_required',
                index => $conf->{index_name},
            };
        }
        elsif($indexer->is_index_status_recreate_required) {
            push @messages, {
                type => 'error',
                code => 'recreate_required',




        $indexer->drop_index() if $indexer->index_exists();
        $indexer->create_index();
    }
    elsif (!$indexer->index_exists) {
        # Create index if does not exist
        $indexer->create_index();
    } elsif ($indexer->is_index_status_ok) {
        # Update mapping unless index is some kind of problematic state
        $indexer->update_mappings();
    } elsif ($indexer->is_index_status_recreate_required) {
        warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;
    }


Lines 115-121 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here

(-)a/t/Koha/SearchEngine/Elasticsearch.t (-2 / +26 lines)
115		115
116	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {	116	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
117		117
118	plan tests => 30;	118	plan tests => 32;
119		119
120	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');	120	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
121		121
Lines 315-320 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
315		315
316	ok(defined $docs->[0][1]->{marc_format}, 'First document marc_format field should be set');	316	ok(defined $docs->[0][1]->{marc_format}, 'First document marc_format field should be set');
317		317
		318	is($docs->[0][1]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');
		319
318	is(scalar @{$docs->[0][1]->{type_of_record}}, 1, 'First document type_of_record field should have one value');	320	is(scalar @{$docs->[0][1]->{type_of_record}}, 1, 'First document type_of_record field should have one value');
319	is_deeply(	321	is_deeply(
320	$docs->[0][1]->{type_of_record},	322	$docs->[0][1]->{type_of_record},
Lines 351-354 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
351		353
352	ok(!(defined $docs->[0][1]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");	354	ok(!(defined $docs->[0][1]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
353		355
		356	# Marc serialization format fallback for records exceeding ISO2709 max record size
		357
		358	my $large_marc_record = MARC::Record->new();
		359	$large_marc_record->leader(' cam 22 a 4500');
		360
		361	$large_marc_record->append_fields(
		362	MARC::Field->new('100', '', '', a => 'Author 1'),
		363	MARC::Field->new('110', '', '', a => 'Corp Author'),
		364	MARC::Field->new('210', '', '', a => 'Title 1'),
		365	MARC::Field->new('245', '', '', a => 'Title:', b => 'large record'),
		366	MARC::Field->new('999', '', '', c => '1234567'),
		367	);
		368
		369	my $item_field = MARC::Field->new('952', '', '', o => '123456789123456789123456789', p => '123456789', z => 'test');
		370	my $items_count = 1638;
		371	while(--$items_count) {
		372	$large_marc_record->append_fields($item_field);
		373	}
		374
		375	$docs = $see->marc_records_to_documents([$large_marc_record]);
		376
		377	is($docs->[0][1]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
		378
354	};	379	};
355	-

Return to bug 19893

Lines 44-88 Koha::SearchEngine::Elasticsearch::Indexer - handles adding new records to the i Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-50 / +162 lines)
44	$indexer->drop_index();	44	$indexer->drop_index();
45	$indexer->update_index(\@biblionumbers, \@records);	45	$indexer->update_index(\@biblionumbers, \@records);
46		46
47	=head1 FUNCTIONS
48		47
49	=head2 $indexer->update_index($biblionums, $records);	48	=head1 CONSTANTS
50		49
51	C<$biblionums> is an arrayref containing the biblionumbers for the records.	50	=over 4
52		51
53	C<$records> is an arrayref containing the L<MARC::Record>s themselves.	52	=item C<Koha::SearchEngine::Elasticsearch::Indexer::INDEX_STATUS_OK>
54		53
55	The values in the arrays must match up, and the 999$c value in the MARC record	54	Represents an index state where index is created and in a working state.
56	will be rewritten using the values in C<$biblionums> to ensure they are correct.
57	If C<$biblionums> is C<undef>, this won't happen, but you should be sure that
58	999$c is correct on your own then.
59		55
60	Note that this will modify the original record if C<$biblionums> is supplied.	56	=item C<Koha::SearchEngine::Elasticsearch::Indexer::INDEX_STATUS_REINDEX_REQUIRED>
61	If that's a problem, clone them first.	57
		58	Not currently used, but could be useful later, for example if can detect when new field or mapping added.
		59
		60	=item C<Koha::SearchEngine::Elasticsearch::Indexer::INDEX_STATUS_RECREATE_REQUIRED>
		61
		62	Representings an index state where index needs to be recreated and is not in a working state.
		63
		64	=back
62		65
63	=cut	66	=cut
64		67
65	use constant {	68	use constant {
66	INDEX_STATUS_OK => 0,	69	INDEX_STATUS_OK => 0,
67	INDEX_STATUS_REINDEX_REQUIRED => 1, # Not currently used, but could be useful later, for example if can detect when new field or mapping added	70	INDEX_STATUS_REINDEX_REQUIRED => 1,
68	INDEX_STATUS_RECREATE_REQUIRED => 2,	71	INDEX_STATUS_RECREATE_REQUIRED => 2,
69	};	72	};
70		73
		74	=head1 FUNCTIONS
		75
		76	=head2 update_index($biblionums, $records)
		77
		78	try {
		79	$self->update_index($biblionums, $records);
		80	} catch {
		81	die("Something whent wrong trying to update index:" . $_[0]);
		82	}
		83
		84	Converts C<MARC::Records> C<$records> to Elasticsearch documents and perform
		85	an update request for these records on the Elasticsearch index.
		86
		87	The values in the arrays must match up, and the 999$c value in the MARC record
		88	will be rewritten using the values in C<$biblionums> to ensure they are correct.
		89	If C<$biblionums> is C<undef>, this won't happen, so in that case you should make
		90	sure that 999$c is correct.
		91
		92	Note that this will modify the original record if C<$biblionums> is supplied.
		93	If that's a problem, clone them first.
		94
		95	=over 4
		96
		97	=item C<$biblionums>
		98
		99	Arrayref of biblio numbers for the C<$records>, the order must be the same
		100	as C<$records>.
		101
		102	=item C<$records>
		103
		104	Arrayref of C<MARC::Record>s.
		105
		106	=back
		107
		108	=cut
		109
71	sub update_index {	110	sub update_index {
72	my ($self, $biblionums, $records) = @_;	111	my ($self, $biblionums, $records) = @_;
73		112
74	# TODO should have a separate path for dealing with a large number
75	# of records at once where we use the bulk update functions in ES.
76	if ($biblionums) {	113	if ($biblionums) {
77	$self->_sanitise_records($biblionums, $records);	114	$self->_sanitise_records($biblionums, $records);
78	}	115	}
79		116
80	$self->bulk_index($records);
81	return 1;
82	}
83
84	sub bulk_index {
85	my ($self, $records) = @_;
86	my $conf = $self->get_elasticsearch_params();	117	my $conf = $self->get_elasticsearch_params();
87	my $elasticsearch = $self->get_elasticsearch();	118	my $elasticsearch = $self->get_elasticsearch();
88	my $documents = $self->marc_records_to_documents($records);	119	my $documents = $self->marc_records_to_documents($records);
Lines 108-134 sub bulk_index { Link Here
108	return 1;	139	return 1;
109	}	140	}
110		141
111	sub index_status_ok {	142	=head2 set_index_status_ok
112	my ($self, $set) = @_;	143
113	return defined $set ?	144	Convenience method for setting index status to C<INDEX_STATUS_OK>.
114	$self->index_status(INDEX_STATUS_OK) :	145
115	$self->index_status == INDEX_STATUS_OK;	146	=cut
		147
		148	sub set_index_status_ok {
		149	my ($self) = @_;
		150	$self->index_status(INDEX_STATUS_OK);
116	}	151	}
117		152
118	sub index_status_reindex_required {	153	=head2 is_index_status_ok
119	my ($self, $set) = @_;	154
120	return defined $set ?	155	Convenience method for checking if index status is C<INDEX_STATUS_OK>.
121	$self->index_status(INDEX_STATUS_REINDEX_REQUIRED) :	156
122	$self->index_status == INDEX_STATUS_REINDEX_REQUIRED;	157	=cut
		158
		159	sub is_index_status_ok {
		160	my ($self) = @_;
		161	return $self->index_status == INDEX_STATUS_OK;
123	}	162	}
124		163
125	sub index_status_recreate_required {	164	=head2 set_index_status_reindex_required
126	my ($self, $set) = @_;	165
127	return defined $set ?	166	Convenience method for setting index status to C<INDEX_REINDEX_REQUIRED>.
128	$self->index_status(INDEX_STATUS_RECREATE_REQUIRED) :	167
129	$self->index_status == INDEX_STATUS_RECREATE_REQUIRED;	168	=cut
		169
		170	sub set_index_status_reindex_required {
		171	my ($self) = @_;
		172	$self->index_status(INDEX_STATUS_REINDEX_REQUIRED);
130	}	173	}
131		174
		175	=head2 is_index_status_reindex_required
		176
		177	Convenience method for checking if index status is C<INDEX_STATUS_REINDEX_REQUIRED>.
		178
		179	=cut
		180
		181	sub is_index_status_reindex_required {
		182	my ($self) = @_;
		183	return $self->index_status == INDEX_STATUS_REINDEX_REQUIRED;
		184	}
		185
		186	=head2 set_index_status_recreate_required
		187
		188	Convenience method for setting index status to C<INDEX_STATUS_RECREATE_REQUIRED>.
		189
		190	=cut
		191
		192	sub set_index_status_recreate_required {
		193	my ($self) = @_;
		194	$self->index_status(INDEX_STATUS_RECREATE_REQUIRED);
		195	}
		196
		197	=head2 is_index_status_recreate_required
		198
		199	Convenience method for checking if index status is C<INDEX_STATUS_RECREATE_REQUIRED>.
		200
		201	=cut
		202
		203	sub is_index_status_recreate_required {
		204	my ($self) = @_;
		205	return $self->index_status == INDEX_STATUS_RECREATE_REQUIRED;
		206	}
		207
		208	=head2 index_status($status)
		209
		210	Will either set the current index status to C<$status> and return C<$status>,
		211	or return the current index status if called with no arguments.
		212
		213	=over 4
		214
		215	=item C<$status>
		216
		217	Optional argument. If passed will set current index status to C<$status> if C<$status> is
		218	a valid status. See L</CONSTANTS>.
		219
		220	=back
		221
		222	=cut
		223
132	sub index_status {	224	sub index_status {
133	my ($self, $status) = @_;	225	my ($self, $status) = @_;
134	my $key = 'ElasticsearchIndexStatus_' . $self->index;	226	my $key = 'ElasticsearchIndexStatus_' . $self->index;
Lines 150-155 sub index_status { Link Here
150	}	242	}
151	}	243	}
152		244
		245	=head2 update_mappings
		246
		247	Generate Elasticsearch mappings from mappings stored in database and
		248	perform a request to update Elasticsearch index mappings. Will throw an
		249	error and set index status to C<INDEX_STATUS_RECREATE_REQUIRED> if update
		250	failes.
		251
		252	=cut
		253
153	sub update_mappings {	254	sub update_mappings {
154	my ($self) = @_;	255	my ($self) = @_;
155	my $conf = $self->get_elasticsearch_params();	256	my $conf = $self->get_elasticsearch_params();
Lines 166-200 sub update_mappings { Link Here
166	}	267	}
167	);	268	);
168	} catch {	269	} catch {
169	$self->index_status_recreate_required(1);	270	$self->set_index_status_recreate_required();
170	my $reason = $_[0]->{vars}->{body}->{error}->{reason};	271	my $reason = $_[0]->{vars}->{body}->{error}->{reason};
171	Koha::Exceptions::Exception->throw(	272	Koha::Exceptions::Exception->throw(
172	error => "Unable to update mappings for index \"$conf->{index_name}\". Reason was: \"$reason\". Index needs to be recreated and reindexed",	273	error => "Unable to update mappings for index \"$conf->{index_name}\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
173	);	274	);
174	};	275	};
175	}	276	}
176	$self->index_status_ok(1);	277	$self->set_index_status_ok();
177	}	278	}
178		279
179	=head2 $indexer->update_index_background($biblionums, $records)	280	=head2 update_index_background($biblionums, $records)
180		281
181	This has exactly the same API as C<update_index_background> however it'll	282	This has exactly the same API as C<update_index> however it'll
182	return immediately. It'll start a background process that does the adding.	283	return immediately. It'll start a background process that does the adding.
183		284
184	If it fails to add to Elasticsearch then it'll add to a queue that will cause	285	If it fails to add to Elasticsearch then it'll add to a queue that will cause
185	it to be updated by a regular index cron job in the future.	286	it to be updated by a regular index cron job in the future.
186		287
		288	=cut
		289
187	# TODO implement in the future - I don't know the best way of doing this yet.	290	# TODO implement in the future - I don't know the best way of doing this yet.
188	# If fork: make sure process group is changed so apache doesn't wait for us.	291	# If fork: make sure process group is changed so apache doesn't wait for us.
189		292
190	=cut
191
192	sub update_index_background {	293	sub update_index_background {
193	my $self = shift;	294	my $self = shift;
194	$self->update_index(@_);	295	$self->update_index(@_);
195	}	296	}
196		297
197	=head2 $indexer->delete_index($biblionums)	298	=head2 delete_index($biblionums)
198		299
199	C<$biblionums> is an arrayref of biblionumbers to delete from the index.	300	C<$biblionums> is an arrayref of biblionumbers to delete from the index.
200		301
Lines 206-212 sub delete_index { Link Here
206	if ( !$self->store ) {	307	if ( !$self->store ) {
207	my $params = $self->get_elasticsearch_params();	308	my $params = $self->get_elasticsearch_params();
208	$self->store(	309	$self->store(
209	Catmandu::Store::ElasticSearch->new(	310	Catmandu::Store::Elasticsearch->new(
210	%$params,	311	%$params,
211	index_settings => $self->get_elasticsearch_settings(),	312	index_settings => $self->get_elasticsearch_settings(),
212	index_mappings => $self->get_elasticsearch_mappings(),	313	index_mappings => $self->get_elasticsearch_mappings(),
Lines 217-239 sub delete_index { Link Here
217	$self->store->bag->commit;	318	$self->store->bag->commit;
218	}	319	}
219		320
220	=head2 $indexer->delete_index_background($biblionums)	321	=head2 delete_index_background($biblionums)
221		322
222	Identical to L<delete_index>, this will return immediately and start a	323	Identical to L</delete_index($biblionums)>
223	background process to do the actual deleting.
224		324
225	=cut	325	=cut
226		326
227	# TODO implement in the future	327	# TODO: Should be made async
228
229	sub delete_index_background {	328	sub delete_index_background {
230	my $self = shift;	329	my $self = shift;
231	$self->delete_index(@_);	330	$self->delete_index(@_);
232	}	331	}
233		332
234	=head2 $indexer->drop_index();	333	=head2 drop_index
235		334
236	Drops the index from the elasticsearch server.	335	Drops the index from the Elasticsearch server.
237		336
238	=cut	337	=cut
239		338
Lines 243-252 sub drop_index { Link Here
243	my $conf = $self->get_elasticsearch_params();	342	my $conf = $self->get_elasticsearch_params();
244	my $elasticsearch = $self->get_elasticsearch();	343	my $elasticsearch = $self->get_elasticsearch();
245	$elasticsearch->indices->delete(index => $conf->{index_name});	344	$elasticsearch->indices->delete(index => $conf->{index_name});
246	$self->index_status_recreate_required(1);	345	$self->set_index_status_recreate_required();
247	}	346	}
248	}	347	}
249		348
		349	=head2 create_index
		350
		351	Creates the index (including mappings) on the Elasticsearch server.
		352
		353	=cut
		354
250	sub create_index {	355	sub create_index {
251	my ($self) = @_;	356	my ($self) = @_;
252	my $conf = $self->get_elasticsearch_params();	357	my $conf = $self->get_elasticsearch_params();
Lines 261-266 sub create_index { Link Here
261	$self->update_mappings();	366	$self->update_mappings();
262	}	367	}
263		368
		369	=head2 index_exists
		370
		371	Checks if index has been created on the ElasticSearch server. Returns C<1> or the
		372	empty string to indicate whether index exists or not.
		373
		374	=cut
		375
264	sub index_exists {	376	sub index_exists {
265	my ($self) = @_;	377	my ($self) = @_;
266	my $conf = $self->get_elasticsearch_params();	378	my $conf = $self->get_elasticsearch_params();

Lines 135-150 my @indexes; Link Here

(-)a/admin/searchengine/elasticsearch/mappings.pl (-3 / +3 lines)
135		135
136	for my $index_name (@index_names) {	136	for my $index_name (@index_names) {
137	my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });	137	my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
138	if (!$indexer->index_status_ok) {	138	if (!$indexer->is_index_status_ok) {
139	my $conf = $indexer->get_elasticsearch_params();	139	my $conf = $indexer->get_elasticsearch_params();
140	if ($indexer->index_status_reindex_required) {	140	if ($indexer->is_index_status_reindex_required) {
141	push @messages, {	141	push @messages, {
142	type => 'error',	142	type => 'error',
143	code => 'reindex_required',	143	code => 'reindex_required',
144	index => $conf->{index_name},	144	index => $conf->{index_name},
145	};	145	};
146	}	146	}
147	elsif($indexer->index_status_recreate_required) {	147	elsif($indexer->is_index_status_recreate_required) {
148	push @messages, {	148	push @messages, {
149	type => 'error',	149	type => 'error',
150	code => 'recreate_required',	150	code => 'recreate_required',

Lines 165-177 sub do_reindex { Link Here

(-)a/misc/search_tools/rebuild_elastic_search.pl (-3 / +3 lines)
165	$indexer->drop_index() if $indexer->index_exists();	165	$indexer->drop_index() if $indexer->index_exists();
166	$indexer->create_index();	166	$indexer->create_index();
167	}	167	}
168	elsif (!$indexer->index_exists()) {	168	elsif (!$indexer->index_exists) {
169	# Create index if does not exist	169	# Create index if does not exist
170	$indexer->create_index();	170	$indexer->create_index();
171	} elsif ($indexer->index_status_ok) {	171	} elsif ($indexer->is_index_status_ok) {
172	# Update mapping unless index is some kind of problematic state	172	# Update mapping unless index is some kind of problematic state
173	$indexer->update_mappings();	173	$indexer->update_mappings();
174	} elsif ($indexer->index_status_recreate_required) {	174	} elsif ($indexer->is_index_status_recreate_required) {
175	warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;	175	warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;
176	}	176	}
177		177