Attachment #132610 for bug #27344

View | Details | Raw Unified | Return to bug 27344
Collapse All | Expand All

Lines 256-261 sub type_to_class_mapping { Link Here

(-)a/Koha/BackgroundJob.pm (+1 lines)
256	batch_item_record_deletion => 'Koha::BackgroundJob::BatchDeleteItem',	256	batch_item_record_deletion => 'Koha::BackgroundJob::BatchDeleteItem',
257	batch_item_record_modification => 'Koha::BackgroundJob::BatchUpdateItem',	257	batch_item_record_modification => 'Koha::BackgroundJob::BatchUpdateItem',
258	batch_hold_cancel => 'Koha::BackgroundJob::BatchCancelHold',	258	batch_hold_cancel => 'Koha::BackgroundJob::BatchCancelHold',
		259	update_elastic_index => 'Koha::BackgroundJob::UpdateElasticIndex',
259	};	260	};
260	}	261	}
261		262




package Koha::BackgroundJob::UpdateElasticIndex;

# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;
use JSON qw( encode_json decode_json );

use Koha::BackgroundJobs;
use Koha::DateUtils qw( dt_from_string );
use C4::Biblio;
use C4::MarcModificationTemplates;

use base 'Koha::BackgroundJob';

=head1 NAME

Koha::BackgroundJob::UpdateElasticIndex - Update Elastic index

This is a subclass of Koha::BackgroundJob.

=head1 API

=head2 Class methods

=head3 job_type

Define the job type of this job: update_elastic_index

=cut

sub job_type {
    return 'update_elastic_index';
}

=head3 process

Process the modification.

=cut

sub process {
    my ( $self, $args ) = @_;

    my $job = Koha::BackgroundJobs->find( $args->{job_id} );

    if ( !exists $args->{job_id} || !$job || $job->status eq 'cancelled' ) {
        return;
    }

    # FIXME If the job has already been started, but started again (worker has been restart for instance)
    # Then we will start from scratch and so double process the same records

    my $job_progress = 0;
    $job->started_on(dt_from_string)
        ->progress($job_progress)
        ->status('started')
        ->store;

    my @record_ids = @{ $args->{record_ids} };
    my $record_server = $args->{record_server};

    my $report = {
        total_records => scalar @record_ids,
        total_success => 0,
    };

    my @messages;
    eval {
        my $es_index =
            $record_server eq "authorityserver"
          ? $Koha::SearchEngine::AUTHORITIES_INDEX
          : $Koha::SearchEngine::BIBLIOS_INDEX;
        my $indexer = Koha::SearchEngine::Indexer->new({ index => $es_index });
        $indexer->update_index(\@record_ids);
    };
    if ( $@ ) {
        push @messages, {
            type => 'error',
            code => 'index_error',
            error => $@,

        }
    } else {
        # FIXME This is not correct if some record_ids have been skipped
        $report->{total_success} = scalar @record_ids;
    }

    my $job_data = decode_json $job->data;
    $job_data->{messages} = \@messages;
    $job_data->{report} = $report;

    $job->ended_on(dt_from_string)
        ->data(encode_json $job_data);
    $job->status('finished') if $job->status ne 'cancelled';
    $job->store;
}

=head3 enqueue

Enqueue the new job

=cut

sub enqueue {
    my ( $self, $args) = @_;

    return unless exists $args->{record_server};
    return unless exists $args->{record_ids};

    my $record_server = $args->{record_server};
    my @record_ids = @{ $args->{record_ids} };

    $self->SUPER::enqueue({
        job_size => scalar @record_ids,
        job_args => {record_server => $record_server, record_ids => \@record_ids,}
    });
}

1;




use Koha::Exceptions;
use Koha::Exceptions::Elasticsearch;
use Koha::SearchEngine::Zebra::Indexer;
use Koha::BackgroundJob::UpdateElasticIndex;
use C4::AuthoritiesMarc qw//;
use C4::Biblio;
use C4::Context;

=cut

sub update_index {
    my ($self, $record_ids, $records) = @_;

    my $index_record_ids;
    unless ( $records && @$records ) {
        for my $record_id ( sort { $a <=> $b } @$record_ids ) {

            next unless $record_id;

            my $record = $self->_get_record( $record_id );
            if( $record ){
                push @$records, $record;
                push @$index_record_ids, $record_id;
            }
        }
    } else {
        $index_record_ids = $record_ids;
    }

    my $documents = $self->marc_records_to_documents($records);
    my @body;
    for (my $i = 0; $i < scalar @$index_record_ids; $i++) {
        my $id = $index_record_ids->[$i];
        my $document = $documents->[$i];
        push @body, {
            index => {

    $self->set_index_status_ok();
}

=head2 update_index_background($record_numbers, $server)

This has exactly the same API as C<update_index> however it'll
return immediately. It'll start a background process that does the adding.


=cut

# TODO implement in the future - I don't know the best way of doing this yet.
# If fork: make sure process group is changed so apache doesn't wait for us.

sub update_index_background {
    my ( $self, $record_numbers, $server ) = @_;

    Koha::BackgroundJob::UpdateElasticIndex->new->enqueue({ record_ids => $record_numbers, record_server => $server });
}

=head2 index_records

    $record_numbers = [$record_numbers] if ref $record_numbers ne 'ARRAY' && defined $record_numbers;
    $records = [$records] if ref $records ne 'ARRAY' && defined $records;
    if ( $op eq 'specialUpdate' ) {
        my $index_record_numbers;
        if ($records){
            $self->update_index( $record_numbers, $records );
        } else {
            $self->update_index_background( $record_numbers, $server );
                my $record = _get_record( $record_number, $server );
                if( $record ){
                    push @$records, $record;
                    push @$index_record_numbers, $record_number;
                }
            }
        }
        $self->update_index_background( $index_record_numbers, $records ) if $index_record_numbers && $records;
    }
    elsif ( $op eq 'recordDelete' ) {
        $self->delete_index_background( $record_numbers );

}

sub _get_record {
    my ( $self, $record_id ) = @_;
    return $self->index eq $Koha::SearchEngine::BIBLIOS_INDEX
        ? C4::Biblio::GetMarcBiblio({ biblionumber => $record_id, embed_items  => 1 })
        : C4::AuthoritiesMarc::GetAuthority($record_id);
}

=head2 delete_index($biblionums)




[% USE Koha %]

[% BLOCK report %]
    [% SET report = job.report %]
    [% IF report %]
        [% IF report.total_records == 1 %]
            [% IF report.total_success == 1 %]
                <div class="dialog message">The records have successfully been reindexed!</div>
            [% END %]
        [% ELSE %]
            <div class="dialog message">
                [% report.total_success | html %] / [% report.total_records | html %] records have successfully been reindexed. Some errors occurred.
                [% IF job.status == 'cancelled' %]The job has been cancelled before it finished.[% END %]
            </div>
        [% END %]
    [% END %]
[% END %]

[% BLOCK detail %]
[% END %]

[% BLOCK js %]
[% END %]

Lines 35-40 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/background_jobs.tt (+2 lines)
35	Batch item record deletion	35	Batch item record deletion
36	[% CASE "batch_hold_cancel" %]	36	[% CASE "batch_hold_cancel" %]
37	Batch hold cancellation	37	Batch hold cancellation
		38	[% CASE 'update_elastic_index' %]
		39	Update Elasticsearch index
38	[% CASE %]Unknown job type '[% job_type \| html %]'	40	[% CASE %]Unknown job type '[% job_type \| html %]'
39	[% END %]	41	[% END %]
40		42

Lines 36-41 my @job_types = qw( Link Here

(-)a/misc/background_jobs_worker.pl (-1 / +1 lines)
36	batch_authority_record_deletion	36	batch_authority_record_deletion
37	batch_item_record_deletion	37	batch_item_record_deletion
38	batch_hold_cancel	38	batch_hold_cancel
		39	update_elastic_index
39	);	40	);
40		41
41	if ( $conn ) {	42	if ( $conn ) {
42	-

Return to bug 27344

Lines 26-31 use base qw(Koha::SearchEngine::Elasticsearch); Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-23 / +30 lines)
26	use Koha::Exceptions;	26	use Koha::Exceptions;
27	use Koha::Exceptions::Elasticsearch;	27	use Koha::Exceptions::Elasticsearch;
28	use Koha::SearchEngine::Zebra::Indexer;	28	use Koha::SearchEngine::Zebra::Indexer;
		29	use Koha::BackgroundJob::UpdateElasticIndex;
29	use C4::AuthoritiesMarc qw//;	30	use C4::AuthoritiesMarc qw//;
30	use C4::Biblio;	31	use C4::Biblio;
31	use C4::Context;	32	use C4::Context;
Lines 97-108 Arrayref of C<MARC::Record>s. Link Here
97	=cut	98	=cut
98		99
99	sub update_index {	100	sub update_index {
100	my ($self, $biblionums, $records) = @_;	101	my ($self, $record_ids, $records) = @_;
		102
		103	my $index_record_ids;
		104	unless ( $records && @$records ) {
		105	for my $record_id ( sort { $a <=> $b } @$record_ids ) {
		106
		107	next unless $record_id;
		108
		109	my $record = $self->_get_record( $record_id );
		110	if( $record ){
		111	push @$records, $record;
		112	push @$index_record_ids, $record_id;
		113	}
		114	}
		115	} else {
		116	$index_record_ids = $record_ids;
		117	}
101		118
102	my $documents = $self->marc_records_to_documents($records);	119	my $documents = $self->marc_records_to_documents($records);
103	my @body;	120	my @body;
104	for (my $i = 0; $i < scalar @$biblionums; $i++) {	121	for (my $i = 0; $i < scalar @$index_record_ids; $i++) {
105	my $id = $biblionums->[$i];	122	my $id = $index_record_ids->[$i];
106	my $document = $documents->[$i];	123	my $document = $documents->[$i];
107	push @body, {	124	push @body, {
108	index => {	125	index => {
Lines 271-277 sub update_mappings { Link Here
271	$self->set_index_status_ok();	288	$self->set_index_status_ok();
272	}	289	}
273		290
274	=head2 update_index_background($biblionums, $records)	291	=head2 update_index_background($record_numbers, $server)
275		292
276	This has exactly the same API as C<update_index> however it'll	293	This has exactly the same API as C<update_index> however it'll
277	return immediately. It'll start a background process that does the adding.	294	return immediately. It'll start a background process that does the adding.
Lines 281-292 it to be updated by a regular index cron job in the future. Link Here
281		298
282	=cut	299	=cut
283		300
284	# TODO implement in the future - I don't know the best way of doing this yet.
285	# If fork: make sure process group is changed so apache doesn't wait for us.
286
287	sub update_index_background {	301	sub update_index_background {
288	my $self = shift;	302	my ( $self, $record_numbers, $server ) = @_;
289	$self->update_index(@_);	303
		304	Koha::BackgroundJob::UpdateElasticIndex->new->enqueue({ record_ids => $record_numbers, record_server => $server });
290	}	305	}
291		306
292	=head2 index_records	307	=head2 index_records
Lines 307-325 sub index_records { Link Here
307	$record_numbers = [$record_numbers] if ref $record_numbers ne 'ARRAY' && defined $record_numbers;	322	$record_numbers = [$record_numbers] if ref $record_numbers ne 'ARRAY' && defined $record_numbers;
308	$records = [$records] if ref $records ne 'ARRAY' && defined $records;	323	$records = [$records] if ref $records ne 'ARRAY' && defined $records;
309	if ( $op eq 'specialUpdate' ) {	324	if ( $op eq 'specialUpdate' ) {
310	my $index_record_numbers;
311	if ($records){	325	if ($records){
312	$index_record_numbers = $record_numbers;	326	$self->update_index( $record_numbers, $records );
313	} else {	327	} else {
314	foreach my $record_number ( @$record_numbers ){	328	$self->update_index_background( $record_numbers, $server );
315	my $record = _get_record( $record_number, $server );
316	if( $record ){
317	push @$records, $record;
318	push @$index_record_numbers, $record_number;
319	}
320	}
321	}	329	}
322	$self->update_index_background( $index_record_numbers, $records ) if $index_record_numbers && $records;
323	}	330	}
324	elsif ( $op eq 'recordDelete' ) {	331	elsif ( $op eq 'recordDelete' ) {
325	$self->delete_index_background( $record_numbers );	332	$self->delete_index_background( $record_numbers );
Lines 329-338 sub index_records { Link Here
329	}	336	}
330		337
331	sub _get_record {	338	sub _get_record {
332	my ( $id, $server ) = @_;	339	my ( $self, $record_id ) = @_;
333	return $server eq 'biblioserver'	340	return $self->index eq $Koha::SearchEngine::BIBLIOS_INDEX
334	? C4::Biblio::GetMarcBiblio({ biblionumber => $id, embed_items => 1 })	341	? C4::Biblio::GetMarcBiblio({ biblionumber => $record_id, embed_items => 1 })
335	: C4::AuthoritiesMarc::GetAuthority($id);	342	: C4::AuthoritiesMarc::GetAuthority($record_id);
336	}	343	}
337		344
338	=head2 delete_index($biblionums)	345	=head2 delete_index($biblionums)

Line 0 Link Here

(-)a/Koha/BackgroundJob/UpdateElasticIndex.pm (+132 lines)
		1	package Koha::BackgroundJob::UpdateElasticIndex;
		2
		3	# This file is part of Koha.
		4	#
		5	# Koha is free software; you can redistribute it and/or modify it
		6	# under the terms of the GNU General Public License as published by
		7	# the Free Software Foundation; either version 3 of the License, or
		8	# (at your option) any later version.
		9	#
		10	# Koha is distributed in the hope that it will be useful, but
		11	# WITHOUT ANY WARRANTY; without even the implied warranty of
		12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		13	# GNU General Public License for more details.
		14	#
		15	# You should have received a copy of the GNU General Public License
		16	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		17
		18	use Modern::Perl;
		19	use JSON qw( encode_json decode_json );
		20
		21	use Koha::BackgroundJobs;
		22	use Koha::DateUtils qw( dt_from_string );
		23	use C4::Biblio;
		24	use C4::MarcModificationTemplates;
		25
		26	use base 'Koha::BackgroundJob';
		27
		28	=head1 NAME
		29
		30	Koha::BackgroundJob::UpdateElasticIndex - Update Elastic index
		31
		32	This is a subclass of Koha::BackgroundJob.
		33
		34	=head1 API
		35
		36	=head2 Class methods
		37
		38	=head3 job_type
		39
		40	Define the job type of this job: update_elastic_index
		41
		42	=cut
		43
		44	sub job_type {
		45	return 'update_elastic_index';
		46	}
		47
		48	=head3 process
		49
		50	Process the modification.
		51
		52	=cut
		53
		54	sub process {
		55	my ( $self, $args ) = @_;
		56
		57	my $job = Koha::BackgroundJobs->find( $args->{job_id} );
		58
		59	if ( !exists $args->{job_id} \|\| !$job \|\| $job->status eq 'cancelled' ) {
		60	return;
		61	}
		62
		63	# FIXME If the job has already been started, but started again (worker has been restart for instance)
		64	# Then we will start from scratch and so double process the same records
65
66	my $job_progress = 0;
67	$job->started_on(dt_from_string)
68	->progress($job_progress)
69	->status('started')
70	->store;
71
72	my @record_ids = @{ $args->{record_ids} };
73	my $record_server = $args->{record_server};
74
75	my $report = {
76	total_records => scalar @record_ids,
77	total_success => 0,
78	};
79
80	my @messages;
81	eval {
82	my $es_index =
83	$record_server eq "authorityserver"
84	? $Koha::SearchEngine::AUTHORITIES_INDEX
85	: $Koha::SearchEngine::BIBLIOS_INDEX;
86	my $indexer = Koha::SearchEngine::Indexer->new({ index => $es_index });
87	$indexer->update_index(\@record_ids);
88	};
89	if ( $@ ) {
90	push @messages, {
91	type => 'error',
92	code => 'index_error',
93	error => $@,
94
95	}
96	} else {
97	# FIXME This is not correct if some record_ids have been skipped
98	$report->{total_success} = scalar @record_ids;
99	}
100
101	my $job_data = decode_json $job->data;
102	$job_data->{messages} = \@messages;
103	$job_data->{report} = $report;
104
105	$job->ended_on(dt_from_string)
106	->data(encode_json $job_data);
107	$job->status('finished') if $job->status ne 'cancelled';
108	$job->store;
109	}
110
111	=head3 enqueue
112
113	Enqueue the new job
114
115	=cut
116
117	sub enqueue {
118	my ( $self, $args) = @_;
119
120	return unless exists $args->{record_server};
121	return unless exists $args->{record_ids};
122
123	my $record_server = $args->{record_server};
124	my @record_ids = @{ $args->{record_ids} };
125
126	$self->SUPER::enqueue({
127	job_size => scalar @record_ids,
128	job_args => {record_server => $record_server, record_ids => \@record_ids,}
129	});
130	}
131
132	1;

Line 0 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs_update_elastic_index.inc (+23 lines)
	1	[% USE Koha %]
	2
	3	[% BLOCK report %]
	4	[% SET report = job.report %]
	5	[% IF report %]
	6	[% IF report.total_records == 1 %]
	7	[% IF report.total_success == 1 %]
	8	<div class="dialog message">The records have successfully been reindexed!</div>
	9	[% END %]
	10	[% ELSE %]
	11	<div class="dialog message">
	12	[% report.total_success \| html %] / [% report.total_records \| html %] records have successfully been reindexed. Some errors occurred.
	13	[% IF job.status == 'cancelled' %]The job has been cancelled before it finished.[% END %]
	14	</div>
	15	[% END %]
	16	[% END %]
	17	[% END %]
	18
	19	[% BLOCK detail %]
	20	[% END %]
	21
	22	[% BLOCK js %]
	23	[% END %]