Attachment #144347 for bug #32370

View | Details | Raw Unified | Return to bug 32370
Collapse All | Expand All





use Modern::Perl;
use Encode qw();
use JSON;
use Carp qw( croak );
use Net::Stomp;
use Try::Tiny qw( catch try );

use Koha::Plugins;
use Koha::Exceptions::BackgroundJob;

use base qw( Koha::Object Koha::Object::JSONFields );

=head1 NAME


    my $job_args    = $params->{job_args};
    my $job_context = $params->{job_context} // C4::Context->userenv;
    my $job_queue   = $params->{job_queue}  // 'default';
    my $json = $self->json;

    my $borrowernumber = (C4::Context->userenv) ? C4::Context->userenv->{number} : undef;
    $job_context->{interface} = C4::Context->interface;

    $self->set_encoded_json_field( { data => $job_context, field => 'context' } );
    $self->set_encoded_json_field( { data => $job_args,    field => 'data' } );

    $self->set(
        {   status         => 'new',
            status         => 'new',
            type           => $job_type,
            queue          => $job_queue,
            size           => $job_size,
            data           => $json_args,
            context        => $json_context,
            enqueued_on    => dt_from_string,
            borrowernumber => $borrowernumber,
        }

    };
    return unless $conn;

    my $json_args = $self->data; # we pass the JSON string
    try {
        # This namespace is wrong, it must be a vhost instead.
        # But to do so it needs to be created on the server => much more work when a new Koha instance is created.

    $args ||= {};

    if ( $self->context ) {
        my $context = $self->decode_json_field({ field => 'context' });
        C4::Context->_new_userenv(-1);
        C4::Context->interface( $context->{interface} );
        C4::Context->set_userenv(

    my ( $self, $data ) = @_;

    $self->status('finished') unless $self->status eq 'cancelled' or $self->status eq 'failed';
    $self->set_encoded_json_field({ data => $data, field => 'data' });

    return $self->set({ ended_on => \'NOW()' })->store;
        {
            ended_on => \'NOW()',
            data     => $self->json->encode($data),
        }
    )->store;
}

=head3 json

   my $JSON_object = $self->json;

Returns a JSON object with utf8 disabled. Encoding to UTF-8 should be
done later.

=cut

sub json {
    my ( $self ) = @_;
    $self->{_json} //= JSON->new->utf8(0); # TODO Should we allow_nonref ?
    return $self->{_json};
}

=head3 decoded_data

sub decoded_data {
    my ($self) = @_;

    return $self->decode_json_field({ field => 'data' });
}

=head3 set_encoded_data

sub set_encoded_data {
    my ( $self, $data ) = @_;

    return $self->set_encoded_json_field({ data => $data, field => 'data' });
}

=head3 job_type

    my ( $self ) = @_;

    my @messages;
    my $data_dump = $self->decode_json_field({ field => 'data' });
    if ( exists $data_dump->{messages} ) {
        @messages = @{ $data_dump->{messages} };
    }

sub report {
    my ( $self ) = @_;

    my $data_dump = $self->decode_json_field({ field => 'data' });
    return $data_dump->{report} || {};
}



    my $json = $self->SUPER::to_api( $params );

    $json->{context} = $self->decode_json_field({ field => 'context' })
      if defined $self->context;
    $json->{data} = $self->decode_json_field({ field => 'data' });

    return $json;
}

Lines 102-108 while (1) { Link Here

(-)a/misc/background_jobs_worker.pl (-1 / +1 lines)
102	} else {	102	} else {
103	my $jobs = Koha::BackgroundJobs->search({ status => 'new', queue => \@queues });	103	my $jobs = Koha::BackgroundJobs->search({ status => 'new', queue => \@queues });
104	while ( my $job = $jobs->next ) {	104	while ( my $job = $jobs->next ) {
105	my $args = $job->json->decode($job->data);	105	my $args = $job->decoded_data;
106	process_job( $job, { job_id => $job->id, %$args } );	106	process_job( $job, { job_id => $job->id, %$args } );
107	}	107	}
108	sleep 10;	108	sleep 10;




    is( $job->size,           3,           'Three steps' );
    is( $job->status,         'new',       'Initial status set correctly' );
    is( $job->borrowernumber, $patron->id, 'Borrowernumber set from userenv' );
    is_deeply( $job->decode_json_field({ field => 'context' }), $job_context, 'Context set from userenv + interface' );

    $schema->storage->txn_rollback;
};


    is( $job->status, 'cancelled', "'finish' leaves 'cancelled' untouched" );
    isnt( $job->ended_on, undef, 'ended_on set' );
    is_deeply( $job->decode_json_field({ field => 'data' }), $data );

    $job->status('started')->store;
    $job->finish( $data );

    is( $job->status, 'finished' );
    isnt( $job->ended_on, undef, 'ended_on set' );
    is_deeply( $job->decode_json_field({ field => 'data' }), $data );

    throws_ok
        { $job->start; }

    is_deeply( C4::Context->interface, 'intranet', "Interface set from job context on process" );

    # Manually add a job (->new->store) without context
    my $json = $job->_json; # sorry, quickly borrowing your json object
    my $data = $json->encode({ a => 'a', b => 'b' });
    my $incomplete_job = t::lib::Koha::BackgroundJob::BatchTest->new(
        {   status         => 'new',


    $job->set_encoded_data( $data );

    is_deeply( $job->_json->decode($job->data), $data, 'decode what we sent' );
    is_deeply( $job->decoded_data, $data, 'check with decoded_data' );

    # Let's get some Unicode stuff into the game

Lines 62-68 my $job_id = t::lib::Koha::BackgroundJob::BatchTest->new->enqueue( Link Here

(-)a/t/db_dependent/Koha/BackgroundJobs.t (-1 / +1 lines)
62	# Enqueuing a new job	62	# Enqueuing a new job
63	my $new_job = Koha::BackgroundJobs->find($job_id);	63	my $new_job = Koha::BackgroundJobs->find($job_id);
64	ok( $new_job, 'New job correctly enqueued' );	64	ok( $new_job, 'New job correctly enqueued' );
65	is_deeply( $new_job->json->decode( $new_job->data ),	65	is_deeply( $new_job->_json->decode( $new_job->data ),
66	$data, 'data retrieved and json encoded correctly' );	66	$data, 'data retrieved and json encoded correctly' );
67	is( t::lib::Dates::compare( $new_job->enqueued_on, dt_from_string ),	67	is( t::lib::Dates::compare( $new_job->enqueued_on, dt_from_string ),
68	0, 'enqueued_on correctly filled with now()' );	68	0, 'enqueued_on correctly filled with now()' );

Lines 51-63 subtest "Exceptions must be stringified" => sub { Link Here

(-)a/t/db_dependent/Koha/BackgroundJobs/BatchUpdateBiblio.t (-3 / +10 lines)
51	type => 'batch_biblio_record_modification',	51	type => 'batch_biblio_record_modification',
52	}	52	}
53	);	53	);
54	my $data = $job->json->encode({ record_ids => [ $biblio->biblionumber ] });	54
55	$job->data( $data )->store;	55	$job->set_encoded_json_field(
		56	{ data => { record_ids => [ $biblio->biblionumber ] },
		57	field => 'data'
		58	}
		59	)->store;
		60
56	$job = Koha::BackgroundJobs->find( $job->id );	61	$job = Koha::BackgroundJobs->find( $job->id );
57	$job->process(	62	$job->process(
58	{ job_id => $job->id, record_ids => [ $biblio->biblionumber ] } );	63	{ job_id => $job->id, record_ids => [ $biblio->biblionumber ] } );
59		64
60	$data = $job->json->decode( $job->get_from_storage->data );	65	$job = $job->get_from_storage;
		66
		67	my $data = $job->decode_json_field({ field => 'data' });
61	is_deeply(	68	is_deeply(
62	$data->{messages}->[0],	69	$data->{messages}->[0],
63	{	70	{




        $self->progress( ++$job_progress )->store;
    }

    my $job_data = $self->_json->decode($self->data);
    $job_data->{messages} = \@messages;
    $job_data->{report} = $report;

    $self->ended_on(dt_from_string)
         ->set_encoded_json_field({ data => $job_data, field => 'data' });
    $self->status('finished') if $self->status ne 'cancelled';
    $self->store;
}
- 

Return to bug 32370

Lines 17-23 package Koha::BackgroundJob; Link Here

(-)a/Koha/BackgroundJob.pm (-38 / +15 lines)
17		17
18	use Modern::Perl;	18	use Modern::Perl;
19	use Encode qw();	19	use Encode qw();
20	use JSON;
21	use Carp qw( croak );	20	use Carp qw( croak );
22	use Net::Stomp;	21	use Net::Stomp;
23	use Try::Tiny qw( catch try );	22	use Try::Tiny qw( catch try );
Lines 28-34 use Koha::Exceptions; Link Here
28	use Koha::Plugins;	27	use Koha::Plugins;
29	use Koha::Exceptions::BackgroundJob;	28	use Koha::Exceptions::BackgroundJob;
30		29
31	use base qw( Koha::Object );	30	use base qw( Koha::Object Koha::Object::JSONFields );
32		31
33	=head1 NAME	32	=head1 NAME
34		33
Lines 101-121 sub enqueue { Link Here
101	my $job_args = $params->{job_args};	100	my $job_args = $params->{job_args};
102	my $job_context = $params->{job_context} // C4::Context->userenv;	101	my $job_context = $params->{job_context} // C4::Context->userenv;
103	my $job_queue = $params->{job_queue} // 'default';	102	my $job_queue = $params->{job_queue} // 'default';
104	my $json = $self->json;
105		103
106	my $borrowernumber = (C4::Context->userenv) ? C4::Context->userenv->{number} : undef;	104	my $borrowernumber = (C4::Context->userenv) ? C4::Context->userenv->{number} : undef;
107	$job_context->{interface} = C4::Context->interface;	105	$job_context->{interface} = C4::Context->interface;
108	my $json_context = $json->encode($job_context);	106
109	my $json_args = $json->encode($job_args);	107	$self->set_encoded_json_field( { data => $job_context, field => 'context' } );
		108	$self->set_encoded_json_field( { data => $job_args, field => 'data' } );
110		109
111	$self->set(	110	$self->set(
112	{	111	{ status => 'new',
113	status => 'new',
114	type => $job_type,	112	type => $job_type,
115	queue => $job_queue,	113	queue => $job_queue,
116	size => $job_size,	114	size => $job_size,
117	data => $json_args,
118	context => $json_context,
119	enqueued_on => dt_from_string,	115	enqueued_on => dt_from_string,
120	borrowernumber => $borrowernumber,	116	borrowernumber => $borrowernumber,
121	}	117	}
Lines 131-137 sub enqueue { Link Here
131	};	127	};
132	return unless $conn;	128	return unless $conn;
133		129
134	$json_args = $json->encode($job_args);	130	my $json_args = $self->data; # we pass the JSON string
135	try {	131	try {
136	# This namespace is wrong, it must be a vhost instead.	132	# This namespace is wrong, it must be a vhost instead.
137	# But to do so it needs to be created on the server => much more work when a new Koha instance is created.	133	# But to do so it needs to be created on the server => much more work when a new Koha instance is created.
Lines 169-175 sub process { Link Here
169	$args \|\|= {};	165	$args \|\|= {};
170		166
171	if ( $self->context ) {	167	if ( $self->context ) {
172	my $context = $self->json->decode($self->context);	168	my $context = $self->decode_json_field({ field => 'context' });
173	C4::Context->_new_userenv(-1);	169	C4::Context->_new_userenv(-1);
174	C4::Context->interface( $context->{interface} );	170	C4::Context->interface( $context->{interface} );
175	C4::Context->set_userenv(	171	C4::Context->set_userenv(
Lines 249-276 sub finish { Link Here
249	my ( $self, $data ) = @_;	245	my ( $self, $data ) = @_;
250		246
251	$self->status('finished') unless $self->status eq 'cancelled' or $self->status eq 'failed';	247	$self->status('finished') unless $self->status eq 'cancelled' or $self->status eq 'failed';
		248	$self->set_encoded_json_field({ data => $data, field => 'data' });
252		249
253	return $self->set(	250	return $self->set({ ended_on => \'NOW()' })->store;
254	{
255	ended_on => \'NOW()',
256	data => $self->json->encode($data),
257	}
258	)->store;
259	}
260
261	=head3 json
262
263	my $JSON_object = $self->json;
264
265	Returns a JSON object with utf8 disabled. Encoding to UTF-8 should be
266	done later.
267
268	=cut
269
270	sub json {
271	my ( $self ) = @_;
272	$self->{_json} //= JSON->new->utf8(0); # TODO Should we allow_nonref ?
273	return $self->{_json};
274	}	251	}
275		252
276	=head3 decoded_data	253	=head3 decoded_data
Lines 284-290 Returns the decoded JSON contents from $self->data. Link Here
284	sub decoded_data {	261	sub decoded_data {
285	my ($self) = @_;	262	my ($self) = @_;
286		263
287	return $self->data ? $self->json->decode( $self->data ) : undef;	264	return $self->decode_json_field({ field => 'data' });
288	}	265	}
289		266
290	=head3 set_encoded_data	267	=head3 set_encoded_data
Lines 298-304 Serializes I<$data> as a JSON string and sets the I<data> attribute with it. Link Here
298	sub set_encoded_data {	275	sub set_encoded_data {
299	my ( $self, $data ) = @_;	276	my ( $self, $data ) = @_;
300		277
301	return $self->data( $data ? $self->json->encode($data) : undef );	278	return $self->set_encoded_json_field({ data => $data, field => 'data' });
302	}	279	}
303		280
304	=head3 job_type	281	=head3 job_type
Lines 319-325 sub messages { Link Here
319	my ( $self ) = @_;	296	my ( $self ) = @_;
320		297
321	my @messages;	298	my @messages;
322	my $data_dump = $self->json->decode($self->data);	299	my $data_dump = $self->decode_json_field({ field => 'data' });
323	if ( exists $data_dump->{messages} ) {	300	if ( exists $data_dump->{messages} ) {
324	@messages = @{ $data_dump->{messages} };	301	@messages = @{ $data_dump->{messages} };
325	}	302	}
Lines 336-342 Report of the job. Link Here
336	sub report {	313	sub report {
337	my ( $self ) = @_;	314	my ( $self ) = @_;
338		315
339	my $data_dump = $self->json->decode($self->data);	316	my $data_dump = $self->decode_json_field({ field => 'data' });
340	return $data_dump->{report} \|\| {};	317	return $data_dump->{report} \|\| {};
341	}	318	}
342		319
Lines 489-497 sub to_api { Link Here
489		466
490	my $json = $self->SUPER::to_api( $params );	467	my $json = $self->SUPER::to_api( $params );
491		468
492	$json->{context} = $self->json->decode($self->context)	469	$json->{context} = $self->decode_json_field({ field => 'context' })
493	if defined $self->context;	470	if defined $self->context;
494	$json->{data} = $self->decoded_data;	471	$json->{data} = $self->decode_json_field({ field => 'data' });
495		472
496	return $json;	473	return $json;
497	}	474	}

Lines 108-114 subtest 'enqueue() tests' => sub { Link Here

(-)a/t/db_dependent/Koha/BackgroundJob.t (-5 / +5 lines)
108	is( $job->size, 3, 'Three steps' );	108	is( $job->size, 3, 'Three steps' );
109	is( $job->status, 'new', 'Initial status set correctly' );	109	is( $job->status, 'new', 'Initial status set correctly' );
110	is( $job->borrowernumber, $patron->id, 'Borrowernumber set from userenv' );	110	is( $job->borrowernumber, $patron->id, 'Borrowernumber set from userenv' );
111	is_deeply( $job->json->decode( $job->context ), $job_context, 'Context set from userenv + interface' );	111	is_deeply( $job->decode_json_field({ field => 'context' }), $job_context, 'Context set from userenv + interface' );
112		112
113	$schema->storage->txn_rollback;	113	$schema->storage->txn_rollback;
114	};	114	};
Lines 150-163 subtest 'start(), step() and finish() tests' => sub { Link Here
150		150
151	is( $job->status, 'cancelled', "'finish' leaves 'cancelled' untouched" );	151	is( $job->status, 'cancelled', "'finish' leaves 'cancelled' untouched" );
152	isnt( $job->ended_on, undef, 'ended_on set' );	152	isnt( $job->ended_on, undef, 'ended_on set' );
153	is_deeply( $job->json->decode( $job->data ), $data );	153	is_deeply( $job->decode_json_field({ field => 'data' }), $data );
154		154
155	$job->status('started')->store;	155	$job->status('started')->store;
156	$job->finish( $data );	156	$job->finish( $data );
157		157
158	is( $job->status, 'finished' );	158	is( $job->status, 'finished' );
159	isnt( $job->ended_on, undef, 'ended_on set' );	159	isnt( $job->ended_on, undef, 'ended_on set' );
160	is_deeply( $job->json->decode( $job->data ), $data );	160	is_deeply( $job->decode_json_field({ field => 'data' }), $data );
161		161
162	throws_ok	162	throws_ok
163	{ $job->start; }	163	{ $job->start; }
Lines 224-230 subtest 'process tests' => sub { Link Here
224	is_deeply( C4::Context->interface, 'intranet', "Interface set from job context on process" );	224	is_deeply( C4::Context->interface, 'intranet', "Interface set from job context on process" );
225		225
226	# Manually add a job (->new->store) without context	226	# Manually add a job (->new->store) without context
227	my $json = $job->json; # sorry, quickly borrowing your json object	227	my $json = $job->_json; # sorry, quickly borrowing your json object
228	my $data = $json->encode({ a => 'a', b => 'b' });	228	my $data = $json->encode({ a => 'a', b => 'b' });
229	my $incomplete_job = t::lib::Koha::BackgroundJob::BatchTest->new(	229	my $incomplete_job = t::lib::Koha::BackgroundJob::BatchTest->new(
230	{ status => 'new',	230	{ status => 'new',
Lines 254-260 subtest 'decoded_data() and set_encoded_data() tests' => sub { Link Here
254		254
255	$job->set_encoded_data( $data );	255	$job->set_encoded_data( $data );
256		256
257	is_deeply( $job->json->decode($job->data), $data, 'decode what we sent' );	257	is_deeply( $job->_json->decode($job->data), $data, 'decode what we sent' );
258	is_deeply( $job->decoded_data, $data, 'check with decoded_data' );	258	is_deeply( $job->decoded_data, $data, 'check with decoded_data' );
259		259
260	# Let's get some Unicode stuff into the game	260	# Let's get some Unicode stuff into the game

Lines 56-67 sub process { Link Here

(-)a/t/lib/Koha/BackgroundJob/BatchTest.pm (-3 / +2 lines)
56	$self->progress( ++$job_progress )->store;	56	$self->progress( ++$job_progress )->store;
57	}	57	}
58		58
59	my $job_data = $self->json->decode($self->data);	59	my $job_data = $self->_json->decode($self->data);
60	$job_data->{messages} = \@messages;	60	$job_data->{messages} = \@messages;
61	$job_data->{report} = $report;	61	$job_data->{report} = $report;
62		62
63	$self->ended_on(dt_from_string)	63	$self->ended_on(dt_from_string)
64	->data( $self->json->encode($job_data) );	64	->set_encoded_json_field({ data => $job_data, field => 'data' });
65	$self->status('finished') if $self->status ne 'cancelled';	65	$self->status('finished') if $self->status ne 'cancelled';
66	$self->store;	66	$self->store;
67	}	67	}
68	-