View | Details | Raw Unified | Return to bug 34788
Collapse All | Expand All

(-)a/t/db_dependent/Koha/BackgroundJob/ImportKBARTFile.t (-1 / +355 lines)
Line 0 Link Here
0
- 
1
#!/usr/bin/perl
2
3
# This file is part of Koha
4
#
5
# Koha is free software; you can redistribute it and/or modify it
6
# under the terms of the GNU General Public License as published by
7
# the Free Software Foundation; either version 3 of the License, or
8
# (at your option) any later version.
9
#
10
# Koha is distributed in the hope that it will be useful, but
11
# WITHOUT ANY WARRANTY; without even the implied warranty of
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
# GNU General Public License for more details.
14
#
15
# You should have received a copy of the GNU General Public License
16
# along with Koha; if not, see <http://www.gnu.org/licenses>.
17
18
use Modern::Perl;
19
20
use Test::More tests => 6;
21
use Test::MockModule;
22
23
use Koha::Database;
24
use Koha::BackgroundJobs;
25
use Koha::BackgroundJob::ImportKBARTFile;
26
27
use MIME::Base64 qw( encode_base64 );
28
29
use t::lib::Mocks;
30
use t::lib::TestBuilder;
31
32
my $schema  = Koha::Database->new->schema;
33
my $builder = t::lib::TestBuilder->new;
34
35
subtest 'enqueue' => sub {
36
37
    plan tests => 1;
38
39
    $schema->storage->txn_begin;
40
41
    # FIXME: Should be an exception
42
    my $job_id = Koha::BackgroundJob::ImportKBARTFile->new->enqueue();
43
    is( $job_id, undef, 'Nothing enqueued if missing file param' );
44
45
    $schema->storage->txn_rollback;
46
};
47
48
subtest 'calculate_chunked_file_size' => sub {
49
50
    plan tests => 2;
51
52
    my $max_number_of_lines =
53
        Koha::BackgroundJob::ImportKBARTFile::calculate_chunked_file_size( 500000, 100000, 50000 );
54
    is( $max_number_of_lines, 9000, 'Number of lines calculated correctly' );
55
    my $max_number_of_lines2 =
56
        Koha::BackgroundJob::ImportKBARTFile::calculate_chunked_file_size( 400000, 100000, 60000 );
57
    is( $max_number_of_lines2, 13500, 'Number of lines calculated correctly' );
58
};
59
60
subtest 'format_title' => sub {
61
62
    plan tests => 4;
63
64
    my $title = {
65
        title_id       => 1,
66
        coverage_notes => 'Test notes'
67
    };
68
69
    my $formatted_title = Koha::BackgroundJob::ImportKBARTFile::format_title($title);
70
71
    is( $formatted_title->{external_id}, 1,            'external_id formatted correctly' );
72
    is( $formatted_title->{notes},       'Test notes', 'notes formatted correctly' );
73
    is( $title->{title_id},              undef,        'title_id has been deleted' );
74
    is( $title->{coverage_notes},        undef,        'coverage_notes has been deleted' );
75
};
76
77
subtest 'format_file' => sub {
78
79
    plan tests => 6;
80
81
    my $file = {
82
        filename     => 'Test_file.txt',
83
        file_content => encode_base64(
84
            'publication_title	print_identifier	online_identifier	date_first_issue_online	num_first_vol_online	num_first_issue_online	date_last_issue_online	num_last_vol_online	num_last_issue_online	title_url	first_author	title_id	embargo_info	coverage_depth	coverage_notes	publisher_name	publication_type	date_monograph_published_print	date_monograph_published_online	monograph_volume	monograph_edition	first_editor	parent_publication_title_id	preceding_publication_title_id	access_type
85
Nature Plants		2055-0278	2015-01	1	1				https://www.nature.com/nplants		4aaa7		fulltext	Hybrid (Open Choice)	Nature Publishing Group UK	serial								P
86
Nature Astronomy		2397-3366	2017-01	1	1				https://www.nature.com/natastron		4bbb0		fulltext	Hybrid (Open Choice)	Nature Publishing Group UK	serial								P'
87
        )
88
    };
89
90
    my ( $column_headers, $lines ) = Koha::BackgroundJob::ImportKBARTFile::format_file($file);
91
92
    is( @{$column_headers},     25,                  '25 column headers found' );
93
    is( @{$column_headers}[0],  'publication_title', 'First header correctly extracted' );
94
    is( @{$column_headers}[10], 'first_author',      'Tenth header correctly extracted' );
95
96
    is( @{$lines}, 2, 'Two lines need processing' );
97
    is(
98
        @{$lines}[0],
99
        'Nature Plants		2055-0278	2015-01	1	1				https://www.nature.com/nplants		4aaa7		fulltext	Hybrid (Open Choice)	Nature Publishing Group UK	serial								P',
100
        'Line correctly identified'
101
    );
102
    is(
103
        @{$lines}[1],
104
        'Nature Astronomy		2397-3366	2017-01	1	1				https://www.nature.com/natastron		4bbb0		fulltext	Hybrid (Open Choice)	Nature Publishing Group UK	serial								P',
105
        'Line correctly identified'
106
    );
107
108
};
109
110
subtest 'create_title_hash_from_line_data' => sub {
111
112
    plan tests => 2;
113
114
    my $file = {
115
        filename     => 'Test_file.txt',
116
        file_content => encode_base64(
117
            'publication_title	print_identifier	online_identifier	date_first_issue_online	num_first_vol_online	num_first_issue_online	date_last_issue_online	num_last_vol_online	num_last_issue_online	title_url	first_author	title_id	embargo_info	coverage_depth	coverage_notes	publisher_name	publication_type	date_monograph_published_print	date_monograph_published_online	monograph_volume	monograph_edition	first_editor	parent_publication_title_id	preceding_publication_title_id	access_type
118
Nature Plants		2055-0278	2015-01	1	1				https://www.nature.com/nplants		4aaa7		fulltext	Hybrid (Open Choice)	Nature Publishing Group UK	serial								P
119
Nature Astronomy		2397-3366	2017-01	1	1				https://www.nature.com/natastron		4bbb0		fulltext	Hybrid (Open Choice)	Nature Publishing Group UK	serial								P'
120
        )
121
    };
122
123
    my ( $column_headers, $lines ) = Koha::BackgroundJob::ImportKBARTFile::format_file($file);
124
125
    my $title_from_line1 =
126
        Koha::BackgroundJob::ImportKBARTFile::create_title_hash_from_line_data( @{$lines}[0], $column_headers );
127
    my $title_from_line2 =
128
        Koha::BackgroundJob::ImportKBARTFile::create_title_hash_from_line_data( @{$lines}[1], $column_headers );
129
130
    my $line1_match = {
131
        'coverage_depth'                  => 'fulltext',
132
        'date_monograph_published_print'  => '',
133
        'date_first_issue_online'         => '2015-01',
134
        'date_last_issue_online'          => '',
135
        'coverage_notes'                  => 'Hybrid (Open Choice)',
136
        'first_editor'                    => '',
137
        'date_monograph_published_online' => '',
138
        'preceding_publication_title_id'  => '',
139
        'num_last_issue_online'           => '',
140
        'embargo_info'                    => '',
141
        'access_type'                     => 'P',
142
        'num_first_issue_online'          => '1',
143
        'online_identifier'               => '2055-0278',
144
        'title_url'                       => 'https://www.nature.com/nplants',
145
        'monograph_volume'                => '',
146
        'first_author'                    => '',
147
        'parent_publication_title_id'     => '',
148
        'num_last_vol_online'             => '',
149
        'publication_title'               => 'Nature Plants',
150
        'num_first_vol_online'            => '1',
151
        'print_identifier'                => '',
152
        'publisher_name'                  => 'Nature Publishing Group UK',
153
        'title_id'                        => '4aaa7',
154
        'publication_type'                => 'serial',
155
        'monograph_edition'               => ''
156
    };
157
    my $line2_match = {
158
        'date_monograph_published_online' => '',
159
        'num_first_vol_online'            => '1',
160
        'num_last_issue_online'           => '',
161
        'preceding_publication_title_id'  => '',
162
        'title_url'                       => 'https://www.nature.com/natastron',
163
        'online_identifier'               => '2397-3366',
164
        'print_identifier'                => '',
165
        'num_last_vol_online'             => '',
166
        'embargo_info'                    => '',
167
        'parent_publication_title_id'     => '',
168
        'publisher_name'                  => 'Nature Publishing Group UK',
169
        'date_first_issue_online'         => '2017-01',
170
        'monograph_volume'                => '',
171
        'monograph_edition'               => '',
172
        'access_type'                     => 'P',
173
        'first_author'                    => '',
174
        'num_first_issue_online'          => '1',
175
        'first_editor'                    => '',
176
        'publication_title'               => 'Nature Astronomy',
177
        'date_monograph_published_print'  => '',
178
        'publication_type'                => 'serial',
179
        'title_id'                        => '4bbb0',
180
        'coverage_depth'                  => 'fulltext',
181
        'coverage_notes'                  => 'Hybrid (Open Choice)',
182
        'date_last_issue_online'          => ''
183
    };
184
185
    is_deeply( $title_from_line1, $line1_match, 'Title hash created correctly' );
186
    is_deeply( $title_from_line2, $line2_match, 'Title hash created correctly' );
187
};
188
189
subtest 'process' => sub {
190
    plan tests => 12;
191
192
    $schema->storage->txn_begin;
193
194
    my $file = {
195
        filename     => 'Test_file.txt',
196
        file_content => encode_base64(
197
            'publication_title	print_identifier	online_identifier	date_first_issue_online	num_first_vol_online	num_first_issue_online	date_last_issue_online	num_last_vol_online	num_last_issue_online	title_url	first_author	title_id	embargo_info	coverage_depth	coverage_notes	publisher_name	publication_type	date_monograph_published_print	date_monograph_published_online	monograph_volume	monograph_edition	first_editor	parent_publication_title_id	preceding_publication_title_id	access_type
198
Nature Plants		2055-0278	2015-01	1	1				https://www.nature.com/nplants		4aaa7		fulltext	Hybrid (Open Choice)	Nature Publishing Group UK	serial								P
199
Nature Astronomy		2397-3366	2017-01	1	1				https://www.nature.com/natastron		4bbb0		fulltext	Hybrid (Open Choice)	Nature Publishing Group UK	serial								P'
200
        )
201
    };
202
203
    my $job = Koha::BackgroundJob::ImportKBARTFile->new(
204
        {
205
            status => 'new',
206
            type   => 'import_from_kbart_file',
207
            size   => 1,
208
        }
209
    )->store;
210
    $job = Koha::BackgroundJobs->find( $job->id );
211
    my $data = { file => $file };
212
    my $json = $job->json->encode($data);
213
    $job->data($json)->store;
214
    $job->process($data);
215
216
    is( $job->report->{titles_imported}, 2, 'Two titles successfully imported' );
217
218
    my $job2 = Koha::BackgroundJob::ImportKBARTFile->new(
219
        {
220
            status => 'new',
221
            type   => 'import_from_kbart_file',
222
            size   => 1,
223
        }
224
    )->store;
225
    $job2 = Koha::BackgroundJobs->find( $job2->id );
226
    $job2->data($json)->store;
227
    $job2->process($data);
228
229
    is( $job2->report->{duplicates_found}, 2, 'Two duplicates found, no titles should be imported' );
230
    is( $job2->report->{titles_imported},  0, 'No titles were imported' );
231
    is_deeply(
232
        $job2->messages,
233
        [
234
            {
235
                'type'          => 'warning',
236
                'title'         => 'Nature Plants',
237
                'code'          => 'title_already_exists',
238
                'error_message' => undef
239
            },
240
            {
241
                'error_message' => undef,
242
                'code'          => 'title_already_exists',
243
                'title'         => 'Nature Astronomy',
244
                'type'          => 'warning'
245
            }
246
        ],
247
        'Two duplicate messages added'
248
    );
249
250
    my $module = Test::MockModule->new('Koha::BackgroundJob::ImportKBARTFile');
251
    $module->mock(
252
        'create_title_hash_from_line_data',
253
        sub {
254
            my ( $line, $column_headers ) = @_;
255
256
            my %new_title;
257
            my @values = split /\t/, $line;
258
259
            @new_title{ @{$column_headers} } = @values;
260
261
            $new_title{title_id}          = '12345' if $new_title{publication_title} eq 'Nature Plants';
262
            $new_title{publication_title} = ''      if $new_title{publication_title} eq 'Nature Plants';
263
264
            return \%new_title;
265
        }
266
    );
267
268
    my $job3 = Koha::BackgroundJob::ImportKBARTFile->new(
269
        {
270
            status => 'new',
271
            type   => 'import_from_kbart_file',
272
            size   => 1,
273
        }
274
    )->store;
275
    $job3 = Koha::BackgroundJobs->find( $job3->id );
276
    $job3->data($json)->store;
277
    $job3->process($data);
278
279
    is( $job3->report->{duplicates_found}, 1, 'One duplicate found' );
280
    is( $job3->report->{titles_imported},  0, 'No titles were imported' );
281
    is( $job3->report->{failed_imports},   1, 'One failure found' );
282
    is_deeply(
283
        $job3->messages,
284
        [
285
            {
286
                'type'          => 'error',
287
                'error_message' => 'No publication_title found for title_id: ',
288
                'code'          => 'title_failed',
289
                'title_id'      => '12345',
290
                'title'         => '(Unknown)'
291
            },
292
            {
293
                'code'          => 'title_already_exists',
294
                'title'         => 'Nature Astronomy',
295
                'error_message' => undef,
296
                'type'          => 'warning'
297
            }
298
        ],
299
        'One duplicate message and one failure message for a missing title'
300
    );
301
302
    $module->mock(
303
        'create_title_hash_from_line_data',
304
        sub {
305
            my ( $line, $column_headers ) = @_;
306
307
            my %new_title;
308
            my @values = split /\t/, $line;
309
310
            @new_title{ @{$column_headers} } = @values;
311
312
            $new_title{title_id}      = 'abcde' if $new_title{publication_title} eq 'Nature Plants';
313
            $new_title{unknown_field} = ''      if $new_title{publication_title} eq 'Nature Plants';
314
315
            return \%new_title;
316
        }
317
    );
318
319
    my $job4 = Koha::BackgroundJob::ImportKBARTFile->new(
320
        {
321
            status => 'new',
322
            type   => 'import_from_kbart_file',
323
            size   => 1,
324
        }
325
    )->store;
326
    $job4 = Koha::BackgroundJobs->find( $job4->id );
327
    $job4->data($json)->store;
328
    $job4->process($data);
329
330
    is( $job4->report->{duplicates_found}, 1, 'One duplicate found' );
331
    is( $job4->report->{titles_imported},  0, 'No titles were imported' );
332
    is( $job4->report->{failed_imports},   1, 'One failure found' );
333
    is_deeply(
334
        $job4->messages,
335
        [
336
            {
337
                'type'  => 'error',
338
                'title' => 'Nature Plants',
339
                'error_message' =>
340
                    'DBIx::Class::Row::store_column(): No such column \'unknown_field\' on Koha::Schema::Result::ErmEholdingsTitle at /kohadevbox/koha/Koha/Object.pm line 79
341
',
342
                'code' => 'title_failed'
343
            },
344
            {
345
                'code'          => 'title_already_exists',
346
                'error_message' => undef,
347
                'title'         => 'Nature Astronomy',
348
                'type'          => 'warning'
349
            }
350
        ],
351
        'One duplicate message and one failure message for an incorrect column'
352
    );
353
354
    $schema->storage->txn_rollback;
355
    }

Return to bug 34788