View | Details | Raw Unified | Return to bug 12598
Collapse All | Expand All

(-)a/Koha/Patrons/Import.pm (+516 lines)
Line 0 Link Here
1
package Koha::Patrons::Import;
2
3
# This file is part of Koha.
4
#
5
# Koha is free software; you can redistribute it and/or modify it under the
6
# terms of the GNU General Public License as published by the Free Software
7
# Foundation; either version 3 of the License, or (at your option) any later
8
# version.
9
#
10
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
11
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
12
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
13
#
14
# You should have received a copy of the GNU General Public License along
15
# with Koha; if not, write to the Free Software Foundation, Inc.,
16
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17
18
use Modern::Perl;
19
use Moo;
20
use namespace::clean;
21
22
use Carp;
23
use Text::CSV;
24
25
use C4::Members;
26
use C4::Branch;
27
use C4::Members::Attributes qw(:all);
28
use C4::Members::AttributeTypes;
29
30
use Koha::DateUtils;
31
32
=head1 NAME
33
34
Koha::Patrons::Import - Perl Module containing import_patrons method exported from import_borrowers script.
35
36
=head1 SYNOPSIS
37
38
use Koha::Patrons::Import;
39
40
=head1 DESCRIPTION
41
42
This module contains one method for importing patrons in bulk.
43
44
=head1 FUNCTIONS
45
46
=head2 import_patrons
47
48
 my $return = Koha::Patrons::Import::import_patrons($params);
49
50
Applies various checks and imports patrons in bulk from a csv file.
51
52
Further pod documentation needed here.
53
54
=cut
55
56
has 'today_iso' => ( is => 'ro', lazy => 1,
57
    default => sub { output_pref( { dt => dt_from_string(), dateonly => 1, dateformat => 'iso' } ); }, );
58
59
has 'text_csv' => ( is => 'rw', lazy => 1,
60
    default => sub { Text::CSV->new( { binary => 1, } ); },  );
61
62
sub import_patrons {
63
    my ($self, $params) = @_;
64
65
    my $handle = $params->{file};
66
    unless( $handle ) { carp('No file handle passed in!'); return; }
67
68
    my $matchpoint           = $params->{matchpoint};
69
    my $defaults             = $params->{defaults};
70
    my $ext_preserve         = $params->{preserve_extended_attributes};
71
    my $overwrite_cardnumber = $params->{overwrite_cardnumber};
72
    my $extended             = C4::Context->preference('ExtendedPatronAttributes');
73
    my $set_messaging_prefs  = C4::Context->preference('EnhancedMessagingPreferences');
74
75
    my @columnkeys = $self->set_column_keys($extended);
76
    my @feedback;
77
    my @errors;
78
79
    my $imported    = 0;
80
    my $alreadyindb = 0;
81
    my $overwritten = 0;
82
    my $invalid     = 0;
83
    my $matchpoint_attr_type = $self->set_attribute_types({ extended => $extended, matchpoint => $matchpoint, });
84
85
    # Use header line to construct key to column map
86
    my %csvkeycol;
87
    my $borrowerline = <$handle>;
88
    my @csvcolumns   = $self->prepare_columns({headerrow => $borrowerline, keycol => \%csvkeycol, errors => \@errors, });
89
    push(@feedback, { feedback => 1, name => 'headerrow', value => join( ', ', @csvcolumns ) });
90
91
    my @criticals = qw( surname );    # there probably should be others - rm branchcode && categorycode
92
  LINE: while ( my $borrowerline = <$handle> ) {
93
        my $line_number = $.;
94
        my %borrower;
95
        my @missing_criticals;
96
97
        my $status  = $self->text_csv->parse($borrowerline);
98
        my @columns = $self->text_csv->fields();
99
        if ( !$status ) {
100
            push @missing_criticals, { badparse => 1, line => $line_number, lineraw => $borrowerline };
101
        }
102
        elsif ( @columns == @columnkeys ) {
103
            @borrower{@columnkeys} = @columns;
104
105
            # MJR: try to fill blanks gracefully by using default values
106
            foreach my $key (@columnkeys) {
107
                if ( $borrower{$key} !~ /\S/ ) {
108
                    $borrower{$key} = $defaults->{$key};
109
                }
110
            }
111
        }
112
        else {
113
            # MJR: try to recover gracefully by using default values
114
            foreach my $key (@columnkeys) {
115
                if ( defined( $csvkeycol{$key} ) and $columns[ $csvkeycol{$key} ] =~ /\S/ ) {
116
                    $borrower{$key} = $columns[ $csvkeycol{$key} ];
117
                }
118
                elsif ( $defaults->{$key} ) {
119
                    $borrower{$key} = $defaults->{$key};
120
                }
121
                elsif ( scalar grep { $key eq $_ } @criticals ) {
122
123
                    # a critical field is undefined
124
                    push @missing_criticals, { key => $key, line => $., lineraw => $borrowerline };
125
                }
126
                else {
127
                    $borrower{$key} = '';
128
                }
129
            }
130
        }
131
132
        # Check if borrower category code exists and if it matches to a known category. Pushing error to missing_criticals otherwise.
133
        $self->check_borrower_category($borrower{categorycode}, $borrowerline, $line_number, \@missing_criticals);
134
135
        # Check if branch code exists and if it matches to a branch name. Pushing error to missing_criticals otherwise.
136
        $self->check_branch_code($borrower{branchcode}, $borrowerline, $line_number, \@missing_criticals);
137
138
        # Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it.
139
        $self->format_dates({borrower => \%borrower, lineraw => $borrowerline, line => $line_number, missing_criticals => \@missing_criticals, });
140
141
        if (@missing_criticals) {
142
            foreach (@missing_criticals) {
143
                $_->{borrowernumber} = $borrower{borrowernumber} || 'UNDEF';
144
                $_->{surname}        = $borrower{surname}        || 'UNDEF';
145
            }
146
            $invalid++;
147
            ( 25 > scalar @errors ) and push @errors, { missing_criticals => \@missing_criticals };
148
149
            # The first 25 errors are enough.  Keeping track of 30,000+ would destroy performance.
150
            next LINE;
151
        }
152
153
        # Set patron attributes if extended.
154
        my $patron_attributes = $self->set_patron_attributes($extended, $borrower{patron_attributes}, \@feedback);
155
        if( $extended ) { delete $borrower{patron_attributes}; } # Not really a field in borrowers.
156
157
        # Default date enrolled and date expiry if not already set.
158
        $borrower{dateenrolled} = $self->today_iso() unless $borrower{dateenrolled};
159
        $borrower{dateexpiry} = GetExpiryDate( $borrower{categorycode}, $borrower{dateenrolled} ) unless $borrower{dateexpiry};
160
161
        my $borrowernumber;
162
        my $member;
163
        if ( defined($matchpoint) && ( $matchpoint eq 'cardnumber' ) && ( $borrower{'cardnumber'} ) ) {
164
            $member = GetMember( 'cardnumber' => $borrower{'cardnumber'} );
165
            if ($member) {
166
                $borrowernumber = $member->{'borrowernumber'};
167
            }
168
        }
169
        elsif ($extended) {
170
            if ( defined($matchpoint_attr_type) ) {
171
                foreach my $attr (@$patron_attributes) {
172
                    if ( $attr->{code} eq $matchpoint and $attr->{value} ne '' ) {
173
                        my @borrowernumbers = $matchpoint_attr_type->get_patrons( $attr->{value} );
174
                        $borrowernumber = $borrowernumbers[0] if scalar(@borrowernumbers) == 1;
175
                        last;
176
                    }
177
                }
178
            }
179
        }
180
181
        if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) {
182
            push @errors,
183
              {
184
                invalid_cardnumber => 1,
185
                borrowernumber     => $borrowernumber,
186
                cardnumber         => $borrower{cardnumber}
187
              };
188
            $invalid++;
189
            next;
190
        }
191
192
        # Check if the userid provided does not exist yet
193
        if (  exists $borrower{userid}
194
                 and $borrower{userid}
195
             and not Check_Userid( $borrower{userid}, $borrower{borrowernumber} ) ) {
196
             push @errors, { duplicate_userid => 1, userid => $borrower{userid} };
197
             $invalid++;
198
             next LINE;
199
        }
200
201
        if ($borrowernumber) {
202
203
            # borrower exists
204
            unless ($overwrite_cardnumber) {
205
                $alreadyindb++;
206
                push(
207
                    @feedback,
208
                    {
209
                        already_in_db => 1,
210
                        value         => $borrower{'surname'} . ' / ' . $borrowernumber
211
                    }
212
                );
213
                next LINE;
214
            }
215
            $borrower{'borrowernumber'} = $borrowernumber;
216
            for my $col ( keys %borrower ) {
217
218
                # use values from extant patron unless our csv file includes this column or we provided a default.
219
                # FIXME : You cannot update a field with a  perl-evaluated false value using the defaults.
220
221
                # The password is always encrypted, skip it!
222
                next if $col eq 'password';
223
224
                unless ( exists( $csvkeycol{$col} ) || $defaults->{$col} ) {
225
                    $borrower{$col} = $member->{$col} if ( $member->{$col} );
226
                }
227
            }
228
229
            unless ( ModMember(%borrower) ) {
230
                $invalid++;
231
232
                push(
233
                    @errors,
234
                    {
235
                        name  => 'lastinvalid',
236
                        value => $borrower{'surname'} . ' / ' . $borrowernumber
237
                    }
238
                );
239
                next LINE;
240
            }
241
            if ( $borrower{debarred} ) {
242
243
                # Check to see if this debarment already exists
244
                my $debarrments = GetDebarments(
245
                    {
246
                        borrowernumber => $borrowernumber,
247
                        expiration     => $borrower{debarred},
248
                        comment        => $borrower{debarredcomment}
249
                    }
250
                );
251
252
                # If it doesn't, then add it!
253
                unless (@$debarrments) {
254
                    AddDebarment(
255
                        {
256
                            borrowernumber => $borrowernumber,
257
                            expiration     => $borrower{debarred},
258
                            comment        => $borrower{debarredcomment}
259
                        }
260
                    );
261
                }
262
            }
263
            if ($extended) {
264
                if ($ext_preserve) {
265
                    my $old_attributes = GetBorrowerAttributes($borrowernumber);
266
                    $patron_attributes = extended_attributes_merge( $old_attributes, $patron_attributes );
267
                }
268
                push @errors, { unknown_error => 1 }
269
                  unless SetBorrowerAttributes( $borrower{'borrowernumber'}, $patron_attributes, 'no_branch_limit' );
270
            }
271
            $overwritten++;
272
            push(
273
                @feedback,
274
                {
275
                    feedback => 1,
276
                    name     => 'lastoverwritten',
277
                    value    => $borrower{'surname'} . ' / ' . $borrowernumber
278
                }
279
            );
280
        }
281
        else {
282
            # FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either.
283
            # At least this is closer to AddMember than in members/memberentry.pl
284
            if ( !$borrower{'cardnumber'} ) {
285
                $borrower{'cardnumber'} = fixup_cardnumber(undef);
286
            }
287
            if ( $borrowernumber = AddMember(%borrower) ) {
288
289
                if ( $borrower{debarred} ) {
290
                    AddDebarment(
291
                        {
292
                            borrowernumber => $borrowernumber,
293
                            expiration     => $borrower{debarred},
294
                            comment        => $borrower{debarredcomment}
295
                        }
296
                    );
297
                }
298
299
                if ($extended) {
300
                    SetBorrowerAttributes( $borrowernumber, $patron_attributes );
301
                }
302
303
                if ($set_messaging_prefs) {
304
                    C4::Members::Messaging::SetMessagingPreferencesFromDefaults(
305
                        {
306
                            borrowernumber => $borrowernumber,
307
                            categorycode   => $borrower{categorycode}
308
                        }
309
                    );
310
                }
311
312
                $imported++;
313
                push(
314
                    @feedback,
315
                    {
316
                        feedback => 1,
317
                        name     => 'lastimported',
318
                        value    => $borrower{'surname'} . ' / ' . $borrowernumber
319
                    }
320
                );
321
            }
322
            else {
323
                $invalid++;
324
                push @errors, { unknown_error => 1 };
325
                push(
326
                    @errors,
327
                    {
328
                        name  => 'lastinvalid',
329
                        value => $borrower{'surname'} . ' / AddMember',
330
                    }
331
                );
332
            }
333
        }
334
    }
335
336
    return {
337
        feedback      => \@feedback,
338
        errors        => \@errors,
339
        imported      => $imported,
340
        overwritten   => $overwritten,
341
        already_in_db => $alreadyindb,
342
        invalid       => $invalid,
343
    };
344
}
345
346
=head2 prepare_columns
347
348
 my @csvcolumns = $self->prepare_columns({headerrow => $borrowerline, keycol => \%csvkeycol, errors => \@errors, });
349
350
Returns an array of all column key and populates a hash of colunm key positions.
351
352
=cut
353
354
sub prepare_columns {
355
    my ($self, $params) = @_;
356
357
    my $status = $self->text_csv->parse($params->{headerrow});
358
    unless( $status ) {
359
        push( @{$params->{errors}}, { badheader => 1, line => 1, lineraw => $params->{headerrow} });
360
        return;
361
    }
362
363
    my @csvcolumns = $self->text_csv->fields();
364
    my $col = 0;
365
    foreach my $keycol (@csvcolumns) {
366
        # columnkeys don't contain whitespace, but some stupid tools add it
367
        $keycol =~ s/ +//g;
368
        $params->{keycol}->{$keycol} = $col++;
369
    }
370
371
    return @csvcolumns;
372
}
373
374
=head2 set_attribute_types
375
376
 my $matchpoint_attr_type = $self->set_attribute_types({ extended => $extended, matchpoint => $matchpoint, });
377
378
Returns an attribute type based on matchpoint parameter.
379
380
=cut
381
382
sub set_attribute_types {
383
    my ($self, $params) = @_;
384
385
    my $attribute_types;
386
    if( $params->{extended} ) {
387
        $attribute_types = C4::Members::AttributeTypes->fetch($params->{matchpoint});
388
    }
389
390
    return $attribute_types;
391
}
392
393
=head2 set_column_keys
394
395
 my @columnkeys = set_column_keys($extended);
396
397
Returns an array of borrowers' table columns.
398
399
=cut
400
401
sub set_column_keys {
402
    my ($self, $extended) = @_;
403
404
    my @columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } C4::Members::columns();
405
    push( @columnkeys, 'patron_attributes' ) if $extended;
406
407
    return @columnkeys;
408
}
409
410
=head2 set_patron_attributes
411
412
 my $patron_attributes = set_patron_attributes($extended, $borrower{patron_attributes}, $feedback);
413
414
Returns a reference to array of hashrefs data structure as expected by SetBorrowerAttributes.
415
416
=cut
417
418
sub set_patron_attributes {
419
    my ($self, $extended, $patron_attributes, $feedback) = @_;
420
421
    unless( $extended ) { return; }
422
    unless( defined($patron_attributes) ) { return; }
423
424
    # Fixup double quotes in case we are passed smart quotes
425
    $patron_attributes =~ s/\xe2\x80\x9c/"/g;
426
    $patron_attributes =~ s/\xe2\x80\x9d/"/g;
427
428
    push (@$feedback, { feedback => 1, name => 'attribute string', value => $patron_attributes });
429
430
    my $result = extended_attributes_code_value_arrayref($patron_attributes);
431
432
    return $result;
433
}
434
435
=head2 check_branch_code
436
437
 check_branch_code($borrower{branchcode}, $borrowerline, $line_number, \@missing_criticals);
438
439
Pushes a 'missing_criticals' error entry if no branch code or branch code does not map to a branch name.
440
441
=cut
442
443
sub check_branch_code {
444
    my ($self, $branchcode, $borrowerline, $line_number, $missing_criticals) = @_;
445
446
    # No branch code
447
    unless( $branchcode ) {
448
        push (@$missing_criticals, { key => 'branchcode', line => $line_number, lineraw => $borrowerline, });
449
        return;
450
    }
451
452
    # look for branch code
453
    my $branch_name = GetBranchName( $branchcode );
454
    unless( $branch_name ) {
455
        push (@$missing_criticals, { key => 'branchcode', line => $line_number, lineraw => $borrowerline,
456
                                     value => $branchcode, branch_map => 1, });
457
    }
458
}
459
460
=head2 check_borrower_category
461
462
 check_borrower_category($borrower{categorycode}, $borrowerline, $line_number, \@missing_criticals);
463
464
Pushes a 'missing_criticals' error entry if no category code or category code does not map to a known category.
465
466
=cut
467
468
sub check_borrower_category {
469
    my ($self, $categorycode, $borrowerline, $line_number, $missing_criticals) = @_;
470
471
    # No branch code
472
    unless( $categorycode ) {
473
        push (@$missing_criticals, { key => 'categorycode', line => $line_number, lineraw => $borrowerline, });
474
        return;
475
    }
476
477
    # Looking for borrower category
478
    my $category = GetBorrowercategory( $categorycode );
479
    unless( $category ) {
480
        push (@$missing_criticals, { key => 'categorycode', line => $line_number, lineraw => $borrowerline,
481
                                     value => $categorycode, category_map => 1, });
482
    }
483
}
484
485
=head2 format_dates
486
487
 format_dates({borrower => \%borrower, lineraw => $lineraw, line => $line_number, missing_criticals => \@missing_criticals, });
488
489
Pushes a 'missing_criticals' error entry for each of the 3 date types dateofbirth, dateenrolled and dateexpiry if it can not
490
be formatted to the chosen date format. Populates the correctly formatted date otherwise.
491
492
=cut
493
494
sub format_dates {
495
    my ($self, $params) = @_;
496
497
    foreach my $date_type (qw(dateofbirth dateenrolled dateexpiry)) {
498
        my $tempdate = $params->{borrower}->{$date_type} or next();
499
        my $formatted_date = eval { output_pref( { dt => dt_from_string( $tempdate ), dateonly => 1, dateformat => 'iso' } ); };
500
501
        if ($formatted_date) {
502
            $params->{borrower}->{$date_type} = $formatted_date;
503
        } else {
504
            $params->{borrower}->{$date_type} = '';
505
            push (@{$params->{missing_criticals}}, { key => $date_type, line => $params->{line}, lineraw => $params->{lineraw}, bad_date => 1 });
506
        }
507
    }
508
}
509
510
1;
511
512
=head1 AUTHOR
513
514
Koha Team
515
516
=cut
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tt (-188 / +242 lines)
Lines 26-196 Link Here
26
     <div class="yui-u first">
26
     <div class="yui-u first">
27
<h1>Import patrons</h1>
27
<h1>Import patrons</h1>
28
[% IF ( uploadborrowers ) %]
28
[% IF ( uploadborrowers ) %]
29
<h5>Import results :</h5>
29
    <h5>Import results :</h5>
30
<ul>
30
    <ul>
31
	<li>[% imported %] imported records [% IF ( lastimported ) %](last was [% lastimported %])[% END %]</li>
31
        <li>[% imported %] imported records [% IF ( lastimported ) %](last was [% lastimported %])[% END %]</li>
32
        [% IF imported and patronlistname %]
32
        [% IF imported and patronlistname %]
33
            <li>Patronlist with imported patrons: [% patronlistname %]</li>
33
            <li>Patronlist with imported patrons: [% patronlistname %]</li>
34
        [% END %]
34
        [% END %]
35
	<li>[% overwritten %] overwritten [% IF ( lastoverwritten ) %](last was [% lastoverwritten %])[% END %]</li>
35
        <li>[% overwritten %] overwritten [% IF ( lastoverwritten ) %](last was [% lastoverwritten %])[% END %]</li>
36
	<li>[% alreadyindb %] not imported because already in borrowers table and overwrite disabled [% IF ( lastalreadyindb ) %](last was [% lastalreadyindb %])[% END %]</li>
36
        <li>[% alreadyindb %] not imported because already in borrowers table and overwrite disabled [% IF ( lastalreadyindb ) %](last was [% lastalreadyindb %])[% END %]</li>
37
	<li>[% invalid %] not imported because they are not in the expected format [% IF ( lastinvalid ) %](last was [% lastinvalid %])[% END %]</li>
37
        <li>[% invalid %] not imported because they are not in the expected format [% IF ( lastinvalid ) %](last was [% lastinvalid %])[% END %]</li>
38
	<li>[% total %] records parsed</li>
38
        <li>[% total %] records parsed</li>
39
	<li><a href="/cgi-bin/koha/tools/tools-home.pl">Back to Tools</a></li>
39
        <li><a href="/cgi-bin/koha/tools/tools-home.pl">Back to Tools</a></li>
40
</ul>
41
  [% IF ( FEEDBACK ) %]
42
  <br /><br />
43
    <div>
44
    <h5>Feedback:</h5>
45
    <ul class="feedback">
46
    [% FOREACH FEEDBAC IN FEEDBACK %]
47
    <li>
48
        [% IF ( FEEDBAC.filename ) %]Parsing upload file <span class="filename">[% FEEDBAC.filename %]</span>
49
        [% ELSIF ( FEEDBAC.backend ) %]Upload parsed using [% FEEDBAC.backend %]
50
        [% ELSIF ( FEEDBAC.headerrow ) %]These fields found: [% FEEDBAC.value %]
51
        [% ELSE %][% FEEDBAC.name %] : [% FEEDBAC.value %]
52
        [% END %]
53
    </li>
54
    [% END %]
55
    </ul>
40
    </ul>
56
    </div>
41
57
  [% END %]
42
    [% IF ( feedback ) %]
58
  [% IF ( ERRORS ) %]
43
        <br /><br />
59
  <br /><br />
44
60
    <div>
45
        <div>
61
    <h5>Error analysis:</h5>
46
            <h5>Feedback:</h5>
62
    <ul>
47
                <ul class="feedback">
63
    [% FOREACH ERROR IN ERRORS %]
48
                    [% FOREACH f IN feedback %]
64
        [% IF ( ERROR.badheader ) %]<li>Header row could not be parsed</li>[% END %]
49
                        <li>
65
        [% FOREACH missing_critical IN ERROR.missing_criticals %]
50
                            [% IF ( f.filename ) %]
66
        <li class="line_error">
51
                                Parsing upload file <span class="filename">[% f.filename %]</span>
67
            Line <span class="linenumber">[% missing_critical.line %]</span>
52
                            [% ELSIF ( f.backend ) %]
68
            [% IF ( missing_critical.badparse ) %]
53
                                Upload parsed using [% f.backend %]
69
                could not be parsed!
54
                            [% ELSIF ( f.headerrow ) %]
70
            [% ELSIF ( missing_critical.bad_date ) %]
55
                                These fields found: [% f.value %]
71
                has &quot;[% missing_critical.key %]&quot; in unrecognized format: &quot;[% missing_critical.value %]&quot;
56
                            [% ELSIF ( f.already_in_db ) %]
72
            [% ELSE %]
57
                                Patron already in database: [% f.value %]
73
                Critical field &quot;[% missing_critical.key %]&quot;
58
                            [% ELSE %]
74
                [% IF ( missing_critical.branch_map ) %]has unrecognized value &quot;[% missing_critical.value %]&quot;
59
                                [% f.name %] : [% f.value %]
75
                [% ELSIF ( missing_critical.category_map ) %]has unrecognized value &quot;[% missing_critical.value %]&quot;
60
                            [% END %]
76
                [% ELSE %]missing
61
                        </li>
62
                    [% END %]
63
                </ul>
64
        </div>
65
    [% END %]
66
67
    [% IF ( errors ) %]
68
        <br /><br />
69
70
        <div>
71
            <h5>Error analysis:</h5>
72
            <ul>
73
                [% FOREACH e IN errors %]
74
                    [% IF ( e.badheader ) %]<li>Header row could not be parsed</li>[% END %]
75
76
                    [% FOREACH missing_critical IN e.missing_criticals %]
77
                        <li class="line_error">
78
                            Line <span class="linenumber">[% missing_critical.line %]</span>
79
80
                            [% IF ( missing_critical.badparse ) %]
81
                                could not be parsed!
82
                            [% ELSIF ( missing_critical.bad_date ) %]
83
                                has &quot;[% missing_critical.key %]&quot; in unrecognized format: &quot;[% missing_critical.value %]&quot;
84
                            [% ELSE %]
85
                                Critical field &quot;[% missing_critical.key %]&quot;
86
87
                                [% IF ( missing_critical.branch_map ) %]
88
                                    has unrecognized value &quot;[% missing_critical.value %]&quot;
89
                                [% ELSIF ( missing_critical.category_map ) %]
90
                                    has unrecognized value &quot;[% missing_critical.value %]&quot;
91
                                [% ELSE %]
92
                                    missing
93
                                [% END %]
94
95
                                (borrowernumber: [% missing_critical.borrowernumber %]; surname: [% missing_critical.surname %]).
96
                            [% END %]
97
98
                            <br/>
99
                            <code>[% missing_critical.lineraw %]</code>
100
                        </li>
101
                    [% END %]
102
103
                    [% IF e.invalid_cardnumber %]
104
                        <li class="line_error">
105
                            Cardnumber [% e.cardnumber %] is not a valid cardnumber
106
                            [% IF e.borrowernumber %] (for patron with borrowernumber [% e.borrowernumber %])[% END %]
107
                        </li>
108
                    [% END %]
109
                    [% IF e.duplicate_userid %]
110
                        <li class="line_error">
111
                            Userid [% e.userid %] is already used by another patron.
112
                        </li>
113
                    [% END %]
77
                [% END %]
114
                [% END %]
78
                (borrowernumber: [% missing_critical.borrowernumber %]; surname: [% missing_critical.surname %]).
115
            </ul>
79
            [% END %]
116
        </div>
80
            <br /><code>[% missing_critical.lineraw %]</code>
81
        </li>
82
        [% END %]
83
        [% IF ERROR.invalid_cardnumber %]
84
            <li class="line_error">
85
                Cardnumber [% ERROR.cardnumber %] is not a valid cardnumber
86
                [% IF ERROR.borrowernumber %] (for patron with borrowernumber [% ERROR.borrowernumber %])[% END %]
87
            </li>
88
        [% END %]
89
        [% IF ERROR.duplicate_userid %]
90
            <li class="line_error">
91
                Userid [% ERROR.userid %] is already used by another patron.
92
            </li>
93
        [% END %]
94
    [% END %]
117
    [% END %]
95
    </ul>
96
    </div>
97
  [% END %]
98
[% ELSE %]
118
[% ELSE %]
99
<ul>
119
    <ul>
100
    <li>Select a file to import into the borrowers table.</li>
120
        <li>Select a file to import into the borrowers table</li>
101
    <li>If a cardnumber exists in the table, you can choose whether to ignore the new one or overwrite the old one.</li>
121
        <li>If a cardnumber exists in the table, you can choose whether to ignore the new one or overwrite the old one.</li>
102
</ul>
122
    </ul>
103
<form method="post" action="[% SCRIPT_NAME %]" enctype="multipart/form-data">
104
<fieldset class="rows">
105
<legend>Import into the borrowers table</legend>
106
<ol>
107
	<li>
108
		<label for="uploadborrowers">Select the file to import: </label>
109
		<input type="file" id="uploadborrowers" name="uploadborrowers" />
110
	</li>
111
        <li>
112
            <label for "createpatronlist">Create patron list: </label>
113
            <input name="createpatronlist" id="createpatronlist" value="1" type="checkbox">
114
            <span class="hint">List name will be file name with timestamp</span>
115
        </li>
116
123
117
</ol></fieldset>
124
    <form method="post" action="[% SCRIPT_NAME %]" enctype="multipart/form-data">
118
    <fieldset class="rows">
125
        <fieldset class="rows">
119
        <legend>Field to use for record matching</legend>
126
            <legend>Import into the borrowers table</legend>
120
        <ol>
127
121
            <li class="radio">
128
            <ol>
122
                <select name="matchpoint" id="matchpoint">
129
                <li>
123
                    <option value="cardnumber">Cardnumber</option>
130
                    <label for="uploadborrowers">Select the file to import: </label>
124
                    <option value="userid">Username</option>
131
                    <input type="file" id="uploadborrowers" name="uploadborrowers" />
125
                    [% FOREACH matchpoint IN matchpoints %]
132
                </li>
126
                        <option value="[% matchpoint.code %]">[% matchpoint.description %]</option>
133
134
                <li>
135
                    <label for "createpatronlist">Create patron list: </label>
136
                    <input name="createpatronlist" id="createpatronlist" value="1" type="checkbox">
137
                    <span class="hint">List name will be file name with timestamp</span>
138
                </li>
139
            </ol>
140
        </fieldset>
141
142
        <fieldset class="rows">
143
            <legend>Field to use for record matching</legend>
144
            <ol>
145
                <li class="radio">
146
                    <select name="matchpoint" id="matchpoint">
147
                        <option value="cardnumber">Cardnumber</option>
148
                        <option value="userid">Username</option>
149
                        [% FOREACH matchpoint IN matchpoints %]
150
                            <option value="[% matchpoint.code %]">[% matchpoint.description %]</option>
151
                        [% END %]
152
                    </select>
153
                </li>
154
            </ol>
155
        </fieldset>
156
157
        <fieldset class="rows">
158
            <legend>Default values</legend>
159
160
            <ol>
161
                [% FOREACH borrower_field IN borrower_fields %]
162
163
                    [% SWITCH borrower_field.field %]
164
                    [% CASE 'branchcode' %]
165
                        <li>
166
                            <label class="description" for="branchcode">[% borrower_field.description %]: </label>
167
                            <select id="branchcode" name="branchcode">
168
                                <option value="" selected="selected"></option>
169
                                [% FOREACH library IN Branches.all() %]
170
                                    <option value="[% library.branchcode %]">[% library.branchname %]</option>
171
                                [% END %]
172
                            </select>
173
                            <span class="field_hint">[% borrower_field.field %]</span>
174
                        </li>
175
                    [% CASE 'categorycode' %]
176
                        <li>
177
                            <label class="description" for="categorycode">[% borrower_field.description %]: </label>
178
                            <select id="categorycode" name="categorycode">
179
                                <option value="" selected="selected"></option>
180
                                [% FOREACH category IN categories %]
181
                                    <option value="[% category.categorycode %]">[% category.description %]</option>
182
                                [% END %]
183
                            </select>
184
                            <span class="field_hint">[% borrower_field.field %]</span>
185
                        </li>
186
                    [% CASE %]
187
                        <li>
188
                            <label class="description" for="[% borrower_field.field %]">[% borrower_field.description %]: </label>
189
                            <input id="[% borrower_field.field %]" name="[% borrower_field.field %]" />
190
                            <span class="field_hint">[% borrower_field.field %]</span>
191
                        </li>
127
                    [% END %]
192
                    [% END %]
128
                </select>
193
                [% END %]
129
            </li>
194
130
        </ol>
195
                [% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
131
    </fieldset>
196
                    <li>
132
<fieldset class="rows">
197
                        <label class="description" for="patron_attributes">Patron attributes: </label>
133
<legend>Default values</legend>
198
                        <input id="patron_attributes" name="patron_attributes" />
134
<ol>
199
                        <span class="field_hint">patron_attributes</span>
135
[% FOREACH borrower_field IN borrower_fields %]
200
                    </li>
136
  [% SWITCH borrower_field.field %]
201
                [% END %]
137
  [% CASE 'branchcode' %]
202
138
    <li>
203
            </ol>
139
        <label class="description" for="branchcode">[% borrower_field.description %]: </label>
204
        </fieldset>
140
        <select id="branchcode" name="branchcode">
205
141
            <option value="" selected="selected"></option>
206
        <fieldset class="rows">
142
        [% FOREACH library IN Branches.all() %]
207
            <legend>If matching record is already in the borrowers table:</legend>
143
            <option value="[% library.branchcode %]">
208
144
                [% library.branchname %]</option>
209
            <ol>
145
        [% END %]
210
                <li class="radio">
146
        </select><span class="field_hint">[% borrower_field.field %]</span>
211
                    <input type="radio" id="overwrite_cardnumberno" name="overwrite_cardnumber" value="0" checked="checked" /><label for="overwrite_cardnumberno">Ignore this one, keep the existing one</label>
147
    </li>
212
                </li>
148
  [% CASE 'categorycode' %]
213
149
    <li>
214
                <li class="radio">
150
        <label class="description" for="categorycode">[% borrower_field.description %]: </label>
215
                    <input type="radio" id="overwrite_cardnumberyes" name="overwrite_cardnumber" value="1" /><label for="overwrite_cardnumberyes">Overwrite the existing one with this</label>
151
        <select id="categorycode" name="categorycode">
216
                </li>
152
            <option value="" selected="selected"></option>
217
            </ol>
153
        [% FOREACH category IN categories %]
218
        </fieldset>
154
            <option value="[% category.categorycode %]">
219
155
                [% category.description %]</option>
220
        [% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
221
            <fieldset class="rows">
222
                <legend>Patron attributes</legend>
223
224
                <ol>
225
                    <li class="radio">
226
                        <input type="radio" id="ext_preserve_0" name="ext_preserve" value="0" checked="checked" /><label for="ext_preserve_0">Replace all patron attributes</label>
227
                    </li>
228
229
                    <li class="radio">
230
                        <input type="radio" id="ext_preserve_1" name="ext_preserve" value="1" /><label for="ext_preserve_1">Replace only included patron attributes</label>
231
                    </li>
232
                </ol>
233
            </fieldset>
156
        [% END %]
234
        [% END %]
157
        </select><span class="field_hint">[% borrower_field.field %]</span>
235
158
    </li>
236
        <fieldset class="action"><input type="submit" value="Import" /></fieldset>
159
  [% CASE %]
237
    </form>
160
    <li>
161
        <label class="description" for="[% borrower_field.field %]">[% borrower_field.description %]: </label>
162
        <input id="[% borrower_field.field %]" name="[% borrower_field.field %]" /><span class="field_hint">[% borrower_field.field %]</span>
163
    </li>
164
  [% END %]
165
[% END %]
166
[% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
167
    <li>
168
        <label class="description" for="patron_attributes">Patron attributes: </label>
169
        <input id="patron_attributes" name="patron_attributes" />
170
        <span class="field_hint">patron_attributes</span>
171
    </li>
172
[% END %]
238
[% END %]
173
</ol></fieldset>
239
174
	<fieldset class="rows">
240
</div>
175
	<legend>If matching record is already in the borrowers table:</legend>
241
176
    <ol><li class="radio">
242
<div class="yui-u">
177
        <input type="radio" id="overwrite_cardnumberno" name="overwrite_cardnumber" value="0" checked="checked" /><label for="overwrite_cardnumberno">Ignore this one, keep the existing one</label>
243
    <h2>Notes:</h2>
244
    <ul>
245
        <li>The first line in the file must be a header row defining which columns you are supplying in the import file.</li>
246
247
        <li><b>Download a starter CSV file with all the columns <a href="?sample=1">here</a>.</b>  Values are comma-separated.</li>
248
249
        <li>
250
            OR choose which fields you want to supply from the following list:
251
            <ul>
252
                <li>
253
                    [% FOREACH columnkey IN borrower_fields %]'[% columnkey.field %]', [% END %]
254
                </li>
255
            </ul>
178
        </li>
256
        </li>
179
        <li class="radio">
257
180
		<input type="radio" id="overwrite_cardnumberyes" name="overwrite_cardnumber" value="1" /><label for="overwrite_cardnumberyes">Overwrite the existing one with this</label>
258
        [% IF ( ExtendedPatronAttributes ) %]
259
            <li>
260
                If loading patron attributes, the 'patron_attributes' field should contain a comma-separated list of attribute types and values. The attribute type code and a colon should precede each value. For example: <b>INSTID:12345,LANG:fr</b> or <b>STARTDATE:January 1 2010,TRACK:Day</b>. If an input record has more than one attribute, the fields should either be entered as an unquoted string (previous examples), or with each field wrapped in separate double quotes and delimited by a comma: <b>&quot;STARTDATE:January 1, 2010&quot;,&quot;TRACK:Day&quot;</b>.  The second syntax would be required if the data might have a comma in it, like a date string.
261
            </li>
262
        [% END %]
263
264
        <li>
265
            The fields 'branchcode' and 'categorycode' are <b>required</b> and <b>must match</b> valid entries in your database.
181
        </li>
266
        </li>
182
    </ol>
267
183
    </fieldset>
268
        <li>
184
    [% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
269
            'password' should be stored in plaintext, and will be converted to a Bcrypt hash (if your passwords are already encrypted, talk to your system administrator about options).
185
	<fieldset class="rows">
186
    <legend>Patron attributes</legend>
187
    <ol><li class="radio">
188
        <input type="radio" id="ext_preserve_0" name="ext_preserve" value="0" checked="checked" /><label for="ext_preserve_0">Replace all patron attributes</label>
189
        </li>
270
        </li>
190
        <li class="radio">
271
191
        <input type="radio" id="ext_preserve_1" name="ext_preserve" value="1" /><label for="ext_preserve_1">Replace only included patron attributes</label>
272
        <li>
273
            Date formats should match your system preference, and <b>must</b> be zero-padded, e.g. '01/02/2008'.  Alternatively,
274
you can supply dates in ISO format (e.g., '2010-10-28').
192
        </li>
275
        </li>
193
    </ol>
276
    </ul>
194
    </fieldset>
277
    </fieldset>
195
    [% END %]
278
    [% END %]
196
    <fieldset class="action">
279
    <fieldset class="action">
Lines 199-242 Link Here
199
    </fieldset>
282
    </fieldset>
200
</form>
283
</form>
201
[% END %]
284
[% END %]
285
286
</div>
287
</div>
288
</div>
289
</div>
290
291
<div class="yui-b noprint">
292
    [% INCLUDE 'tools-menu.inc' %]
293
</div>
202
</div>
294
</div>
203
<div class="yui-u">
204
<h2>Notes:</h2>
205
<ul>
206
<li><b>Header: </b>The first line in the file must be a header row defining which columns you are supplying in the import file.</li>
207
<li><b>Separator: </b>Values are comma-separated.</li>
208
<li><b>Starter CSV: </b> Koha provides a starter CSV with all the columns.
209
    <ul><li><a href="?sample=1">Download starter CSV</a></li></ul>
210
</li>
211
<li><b>Field list: </b>Alternatively, you can create your own CSV and choose which fields you want to supply from the following list:
212
    <ul><li>
213
       [% FOREACH columnkey IN borrower_fields %]'[% columnkey.field %]', [% END %]
214
    </li></ul>
215
</li>
216
[% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
217
<li><b>Extended patron attributes: </b>If loading patron attributes, the 'patron_attributes' field should contain a comma-separated list of attribute types and values. The attribute type code and a colon should precede each value.
218
    <ul><li>Example 1: INSTID:12345,LANG:fr</li><li>Example 2: STARTDATE:January 1 2010,TRACK:Day</li></ul>
219
If an input record has more than one attribute, the fields should either be entered as an unquoted string (previous examples), or with each field wrapped in separate double quotes and delimited by a comma:
220
    <ul><li>Example 3: &quot;STARTDATE:January 1, 2010&quot;,&quot;TRACK:Day&quot;</li></ul>
221
The second syntax would be required if the data might have a comma in it, like a date string.</li>
222
[% END %]
223
<li><b>Required fields: </b>The fields 'branchcode' and 'categorycode' are required and must match valid entries in your database.</li>
224
<li><b>Password: </b>Values for the field 'password' should be stored in plaintext, and will be converted to a Bcrypt hash (if your passwords are already encrypted, talk to your system administrator about options).</li>
225
<li><b>Date formats: </b> Date values should match your system preference, and must be zero-padded.
226
    <ul><li>Example: '01/02/2008'</li></ul>
227
Alternatively, you can supply dates in ISO format.
228
    <ul><li>Example: '2010-10-28'</li></ul>
229
</li>
230
</ul>
231
232
     </div>
233
    </div>
234
   </div>
235
  </div>
236
  <div class="yui-b noprint">
237
[% INCLUDE 'tools-menu.inc' %]
238
  </div>
239
 </div>
240
295
241
[% MACRO jsinclude BLOCK %]
296
[% MACRO jsinclude BLOCK %]
242
    [% INCLUDE 'calendar.inc' %]
297
    [% INCLUDE 'calendar.inc' %]
Lines 249-253 Alternatively, you can supply dates in ISO format. Link Here
249
        });
304
        });
250
    </script>
305
    </script>
251
[% END %]
306
[% END %]
252
253
[% INCLUDE 'intranet-bottom.inc' %]
307
[% INCLUDE 'intranet-bottom.inc' %]
(-)a/misc/import_patrons.pl (+103 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Parts copyright 2014 ByWater Solutions
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
use Modern::Perl;
21
22
use Getopt::Long;
23
24
use Koha::Patrons::Import;
25
my $Import = Koha::Patrons::Import->new();
26
27
my $csv_file;
28
my $matchpoint;
29
my $overwrite_cardnumber;
30
my %defaults;
31
my $ext_preserve = 0;
32
my $confirm;
33
my $verbose      = 0;
34
my $help;
35
36
GetOptions(
37
    'c|confirm'                     => \$confirm,
38
    'f|file=s'                      => \$csv_file,
39
    'm|matchpoint=s'                => \$matchpoint,
40
    'd|default=s'                   => \%defaults,
41
    'o|overwrite'                   => \$overwrite_cardnumber,
42
    'p|preserve-extended-atributes' => \$ext_preserve,
43
    'v|verbose+'                    => \$verbose,
44
    'h|help|?'                      => \$help,
45
);
46
47
print_help() if ( $help || !$csv_file || !$matchpoint || !$confirm );
48
49
my $handle;
50
open( $handle, "<", $csv_file ) or die $!;
51
52
my $return = $Import->import_patrons(
53
    {
54
        file                         => $handle,
55
        defaults                     => \%defaults,
56
        matchpoint                   => $matchpoint,
57
        overwrite_cardnumber         => $overwrite_cardnumber,
58
        preserve_extended_attributes => $ext_preserve,
59
    }
60
);
61
62
my $feedback    = $return->{feedback};
63
my $errors      = $return->{errors};
64
my $imported    = $return->{imported};
65
my $overwritten = $return->{overwritten};
66
my $alreadyindb = $return->{already_in_db};
67
my $invalid     = $return->{invalid};
68
69
if ($verbose) {
70
    my $total = $imported + $alreadyindb + $invalid + $overwritten;
71
    say q{};
72
    say "Import complete:";
73
    say "Imported:    $imported";
74
    say "Overwritten: $overwritten";
75
    say "Skipped:     $alreadyindb";
76
    say "Invalid:     $invalid";
77
    say "Total:       $total";
78
    say q{};
79
}
80
81
if ($verbose > 1 ) {
82
    say "Errors:";
83
    say Data::Dumper::Dumper( $errors );
84
}
85
86
if ($verbose > 2 ) {
87
    say "Feedback:";
88
    say Data::Dumper::Dumper( $feedback );
89
}
90
91
sub print_help {
92
    print <<_USAGE_;
93
import_patrons.pl -c /path/to/patrons.csv -m cardnumber
94
    -c --confirm                        Confirms you really want to import these patrons, otherwise prints this help
95
    -f --file                           Path to the CSV file of patrons to import
96
    -m --matchpoint                     Field on which to match incoming patrons to existing patrons
97
    -d --default                        Set defaults to patron fields, repeatable e.g. --default branchcode=MPL --default categorycode=PT
98
    -p --preserve-extended-atributes    Retain extended patron attributes for existing patrons being overwritten
99
    -o --overwrite                      Overwrite existing patrons with new data if a match is found
100
    -v --verbose                        Be verbose
101
_USAGE_
102
    exit;
103
}
(-)a/t/db_dependent/Koha/Patrons/Import.t (+632 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright 2015 Koha Development team
4
#
5
# This file is part of Koha
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
use Modern::Perl;
21
use Test::More tests => 124;
22
use Test::Warn;
23
24
# To be replaced by t::lib::Mock
25
use Test::MockModule;
26
use Koha::Database;
27
28
use File::Temp qw(tempfile tempdir);
29
my $temp_dir = tempdir('Koha_patrons_import_test_XXXX', CLEANUP => 1, TMPDIR => 1);
30
31
use t::lib::TestBuilder;
32
my $builder = t::lib::TestBuilder->new;
33
34
my $schema = Koha::Database->new->schema;
35
$schema->storage->txn_begin;
36
37
# ########## Tests start here #############################
38
# Given ... we can use the module
39
BEGIN { use_ok('Koha::Patrons::Import'); }
40
41
my $patrons_import = new_ok('Koha::Patrons::Import');
42
43
subtest 'test_methods' => sub {
44
    plan tests => 1;
45
46
    # Given ... we can reach the method(s)
47
    my @methods = ('import_patrons',
48
                   'set_attribute_types',
49
                   'prepare_columns',
50
                   'set_column_keys',
51
                   'set_patron_attributes',
52
                   'check_branch_code',
53
                   'format_dates',
54
                  );
55
    can_ok('Koha::Patrons::Import', @methods);
56
};
57
58
subtest 'test_attributes' => sub {
59
    plan tests => 1;
60
61
    my @attributes = ('today_iso', 'text_csv');
62
    can_ok('Koha::Patrons::Import', @attributes);
63
};
64
65
# Tests for Koha::Patrons::Import::import_patrons()
66
# Given ... nothing much. When ... Then ...
67
my $result;
68
warning_is { $result = $patrons_import->import_patrons(undef) }
69
           { carped => 'No file handle passed in!' },
70
           " Koha::Patrons::Import->import_patrons carps if no file handle is passed";
71
is($result, undef, 'Got the expected undef from import_patrons with nothing much');
72
73
# Given ... some params but no file handle.
74
my $params_0 = { some_stuff => 'random stuff', };
75
76
# When ... Then ...
77
my $result_0;
78
warning_is { $result_0 = $patrons_import->import_patrons($params_0) }
79
           { carped => 'No file handle passed in!' },
80
           " Koha::Patrons::Import->import_patrons carps if no file handle is passed";
81
is($result_0, undef, 'Got the expected undef from import_patrons with no file handle');
82
83
# Given ... a file handle to file with headers only.
84
my $ExtendedPatronAttributes = 0;
85
my $context = Test::MockModule->new('C4::Context'); # Necessary mocking for consistent test results.
86
$context->mock('preference', sub { my ($mod, $meth) = @_;
87
                                    if ( $meth eq 'ExtendedPatronAttributes' ) { return $ExtendedPatronAttributes; }
88
                                    if ( $meth eq 'dateformat' ) { return 'us'; }
89
                                });
90
91
92
my $csv_headers  = 'cardnumber,surname,firstname,title,othernames,initials,streetnumber,streettype,address,address2,city,state,zipcode,country,email,phone,mobile,fax,dateofbirth,branchcode,categorycode,dateenrolled,dateexpiry,userid,password';
93
my $res_header   = 'cardnumber, surname, firstname, title, othernames, initials, streetnumber, streettype, address, address2, city, state, zipcode, country, email, phone, mobile, fax, dateofbirth, branchcode, categorycode, dateenrolled, dateexpiry, userid, password';
94
my $csv_one_line = '1000,Nancy,Jenkins,Dr,,NJ,78,Circle,Bunting,El Paso,Henderson,Texas,79984,United States,ajenkins0@sourceforge.net,7-(388)559-6763,3-(373)151-4471,8-(509)286-4001,10/16/1965,CPL,PT,12/28/2014,07/01/2015,jjenkins0,DPQILy';
95
96
my $filename_1 = make_csv($temp_dir, $csv_headers, $csv_one_line);
97
open(my $handle_1, "<", $filename_1) or die "cannot open < $filename_1: $!";
98
my $params_1 = { file => $handle_1, };
99
100
# When ...
101
my $result_1 = $patrons_import->import_patrons($params_1);
102
103
# Then ...
104
is($result_1->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons with no matchpoint defined');
105
is(scalar @{$result_1->{errors}}, 0, 'Got the expected 0 size error array from import_patrons with no matchpoint defined');
106
107
is($result_1->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons with no matchpoint defined');
108
is($result_1->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons with no matchpoint defined');
109
is($result_1->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons with no matchpoint defined');
110
111
is($result_1->{feedback}->[1]->{feedback}, 1, 'Got the expected second feedback from import_patrons with no matchpoint defined');
112
is($result_1->{feedback}->[1]->{name}, 'lastimported', 'Got the expected last imported name from import_patrons with no matchpoint defined');
113
like($result_1->{feedback}->[1]->{value}, qr/^Nancy \/ \d+/, 'Got the expected second header row value from import_patrons with no matchpoint defined');
114
115
is($result_1->{imported}, 1, 'Got the expected 1 imported result from import_patrons with no matchpoint defined');
116
is($result_1->{invalid}, 0, 'Got the expected 0 invalid result from import_patrons with no matchpoint defined');
117
is($result_1->{overwritten}, 0, 'Got the expected 0 overwritten result from import_patrons with no matchpoint defined');
118
119
# Given ... a valid file handle, a bad matchpoint resulting in invalid card number
120
my $filename_2 = make_csv($temp_dir, $csv_headers, $csv_one_line);
121
open(my $handle_2, "<", $filename_2) or die "cannot open < $filename_2: $!";
122
my $params_2 = { file => $handle_2, matchpoint => 'SHOW_BCODE', };
123
124
# When ...
125
my $result_2 = $patrons_import->import_patrons($params_2);
126
127
# Then ...
128
is($result_2->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons with invalid card number');
129
is($result_2->{errors}->[0]->{borrowernumber}, undef, 'Got the expected undef borrower number from import patrons with invalid card number');
130
is($result_2->{errors}->[0]->{cardnumber}, 1000, 'Got the expected 1000 card number from import patrons with invalid card number');
131
is($result_2->{errors}->[0]->{invalid_cardnumber}, 1, 'Got the expected invalid card number from import patrons with invalid card number');
132
133
is($result_2->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons with invalid card number');
134
is($result_2->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons with invalid card number');
135
is($result_2->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons with invalid card number');
136
137
is($result_2->{imported}, 0, 'Got the expected 0 imported result from import_patrons with invalid card number');
138
is($result_2->{invalid}, 1, 'Got the expected 1 invalid result from import_patrons with invalid card number');
139
is($result_2->{overwritten}, 0, 'Got the expected 0 overwritten result from import_patrons with invalid card number');
140
141
# Given ... valid file handle, good matchpoint but same input as prior test.
142
my $filename_3 = make_csv($temp_dir, $csv_headers, $csv_one_line);
143
open(my $handle_3, "<", $filename_3) or die "cannot open < $filename_3: $!";
144
my $params_3 = { file => $handle_3, matchpoint => 'cardnumber', };
145
146
# When ...
147
my $result_3 = $patrons_import->import_patrons($params_3);
148
149
# Then ...
150
is($result_3->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons with duplicate userid');
151
is($result_3->{errors}->[0]->{duplicate_userid}, 1, 'Got the expected duplicate userid error from import patrons with duplicate userid');
152
is($result_3->{errors}->[0]->{userid}, 'jjenkins0', 'Got the expected userid error from import patrons with duplicate userid');
153
154
is($result_3->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons with duplicate userid');
155
is($result_3->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons with duplicate userid');
156
is($result_3->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons with duplicate userid');
157
158
is($result_3->{imported}, 0, 'Got the expected 0 imported result from import_patrons with duplicate userid');
159
is($result_3->{invalid}, 1, 'Got the expected 1 invalid result from import_patrons with duplicate userid');
160
is($result_3->{overwritten}, 0, 'Got the expected 0 overwritten result from import_patrons with duplicate userid');
161
162
# Given ... a new input and mocked C4::Context
163
$ExtendedPatronAttributes = 1; # Updates mocked C4::Preferences result.
164
165
my $new_input_line = '1001,Donna,Sullivan,Mrs,Henry,DS,59,Court,Burrows,Reading,Salt Lake City,Pennsylvania,19605,United States,hsullivan1@purevolume.com,3-(864)009-3006,7-(291)885-8423,1-(879)095-5038,09/19/1970,LPL,PT,03/04/2015,07/01/2015,hsullivan1,8j6P6Dmap';
166
my $filename_4 = make_csv($temp_dir, $csv_headers, $new_input_line);
167
open(my $handle_4, "<", $filename_4) or die "cannot open < $filename_4: $!";
168
my $params_4 = { file => $handle_4, matchpoint => 'SHOW_BCODE', };
169
170
# When ...
171
my $result_4 = $patrons_import->import_patrons($params_4);
172
173
# Then ...
174
is($result_4->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons with extended user');
175
is(scalar @{$result_4->{errors}}, 0, 'Got the expected 0 size error array from import_patrons with extended user');
176
177
is($result_4->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons with extended user');
178
is($result_4->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons with extended user');
179
is($result_4->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons with extended user');
180
181
is($result_4->{feedback}->[1]->{feedback}, 1, 'Got the expected second feedback from import_patrons with extended user');
182
is($result_4->{feedback}->[1]->{name}, 'attribute string', 'Got the expected attribute string from import_patrons with extended user');
183
is($result_4->{feedback}->[1]->{value}, '', 'Got the expected second feedback value from import_patrons with extended user');
184
185
is($result_4->{feedback}->[2]->{feedback}, 1, 'Got the expected third feedback from import_patrons with extended user');
186
is($result_4->{feedback}->[2]->{name}, 'lastimported', 'Got the expected last imported name from import_patrons with extended user');
187
like($result_4->{feedback}->[2]->{value}, qr/^Donna \/ \d+/, 'Got the expected third feedback value from import_patrons with extended user');
188
189
is($result_4->{imported}, 1, 'Got the expected 1 imported result from import_patrons with extended user');
190
is($result_4->{invalid}, 0, 'Got the expected 0 invalid result from import_patrons with extended user');
191
is($result_4->{overwritten}, 0, 'Got the expected 0 overwritten result from import_patrons with extended user');
192
193
$context->unmock('preference');
194
195
# Given ... 3 new inputs. One with no branch code, one with unexpected branch code.
196
my $input_no_branch   = '1002,Johnny,Reynolds,Mr,Patricia,JR,12,Hill,Kennedy,Saint Louis,Colorado Springs,Missouri,63131,United States,preynolds2@washington.edu,7-(925)314-9514,0-(315)973-8956,4-(510)556-2323,09/18/1967,,PT,05/07/2015,07/01/2015,preynolds2,K3HiDzl';
197
my $input_good_branch = '1003,Linda,Richardson,Mr,Kimberly,LR,90,Place,Bayside,Atlanta,Erie,Georgia,31190,United States,krichardson3@pcworld.com,8-(035)185-0387,4-(796)518-3676,3-(644)960-3789,04/13/1954,RPL,PT,06/06/2015,07/01/2015,krichardson3,P3EO0MVRPXbM';
198
my $input_na_branch   = '1005,Ruth,Greene,Mr,Michael,RG,3,Avenue,Grim,Peoria,Jacksonville,Illinois,61614,United States,mgreene5@seesaa.net,3-(941)565-5752,1-(483)885-8138,4-(979)577-6908,02/09/1957,ZZZ,ST,04/02/2015,07/01/2015,mgreene5,or4ORT6JH';
199
200
my $filename_5 = make_csv($temp_dir, $csv_headers, $input_no_branch, $input_good_branch, $input_na_branch);
201
open(my $handle_5, "<", $filename_5) or die "cannot open < $filename_5: $!";
202
my $params_5 = { file => $handle_5, matchpoint => 'cardnumber', };
203
204
# When ...
205
my $result_5 = $patrons_import->import_patrons($params_5);
206
207
# Then ...
208
is($result_5->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons for branch tests');
209
210
is($result_5->{errors}->[0]->{missing_criticals}->[0]->{borrowernumber}, 'UNDEF', 'Got the expected undef borrower number error from import patrons for branch tests');
211
is($result_5->{errors}->[0]->{missing_criticals}->[0]->{key}, 'branchcode', 'Got the expected branch code key from import patrons for branch tests');
212
is($result_5->{errors}->[0]->{missing_criticals}->[0]->{line}, 2, 'Got the expected 2 line number error from import patrons for branch tests');
213
is($result_5->{errors}->[0]->{missing_criticals}->[0]->{lineraw}, $input_no_branch."\r\n", 'Got the expected lineraw error from import patrons for branch tests');
214
is($result_5->{errors}->[0]->{missing_criticals}->[0]->{surname}, 'Johnny', 'Got the expected surname error from import patrons for branch tests');
215
216
is($result_5->{errors}->[1]->{missing_criticals}->[0]->{borrowernumber}, 'UNDEF', 'Got the expected undef borrower number error from import patrons for branch tests');
217
is($result_5->{errors}->[1]->{missing_criticals}->[0]->{branch_map}, 1, 'Got the expected 1 branchmap error from import patrons for branch tests');
218
is($result_5->{errors}->[1]->{missing_criticals}->[0]->{key}, 'branchcode', 'Got the expected branch code key from import patrons for branch tests');
219
is($result_5->{errors}->[1]->{missing_criticals}->[0]->{line}, 4, 'Got the expected 4 line number error from import patrons for branch tests');
220
is($result_5->{errors}->[1]->{missing_criticals}->[0]->{lineraw}, $input_na_branch."\r\n", 'Got the expected lineraw error from import patrons for branch tests');
221
is($result_5->{errors}->[1]->{missing_criticals}->[0]->{surname}, 'Ruth', 'Got the expected surname error from import patrons for branch tests');
222
is($result_5->{errors}->[1]->{missing_criticals}->[0]->{value}, 'ZZZ', 'Got the expected ZZZ value error from import patrons for branch tests');
223
224
is($result_5->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for branch tests');
225
is($result_5->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons for branch tests');
226
is($result_5->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons for branch tests');
227
228
is($result_5->{feedback}->[1]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for branch tests');
229
is($result_5->{feedback}->[1]->{name}, 'lastimported', 'Got the expected lastimported name from import_patrons for branch tests');
230
like($result_5->{feedback}->[1]->{value},  qr/^Linda \/ \d+/, 'Got the expected last imported value from import_patrons with for branch tests');
231
232
is($result_5->{imported}, 1, 'Got the expected 1 imported result from import patrons for branch tests');
233
is($result_5->{invalid}, 2, 'Got the expected 2 invalid result from import patrons for branch tests');
234
is($result_5->{overwritten}, 0, 'Got the expected 0 overwritten result from import patrons for branch tests');
235
236
# Given ... 3 new inputs. One with no category code, one with unexpected category code.
237
my $input_no_category   = '1006,Christina,Olson,Rev,Kimberly,CO,8,Avenue,Northridge,Lexington,Wilmington,Kentucky,40510,United States,kolson6@dropbox.com,7-(810)636-6048,1-(052)012-8984,8-(567)232-7818,03/26/1952,FFL,,09/07/2014,01/07/2015,kolson6,x5D3qGbLlptx';
238
my $input_good_category = '1007,Peter,Peters,Mrs,Lawrence,PP,6,Trail,South,Oklahoma City,Topeka,Oklahoma,73135,United States,lpeters7@bandcamp.com,5-(992)205-9318,0-(732)586-9365,3-(448)146-7936,08/16/1983,PVL,T,03/24/2015,07/01/2015,lpeters7,Z19BrQ4';
239
my $input_na_category   = '1008,Emily,Richards,Ms,Judy,ER,73,Way,Kedzie,Fort Wayne,Phoenix,Indiana,46825,United States,jrichards8@arstechnica.com,5-(266)658-8957,3-(550)500-9107,7-(816)675-9822,08/09/1984,FFL,ZZ,11/09/2014,07/01/2015,jrichards8,D5PvU6H2R';
240
241
my $filename_6 = make_csv($temp_dir, $csv_headers, $input_no_category, $input_good_category, $input_na_category);
242
open(my $handle_6, "<", $filename_6) or die "cannot open < $filename_6: $!";
243
my $params_6 = { file => $handle_6, matchpoint => 'cardnumber', };
244
245
# When ...
246
my $result_6 = $patrons_import->import_patrons($params_6);
247
248
# Then ...
249
is($result_6->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons for category tests');
250
251
is($result_6->{errors}->[0]->{missing_criticals}->[0]->{borrowernumber}, 'UNDEF', 'Got the expected undef borrower number error from import patrons for category tests');
252
is($result_6->{errors}->[0]->{missing_criticals}->[0]->{key}, 'categorycode', 'Got the expected category code key from import patrons for category tests');
253
is($result_6->{errors}->[0]->{missing_criticals}->[0]->{line}, 2, 'Got the expected 2 line number error from import patrons for category tests');
254
is($result_6->{errors}->[0]->{missing_criticals}->[0]->{lineraw}, $input_no_category."\r\n", 'Got the expected lineraw error from import patrons for category tests');
255
is($result_6->{errors}->[0]->{missing_criticals}->[0]->{surname}, 'Christina', 'Got the expected surname error from import patrons for category tests');
256
257
is($result_6->{errors}->[1]->{missing_criticals}->[0]->{borrowernumber}, 'UNDEF', 'Got the expected undef borrower number error from import patrons for category tests');
258
is($result_6->{errors}->[1]->{missing_criticals}->[0]->{category_map}, 1, 'Got the expected 1 category_map error from import patrons for category tests');
259
is($result_6->{errors}->[1]->{missing_criticals}->[0]->{key}, 'categorycode', 'Got the expected category code key from import patrons for category tests');
260
is($result_6->{errors}->[1]->{missing_criticals}->[0]->{line}, 4, 'Got the expected 4 line number error from import patrons for category tests');
261
is($result_6->{errors}->[1]->{missing_criticals}->[0]->{lineraw}, $input_na_category."\r\n", 'Got the expected lineraw error from import patrons for category tests');
262
is($result_6->{errors}->[1]->{missing_criticals}->[0]->{surname}, 'Emily', 'Got the expected surname error from import patrons for category tests');
263
is($result_6->{errors}->[1]->{missing_criticals}->[0]->{value}, 'ZZ', 'Got the expected ZZ value error from import patrons for category tests');
264
265
is($result_6->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for category tests');
266
is($result_6->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons for category tests');
267
is($result_6->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons for category tests');
268
269
is($result_6->{feedback}->[1]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for category tests');
270
is($result_6->{feedback}->[1]->{name}, 'lastimported', 'Got the expected lastimported name from import_patrons for category tests');
271
like($result_6->{feedback}->[1]->{value},  qr/^Peter \/ \d+/, 'Got the expected last imported value from import_patrons with for category tests');
272
273
is($result_6->{imported}, 1, 'Got the expected 1 imported result from import patrons for category tests');
274
is($result_6->{invalid}, 2, 'Got the expected 2 invalid result from import patrons for category tests');
275
is($result_6->{overwritten}, 0, 'Got the expected 0 overwritten result from import patrons for category tests');
276
277
# Given ... 2 new inputs. One without dateofbirth, dateenrolled and dateexpiry values.
278
my $input_complete = '1009,Christina,Harris,Dr,Philip,CH,99,Street,Grayhawk,Baton Rouge,Dallas,Louisiana,70810,United States,pharris9@hp.com,9-(317)603-5513,7-(005)062-7593,8-(349)134-1627,06/19/1969,IPT,PT,04/09/2015,07/01/2015,pharris9,NcAhcvvnB';
279
my $input_no_date  = '1010,Ralph,Warren,Ms,Linda,RW,6,Way,Barby,Orlando,Albany,Florida,32803,United States,lwarrena@multiply.com,7-(579)753-7752,6-(847)086-7566,9-(122)729-8226,26/01/2001,LPL,T,25/01/2001,24/01/2001,lwarrena,tJ56RD4uV';
280
281
my $filename_7 = make_csv($temp_dir, $csv_headers, $input_complete, $input_no_date);
282
open(my $handle_7, "<", $filename_7) or die "cannot open < $filename_7: $!";
283
my $params_7 = { file => $handle_7, matchpoint => 'cardnumber', };
284
285
# When ...
286
my $result_7 = $patrons_import->import_patrons($params_7);
287
288
# Then ...
289
is($result_7->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons for dates tests');
290
is(scalar @{$result_7->{errors}}, 1, 'Got the expected 1 error array size from import_patrons for dates tests');
291
is(scalar @{$result_7->{errors}->[0]->{missing_criticals}}, 3, 'Got the expected 3 missing critical errors from import_patrons for dates tests');
292
293
is($result_7->{errors}->[0]->{missing_criticals}->[0]->{bad_date}, 1, 'Got the expected 1 bad_date error from import patrons for dates tests');
294
is($result_7->{errors}->[0]->{missing_criticals}->[0]->{borrowernumber}, 'UNDEF', 'Got the expected undef borrower number error from import patrons for dates tests');
295
is($result_7->{errors}->[0]->{missing_criticals}->[0]->{key}, 'dateofbirth', 'Got the expected dateofbirth key from import patrons for dates tests');
296
is($result_7->{errors}->[0]->{missing_criticals}->[0]->{line}, 3, 'Got the expected 2 line number error from import patrons for dates tests');
297
is($result_7->{errors}->[0]->{missing_criticals}->[0]->{lineraw}, $input_no_date."\r\n", 'Got the expected lineraw error from import patrons for dates tests');
298
is($result_7->{errors}->[0]->{missing_criticals}->[0]->{surname}, 'Ralph', 'Got the expected surname error from import patrons for dates tests');
299
300
is($result_7->{errors}->[0]->{missing_criticals}->[1]->{key}, 'dateenrolled', 'Got the expected dateenrolled key from import patrons for dates tests');
301
is($result_7->{errors}->[0]->{missing_criticals}->[2]->{key}, 'dateexpiry', 'Got the expected dateexpiry key from import patrons for dates tests');
302
303
is(scalar @{$result_7->{feedback}}, 2, 'Got the expected 2 feedback from import patrons for dates tests');
304
is($result_7->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for dates tests');
305
is($result_7->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons for dates tests');
306
is($result_7->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons for dates tests');
307
308
is($result_7->{feedback}->[1]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for dates tests');
309
is($result_7->{feedback}->[1]->{name}, 'lastimported', 'Got the expected lastimported from import_patrons for dates tests');
310
like($result_7->{feedback}->[1]->{value}, qr/^Christina \/ \d+/, 'Got the expected lastimported value from import_patrons for dates tests');
311
312
is($result_7->{imported}, 1, 'Got the expected 1 imported result from import patrons for dates tests');
313
is($result_7->{invalid}, 1, 'Got the expected 1 invalid result from import patrons for dates tests');
314
is($result_7->{overwritten}, 0, 'Got the expected 0 overwritten result from import patrons for dates tests');
315
316
subtest 'test_prepare_columns' => sub {
317
    plan tests => 16;
318
319
    # Given ... no header row
320
    my $headerrow_0;
321
    my %csvkeycol_0;
322
    my @errors_0;
323
324
    # When ...
325
    my @csvcolumns_0 = $patrons_import->prepare_columns({headerrow => undef, keycol => \%csvkeycol_0, errors => \@errors_0, });
326
327
    # Then ...
328
    is(scalar @csvcolumns_0, 0, 'Got the expected empty column array from prepare columns with no header row');
329
330
    is(scalar @errors_0, 1, 'Got the expected 1 entry in error array from prepare columns with no header row');
331
    is($errors_0[0]->{badheader}, 1, 'Got the expected 1 badheader from prepare columns with no header row');
332
    is($errors_0[0]->{line}, 1, 'Got the expected 1 line from prepare columns with no header row');
333
    is($errors_0[0]->{lineraw}, undef, 'Got the expected undef lineraw from prepare columns with no header row');
334
335
    # Given ... a good header row with plenty of whitespaces
336
    my $headerrow_1 = 'a,    b ,        c,  ,   d';
337
    my %csvkeycol_1;
338
    my @errors_1;
339
340
    # When ...
341
    my @csvcolumns_1 = $patrons_import->prepare_columns({headerrow => $headerrow_1, keycol => \%csvkeycol_1, errors => \@errors_1, });
342
343
    # Then ...
344
    is(scalar @csvcolumns_1, 5, 'Got the expected 5 column array from prepare columns');
345
    is($csvcolumns_1[0], 'a', 'Got the expected a header from prepare columns');
346
    is($csvcolumns_1[1], 'b', 'Got the expected b header from prepare columns');
347
    is($csvcolumns_1[2], 'c', 'Got the expected c header from prepare columns');
348
    is($csvcolumns_1[3], '', 'Got the expected empty header from prepare columns');
349
    is($csvcolumns_1[4], 'd', 'Got the expected d header from prepare columns');
350
351
    is($csvkeycol_1{a}, 0, 'Got the expected 0 value for key a from prepare columns hash');
352
    is($csvkeycol_1{b}, 1, 'Got the expected 1 value for key b from prepare columns hash');
353
    is($csvkeycol_1{c}, 2, 'Got the expected 2 value for key c from prepare columns hash');
354
    is($csvkeycol_1{''}, 3, 'Got the expected 3 value for empty string key from prepare columns hash');
355
    is($csvkeycol_1{d}, 4, 'Got the expected 4 value for key d from prepare columns hash');
356
};
357
358
subtest 'test_set_column_keys' => sub {
359
    plan tests => 5;
360
361
    # Given ... nothing at all
362
    # When ... Then ...
363
    my $attr_type_0 = $patrons_import->set_attribute_types(undef);
364
    is($attr_type_0, undef, 'Got the expected undef attribute type from set attribute types with nothing');
365
366
    # Given ... extended but not matchpoint
367
    my $params_1 = { extended => 1, matchpoint => undef, };
368
369
    # When ... Then ...
370
    my $attr_type_1 = $patrons_import->set_attribute_types($params_1);
371
    is($attr_type_1, undef, 'Got the expected undef attribute type from set attribute types with no matchpoint');
372
373
    # Given ... extended and unexpected matchpoint
374
    my $params_2 = { extended => 1, matchpoint => 'unexpected', };
375
376
    # When ... Then ...
377
    my $attr_type_2 = $patrons_import->set_attribute_types($params_2);
378
    is($attr_type_2, undef, 'Got the expected undef attribute type from set attribute types with unexpected matchpoint');
379
380
    # Given ...
381
    my $code_3   = 'SHOW_BCODE';
382
    my $params_3 = { extended => 1, matchpoint => $code_3, };
383
384
    # When ...
385
    my $attr_type_3 = $patrons_import->set_attribute_types($params_3);
386
387
    # Then ...
388
    isa_ok($attr_type_3, 'C4::Members::AttributeTypes');
389
    is($attr_type_3->{code}, $code_3, 'Got the expected code attribute type from set attribute types');
390
};
391
392
subtest 'test_set_column_keys' => sub {
393
    plan tests => 2;
394
395
    # Given ... nothing at all
396
    # When ... Then ...
397
    my @columnkeys_0 = $patrons_import->set_column_keys(undef);
398
    is(scalar @columnkeys_0, 66, 'Got the expected array size from set column keys with undef extended');
399
400
    # Given ... extended.
401
    my $extended = 1;
402
403
    # When ... Then ...
404
    my @columnkeys_1 = $patrons_import->set_column_keys($extended);
405
    is(scalar @columnkeys_1, 67, 'Got the expected array size from set column keys with extended');
406
};
407
408
subtest 'test_set_patron_attributes' => sub {
409
    plan tests => 13;
410
411
    # Given ... nothing at all
412
    # When ... Then ...
413
    my $result_0 = $patrons_import->set_patron_attributes(undef, undef, undef);
414
    is($result_0, undef, 'Got the expected undef from set patron attributes with nothing');
415
416
    # Given ... not extended.
417
    my $extended_1 = 0;
418
419
    # When ... Then ...
420
    my $result_1 = $patrons_import->set_patron_attributes($extended_1, undef, undef);
421
    is($result_1, undef, 'Got the expected undef from set patron attributes with not extended');
422
423
    # Given ... NO patrons attributes
424
    my $extended_2          = 1;
425
    my $patron_attributes_2 = undef;
426
    my @feedback_2;
427
428
    # When ...
429
    my $result_2 = $patrons_import->set_patron_attributes($extended_2, $patron_attributes_2, \@feedback_2);
430
431
    # Then ...
432
    is($result_2, undef, 'Got the expected undef from set patron attributes with no patrons attributes');
433
    is(scalar @feedback_2, 0, 'Got the expected 0 size feedback array from set patron attributes with no patrons attributes');
434
435
    # Given ... some patrons attributes
436
    my $patron_attributes_3 = "homeroom:1150605,grade:01";
437
    my @feedback_3;
438
439
    # When ...
440
    my $result_3 = $patrons_import->set_patron_attributes($extended_2, $patron_attributes_3, \@feedback_3);
441
442
    # Then ...
443
    ok($result_3, 'Got some data back from set patron attributes');
444
    is($result_3->[0]->{code}, 'grade', 'Got the expected first code from set patron attributes');
445
    is($result_3->[0]->{value}, '01', 'Got the expected first value from set patron attributes');
446
447
    is($result_3->[1]->{code}, 'homeroom', 'Got the expected second code from set patron attributes');
448
    is($result_3->[1]->{value}, 1150605, 'Got the expected second value from set patron attributes');
449
450
    is(scalar @feedback_3, 1, 'Got the expected 1 array size from set patron attributes with extended user');
451
    is($feedback_3[0]->{feedback}, 1, 'Got the expected second feedback from set patron attributes with extended user');
452
    is($feedback_3[0]->{name}, 'attribute string', 'Got the expected attribute string from set patron attributes with extended user');
453
    is($feedback_3[0]->{value}, 'homeroom:1150605,grade:01', 'Got the expected feedback value from set patron attributes with extended user');
454
};
455
456
subtest 'test_check_branch_code' => sub {
457
    plan tests => 11;
458
459
    # Given ... no branch code.
460
    my $borrowerline      = 'some, line';
461
    my $line_number       = 78;
462
    my @missing_criticals = ();
463
464
    # When ...
465
    $patrons_import->check_branch_code(undef, $borrowerline, $line_number, \@missing_criticals);
466
467
    # Then ...
468
    is(scalar @missing_criticals, 1, 'Got the expected missing critical array size of 1 from check_branch_code with no branch code');
469
470
    is($missing_criticals[0]->{key}, 'branchcode', 'Got the expected branchcode key from check_branch_code with no branch code');
471
    is($missing_criticals[0]->{line}, $line_number, 'Got the expected line number from check_branch_code with no branch code');
472
    is($missing_criticals[0]->{lineraw}, $borrowerline, 'Got the expected lineraw value from check_branch_code with no branch code');
473
474
    # Given ... unknown branch code
475
    my $branchcode_1        = 'unexpected';
476
    my $borrowerline_1      = 'some, line,'.$branchcode_1;
477
    my $line_number_1       = 79;
478
    my @missing_criticals_1 = ();
479
480
    # When ...
481
    $patrons_import->check_branch_code($branchcode_1, $borrowerline_1, $line_number_1, \@missing_criticals_1);
482
483
    # Then ...
484
    is(scalar @missing_criticals_1, 1, 'Got the expected missing critical array size of 1 from check_branch_code with unexpected branch code');
485
486
    is($missing_criticals_1[0]->{branch_map}, 1, 'Got the expected 1 branch_map from check_branch_code with unexpected branch code');
487
    is($missing_criticals_1[0]->{key}, 'branchcode', 'Got the expected branchcode key from check_branch_code with unexpected branch code');
488
    is($missing_criticals_1[0]->{line}, $line_number_1, 'Got the expected line number from check_branch_code with unexpected branch code');
489
    is($missing_criticals_1[0]->{lineraw}, $borrowerline_1, 'Got the expected lineraw value from check_branch_code with unexpected branch code');
490
    is($missing_criticals_1[0]->{value}, $branchcode_1, 'Got the expected value from check_branch_code with unexpected branch code');
491
492
    # Given ... a known branch code. Relies on database sample data
493
    my $branchcode_2        = 'FFL';
494
    my $borrowerline_2      = 'some, line,'.$branchcode_2;
495
    my $line_number_2       = 80;
496
    my @missing_criticals_2 = ();
497
498
    # When ...
499
    $patrons_import->check_branch_code($branchcode_2, $borrowerline_2, $line_number_2, \@missing_criticals_2);
500
501
    # Then ...
502
    is(scalar @missing_criticals_2, 0, 'Got the expected missing critical array size of 0 from check_branch_code');
503
};
504
505
subtest 'test_check_borrower_category' => sub {
506
    plan tests => 11;
507
508
    # Given ... no category code.
509
    my $borrowerline      = 'some, line';
510
    my $line_number       = 781;
511
    my @missing_criticals = ();
512
513
    # When ...
514
    $patrons_import->check_borrower_category(undef, $borrowerline, $line_number, \@missing_criticals);
515
516
    # Then ...
517
    is(scalar @missing_criticals, 1, 'Got the expected missing critical array size of 1 from check_branch_code with no category code');
518
519
    is($missing_criticals[0]->{key}, 'categorycode', 'Got the expected categorycode key from check_branch_code with no category code');
520
    is($missing_criticals[0]->{line}, $line_number, 'Got the expected line number from check_branch_code with no category code');
521
    is($missing_criticals[0]->{lineraw}, $borrowerline, 'Got the expected lineraw value from check_branch_code with no category code');
522
523
    # Given ... unknown category code
524
    my $categorycode_1      = 'unexpected';
525
    my $borrowerline_1      = 'some, line, line, '.$categorycode_1;
526
    my $line_number_1       = 791;
527
    my @missing_criticals_1 = ();
528
529
    # When ...
530
    $patrons_import->check_borrower_category($categorycode_1, $borrowerline_1, $line_number_1, \@missing_criticals_1);
531
532
    # Then ...
533
    is(scalar @missing_criticals_1, 1, 'Got the expected missing critical array size of 1 from check_branch_code with unexpected category code');
534
535
    is($missing_criticals_1[0]->{category_map}, 1, 'Got the expected 1 category_map from check_branch_code with unexpected category code');
536
    is($missing_criticals_1[0]->{key}, 'categorycode', 'Got the expected branchcode key from check_branch_code with unexpected category code');
537
    is($missing_criticals_1[0]->{line}, $line_number_1, 'Got the expected line number from check_branch_code with unexpected category code');
538
    is($missing_criticals_1[0]->{lineraw}, $borrowerline_1, 'Got the expected lineraw value from check_branch_code with unexpected category code');
539
    is($missing_criticals_1[0]->{value}, $categorycode_1, 'Got the expected value from check_branch_code with unexpected category code');
540
541
    # Given ... a known category code. Relies on database sample data.
542
    my $categorycode_2      = 'T';
543
    my $borrowerline_2      = 'some, line,'.$categorycode_2;
544
    my $line_number_2       = 801;
545
    my @missing_criticals_2 = ();
546
547
    # When ...
548
    $patrons_import->check_borrower_category($categorycode_2, $borrowerline_2, $line_number_2, \@missing_criticals_2);
549
550
    # Then ...
551
    is(scalar @missing_criticals_2, 0, 'Got the expected missing critical array size of 0 from check_branch_code');
552
};
553
554
subtest 'test_format_dates' => sub {
555
    plan tests => 22;
556
557
    # Given ... no borrower data.
558
    my $borrowerline      = 'another line';
559
    my $line_number       = 987;
560
    my @missing_criticals = ();
561
    my %borrower;
562
    my $params = {borrower => \%borrower, lineraw => $borrowerline, line => $line_number, missing_criticals => \@missing_criticals, };
563
564
    # When ...
565
    $patrons_import->format_dates($params);
566
567
    # Then ...
568
    ok( not(%borrower), 'Got the expected no borrower from format_dates with no dates');
569
    is(scalar @missing_criticals, 0, 'Got the expected missing critical array size of 0 from format_dates with no dates');
570
571
    # Given ... some good dates
572
    my @missing_criticals_1 = ();
573
    my $dateofbirth_1  = '2016-05-03';
574
    my $dateenrolled_1 = '2016-05-04';
575
    my $dateexpiry_1   = '2016-05-06';
576
    my $borrower_1     = { dateofbirth => $dateofbirth_1, dateenrolled => $dateenrolled_1, dateexpiry => $dateexpiry_1, };
577
    my $params_1       = {borrower => $borrower_1, lineraw => $borrowerline, line => $line_number, missing_criticals => \@missing_criticals_1, };
578
579
    # When ...
580
    $patrons_import->format_dates($params_1);
581
582
    # Then ...
583
    is($borrower_1->{dateofbirth}, $dateofbirth_1, 'Got the expected date of birth from format_dates with good dates');
584
    is($borrower_1->{dateenrolled}, $dateenrolled_1, 'Got the expected date of birth from format_dates with good dates');
585
    is($borrower_1->{dateexpiry}, $dateexpiry_1, 'Got the expected date of birth from format_dates with good dates');
586
    is(scalar @missing_criticals_1, 0, 'Got the expected missing critical array size of 0 from check_branch_code with good dates');
587
588
    # Given ... some very bad dates
589
    my @missing_criticals_2 = ();
590
    my $dateofbirth_2  = '03-2016-05';
591
    my $dateenrolled_2 = '04-2016-05';
592
    my $dateexpiry_2   = '06-2016-05';
593
    my $borrower_2     = { dateofbirth => $dateofbirth_2, dateenrolled => $dateenrolled_2, dateexpiry => $dateexpiry_2, };
594
    my $params_2       = {borrower => $borrower_2, lineraw => $borrowerline, line => $line_number, missing_criticals => \@missing_criticals_2, };
595
596
    # When ...
597
    $patrons_import->format_dates($params_2);
598
599
    # Then ...
600
    is($borrower_2->{dateofbirth}, '', 'Got the expected empty date of birth from format_dates with bad dates');
601
    is($borrower_2->{dateenrolled}, '', 'Got the expected emptydate of birth from format_dates with bad dates');
602
    is($borrower_2->{dateexpiry}, '', 'Got the expected empty date of birth from format_dates with bad dates');
603
604
    is(scalar @missing_criticals_2, 3, 'Got the expected missing critical array size of 3 from check_branch_code with bad dates');
605
    is($missing_criticals_2[0]->{bad_date}, 1, 'Got the expected first bad date flag from check_branch_code with bad dates');
606
    is($missing_criticals_2[0]->{key}, 'dateofbirth', 'Got the expected dateofbirth key from check_branch_code with bad dates');
607
    is($missing_criticals_2[0]->{line}, $line_number, 'Got the expected first line from check_branch_code with bad dates');
608
    is($missing_criticals_2[0]->{lineraw}, $borrowerline, 'Got the expected first lineraw from check_branch_code with bad dates');
609
610
    is($missing_criticals_2[1]->{bad_date}, 1, 'Got the expected second bad date flag from check_branch_code with bad dates');
611
    is($missing_criticals_2[1]->{key}, 'dateenrolled', 'Got the expected dateenrolled key from check_branch_code with bad dates');
612
    is($missing_criticals_2[1]->{line}, $line_number, 'Got the expected second line from check_branch_code with bad dates');
613
    is($missing_criticals_2[1]->{lineraw}, $borrowerline, 'Got the expected second lineraw from check_branch_code with bad dates');
614
615
    is($missing_criticals_2[2]->{bad_date}, 1, 'Got the expected third bad date flag from check_branch_code with bad dates');
616
    is($missing_criticals_2[2]->{key}, 'dateexpiry', 'Got the expected dateexpiry key from check_branch_code with bad dates');
617
    is($missing_criticals_2[2]->{line}, $line_number, 'Got the expected third line from check_branch_code with bad dates');
618
    is($missing_criticals_2[2]->{lineraw}, $borrowerline, 'Got the expected third lineraw from check_branch_code with bad dates');
619
};
620
621
# ###### Test utility ###########
622
sub make_csv {
623
    my ($temp_dir, @lines) = @_;
624
625
    my ($fh, $filename) = tempfile( DIR => $temp_dir) or die $!;
626
    print $fh $_."\r\n" foreach @lines;
627
    close $fh or die $!;
628
629
    return $filename;
630
}
631
632
1;
(-)a/tools/import_borrowers.pl (-285 / +52 lines)
Lines 38-49 use Modern::Perl; Link Here
38
38
39
use C4::Auth;
39
use C4::Auth;
40
use C4::Output;
40
use C4::Output;
41
use C4::Context;
42
use C4::Members;
43
use C4::Members::Attributes qw(:all);
44
use C4::Members::AttributeTypes;
45
use C4::Members::Messaging;
46
use C4::Reports::Guided;
47
use C4::Templates;
41
use C4::Templates;
48
use Koha::Patron::Debarments;
42
use Koha::Patron::Debarments;
49
use Koha::Patrons;
43
use Koha::Patrons;
Lines 53-86 use Koha::Libraries; Link Here
53
use Koha::Patron::Categories;
47
use Koha::Patron::Categories;
54
use Koha::List::Patron;
48
use Koha::List::Patron;
55
49
50
use Koha::Patrons::Import;
51
my $Import = Koha::Patrons::Import->new();
52
56
use Text::CSV;
53
use Text::CSV;
54
57
# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals:
55
# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals:
58
# ė
56
# ė
59
# č
57
# č
60
58
61
use CGI qw ( -utf8 );
59
use CGI qw ( -utf8 );
62
60
63
my (@errors, @feedback);
61
my ( @errors, @feedback );
64
my $extended = C4::Context->preference('ExtendedPatronAttributes');
62
my $extended = C4::Context->preference('ExtendedPatronAttributes');
65
my $set_messaging_prefs = C4::Context->preference('EnhancedMessagingPreferences');
63
66
my @columnkeys = Koha::Patrons->columns();
64
my @columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } Koha::Patrons->columns();
67
@columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } @columnkeys;
65
push( @columnkeys, 'patron_attributes' ) if $extended;
68
if ($extended) {
69
    push @columnkeys, 'patron_attributes';
70
}
71
66
72
my $input = CGI->new();
67
my $input = CGI->new();
73
our $csv  = Text::CSV->new({binary => 1});  # binary needed for non-ASCII Unicode
68
74
#push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend}; #XXX
69
#push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend}; #XXX
75
70
76
my ( $template, $loggedinuser, $cookie ) = get_template_and_user({
71
my ( $template, $loggedinuser, $cookie ) = get_template_and_user(
72
    {
77
        template_name   => "tools/import_borrowers.tt",
73
        template_name   => "tools/import_borrowers.tt",
78
        query           => $input,
74
        query           => $input,
79
        type            => "intranet",
75
        type            => "intranet",
80
        authnotrequired => 0,
76
        authnotrequired => 0,
81
        flagsrequired   => { tools => 'import_patrons' },
77
        flagsrequired   => { tools => 'import_patrons' },
82
        debug           => 1,
78
        debug           => 1,
83
});
79
    }
80
);
84
81
85
# get the patron categories and pass them to the template
82
# get the patron categories and pass them to the template
86
my @patron_categories = Koha::Patron::Categories->search_limited({}, {order_by => ['description']});
83
my @patron_categories = Koha::Patron::Categories->search_limited({}, {order_by => ['description']});
Lines 89-109 my $columns = C4::Templates::GetColumnDefs( $input )->{borrowers}; Link Here
89
$columns = [ grep { $_->{field} ne 'borrowernumber' ? $_ : () } @$columns ];
86
$columns = [ grep { $_->{field} ne 'borrowernumber' ? $_ : () } @$columns ];
90
$template->param( borrower_fields => $columns );
87
$template->param( borrower_fields => $columns );
91
88
92
if ($input->param('sample')) {
89
if ( $input->param('sample') ) {
90
    our $csv = Text::CSV->new( { binary => 1 } );    # binary needed for non-ASCII Unicode
93
    print $input->header(
91
    print $input->header(
94
        -type       => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ?
92
        -type       => 'application/vnd.sun.xml.calc',    # 'application/vnd.ms-excel' ?
95
        -attachment => 'patron_import.csv',
93
        -attachment => 'patron_import.csv',
96
    );
94
    );
97
    $csv->combine(@columnkeys);
95
    $csv->combine(@columnkeys);
98
    print $csv->string, "\n";
96
    print $csv->string, "\n";
99
    exit 0;
97
    exit 0;
100
}
98
}
99
101
my $uploadborrowers = $input->param('uploadborrowers');
100
my $uploadborrowers = $input->param('uploadborrowers');
102
my $matchpoint      = $input->param('matchpoint');
101
my $matchpoint      = $input->param('matchpoint');
103
if ($matchpoint) {
102
if ($matchpoint) {
104
    $matchpoint =~ s/^patron_attribute_//;
103
    $matchpoint =~ s/^patron_attribute_//;
105
}
104
}
106
my $overwrite_cardnumber = $input->param('overwrite_cardnumber');
107
105
108
#create a patronlist
106
#create a patronlist
109
my $createpatronlist = $input->param('createpatronlist') || 0;
107
my $createpatronlist = $input->param('createpatronlist') || 0;
Lines 120-406 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
120
            token  => scalar $input->param('csrf_token'),
118
            token  => scalar $input->param('csrf_token'),
121
        });
119
        });
122
120
123
    push @feedback, {feedback=>1, name=>'filename', value=>$uploadborrowers, filename=>$uploadborrowers};
121
    my $handle   = $input->upload('uploadborrowers');
124
    my $handle = $input->upload('uploadborrowers');
125
    my $uploadinfo = $input->uploadInfo($uploadborrowers);
126
    foreach (keys %$uploadinfo) {
127
        push @feedback, {feedback=>1, name=>$_, value=>$uploadinfo->{$_}, $_=>$uploadinfo->{$_}};
128
    }
129
130
    my $imported    = 0;
131
    my @imported_borrowers;
132
    my $alreadyindb = 0;
133
    my $overwritten = 0;
134
    my $invalid     = 0;
135
    my $matchpoint_attr_type; 
136
    my %defaults = $input->Vars;
122
    my %defaults = $input->Vars;
137
123
138
    # use header line to construct key to column map
124
    my $return = $Import->import_patrons(
139
    my $borrowerline = <$handle>;
125
        {
140
    my $status = $csv->parse($borrowerline);
126
            file                         => $handle,
141
    ($status) or push @errors, {badheader=>1,line=>$., lineraw=>$borrowerline};
127
            defaults                     => \%defaults,
142
    my @csvcolumns = $csv->fields();
128
            matchpoint                   => $matchpoint,
143
    my %csvkeycol;
129
            overwrite_cardnumber         => $input->param('overwrite_cardnumber'),
144
    my $col = 0;
130
            preserve_extended_attributes => $input->param('ext_preserve') || 0,
145
    foreach my $keycol (@csvcolumns) {
146
    	# columnkeys don't contain whitespace, but some stupid tools add it
147
    	$keycol =~ s/ +//g;
148
        $csvkeycol{$keycol} = $col++;
149
    }
150
    #warn($borrowerline);
151
    my $ext_preserve = $input->param('ext_preserve') || 0;
152
    if ($extended) {
153
        $matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint);
154
    }
155
156
    push @feedback, {feedback=>1, name=>'headerrow', value=>join(', ', @csvcolumns)};
157
    my $today = output_pref;
158
    my @criticals = qw(surname branchcode categorycode);    # there probably should be others
159
    my @bad_dates;  # I've had a few.
160
    LINE: while ( my $borrowerline = <$handle> ) {
161
        my %borrower;
162
        my @missing_criticals;
163
        my $patron_attributes;
164
        my $status  = $csv->parse($borrowerline);
165
        my @columns = $csv->fields();
166
        if (! $status) {
167
            push @missing_criticals, {badparse=>1, line=>$., lineraw=>$borrowerline};
168
        } elsif (@columns == @columnkeys) {
169
            @borrower{@columnkeys} = @columns;
170
            # MJR: try to fill blanks gracefully by using default values
171
            foreach my $key (@columnkeys) {
172
                if ($borrower{$key} !~ /\S/) {
173
                    $borrower{$key} = $defaults{$key};
174
                }
175
            } 
176
        } else {
177
            # MJR: try to recover gracefully by using default values
178
            foreach my $key (@columnkeys) {
179
            	if (defined($csvkeycol{$key}) and $columns[$csvkeycol{$key}] =~ /\S/) { 
180
            	    $borrower{$key} = $columns[$csvkeycol{$key}];
181
            	} elsif ( $defaults{$key} ) {
182
            	    $borrower{$key} = $defaults{$key};
183
            	} elsif ( scalar grep {$key eq $_} @criticals ) {
184
            	    # a critical field is undefined
185
            	    push @missing_criticals, {key=>$key, line=>$., lineraw=>$borrowerline};
186
            	} else {
187
            		$borrower{$key} = '';
188
            	}
189
            }
190
        }
191
        #warn join(':',%borrower);
192
        if ($borrower{categorycode}) {
193
            push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline, value=>$borrower{categorycode}, category_map=>1}
194
                unless Koha::Patron::Categories->find($borrower{categorycode});
195
        } else {
196
            push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline};
197
        }
198
        if ($borrower{branchcode}) {
199
            push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline, value=>$borrower{branchcode}, branch_map=>1}
200
                unless Koha::Libraries->find($borrower{branchcode});
201
        } else {
202
            push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline};
203
        }
204
        if (@missing_criticals) {
205
            foreach (@missing_criticals) {
206
                $_->{borrowernumber} = $borrower{borrowernumber} || 'UNDEF';
207
                $_->{surname}        = $borrower{surname} || 'UNDEF';
208
            }
209
            $invalid++;
210
            (25 > scalar @errors) and push @errors, {missing_criticals=>\@missing_criticals};
211
            # The first 25 errors are enough.  Keeping track of 30,000+ would destroy performance.
212
            next LINE;
213
        }
214
        if ($extended) {
215
            my $attr_str = $borrower{patron_attributes};
216
            $attr_str =~ s/\xe2\x80\x9c/"/g; # fixup double quotes in case we are passed smart quotes
217
            $attr_str =~ s/\xe2\x80\x9d/"/g;
218
            push @feedback, {feedback=>1, name=>'attribute string', value=>$attr_str, filename=>$uploadborrowers};
219
            delete $borrower{patron_attributes};    # not really a field in borrowers, so we don't want to pass it to ModMember.
220
            $patron_attributes = extended_attributes_code_value_arrayref($attr_str); 
221
        }
222
	# Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it.
223
        foreach (qw(dateofbirth dateenrolled dateexpiry)) {
224
            my $tempdate = $borrower{$_} or next;
225
            $tempdate = eval { output_pref( { dt => dt_from_string( $tempdate ), dateonly => 1, dateformat => 'iso' } ); };
226
            if ($tempdate) {
227
                $borrower{$_} = $tempdate;
228
            } else {
229
                $borrower{$_} = '';
230
                push @missing_criticals, {key=>$_, line=>$. , lineraw=>$borrowerline, bad_date=>1};
231
            }
232
        }
233
        $borrower{dateenrolled} ||= $today;
234
        $borrower{dateexpiry}   ||= Koha::Patron::Categories->find( $borrower{categorycode} )->get_expiry_date( $borrower{dateenrolled} );
235
        my $borrowernumber;
236
        my $member;
237
        if ( ($matchpoint eq 'cardnumber') && ($borrower{'cardnumber'}) ) {
238
            $member = Koha::Patrons->find( { cardnumber => $borrower{'cardnumber'} } );
239
        } elsif ( ($matchpoint eq 'userid') && ($borrower{'userid'}) ) {
240
            $member = Koha::Patrons->find( { userid => $borrower{'userid'} } );
241
        } elsif ($extended) {
242
            if (defined($matchpoint_attr_type)) {
243
                foreach my $attr (@$patron_attributes) {
244
                    if ($attr->{code} eq $matchpoint and $attr->{value} ne '') {
245
                        my @borrowernumbers = $matchpoint_attr_type->get_patrons($attr->{value});
246
                        $borrowernumber = $borrowernumbers[0] if scalar(@borrowernumbers) == 1;
247
                        last;
248
                    }
249
                }
250
            }
251
        }
252
253
        if ($member) {
254
            $member = $member->unblessed;
255
            $borrowernumber = $member->{'borrowernumber'};
256
        } else {
257
            $member = {};
258
        }
259
260
        if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) {
261
            push @errors, {
262
                invalid_cardnumber => 1,
263
                borrowernumber => $borrowernumber,
264
                cardnumber => $borrower{cardnumber}
265
            };
266
            $invalid++;
267
            next;
268
        }
131
        }
132
    );
269
133
270
        if ($borrowernumber) {
134
    my $feedback    = $return->{feedback};
271
            # borrower exists
135
    my $errors      = $return->{errors};
272
            unless ($overwrite_cardnumber) {
136
    my $imported    = $return->{imported};
273
                $alreadyindb++;
137
    my $overwritten = $return->{overwritten};
274
                $template->param('lastalreadyindb'=>$borrower{'surname'}.' / '.$borrowernumber);
138
    my $alreadyindb = $return->{already_in_db};
275
                next LINE;
139
    my $invalid     = $return->{invalid};
276
            }
277
            $borrower{'borrowernumber'} = $borrowernumber;
278
            for my $col (keys %borrower) {
279
                # use values from extant patron unless our csv file includes this column or we provided a default.
280
                # FIXME : You cannot update a field with a  perl-evaluated false value using the defaults.
281
282
                # The password is always encrypted, skip it!
283
                next if $col eq 'password';
284
285
                unless(exists($csvkeycol{$col}) || $defaults{$col}) {
286
                    $borrower{$col} = $member->{$col} if($member->{$col}) ;
287
                }
288
            }
289
290
            # Check if the userid provided does not exist yet
291
            if (  exists $borrower{userid}
292
                     and $borrower{userid}
293
                 and not Check_Userid( $borrower{userid}, $borrower{borrowernumber} ) ) {
294
                push @errors, { duplicate_userid => 1, userid => $borrower{userid} };
295
                $invalid++;
296
                next LINE;
297
            }
298
299
            unless (ModMember(%borrower)) {
300
                $invalid++;
301
                # until we have better error trapping, we have no way of knowing why ModMember errored out...
302
                push @errors, {unknown_error => 1};
303
                $template->param('lastinvalid'=>$borrower{'surname'}.' / '.$borrowernumber);
304
                next LINE;
305
            }
306
307
            # Don't add a new restriction if the existing 'combined' restriction matches this one
308
            if ( $borrower{debarred} && ( ( $borrower{debarred} ne $member->{debarred} ) || ( $borrower{debarredcomment} ne $member->{debarredcomment} ) ) ) {
309
                # Check to see if this debarment already exists
310
                my $debarrments = GetDebarments(
311
                    {
312
                        borrowernumber => $borrowernumber,
313
                        expiration     => $borrower{debarred},
314
                        comment        => $borrower{debarredcomment}
315
                    }
316
                );
317
                # If it doesn't, then add it!
318
                unless (@$debarrments) {
319
                    AddDebarment(
320
                        {
321
                            borrowernumber => $borrowernumber,
322
                            expiration     => $borrower{debarred},
323
                            comment        => $borrower{debarredcomment}
324
                        }
325
                    );
326
                }
327
            }
328
329
            if ($extended) {
330
                if ($ext_preserve) {
331
                    my $old_attributes = GetBorrowerAttributes($borrowernumber);
332
                    $patron_attributes = extended_attributes_merge($old_attributes, $patron_attributes);  #TODO: expose repeatable options in template
333
                }
334
                push @errors, {unknown_error => 1} unless SetBorrowerAttributes($borrower{'borrowernumber'}, $patron_attributes, 'no_branch_limit' );
335
            }
336
            $overwritten++;
337
            $template->param('lastoverwritten'=>$borrower{'surname'}.' / '.$borrowernumber);
338
        } else {
339
            # FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either.
340
            # At least this is closer to AddMember than in members/memberentry.pl
341
            if (!$borrower{'cardnumber'}) {
342
                $borrower{'cardnumber'} = fixup_cardnumber(undef);
343
            }
344
            if ($borrowernumber = AddMember(%borrower)) {
345
346
                if ( $borrower{debarred} ) {
347
                    AddDebarment(
348
                        {
349
                            borrowernumber => $borrowernumber,
350
                            expiration     => $borrower{debarred},
351
                            comment        => $borrower{debarredcomment}
352
                        }
353
                    );
354
                }
355
356
                if ($extended) {
357
                    SetBorrowerAttributes($borrowernumber, $patron_attributes);
358
                }
359
360
                if ($set_messaging_prefs) {
361
                    C4::Members::Messaging::SetMessagingPreferencesFromDefaults({ borrowernumber => $borrowernumber,
362
                                                                                  categorycode => $borrower{categorycode} });
363
                }
364
140
365
                $imported++;
141
    my $uploadinfo = $input->uploadInfo($uploadborrowers);
366
                $template->param('lastimported'=>$borrower{'surname'}.' / '.$borrowernumber);
142
    foreach ( keys %$uploadinfo ) {
367
                push @imported_borrowers, $borrowernumber; #for patronlist
143
        push @$feedback, { feedback => 1, name => $_, value => $uploadinfo->{$_}, $_ => $uploadinfo->{$_} };
368
            } else {
369
                $invalid++;
370
                push @errors, {unknown_error => 1};
371
                $template->param('lastinvalid'=>$borrower{'surname'}.' / AddMember');
372
            }
373
        }
374
    }
144
    }
375
145
376
    if ( $imported && $createpatronlist ) {
146
    push @$feedback, { feedback => 1, name => 'filename', value => $uploadborrowers, filename => $uploadborrowers };
377
        my $patronlist = AddPatronList({ name => $patronlistname });
378
        AddPatronsToList({ list => $patronlist, borrowernumbers => \@imported_borrowers });
379
        $template->param('patronlistname' => $patronlistname);
380
    }
381
147
382
    (@errors  ) and $template->param(  ERRORS=>\@errors  );
383
    (@feedback) and $template->param(FEEDBACK=>\@feedback);
384
    $template->param(
148
    $template->param(
385
        'uploadborrowers' => 1,
149
        uploadborrowers => 1,
386
        'imported'        => $imported,
150
        errors          => $errors,
387
        'overwritten'     => $overwritten,
151
        feedback        => $feedback,
388
        'alreadyindb'     => $alreadyindb,
152
        imported        => $imported,
389
        'invalid'         => $invalid,
153
        overwritten     => $overwritten,
390
        'total'           => $imported + $alreadyindb + $invalid + $overwritten,
154
        alreadyindb     => $alreadyindb,
155
        invalid         => $invalid,
156
        total           => $imported + $alreadyindb + $invalid + $overwritten,
391
    );
157
    );
392
158
393
} else {
159
}
160
else {
394
    if ($extended) {
161
    if ($extended) {
395
        my @matchpoints = ();
162
        my @matchpoints = ();
396
        my @attr_types = C4::Members::AttributeTypes::GetAttributeTypes(undef, 1);
163
        my @attr_types = C4::Members::AttributeTypes::GetAttributeTypes( undef, 1 );
397
        foreach my $type (@attr_types) {
164
        foreach my $type (@attr_types) {
398
            my $attr_type = C4::Members::AttributeTypes->fetch($type->{code});
165
            my $attr_type = C4::Members::AttributeTypes->fetch( $type->{code} );
399
            if ($attr_type->unique_id()) {
166
            if ( $attr_type->unique_id() ) {
400
            push @matchpoints, { code =>  "patron_attribute_" . $attr_type->code(), description => $attr_type->description() };
167
                push @matchpoints,
168
                  { code => "patron_attribute_" . $attr_type->code(), description => $attr_type->description() };
401
            }
169
            }
402
        }
170
        }
403
        $template->param(matchpoints => \@matchpoints);
171
        $template->param( matchpoints => \@matchpoints );
404
    }
172
    }
405
173
406
    $template->param(
174
    $template->param(
407
- 

Return to bug 12598