View | Details | Raw Unified | Return to bug 12598
Collapse All | Expand All

(-)a/Koha/Patrons/Import.pm (+516 lines)
Line 0 Link Here
1
package Koha::Patrons::Import;
2
3
# This file is part of Koha.
4
#
5
# Koha is free software; you can redistribute it and/or modify it under the
6
# terms of the GNU General Public License as published by the Free Software
7
# Foundation; either version 3 of the License, or (at your option) any later
8
# version.
9
#
10
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
11
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
12
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
13
#
14
# You should have received a copy of the GNU General Public License along
15
# with Koha; if not, write to the Free Software Foundation, Inc.,
16
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17
18
use Modern::Perl;
19
use Moo;
20
use namespace::clean;
21
22
use Carp;
23
use Text::CSV;
24
25
use C4::Members;
26
use C4::Branch;
27
use C4::Members::Attributes qw(:all);
28
use C4::Members::AttributeTypes;
29
30
use Koha::DateUtils;
31
32
=head1 NAME
33
34
Koha::Patrons::Import - Perl Module containing import_patrons method exported from import_borrowers script.
35
36
=head1 SYNOPSIS
37
38
use Koha::Patrons::Import;
39
40
=head1 DESCRIPTION
41
42
This module contains one method for importing patrons in bulk.
43
44
=head1 FUNCTIONS
45
46
=head2 import_patrons
47
48
 my $return = Koha::Patrons::Import::import_patrons($params);
49
50
Applies various checks and imports patrons in bulk from a csv file.
51
52
Further pod documentation needed here.
53
54
=cut
55
56
has 'today_iso' => ( is => 'ro', lazy => 1,
57
    default => sub { output_pref( { dt => dt_from_string(), dateonly => 1, dateformat => 'iso' } ); }, );
58
59
has 'text_csv' => ( is => 'rw', lazy => 1,
60
    default => sub { Text::CSV->new( { binary => 1, } ); },  );
61
62
sub import_patrons {
63
    my ($self, $params) = @_;
64
65
    my $handle = $params->{file};
66
    unless( $handle ) { carp('No file handle passed in!'); return; }
67
68
    my $matchpoint           = $params->{matchpoint};
69
    my $defaults             = $params->{defaults};
70
    my $ext_preserve         = $params->{preserve_extended_attributes};
71
    my $overwrite_cardnumber = $params->{overwrite_cardnumber};
72
    my $extended             = C4::Context->preference('ExtendedPatronAttributes');
73
    my $set_messaging_prefs  = C4::Context->preference('EnhancedMessagingPreferences');
74
75
    my @columnkeys = $self->set_column_keys($extended);
76
    my @feedback;
77
    my @errors;
78
79
    my $imported    = 0;
80
    my $alreadyindb = 0;
81
    my $overwritten = 0;
82
    my $invalid     = 0;
83
    my $matchpoint_attr_type = $self->set_attribute_types({ extended => $extended, matchpoint => $matchpoint, });
84
85
    # Use header line to construct key to column map
86
    my %csvkeycol;
87
    my $borrowerline = <$handle>;
88
    my @csvcolumns   = $self->prepare_columns({headerrow => $borrowerline, keycol => \%csvkeycol, errors => \@errors, });
89
    push(@feedback, { feedback => 1, name => 'headerrow', value => join( ', ', @csvcolumns ) });
90
91
    my @criticals = qw( surname );    # there probably should be others - rm branchcode && categorycode
92
  LINE: while ( my $borrowerline = <$handle> ) {
93
        my $line_number = $.;
94
        my %borrower;
95
        my @missing_criticals;
96
97
        my $status  = $self->text_csv->parse($borrowerline);
98
        my @columns = $self->text_csv->fields();
99
        if ( !$status ) {
100
            push @missing_criticals, { badparse => 1, line => $line_number, lineraw => $borrowerline };
101
        }
102
        elsif ( @columns == @columnkeys ) {
103
            @borrower{@columnkeys} = @columns;
104
105
            # MJR: try to fill blanks gracefully by using default values
106
            foreach my $key (@columnkeys) {
107
                if ( $borrower{$key} !~ /\S/ ) {
108
                    $borrower{$key} = $defaults->{$key};
109
                }
110
            }
111
        }
112
        else {
113
            # MJR: try to recover gracefully by using default values
114
            foreach my $key (@columnkeys) {
115
                if ( defined( $csvkeycol{$key} ) and $columns[ $csvkeycol{$key} ] =~ /\S/ ) {
116
                    $borrower{$key} = $columns[ $csvkeycol{$key} ];
117
                }
118
                elsif ( $defaults->{$key} ) {
119
                    $borrower{$key} = $defaults->{$key};
120
                }
121
                elsif ( scalar grep { $key eq $_ } @criticals ) {
122
123
                    # a critical field is undefined
124
                    push @missing_criticals, { key => $key, line => $., lineraw => $borrowerline };
125
                }
126
                else {
127
                    $borrower{$key} = '';
128
                }
129
            }
130
        }
131
132
        # Check if borrower category code exists and if it matches to a known category. Pushing error to missing_criticals otherwise.
133
        $self->check_borrower_category($borrower{categorycode}, $borrowerline, $line_number, \@missing_criticals);
134
135
        # Check if branch code exists and if it matches to a branch name. Pushing error to missing_criticals otherwise.
136
        $self->check_branch_code($borrower{branchcode}, $borrowerline, $line_number, \@missing_criticals);
137
138
        # Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it.
139
        $self->format_dates({borrower => \%borrower, lineraw => $borrowerline, line => $line_number, missing_criticals => \@missing_criticals, });
140
141
        if (@missing_criticals) {
142
            foreach (@missing_criticals) {
143
                $_->{borrowernumber} = $borrower{borrowernumber} || 'UNDEF';
144
                $_->{surname}        = $borrower{surname}        || 'UNDEF';
145
            }
146
            $invalid++;
147
            ( 25 > scalar @errors ) and push @errors, { missing_criticals => \@missing_criticals };
148
149
            # The first 25 errors are enough.  Keeping track of 30,000+ would destroy performance.
150
            next LINE;
151
        }
152
153
        # Set patron attributes if extended.
154
        my $patron_attributes = $self->set_patron_attributes($extended, $borrower{patron_attributes}, \@feedback);
155
        if( $extended ) { delete $borrower{patron_attributes}; } # Not really a field in borrowers.
156
157
        # Default date enrolled and date expiry if not already set.
158
        $borrower{dateenrolled} = $self->today_iso() unless $borrower{dateenrolled};
159
        $borrower{dateexpiry} = GetExpiryDate( $borrower{categorycode}, $borrower{dateenrolled} ) unless $borrower{dateexpiry};
160
161
        my $borrowernumber;
162
        my $member;
163
        if ( defined($matchpoint) && ( $matchpoint eq 'cardnumber' ) && ( $borrower{'cardnumber'} ) ) {
164
            $member = GetMember( 'cardnumber' => $borrower{'cardnumber'} );
165
            if ($member) {
166
                $borrowernumber = $member->{'borrowernumber'};
167
            }
168
        }
169
        elsif ($extended) {
170
            if ( defined($matchpoint_attr_type) ) {
171
                foreach my $attr (@$patron_attributes) {
172
                    if ( $attr->{code} eq $matchpoint and $attr->{value} ne '' ) {
173
                        my @borrowernumbers = $matchpoint_attr_type->get_patrons( $attr->{value} );
174
                        $borrowernumber = $borrowernumbers[0] if scalar(@borrowernumbers) == 1;
175
                        last;
176
                    }
177
                }
178
            }
179
        }
180
181
        if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) {
182
            push @errors,
183
              {
184
                invalid_cardnumber => 1,
185
                borrowernumber     => $borrowernumber,
186
                cardnumber         => $borrower{cardnumber}
187
              };
188
            $invalid++;
189
            next;
190
        }
191
192
        # Check if the userid provided does not exist yet
193
        if (  exists $borrower{userid}
194
                 and $borrower{userid}
195
             and not Check_Userid( $borrower{userid}, $borrower{borrowernumber} ) ) {
196
             push @errors, { duplicate_userid => 1, userid => $borrower{userid} };
197
             $invalid++;
198
             next LINE;
199
        }
200
201
        if ($borrowernumber) {
202
203
            # borrower exists
204
            unless ($overwrite_cardnumber) {
205
                $alreadyindb++;
206
                push(
207
                    @feedback,
208
                    {
209
                        already_in_db => 1,
210
                        value         => $borrower{'surname'} . ' / ' . $borrowernumber
211
                    }
212
                );
213
                next LINE;
214
            }
215
            $borrower{'borrowernumber'} = $borrowernumber;
216
            for my $col ( keys %borrower ) {
217
218
                # use values from extant patron unless our csv file includes this column or we provided a default.
219
                # FIXME : You cannot update a field with a  perl-evaluated false value using the defaults.
220
221
                # The password is always encrypted, skip it!
222
                next if $col eq 'password';
223
224
                unless ( exists( $csvkeycol{$col} ) || $defaults->{$col} ) {
225
                    $borrower{$col} = $member->{$col} if ( $member->{$col} );
226
                }
227
            }
228
229
            unless ( ModMember(%borrower) ) {
230
                $invalid++;
231
232
                push(
233
                    @errors,
234
                    {
235
                        name  => 'lastinvalid',
236
                        value => $borrower{'surname'} . ' / ' . $borrowernumber
237
                    }
238
                );
239
                next LINE;
240
            }
241
            if ( $borrower{debarred} ) {
242
243
                # Check to see if this debarment already exists
244
                my $debarrments = GetDebarments(
245
                    {
246
                        borrowernumber => $borrowernumber,
247
                        expiration     => $borrower{debarred},
248
                        comment        => $borrower{debarredcomment}
249
                    }
250
                );
251
252
                # If it doesn't, then add it!
253
                unless (@$debarrments) {
254
                    AddDebarment(
255
                        {
256
                            borrowernumber => $borrowernumber,
257
                            expiration     => $borrower{debarred},
258
                            comment        => $borrower{debarredcomment}
259
                        }
260
                    );
261
                }
262
            }
263
            if ($extended) {
264
                if ($ext_preserve) {
265
                    my $old_attributes = GetBorrowerAttributes($borrowernumber);
266
                    $patron_attributes = extended_attributes_merge( $old_attributes, $patron_attributes );
267
                }
268
                push @errors, { unknown_error => 1 }
269
                  unless SetBorrowerAttributes( $borrower{'borrowernumber'}, $patron_attributes, 'no_branch_limit' );
270
            }
271
            $overwritten++;
272
            push(
273
                @feedback,
274
                {
275
                    feedback => 1,
276
                    name     => 'lastoverwritten',
277
                    value    => $borrower{'surname'} . ' / ' . $borrowernumber
278
                }
279
            );
280
        }
281
        else {
282
            # FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either.
283
            # At least this is closer to AddMember than in members/memberentry.pl
284
            if ( !$borrower{'cardnumber'} ) {
285
                $borrower{'cardnumber'} = fixup_cardnumber(undef);
286
            }
287
            if ( $borrowernumber = AddMember(%borrower) ) {
288
289
                if ( $borrower{debarred} ) {
290
                    AddDebarment(
291
                        {
292
                            borrowernumber => $borrowernumber,
293
                            expiration     => $borrower{debarred},
294
                            comment        => $borrower{debarredcomment}
295
                        }
296
                    );
297
                }
298
299
                if ($extended) {
300
                    SetBorrowerAttributes( $borrowernumber, $patron_attributes );
301
                }
302
303
                if ($set_messaging_prefs) {
304
                    C4::Members::Messaging::SetMessagingPreferencesFromDefaults(
305
                        {
306
                            borrowernumber => $borrowernumber,
307
                            categorycode   => $borrower{categorycode}
308
                        }
309
                    );
310
                }
311
312
                $imported++;
313
                push(
314
                    @feedback,
315
                    {
316
                        feedback => 1,
317
                        name     => 'lastimported',
318
                        value    => $borrower{'surname'} . ' / ' . $borrowernumber
319
                    }
320
                );
321
            }
322
            else {
323
                $invalid++;
324
                push @errors, { unknown_error => 1 };
325
                push(
326
                    @errors,
327
                    {
328
                        name  => 'lastinvalid',
329
                        value => $borrower{'surname'} . ' / AddMember',
330
                    }
331
                );
332
            }
333
        }
334
    }
335
336
    return {
337
        feedback      => \@feedback,
338
        errors        => \@errors,
339
        imported      => $imported,
340
        overwritten   => $overwritten,
341
        already_in_db => $alreadyindb,
342
        invalid       => $invalid,
343
    };
344
}
345
346
=head2 prepare_columns
347
348
 my @csvcolumns = $self->prepare_columns({headerrow => $borrowerline, keycol => \%csvkeycol, errors => \@errors, });
349
350
Returns an array of all column key and populates a hash of colunm key positions.
351
352
=cut
353
354
sub prepare_columns {
355
    my ($self, $params) = @_;
356
357
    my $status = $self->text_csv->parse($params->{headerrow});
358
    unless( $status ) {
359
        push( @{$params->{errors}}, { badheader => 1, line => 1, lineraw => $params->{headerrow} });
360
        return;
361
    }
362
363
    my @csvcolumns = $self->text_csv->fields();
364
    my $col = 0;
365
    foreach my $keycol (@csvcolumns) {
366
        # columnkeys don't contain whitespace, but some stupid tools add it
367
        $keycol =~ s/ +//g;
368
        $params->{keycol}->{$keycol} = $col++;
369
    }
370
371
    return @csvcolumns;
372
}
373
374
=head2 set_attribute_types
375
376
 my $matchpoint_attr_type = $self->set_attribute_types({ extended => $extended, matchpoint => $matchpoint, });
377
378
Returns an attribute type based on matchpoint parameter.
379
380
=cut
381
382
sub set_attribute_types {
383
    my ($self, $params) = @_;
384
385
    my $attribute_types;
386
    if( $params->{extended} ) {
387
        $attribute_types = C4::Members::AttributeTypes->fetch($params->{matchpoint});
388
    }
389
390
    return $attribute_types;
391
}
392
393
=head2 set_column_keys
394
395
 my @columnkeys = set_column_keys($extended);
396
397
Returns an array of borrowers' table columns.
398
399
=cut
400
401
sub set_column_keys {
402
    my ($self, $extended) = @_;
403
404
    my @columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } C4::Members::columns();
405
    push( @columnkeys, 'patron_attributes' ) if $extended;
406
407
    return @columnkeys;
408
}
409
410
=head2 set_patron_attributes
411
412
 my $patron_attributes = set_patron_attributes($extended, $borrower{patron_attributes}, $feedback);
413
414
Returns a reference to array of hashrefs data structure as expected by SetBorrowerAttributes.
415
416
=cut
417
418
sub set_patron_attributes {
419
    my ($self, $extended, $patron_attributes, $feedback) = @_;
420
421
    unless( $extended ) { return; }
422
    unless( defined($patron_attributes) ) { return; }
423
424
    # Fixup double quotes in case we are passed smart quotes
425
    $patron_attributes =~ s/\xe2\x80\x9c/"/g;
426
    $patron_attributes =~ s/\xe2\x80\x9d/"/g;
427
428
    push (@$feedback, { feedback => 1, name => 'attribute string', value => $patron_attributes });
429
430
    my $result = extended_attributes_code_value_arrayref($patron_attributes);
431
432
    return $result;
433
}
434
435
=head2 check_branch_code
436
437
 check_branch_code($borrower{branchcode}, $borrowerline, $line_number, \@missing_criticals);
438
439
Pushes a 'missing_criticals' error entry if no branch code or branch code does not map to a branch name.
440
441
=cut
442
443
sub check_branch_code {
444
    my ($self, $branchcode, $borrowerline, $line_number, $missing_criticals) = @_;
445
446
    # No branch code
447
    unless( $branchcode ) {
448
        push (@$missing_criticals, { key => 'branchcode', line => $line_number, lineraw => $borrowerline, });
449
        return;
450
    }
451
452
    # look for branch code
453
    my $branch_name = GetBranchName( $branchcode );
454
    unless( $branch_name ) {
455
        push (@$missing_criticals, { key => 'branchcode', line => $line_number, lineraw => $borrowerline,
456
                                     value => $branchcode, branch_map => 1, });
457
    }
458
}
459
460
=head2 check_borrower_category
461
462
 check_borrower_category($borrower{categorycode}, $borrowerline, $line_number, \@missing_criticals);
463
464
Pushes a 'missing_criticals' error entry if no category code or category code does not map to a known category.
465
466
=cut
467
468
sub check_borrower_category {
469
    my ($self, $categorycode, $borrowerline, $line_number, $missing_criticals) = @_;
470
471
    # No branch code
472
    unless( $categorycode ) {
473
        push (@$missing_criticals, { key => 'categorycode', line => $line_number, lineraw => $borrowerline, });
474
        return;
475
    }
476
477
    # Looking for borrower category
478
    my $category = GetBorrowercategory( $categorycode );
479
    unless( $category ) {
480
        push (@$missing_criticals, { key => 'categorycode', line => $line_number, lineraw => $borrowerline,
481
                                     value => $categorycode, category_map => 1, });
482
    }
483
}
484
485
=head2 format_dates
486
487
 format_dates({borrower => \%borrower, lineraw => $lineraw, line => $line_number, missing_criticals => \@missing_criticals, });
488
489
Pushes a 'missing_criticals' error entry for each of the 3 date types dateofbirth, dateenrolled and dateexpiry if it can not
490
be formatted to the chosen date format. Populates the correctly formatted date otherwise.
491
492
=cut
493
494
sub format_dates {
495
    my ($self, $params) = @_;
496
497
    foreach my $date_type (qw(dateofbirth dateenrolled dateexpiry)) {
498
        my $tempdate = $params->{borrower}->{$date_type} or next();
499
        my $formatted_date = eval { output_pref( { dt => dt_from_string( $tempdate ), dateonly => 1, dateformat => 'iso' } ); };
500
501
        if ($formatted_date) {
502
            $params->{borrower}->{$date_type} = $formatted_date;
503
        } else {
504
            $params->{borrower}->{$date_type} = '';
505
            push (@{$params->{missing_criticals}}, { key => $date_type, line => $params->{line}, lineraw => $params->{lineraw}, bad_date => 1 });
506
        }
507
    }
508
}
509
510
1;
511
512
=head1 AUTHOR
513
514
Koha Team
515
516
=cut
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tt (-187 / +243 lines)
Lines 34-204 Link Here
34
     <div class="yui-u first">
34
     <div class="yui-u first">
35
<h1>Import patrons</h1>
35
<h1>Import patrons</h1>
36
[% IF ( uploadborrowers ) %]
36
[% IF ( uploadborrowers ) %]
37
<h5>Import results :</h5>
37
    <h5>Import results :</h5>
38
<ul>
38
    <ul>
39
	<li>[% imported %] imported records [% IF ( lastimported ) %](last was [% lastimported %])[% END %]</li>
39
        <li>[% imported %] imported records [% IF ( lastimported ) %](last was [% lastimported %])[% END %]</li>
40
        [% IF imported and patronlistname %]
40
        [% IF imported and patronlistname %]
41
            <li>Patronlist with imported patrons: [% patronlistname %]</li>
41
            <li>Patronlist with imported patrons: [% patronlistname %]</li>
42
        [% END %]
42
        [% END %]
43
	<li>[% overwritten %] overwritten [% IF ( lastoverwritten ) %](last was [% lastoverwritten %])[% END %]</li>
43
        <li>[% overwritten %] overwritten [% IF ( lastoverwritten ) %](last was [% lastoverwritten %])[% END %]</li>
44
	<li>[% alreadyindb %] not imported because already in borrowers table and overwrite disabled [% IF ( lastalreadyindb ) %](last was [% lastalreadyindb %])[% END %]</li>
44
        <li>[% alreadyindb %] not imported because already in borrowers table and overwrite disabled [% IF ( lastalreadyindb ) %](last was [% lastalreadyindb %])[% END %]</li>
45
	<li>[% invalid %] not imported because they are not in the expected format [% IF ( lastinvalid ) %](last was [% lastinvalid %])[% END %]</li>
45
        <li>[% invalid %] not imported because they are not in the expected format [% IF ( lastinvalid ) %](last was [% lastinvalid %])[% END %]</li>
46
	<li>[% total %] records parsed</li>
46
        <li>[% total %] records parsed</li>
47
	<li><a href="/cgi-bin/koha/tools/tools-home.pl">Back to Tools</a></li>
47
        <li><a href="/cgi-bin/koha/tools/tools-home.pl">Back to Tools</a></li>
48
</ul>
49
  [% IF ( FEEDBACK ) %]
50
  <br /><br />
51
    <div>
52
    <h5>Feedback:</h5>
53
    <ul class="feedback">
54
    [% FOREACH FEEDBAC IN FEEDBACK %]
55
    <li>
56
        [% IF ( FEEDBAC.filename ) %]Parsing upload file <span class="filename">[% FEEDBAC.filename %]</span>
57
        [% ELSIF ( FEEDBAC.backend ) %]Upload parsed using [% FEEDBAC.backend %]
58
        [% ELSIF ( FEEDBAC.headerrow ) %]These fields found: [% FEEDBAC.value %]
59
        [% ELSE %][% FEEDBAC.name %] : [% FEEDBAC.value %]
60
        [% END %]
61
    </li>
62
    [% END %]
63
    </ul>
48
    </ul>
64
    </div>
49
65
  [% END %]
50
    [% IF ( feedback ) %]
66
  [% IF ( ERRORS ) %]
51
        <br /><br />
67
  <br /><br />
52
68
    <div>
53
        <div>
69
    <h5>Error analysis:</h5>
54
            <h5>Feedback:</h5>
70
    <ul>
55
                <ul class="feedback">
71
    [% FOREACH ERROR IN ERRORS %]
56
                    [% FOREACH f IN feedback %]
72
        [% IF ( ERROR.badheader ) %]<li>Header row could not be parsed</li>[% END %]
57
                        <li>
73
        [% FOREACH missing_critical IN ERROR.missing_criticals %]
58
                            [% IF ( f.filename ) %]
74
        <li class="line_error">
59
                                Parsing upload file <span class="filename">[% f.filename %]</span>
75
            Line <span class="linenumber">[% missing_critical.line %]</span>
60
                            [% ELSIF ( f.backend ) %]
76
            [% IF ( missing_critical.badparse ) %]
61
                                Upload parsed using [% f.backend %]
77
                could not be parsed!
62
                            [% ELSIF ( f.headerrow ) %]
78
            [% ELSIF ( missing_critical.bad_date ) %]
63
                                These fields found: [% f.value %]
79
                has &quot;[% missing_critical.key %]&quot; in unrecognized format: &quot;[% missing_critical.value %]&quot;
64
                            [% ELSIF ( f.already_in_db ) %]
80
            [% ELSE %]
65
                                Patron already in database: [% f.value %]
81
                Critical field &quot;[% missing_critical.key %]&quot;
66
                            [% ELSE %]
82
                [% IF ( missing_critical.branch_map ) %]has unrecognized value &quot;[% missing_critical.value %]&quot;
67
                                [% f.name %] : [% f.value %]
83
                [% ELSIF ( missing_critical.category_map ) %]has unrecognized value &quot;[% missing_critical.value %]&quot;
68
                            [% END %]
84
                [% ELSE %]missing
69
                        </li>
70
                    [% END %]
71
                </ul>
72
        </div>
73
    [% END %]
74
75
    [% IF ( errors ) %]
76
        <br /><br />
77
78
        <div>
79
            <h5>Error analysis:</h5>
80
            <ul>
81
                [% FOREACH e IN errors %]
82
                    [% IF ( e.badheader ) %]<li>Header row could not be parsed</li>[% END %]
83
84
                    [% FOREACH missing_critical IN e.missing_criticals %]
85
                        <li class="line_error">
86
                            Line <span class="linenumber">[% missing_critical.line %]</span>
87
88
                            [% IF ( missing_critical.badparse ) %]
89
                                could not be parsed!
90
                            [% ELSIF ( missing_critical.bad_date ) %]
91
                                has &quot;[% missing_critical.key %]&quot; in unrecognized format: &quot;[% missing_critical.value %]&quot;
92
                            [% ELSE %]
93
                                Critical field &quot;[% missing_critical.key %]&quot;
94
95
                                [% IF ( missing_critical.branch_map ) %]
96
                                    has unrecognized value &quot;[% missing_critical.value %]&quot;
97
                                [% ELSIF ( missing_critical.category_map ) %]
98
                                    has unrecognized value &quot;[% missing_critical.value %]&quot;
99
                                [% ELSE %]
100
                                    missing
101
                                [% END %]
102
103
                                (borrowernumber: [% missing_critical.borrowernumber %]; surname: [% missing_critical.surname %]).
104
                            [% END %]
105
106
                            <br/>
107
                            <code>[% missing_critical.lineraw %]</code>
108
                        </li>
109
                    [% END %]
110
111
                    [% IF e.invalid_cardnumber %]
112
                        <li class="line_error">
113
                            Cardnumber [% e.cardnumber %] is not a valid cardnumber
114
                            [% IF e.borrowernumber %] (for patron with borrowernumber [% e.borrowernumber %])[% END %]
115
                        </li>
116
                    [% END %]
117
                    [% IF e.duplicate_userid %]
118
                        <li class="line_error">
119
                            Userid [% e.userid %] is already used by another patron.
120
                        </li>
121
                    [% END %]
85
                [% END %]
122
                [% END %]
86
                (borrowernumber: [% missing_critical.borrowernumber %]; surname: [% missing_critical.surname %]).
123
            </ul>
87
            [% END %]
124
        </div>
88
            <br /><code>[% missing_critical.lineraw %]</code>
89
        </li>
90
        [% END %]
91
        [% IF ERROR.invalid_cardnumber %]
92
            <li class="line_error">
93
                Cardnumber [% ERROR.cardnumber %] is not a valid cardnumber
94
                [% IF ERROR.borrowernumber %] (for patron with borrowernumber [% ERROR.borrowernumber %])[% END %]
95
            </li>
96
        [% END %]
97
        [% IF ERROR.duplicate_userid %]
98
            <li class="line_error">
99
                Userid [% ERROR.userid %] is already used by another patron.
100
            </li>
101
        [% END %]
102
    [% END %]
125
    [% END %]
103
    </ul>
104
    </div>
105
  [% END %]
106
[% ELSE %]
126
[% ELSE %]
107
<ul>
127
    <ul>
108
    <li>Select a file to import into the borrowers table.</li>
128
        <li>Select a file to import into the borrowers table</li>
109
    <li>If a cardnumber exists in the table, you can choose whether to ignore the new one or overwrite the old one.</li>
129
        <li>If a cardnumber exists in the table, you can choose whether to ignore the new one or overwrite the old one.</li>
110
</ul>
130
    </ul>
111
<form method="post" action="[% SCRIPT_NAME %]" enctype="multipart/form-data">
112
<fieldset class="rows">
113
<legend>Import into the borrowers table</legend>
114
<ol>
115
	<li>
116
		<label for="uploadborrowers">Select the file to import: </label>
117
		<input type="file" id="uploadborrowers" name="uploadborrowers" />
118
	</li>
119
        <li>
120
            <label for "createpatronlist">Create patron list: </label>
121
            <input name="createpatronlist" id="createpatronlist" value="1" type="checkbox">
122
            <span class="hint">List name will be file name with timestamp</span>
123
        </li>
124
131
125
</ol></fieldset>
132
    <form method="post" action="[% SCRIPT_NAME %]" enctype="multipart/form-data">
126
    <fieldset class="rows">
133
        <fieldset class="rows">
127
        <legend>Field to use for record matching</legend>
134
            <legend>Import into the borrowers table</legend>
128
        <ol>
135
129
            <li class="radio">
136
            <ol>
130
                <select name="matchpoint" id="matchpoint">
137
                <li>
131
                    <option value="cardnumber">Cardnumber</option>
138
                    <label for="uploadborrowers">Select the file to import: </label>
132
                    <option value="userid">Username</option>
139
                    <input type="file" id="uploadborrowers" name="uploadborrowers" />
133
                    [% FOREACH matchpoint IN matchpoints %]
140
                </li>
134
                        <option value="[% matchpoint.code %]">[% matchpoint.description %]</option>
141
142
                <li>
143
                    <label for "createpatronlist">Create patron list: </label>
144
                    <input name="createpatronlist" id="createpatronlist" value="1" type="checkbox">
145
                    <span class="hint">List name will be file name with timestamp</span>
146
                </li>
147
            </ol>
148
        </fieldset>
149
150
        <fieldset class="rows">
151
            <legend>Field to use for record matching</legend>
152
            <ol>
153
                <li class="radio">
154
                    <select name="matchpoint" id="matchpoint">
155
                        <option value="cardnumber">Cardnumber</option>
156
                        <option value="userid">Username</option>
157
                        [% FOREACH matchpoint IN matchpoints %]
158
                            <option value="[% matchpoint.code %]">[% matchpoint.description %]</option>
159
                        [% END %]
160
                    </select>
161
                </li>
162
            </ol>
163
        </fieldset>
164
165
        <fieldset class="rows">
166
            <legend>Default values</legend>
167
168
            <ol>
169
                [% FOREACH borrower_field IN borrower_fields %]
170
171
                    [% SWITCH borrower_field.field %]
172
                    [% CASE 'branchcode' %]
173
                        <li>
174
                            <label class="description" for="branchcode">[% borrower_field.description %]: </label>
175
                            <select id="branchcode" name="branchcode">
176
                                <option value="" selected="selected"></option>
177
                                [% FOREACH library IN Branches.all() %]
178
                                    <option value="[% library.branchcode %]">[% library.branchname %]</option>
179
                                [% END %]
180
                            </select>
181
                            <span class="field_hint">[% borrower_field.field %]</span>
182
                        </li>
183
                    [% CASE 'categorycode' %]
184
                        <li>
185
                            <label class="description" for="categorycode">[% borrower_field.description %]: </label>
186
                            <select id="categorycode" name="categorycode">
187
                                <option value="" selected="selected"></option>
188
                                [% FOREACH category IN categories %]
189
                                    <option value="[% category.categorycode %]">[% category.description %]</option>
190
                                [% END %]
191
                            </select>
192
                            <span class="field_hint">[% borrower_field.field %]</span>
193
                        </li>
194
                    [% CASE %]
195
                        <li>
196
                            <label class="description" for="[% borrower_field.field %]">[% borrower_field.description %]: </label>
197
                            <input id="[% borrower_field.field %]" name="[% borrower_field.field %]" />
198
                            <span class="field_hint">[% borrower_field.field %]</span>
199
                        </li>
135
                    [% END %]
200
                    [% END %]
136
                </select>
201
                [% END %]
137
            </li>
202
138
        </ol>
203
                [% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
139
    </fieldset>
204
                    <li>
140
<fieldset class="rows">
205
                        <label class="description" for="patron_attributes">Patron attributes: </label>
141
<legend>Default values</legend>
206
                        <input id="patron_attributes" name="patron_attributes" />
142
<ol>
207
                        <span class="field_hint">patron_attributes</span>
143
[% FOREACH borrower_field IN borrower_fields %]
208
                    </li>
144
  [% SWITCH borrower_field.field %]
209
                [% END %]
145
  [% CASE 'branchcode' %]
210
146
    <li>
211
            </ol>
147
        <label class="description" for="branchcode">[% borrower_field.description %]: </label>
212
        </fieldset>
148
        <select id="branchcode" name="branchcode">
213
149
            <option value="" selected="selected"></option>
214
        <fieldset class="rows">
150
        [% FOREACH library IN Branches.all() %]
215
            <legend>If matching record is already in the borrowers table:</legend>
151
            <option value="[% library.branchcode %]">
216
152
                [% library.branchname %]</option>
217
            <ol>
153
        [% END %]
218
                <li class="radio">
154
        </select><span class="field_hint">[% borrower_field.field %]</span>
219
                    <input type="radio" id="overwrite_cardnumberno" name="overwrite_cardnumber" value="0" checked="checked" /><label for="overwrite_cardnumberno">Ignore this one, keep the existing one</label>
155
    </li>
220
                </li>
156
  [% CASE 'categorycode' %]
221
157
    <li>
222
                <li class="radio">
158
        <label class="description" for="categorycode">[% borrower_field.description %]: </label>
223
                    <input type="radio" id="overwrite_cardnumberyes" name="overwrite_cardnumber" value="1" /><label for="overwrite_cardnumberyes">Overwrite the existing one with this</label>
159
        <select id="categorycode" name="categorycode">
224
                </li>
160
            <option value="" selected="selected"></option>
225
            </ol>
161
        [% FOREACH category IN categories %]
226
        </fieldset>
162
            <option value="[% category.categorycode %]">
227
163
                [% category.description %]</option>
228
        [% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
229
            <fieldset class="rows">
230
                <legend>Patron attributes</legend>
231
232
                <ol>
233
                    <li class="radio">
234
                        <input type="radio" id="ext_preserve_0" name="ext_preserve" value="0" checked="checked" /><label for="ext_preserve_0">Replace all patron attributes</label>
235
                    </li>
236
237
                    <li class="radio">
238
                        <input type="radio" id="ext_preserve_1" name="ext_preserve" value="1" /><label for="ext_preserve_1">Replace only included patron attributes</label>
239
                    </li>
240
                </ol>
241
            </fieldset>
164
        [% END %]
242
        [% END %]
165
        </select><span class="field_hint">[% borrower_field.field %]</span>
243
166
    </li>
244
        <fieldset class="action"><input type="submit" value="Import" /></fieldset>
167
  [% CASE %]
245
    </form>
168
    <li>
169
        <label class="description" for="[% borrower_field.field %]">[% borrower_field.description %]: </label>
170
        <input id="[% borrower_field.field %]" name="[% borrower_field.field %]" /><span class="field_hint">[% borrower_field.field %]</span>
171
    </li>
172
  [% END %]
173
[% END %]
174
[% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
175
    <li>
176
        <label class="description" for="patron_attributes">Patron attributes: </label>
177
        <input id="patron_attributes" name="patron_attributes" />
178
        <span class="field_hint">patron_attributes</span>
179
    </li>
180
[% END %]
246
[% END %]
181
</ol></fieldset>
247
182
	<fieldset class="rows">
248
</div>
183
	<legend>If matching record is already in the borrowers table:</legend>
249
184
    <ol><li class="radio">
250
<div class="yui-u">
185
        <input type="radio" id="overwrite_cardnumberno" name="overwrite_cardnumber" value="0" checked="checked" /><label for="overwrite_cardnumberno">Ignore this one, keep the existing one</label>
251
    <h2>Notes:</h2>
252
    <ul>
253
        <li>The first line in the file must be a header row defining which columns you are supplying in the import file.</li>
254
255
        <li><b>Download a starter CSV file with all the columns <a href="?sample=1">here</a>.</b>  Values are comma-separated.</li>
256
257
        <li>
258
            OR choose which fields you want to supply from the following list:
259
            <ul>
260
                <li>
261
                    [% FOREACH columnkey IN borrower_fields %]'[% columnkey.field %]', [% END %]
262
                </li>
263
            </ul>
186
        </li>
264
        </li>
187
        <li class="radio">
265
188
		<input type="radio" id="overwrite_cardnumberyes" name="overwrite_cardnumber" value="1" /><label for="overwrite_cardnumberyes">Overwrite the existing one with this</label>
266
        [% IF ( ExtendedPatronAttributes ) %]
267
            <li>
268
                If loading patron attributes, the 'patron_attributes' field should contain a comma-separated list of attribute types and values. The attribute type code and a colon should precede each value. For example: <b>INSTID:12345,LANG:fr</b> or <b>STARTDATE:January 1 2010,TRACK:Day</b>. If an input record has more than one attribute, the fields should either be entered as an unquoted string (previous examples), or with each field wrapped in separate double quotes and delimited by a comma: <b>&quot;STARTDATE:January 1, 2010&quot;,&quot;TRACK:Day&quot;</b>.  The second syntax would be required if the data might have a comma in it, like a date string.
269
            </li>
270
        [% END %]
271
272
        <li>
273
            The fields 'branchcode' and 'categorycode' are <b>required</b> and <b>must match</b> valid entries in your database.
189
        </li>
274
        </li>
190
    </ol>
275
191
    </fieldset>
276
        <li>
192
    [% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
277
            'password' should be stored in plaintext, and will be converted to a Bcrypt hash (if your passwords are already encrypted, talk to your system administrator about options).
193
	<fieldset class="rows">
194
    <legend>Patron attributes</legend>
195
    <ol><li class="radio">
196
        <input type="radio" id="ext_preserve_0" name="ext_preserve" value="0" checked="checked" /><label for="ext_preserve_0">Replace all patron attributes</label>
197
        </li>
278
        </li>
198
        <li class="radio">
279
199
        <input type="radio" id="ext_preserve_1" name="ext_preserve" value="1" /><label for="ext_preserve_1">Replace only included patron attributes</label>
280
        <li>
281
            Date formats should match your system preference, and <b>must</b> be zero-padded, e.g. '01/02/2008'.  Alternatively,
282
you can supply dates in ISO format (e.g., '2010-10-28').
200
        </li>
283
        </li>
201
    </ol>
284
    </ul>
202
    </fieldset>
285
    </fieldset>
203
    [% END %]
286
    [% END %]
204
    <fieldset class="action">
287
    <fieldset class="action">
Lines 207-248 Link Here
207
    </fieldset>
290
    </fieldset>
208
</form>
291
</form>
209
[% END %]
292
[% END %]
293
210
</div>
294
</div>
211
<div class="yui-u">
295
</div>
212
<h2>Notes:</h2>
296
</div>
213
<ul>
297
</div>
214
<li><b>Header: </b>The first line in the file must be a header row defining which columns you are supplying in the import file.</li>
298
215
<li><b>Separator: </b>Values are comma-separated.</li>
299
<div class="yui-b noprint">
216
<li><b>Starter CSV: </b> Koha provides a starter CSV with all the columns.
300
    [% INCLUDE 'tools-menu.inc' %]
217
    <ul><li><a href="?sample=1">Download starter CSV</a></li></ul>
301
</div>
218
</li>
302
</div>
219
<li><b>Field list: </b>Alternatively, you can create your own CSV and choose which fields you want to supply from the following list:
303
220
    <ul><li>
221
       [% FOREACH columnkey IN borrower_fields %]'[% columnkey.field %]', [% END %]
222
    </li></ul>
223
</li>
224
[% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
225
<li><b>Extended patron attributes: </b>If loading patron attributes, the 'patron_attributes' field should contain a comma-separated list of attribute types and values. The attribute type code and a colon should precede each value.
226
    <ul><li>Example 1: INSTID:12345,LANG:fr</li><li>Example 2: STARTDATE:January 1 2010,TRACK:Day</li></ul>
227
If an input record has more than one attribute, the fields should either be entered as an unquoted string (previous examples), or with each field wrapped in separate double quotes and delimited by a comma:
228
    <ul><li>Example 3: &quot;STARTDATE:January 1, 2010&quot;,&quot;TRACK:Day&quot;</li></ul>
229
The second syntax would be required if the data might have a comma in it, like a date string.</li>
230
[% END %]
231
<li><b>Required fields: </b>The fields 'branchcode' and 'categorycode' are required and must match valid entries in your database.</li>
232
<li><b>Password: </b>Values for the field 'password' should be stored in plaintext, and will be converted to a Bcrypt hash (if your passwords are already encrypted, talk to your system administrator about options).</li>
233
<li><b>Date formats: </b> Date values should match your system preference, and must be zero-padded.
234
    <ul><li>Example: '01/02/2008'</li></ul>
235
Alternatively, you can supply dates in ISO format.
236
    <ul><li>Example: '2010-10-28'</li></ul>
237
</li>
238
</ul>
239
240
     </div>
241
    </div>
242
   </div>
243
  </div>
244
  <div class="yui-b noprint">
245
[% INCLUDE 'tools-menu.inc' %]
246
  </div>
247
 </div>
248
[% INCLUDE 'intranet-bottom.inc' %]
304
[% INCLUDE 'intranet-bottom.inc' %]
(-)a/misc/import_patrons.pl (+103 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Parts copyright 2014 ByWater Solutions
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
use Modern::Perl;
21
22
use Getopt::Long;
23
24
use Koha::Patrons::Import;
25
my $Import = Koha::Patrons::Import->new();
26
27
my $csv_file;
28
my $matchpoint;
29
my $overwrite_cardnumber;
30
my %defaults;
31
my $ext_preserve = 0;
32
my $confirm;
33
my $verbose      = 0;
34
my $help;
35
36
GetOptions(
37
    'c|confirm'                     => \$confirm,
38
    'f|file=s'                      => \$csv_file,
39
    'm|matchpoint=s'                => \$matchpoint,
40
    'd|default=s'                   => \%defaults,
41
    'o|overwrite'                   => \$overwrite_cardnumber,
42
    'p|preserve-extended-atributes' => \$ext_preserve,
43
    'v|verbose+'                    => \$verbose,
44
    'h|help|?'                      => \$help,
45
);
46
47
print_help() if ( $help || !$csv_file || !$matchpoint || !$confirm );
48
49
my $handle;
50
open( $handle, "<", $csv_file ) or die $!;
51
52
my $return = $Import->import_patrons(
53
    {
54
        file                         => $handle,
55
        defaults                     => \%defaults,
56
        matchpoint                   => $matchpoint,
57
        overwrite_cardnumber         => $overwrite_cardnumber,
58
        preserve_extended_attributes => $ext_preserve,
59
    }
60
);
61
62
my $feedback    = $return->{feedback};
63
my $errors      = $return->{errors};
64
my $imported    = $return->{imported};
65
my $overwritten = $return->{overwritten};
66
my $alreadyindb = $return->{already_in_db};
67
my $invalid     = $return->{invalid};
68
69
if ($verbose) {
70
    my $total = $imported + $alreadyindb + $invalid + $overwritten;
71
    say q{};
72
    say "Import complete:";
73
    say "Imported:    $imported";
74
    say "Overwritten: $overwritten";
75
    say "Skipped:     $alreadyindb";
76
    say "Invalid:     $invalid";
77
    say "Total:       $total";
78
    say q{};
79
}
80
81
if ($verbose > 1 ) {
82
    say "Errors:";
83
    say Data::Dumper::Dumper( $errors );
84
}
85
86
if ($verbose > 2 ) {
87
    say "Feedback:";
88
    say Data::Dumper::Dumper( $feedback );
89
}
90
91
sub print_help {
92
    print <<_USAGE_;
93
import_patrons.pl -c /path/to/patrons.csv -m cardnumber
94
    -c --confirm                        Confirms you really want to import these patrons, otherwise prints this help
95
    -f --file                           Path to the CSV file of patrons to import
96
    -m --matchpoint                     Field on which to match incoming patrons to existing patrons
97
    -d --default                        Set defaults to patron fields, repeatable e.g. --default branchcode=MPL --default categorycode=PT
98
    -p --preserve-extended-atributes    Retain extended patron attributes for existing patrons being overwritten
99
    -o --overwrite                      Overwrite existing patrons with new data if a match is found
100
    -v --verbose                        Be verbose
101
_USAGE_
102
    exit;
103
}
(-)a/t/db_dependent/Koha/Patrons/Import.t (+632 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright 2015 Koha Development team
4
#
5
# This file is part of Koha
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
use Modern::Perl;
21
use Test::More tests => 124;
22
use Test::Warn;
23
24
# To be replaced by t::lib::Mock
25
use Test::MockModule;
26
use Koha::Database;
27
28
use File::Temp qw(tempfile tempdir);
29
my $temp_dir = tempdir('Koha_patrons_import_test_XXXX', CLEANUP => 1, TMPDIR => 1);
30
31
use t::lib::TestBuilder;
32
my $builder = t::lib::TestBuilder->new;
33
34
my $schema = Koha::Database->new->schema;
35
$schema->storage->txn_begin;
36
37
# ########## Tests start here #############################
38
# Given ... we can use the module
39
BEGIN { use_ok('Koha::Patrons::Import'); }
40
41
my $patrons_import = new_ok('Koha::Patrons::Import');
42
43
subtest 'test_methods' => sub {
44
    plan tests => 1;
45
46
    # Given ... we can reach the method(s)
47
    my @methods = ('import_patrons',
48
                   'set_attribute_types',
49
                   'prepare_columns',
50
                   'set_column_keys',
51
                   'set_patron_attributes',
52
                   'check_branch_code',
53
                   'format_dates',
54
                  );
55
    can_ok('Koha::Patrons::Import', @methods);
56
};
57
58
subtest 'test_attributes' => sub {
59
    plan tests => 1;
60
61
    my @attributes = ('today_iso', 'text_csv');
62
    can_ok('Koha::Patrons::Import', @attributes);
63
};
64
65
# Tests for Koha::Patrons::Import::import_patrons()
66
# Given ... nothing much. When ... Then ...
67
my $result;
68
warning_is { $result = $patrons_import->import_patrons(undef) }
69
           { carped => 'No file handle passed in!' },
70
           " Koha::Patrons::Import->import_patrons carps if no file handle is passed";
71
is($result, undef, 'Got the expected undef from import_patrons with nothing much');
72
73
# Given ... some params but no file handle.
74
my $params_0 = { some_stuff => 'random stuff', };
75
76
# When ... Then ...
77
my $result_0;
78
warning_is { $result_0 = $patrons_import->import_patrons($params_0) }
79
           { carped => 'No file handle passed in!' },
80
           " Koha::Patrons::Import->import_patrons carps if no file handle is passed";
81
is($result_0, undef, 'Got the expected undef from import_patrons with no file handle');
82
83
# Given ... a file handle to file with headers only.
84
my $ExtendedPatronAttributes = 0;
85
my $context = Test::MockModule->new('C4::Context'); # Necessary mocking for consistent test results.
86
$context->mock('preference', sub { my ($mod, $meth) = @_;
87
                                    if ( $meth eq 'ExtendedPatronAttributes' ) { return $ExtendedPatronAttributes; }
88
                                    if ( $meth eq 'dateformat' ) { return 'us'; }
89
                                });
90
91
92
my $csv_headers  = 'cardnumber,surname,firstname,title,othernames,initials,streetnumber,streettype,address,address2,city,state,zipcode,country,email,phone,mobile,fax,dateofbirth,branchcode,categorycode,dateenrolled,dateexpiry,userid,password';
93
my $res_header   = 'cardnumber, surname, firstname, title, othernames, initials, streetnumber, streettype, address, address2, city, state, zipcode, country, email, phone, mobile, fax, dateofbirth, branchcode, categorycode, dateenrolled, dateexpiry, userid, password';
94
my $csv_one_line = '1000,Nancy,Jenkins,Dr,,NJ,78,Circle,Bunting,El Paso,Henderson,Texas,79984,United States,ajenkins0@sourceforge.net,7-(388)559-6763,3-(373)151-4471,8-(509)286-4001,10/16/1965,CPL,PT,12/28/2014,07/01/2015,jjenkins0,DPQILy';
95
96
my $filename_1 = make_csv($temp_dir, $csv_headers, $csv_one_line);
97
open(my $handle_1, "<", $filename_1) or die "cannot open < $filename_1: $!";
98
my $params_1 = { file => $handle_1, };
99
100
# When ...
101
my $result_1 = $patrons_import->import_patrons($params_1);
102
103
# Then ...
104
is($result_1->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons with no matchpoint defined');
105
is(scalar @{$result_1->{errors}}, 0, 'Got the expected 0 size error array from import_patrons with no matchpoint defined');
106
107
is($result_1->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons with no matchpoint defined');
108
is($result_1->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons with no matchpoint defined');
109
is($result_1->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons with no matchpoint defined');
110
111
is($result_1->{feedback}->[1]->{feedback}, 1, 'Got the expected second feedback from import_patrons with no matchpoint defined');
112
is($result_1->{feedback}->[1]->{name}, 'lastimported', 'Got the expected last imported name from import_patrons with no matchpoint defined');
113
like($result_1->{feedback}->[1]->{value}, qr/^Nancy \/ \d+/, 'Got the expected second header row value from import_patrons with no matchpoint defined');
114
115
is($result_1->{imported}, 1, 'Got the expected 1 imported result from import_patrons with no matchpoint defined');
116
is($result_1->{invalid}, 0, 'Got the expected 0 invalid result from import_patrons with no matchpoint defined');
117
is($result_1->{overwritten}, 0, 'Got the expected 0 overwritten result from import_patrons with no matchpoint defined');
118
119
# Given ... a valid file handle, a bad matchpoint resulting in invalid card number
120
my $filename_2 = make_csv($temp_dir, $csv_headers, $csv_one_line);
121
open(my $handle_2, "<", $filename_2) or die "cannot open < $filename_2: $!";
122
my $params_2 = { file => $handle_2, matchpoint => 'SHOW_BCODE', };
123
124
# When ...
125
my $result_2 = $patrons_import->import_patrons($params_2);
126
127
# Then ...
128
is($result_2->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons with invalid card number');
129
is($result_2->{errors}->[0]->{borrowernumber}, undef, 'Got the expected undef borrower number from import patrons with invalid card number');
130
is($result_2->{errors}->[0]->{cardnumber}, 1000, 'Got the expected 1000 card number from import patrons with invalid card number');
131
is($result_2->{errors}->[0]->{invalid_cardnumber}, 1, 'Got the expected invalid card number from import patrons with invalid card number');
132
133
is($result_2->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons with invalid card number');
134
is($result_2->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons with invalid card number');
135
is($result_2->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons with invalid card number');
136
137
is($result_2->{imported}, 0, 'Got the expected 0 imported result from import_patrons with invalid card number');
138
is($result_2->{invalid}, 1, 'Got the expected 1 invalid result from import_patrons with invalid card number');
139
is($result_2->{overwritten}, 0, 'Got the expected 0 overwritten result from import_patrons with invalid card number');
140
141
# Given ... valid file handle, good matchpoint but same input as prior test.
142
my $filename_3 = make_csv($temp_dir, $csv_headers, $csv_one_line);
143
open(my $handle_3, "<", $filename_3) or die "cannot open < $filename_3: $!";
144
my $params_3 = { file => $handle_3, matchpoint => 'cardnumber', };
145
146
# When ...
147
my $result_3 = $patrons_import->import_patrons($params_3);
148
149
# Then ...
150
is($result_3->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons with duplicate userid');
151
is($result_3->{errors}->[0]->{duplicate_userid}, 1, 'Got the expected duplicate userid error from import patrons with duplicate userid');
152
is($result_3->{errors}->[0]->{userid}, 'jjenkins0', 'Got the expected userid error from import patrons with duplicate userid');
153
154
is($result_3->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons with duplicate userid');
155
is($result_3->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons with duplicate userid');
156
is($result_3->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons with duplicate userid');
157
158
is($result_3->{imported}, 0, 'Got the expected 0 imported result from import_patrons with duplicate userid');
159
is($result_3->{invalid}, 1, 'Got the expected 1 invalid result from import_patrons with duplicate userid');
160
is($result_3->{overwritten}, 0, 'Got the expected 0 overwritten result from import_patrons with duplicate userid');
161
162
# Given ... a new input and mocked C4::Context
163
$ExtendedPatronAttributes = 1; # Updates mocked C4::Preferences result.
164
165
my $new_input_line = '1001,Donna,Sullivan,Mrs,Henry,DS,59,Court,Burrows,Reading,Salt Lake City,Pennsylvania,19605,United States,hsullivan1@purevolume.com,3-(864)009-3006,7-(291)885-8423,1-(879)095-5038,09/19/1970,LPL,PT,03/04/2015,07/01/2015,hsullivan1,8j6P6Dmap';
166
my $filename_4 = make_csv($temp_dir, $csv_headers, $new_input_line);
167
open(my $handle_4, "<", $filename_4) or die "cannot open < $filename_4: $!";
168
my $params_4 = { file => $handle_4, matchpoint => 'SHOW_BCODE', };
169
170
# When ...
171
my $result_4 = $patrons_import->import_patrons($params_4);
172
173
# Then ...
174
is($result_4->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons with extended user');
175
is(scalar @{$result_4->{errors}}, 0, 'Got the expected 0 size error array from import_patrons with extended user');
176
177
is($result_4->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons with extended user');
178
is($result_4->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons with extended user');
179
is($result_4->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons with extended user');
180
181
is($result_4->{feedback}->[1]->{feedback}, 1, 'Got the expected second feedback from import_patrons with extended user');
182
is($result_4->{feedback}->[1]->{name}, 'attribute string', 'Got the expected attribute string from import_patrons with extended user');
183
is($result_4->{feedback}->[1]->{value}, '', 'Got the expected second feedback value from import_patrons with extended user');
184
185
is($result_4->{feedback}->[2]->{feedback}, 1, 'Got the expected third feedback from import_patrons with extended user');
186
is($result_4->{feedback}->[2]->{name}, 'lastimported', 'Got the expected last imported name from import_patrons with extended user');
187
like($result_4->{feedback}->[2]->{value}, qr/^Donna \/ \d+/, 'Got the expected third feedback value from import_patrons with extended user');
188
189
is($result_4->{imported}, 1, 'Got the expected 1 imported result from import_patrons with extended user');
190
is($result_4->{invalid}, 0, 'Got the expected 0 invalid result from import_patrons with extended user');
191
is($result_4->{overwritten}, 0, 'Got the expected 0 overwritten result from import_patrons with extended user');
192
193
$context->unmock('preference');
194
195
# Given ... 3 new inputs. One with no branch code, one with unexpected branch code.
196
my $input_no_branch   = '1002,Johnny,Reynolds,Mr,Patricia,JR,12,Hill,Kennedy,Saint Louis,Colorado Springs,Missouri,63131,United States,preynolds2@washington.edu,7-(925)314-9514,0-(315)973-8956,4-(510)556-2323,09/18/1967,,PT,05/07/2015,07/01/2015,preynolds2,K3HiDzl';
197
my $input_good_branch = '1003,Linda,Richardson,Mr,Kimberly,LR,90,Place,Bayside,Atlanta,Erie,Georgia,31190,United States,krichardson3@pcworld.com,8-(035)185-0387,4-(796)518-3676,3-(644)960-3789,04/13/1954,RPL,PT,06/06/2015,07/01/2015,krichardson3,P3EO0MVRPXbM';
198
my $input_na_branch   = '1005,Ruth,Greene,Mr,Michael,RG,3,Avenue,Grim,Peoria,Jacksonville,Illinois,61614,United States,mgreene5@seesaa.net,3-(941)565-5752,1-(483)885-8138,4-(979)577-6908,02/09/1957,ZZZ,ST,04/02/2015,07/01/2015,mgreene5,or4ORT6JH';
199
200
my $filename_5 = make_csv($temp_dir, $csv_headers, $input_no_branch, $input_good_branch, $input_na_branch);
201
open(my $handle_5, "<", $filename_5) or die "cannot open < $filename_5: $!";
202
my $params_5 = { file => $handle_5, matchpoint => 'cardnumber', };
203
204
# When ...
205
my $result_5 = $patrons_import->import_patrons($params_5);
206
207
# Then ...
208
is($result_5->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons for branch tests');
209
210
is($result_5->{errors}->[0]->{missing_criticals}->[0]->{borrowernumber}, 'UNDEF', 'Got the expected undef borrower number error from import patrons for branch tests');
211
is($result_5->{errors}->[0]->{missing_criticals}->[0]->{key}, 'branchcode', 'Got the expected branch code key from import patrons for branch tests');
212
is($result_5->{errors}->[0]->{missing_criticals}->[0]->{line}, 2, 'Got the expected 2 line number error from import patrons for branch tests');
213
is($result_5->{errors}->[0]->{missing_criticals}->[0]->{lineraw}, $input_no_branch."\r\n", 'Got the expected lineraw error from import patrons for branch tests');
214
is($result_5->{errors}->[0]->{missing_criticals}->[0]->{surname}, 'Johnny', 'Got the expected surname error from import patrons for branch tests');
215
216
is($result_5->{errors}->[1]->{missing_criticals}->[0]->{borrowernumber}, 'UNDEF', 'Got the expected undef borrower number error from import patrons for branch tests');
217
is($result_5->{errors}->[1]->{missing_criticals}->[0]->{branch_map}, 1, 'Got the expected 1 branchmap error from import patrons for branch tests');
218
is($result_5->{errors}->[1]->{missing_criticals}->[0]->{key}, 'branchcode', 'Got the expected branch code key from import patrons for branch tests');
219
is($result_5->{errors}->[1]->{missing_criticals}->[0]->{line}, 4, 'Got the expected 4 line number error from import patrons for branch tests');
220
is($result_5->{errors}->[1]->{missing_criticals}->[0]->{lineraw}, $input_na_branch."\r\n", 'Got the expected lineraw error from import patrons for branch tests');
221
is($result_5->{errors}->[1]->{missing_criticals}->[0]->{surname}, 'Ruth', 'Got the expected surname error from import patrons for branch tests');
222
is($result_5->{errors}->[1]->{missing_criticals}->[0]->{value}, 'ZZZ', 'Got the expected ZZZ value error from import patrons for branch tests');
223
224
is($result_5->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for branch tests');
225
is($result_5->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons for branch tests');
226
is($result_5->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons for branch tests');
227
228
is($result_5->{feedback}->[1]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for branch tests');
229
is($result_5->{feedback}->[1]->{name}, 'lastimported', 'Got the expected lastimported name from import_patrons for branch tests');
230
like($result_5->{feedback}->[1]->{value},  qr/^Linda \/ \d+/, 'Got the expected last imported value from import_patrons with for branch tests');
231
232
is($result_5->{imported}, 1, 'Got the expected 1 imported result from import patrons for branch tests');
233
is($result_5->{invalid}, 2, 'Got the expected 2 invalid result from import patrons for branch tests');
234
is($result_5->{overwritten}, 0, 'Got the expected 0 overwritten result from import patrons for branch tests');
235
236
# Given ... 3 new inputs. One with no category code, one with unexpected category code.
237
my $input_no_category   = '1006,Christina,Olson,Rev,Kimberly,CO,8,Avenue,Northridge,Lexington,Wilmington,Kentucky,40510,United States,kolson6@dropbox.com,7-(810)636-6048,1-(052)012-8984,8-(567)232-7818,03/26/1952,FFL,,09/07/2014,01/07/2015,kolson6,x5D3qGbLlptx';
238
my $input_good_category = '1007,Peter,Peters,Mrs,Lawrence,PP,6,Trail,South,Oklahoma City,Topeka,Oklahoma,73135,United States,lpeters7@bandcamp.com,5-(992)205-9318,0-(732)586-9365,3-(448)146-7936,08/16/1983,PVL,T,03/24/2015,07/01/2015,lpeters7,Z19BrQ4';
239
my $input_na_category   = '1008,Emily,Richards,Ms,Judy,ER,73,Way,Kedzie,Fort Wayne,Phoenix,Indiana,46825,United States,jrichards8@arstechnica.com,5-(266)658-8957,3-(550)500-9107,7-(816)675-9822,08/09/1984,FFL,ZZ,11/09/2014,07/01/2015,jrichards8,D5PvU6H2R';
240
241
my $filename_6 = make_csv($temp_dir, $csv_headers, $input_no_category, $input_good_category, $input_na_category);
242
open(my $handle_6, "<", $filename_6) or die "cannot open < $filename_6: $!";
243
my $params_6 = { file => $handle_6, matchpoint => 'cardnumber', };
244
245
# When ...
246
my $result_6 = $patrons_import->import_patrons($params_6);
247
248
# Then ...
249
is($result_6->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons for category tests');
250
251
is($result_6->{errors}->[0]->{missing_criticals}->[0]->{borrowernumber}, 'UNDEF', 'Got the expected undef borrower number error from import patrons for category tests');
252
is($result_6->{errors}->[0]->{missing_criticals}->[0]->{key}, 'categorycode', 'Got the expected category code key from import patrons for category tests');
253
is($result_6->{errors}->[0]->{missing_criticals}->[0]->{line}, 2, 'Got the expected 2 line number error from import patrons for category tests');
254
is($result_6->{errors}->[0]->{missing_criticals}->[0]->{lineraw}, $input_no_category."\r\n", 'Got the expected lineraw error from import patrons for category tests');
255
is($result_6->{errors}->[0]->{missing_criticals}->[0]->{surname}, 'Christina', 'Got the expected surname error from import patrons for category tests');
256
257
is($result_6->{errors}->[1]->{missing_criticals}->[0]->{borrowernumber}, 'UNDEF', 'Got the expected undef borrower number error from import patrons for category tests');
258
is($result_6->{errors}->[1]->{missing_criticals}->[0]->{category_map}, 1, 'Got the expected 1 category_map error from import patrons for category tests');
259
is($result_6->{errors}->[1]->{missing_criticals}->[0]->{key}, 'categorycode', 'Got the expected category code key from import patrons for category tests');
260
is($result_6->{errors}->[1]->{missing_criticals}->[0]->{line}, 4, 'Got the expected 4 line number error from import patrons for category tests');
261
is($result_6->{errors}->[1]->{missing_criticals}->[0]->{lineraw}, $input_na_category."\r\n", 'Got the expected lineraw error from import patrons for category tests');
262
is($result_6->{errors}->[1]->{missing_criticals}->[0]->{surname}, 'Emily', 'Got the expected surname error from import patrons for category tests');
263
is($result_6->{errors}->[1]->{missing_criticals}->[0]->{value}, 'ZZ', 'Got the expected ZZ value error from import patrons for category tests');
264
265
is($result_6->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for category tests');
266
is($result_6->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons for category tests');
267
is($result_6->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons for category tests');
268
269
is($result_6->{feedback}->[1]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for category tests');
270
is($result_6->{feedback}->[1]->{name}, 'lastimported', 'Got the expected lastimported name from import_patrons for category tests');
271
like($result_6->{feedback}->[1]->{value},  qr/^Peter \/ \d+/, 'Got the expected last imported value from import_patrons with for category tests');
272
273
is($result_6->{imported}, 1, 'Got the expected 1 imported result from import patrons for category tests');
274
is($result_6->{invalid}, 2, 'Got the expected 2 invalid result from import patrons for category tests');
275
is($result_6->{overwritten}, 0, 'Got the expected 0 overwritten result from import patrons for category tests');
276
277
# Given ... 2 new inputs. One without dateofbirth, dateenrolled and dateexpiry values.
278
my $input_complete = '1009,Christina,Harris,Dr,Philip,CH,99,Street,Grayhawk,Baton Rouge,Dallas,Louisiana,70810,United States,pharris9@hp.com,9-(317)603-5513,7-(005)062-7593,8-(349)134-1627,06/19/1969,IPT,PT,04/09/2015,07/01/2015,pharris9,NcAhcvvnB';
279
my $input_no_date  = '1010,Ralph,Warren,Ms,Linda,RW,6,Way,Barby,Orlando,Albany,Florida,32803,United States,lwarrena@multiply.com,7-(579)753-7752,6-(847)086-7566,9-(122)729-8226,26/01/2001,LPL,T,25/01/2001,24/01/2001,lwarrena,tJ56RD4uV';
280
281
my $filename_7 = make_csv($temp_dir, $csv_headers, $input_complete, $input_no_date);
282
open(my $handle_7, "<", $filename_7) or die "cannot open < $filename_7: $!";
283
my $params_7 = { file => $handle_7, matchpoint => 'cardnumber', };
284
285
# When ...
286
my $result_7 = $patrons_import->import_patrons($params_7);
287
288
# Then ...
289
is($result_7->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons for dates tests');
290
is(scalar @{$result_7->{errors}}, 1, 'Got the expected 1 error array size from import_patrons for dates tests');
291
is(scalar @{$result_7->{errors}->[0]->{missing_criticals}}, 3, 'Got the expected 3 missing critical errors from import_patrons for dates tests');
292
293
is($result_7->{errors}->[0]->{missing_criticals}->[0]->{bad_date}, 1, 'Got the expected 1 bad_date error from import patrons for dates tests');
294
is($result_7->{errors}->[0]->{missing_criticals}->[0]->{borrowernumber}, 'UNDEF', 'Got the expected undef borrower number error from import patrons for dates tests');
295
is($result_7->{errors}->[0]->{missing_criticals}->[0]->{key}, 'dateofbirth', 'Got the expected dateofbirth key from import patrons for dates tests');
296
is($result_7->{errors}->[0]->{missing_criticals}->[0]->{line}, 3, 'Got the expected 2 line number error from import patrons for dates tests');
297
is($result_7->{errors}->[0]->{missing_criticals}->[0]->{lineraw}, $input_no_date."\r\n", 'Got the expected lineraw error from import patrons for dates tests');
298
is($result_7->{errors}->[0]->{missing_criticals}->[0]->{surname}, 'Ralph', 'Got the expected surname error from import patrons for dates tests');
299
300
is($result_7->{errors}->[0]->{missing_criticals}->[1]->{key}, 'dateenrolled', 'Got the expected dateenrolled key from import patrons for dates tests');
301
is($result_7->{errors}->[0]->{missing_criticals}->[2]->{key}, 'dateexpiry', 'Got the expected dateexpiry key from import patrons for dates tests');
302
303
is(scalar @{$result_7->{feedback}}, 2, 'Got the expected 2 feedback from import patrons for dates tests');
304
is($result_7->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for dates tests');
305
is($result_7->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons for dates tests');
306
is($result_7->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons for dates tests');
307
308
is($result_7->{feedback}->[1]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for dates tests');
309
is($result_7->{feedback}->[1]->{name}, 'lastimported', 'Got the expected lastimported from import_patrons for dates tests');
310
like($result_7->{feedback}->[1]->{value}, qr/^Christina \/ \d+/, 'Got the expected lastimported value from import_patrons for dates tests');
311
312
is($result_7->{imported}, 1, 'Got the expected 1 imported result from import patrons for dates tests');
313
is($result_7->{invalid}, 1, 'Got the expected 1 invalid result from import patrons for dates tests');
314
is($result_7->{overwritten}, 0, 'Got the expected 0 overwritten result from import patrons for dates tests');
315
316
subtest 'test_prepare_columns' => sub {
317
    plan tests => 16;
318
319
    # Given ... no header row
320
    my $headerrow_0;
321
    my %csvkeycol_0;
322
    my @errors_0;
323
324
    # When ...
325
    my @csvcolumns_0 = $patrons_import->prepare_columns({headerrow => undef, keycol => \%csvkeycol_0, errors => \@errors_0, });
326
327
    # Then ...
328
    is(scalar @csvcolumns_0, 0, 'Got the expected empty column array from prepare columns with no header row');
329
330
    is(scalar @errors_0, 1, 'Got the expected 1 entry in error array from prepare columns with no header row');
331
    is($errors_0[0]->{badheader}, 1, 'Got the expected 1 badheader from prepare columns with no header row');
332
    is($errors_0[0]->{line}, 1, 'Got the expected 1 line from prepare columns with no header row');
333
    is($errors_0[0]->{lineraw}, undef, 'Got the expected undef lineraw from prepare columns with no header row');
334
335
    # Given ... a good header row with plenty of whitespaces
336
    my $headerrow_1 = 'a,    b ,        c,  ,   d';
337
    my %csvkeycol_1;
338
    my @errors_1;
339
340
    # When ...
341
    my @csvcolumns_1 = $patrons_import->prepare_columns({headerrow => $headerrow_1, keycol => \%csvkeycol_1, errors => \@errors_1, });
342
343
    # Then ...
344
    is(scalar @csvcolumns_1, 5, 'Got the expected 5 column array from prepare columns');
345
    is($csvcolumns_1[0], 'a', 'Got the expected a header from prepare columns');
346
    is($csvcolumns_1[1], 'b', 'Got the expected b header from prepare columns');
347
    is($csvcolumns_1[2], 'c', 'Got the expected c header from prepare columns');
348
    is($csvcolumns_1[3], '', 'Got the expected empty header from prepare columns');
349
    is($csvcolumns_1[4], 'd', 'Got the expected d header from prepare columns');
350
351
    is($csvkeycol_1{a}, 0, 'Got the expected 0 value for key a from prepare columns hash');
352
    is($csvkeycol_1{b}, 1, 'Got the expected 1 value for key b from prepare columns hash');
353
    is($csvkeycol_1{c}, 2, 'Got the expected 2 value for key c from prepare columns hash');
354
    is($csvkeycol_1{''}, 3, 'Got the expected 3 value for empty string key from prepare columns hash');
355
    is($csvkeycol_1{d}, 4, 'Got the expected 4 value for key d from prepare columns hash');
356
};
357
358
subtest 'test_set_column_keys' => sub {
359
    plan tests => 5;
360
361
    # Given ... nothing at all
362
    # When ... Then ...
363
    my $attr_type_0 = $patrons_import->set_attribute_types(undef);
364
    is($attr_type_0, undef, 'Got the expected undef attribute type from set attribute types with nothing');
365
366
    # Given ... extended but not matchpoint
367
    my $params_1 = { extended => 1, matchpoint => undef, };
368
369
    # When ... Then ...
370
    my $attr_type_1 = $patrons_import->set_attribute_types($params_1);
371
    is($attr_type_1, undef, 'Got the expected undef attribute type from set attribute types with no matchpoint');
372
373
    # Given ... extended and unexpected matchpoint
374
    my $params_2 = { extended => 1, matchpoint => 'unexpected', };
375
376
    # When ... Then ...
377
    my $attr_type_2 = $patrons_import->set_attribute_types($params_2);
378
    is($attr_type_2, undef, 'Got the expected undef attribute type from set attribute types with unexpected matchpoint');
379
380
    # Given ...
381
    my $code_3   = 'SHOW_BCODE';
382
    my $params_3 = { extended => 1, matchpoint => $code_3, };
383
384
    # When ...
385
    my $attr_type_3 = $patrons_import->set_attribute_types($params_3);
386
387
    # Then ...
388
    isa_ok($attr_type_3, 'C4::Members::AttributeTypes');
389
    is($attr_type_3->{code}, $code_3, 'Got the expected code attribute type from set attribute types');
390
};
391
392
subtest 'test_set_column_keys' => sub {
393
    plan tests => 2;
394
395
    # Given ... nothing at all
396
    # When ... Then ...
397
    my @columnkeys_0 = $patrons_import->set_column_keys(undef);
398
    is(scalar @columnkeys_0, 66, 'Got the expected array size from set column keys with undef extended');
399
400
    # Given ... extended.
401
    my $extended = 1;
402
403
    # When ... Then ...
404
    my @columnkeys_1 = $patrons_import->set_column_keys($extended);
405
    is(scalar @columnkeys_1, 67, 'Got the expected array size from set column keys with extended');
406
};
407
408
subtest 'test_set_patron_attributes' => sub {
409
    plan tests => 13;
410
411
    # Given ... nothing at all
412
    # When ... Then ...
413
    my $result_0 = $patrons_import->set_patron_attributes(undef, undef, undef);
414
    is($result_0, undef, 'Got the expected undef from set patron attributes with nothing');
415
416
    # Given ... not extended.
417
    my $extended_1 = 0;
418
419
    # When ... Then ...
420
    my $result_1 = $patrons_import->set_patron_attributes($extended_1, undef, undef);
421
    is($result_1, undef, 'Got the expected undef from set patron attributes with not extended');
422
423
    # Given ... NO patrons attributes
424
    my $extended_2          = 1;
425
    my $patron_attributes_2 = undef;
426
    my @feedback_2;
427
428
    # When ...
429
    my $result_2 = $patrons_import->set_patron_attributes($extended_2, $patron_attributes_2, \@feedback_2);
430
431
    # Then ...
432
    is($result_2, undef, 'Got the expected undef from set patron attributes with no patrons attributes');
433
    is(scalar @feedback_2, 0, 'Got the expected 0 size feedback array from set patron attributes with no patrons attributes');
434
435
    # Given ... some patrons attributes
436
    my $patron_attributes_3 = "homeroom:1150605,grade:01";
437
    my @feedback_3;
438
439
    # When ...
440
    my $result_3 = $patrons_import->set_patron_attributes($extended_2, $patron_attributes_3, \@feedback_3);
441
442
    # Then ...
443
    ok($result_3, 'Got some data back from set patron attributes');
444
    is($result_3->[0]->{code}, 'grade', 'Got the expected first code from set patron attributes');
445
    is($result_3->[0]->{value}, '01', 'Got the expected first value from set patron attributes');
446
447
    is($result_3->[1]->{code}, 'homeroom', 'Got the expected second code from set patron attributes');
448
    is($result_3->[1]->{value}, 1150605, 'Got the expected second value from set patron attributes');
449
450
    is(scalar @feedback_3, 1, 'Got the expected 1 array size from set patron attributes with extended user');
451
    is($feedback_3[0]->{feedback}, 1, 'Got the expected second feedback from set patron attributes with extended user');
452
    is($feedback_3[0]->{name}, 'attribute string', 'Got the expected attribute string from set patron attributes with extended user');
453
    is($feedback_3[0]->{value}, 'homeroom:1150605,grade:01', 'Got the expected feedback value from set patron attributes with extended user');
454
};
455
456
subtest 'test_check_branch_code' => sub {
457
    plan tests => 11;
458
459
    # Given ... no branch code.
460
    my $borrowerline      = 'some, line';
461
    my $line_number       = 78;
462
    my @missing_criticals = ();
463
464
    # When ...
465
    $patrons_import->check_branch_code(undef, $borrowerline, $line_number, \@missing_criticals);
466
467
    # Then ...
468
    is(scalar @missing_criticals, 1, 'Got the expected missing critical array size of 1 from check_branch_code with no branch code');
469
470
    is($missing_criticals[0]->{key}, 'branchcode', 'Got the expected branchcode key from check_branch_code with no branch code');
471
    is($missing_criticals[0]->{line}, $line_number, 'Got the expected line number from check_branch_code with no branch code');
472
    is($missing_criticals[0]->{lineraw}, $borrowerline, 'Got the expected lineraw value from check_branch_code with no branch code');
473
474
    # Given ... unknown branch code
475
    my $branchcode_1        = 'unexpected';
476
    my $borrowerline_1      = 'some, line,'.$branchcode_1;
477
    my $line_number_1       = 79;
478
    my @missing_criticals_1 = ();
479
480
    # When ...
481
    $patrons_import->check_branch_code($branchcode_1, $borrowerline_1, $line_number_1, \@missing_criticals_1);
482
483
    # Then ...
484
    is(scalar @missing_criticals_1, 1, 'Got the expected missing critical array size of 1 from check_branch_code with unexpected branch code');
485
486
    is($missing_criticals_1[0]->{branch_map}, 1, 'Got the expected 1 branch_map from check_branch_code with unexpected branch code');
487
    is($missing_criticals_1[0]->{key}, 'branchcode', 'Got the expected branchcode key from check_branch_code with unexpected branch code');
488
    is($missing_criticals_1[0]->{line}, $line_number_1, 'Got the expected line number from check_branch_code with unexpected branch code');
489
    is($missing_criticals_1[0]->{lineraw}, $borrowerline_1, 'Got the expected lineraw value from check_branch_code with unexpected branch code');
490
    is($missing_criticals_1[0]->{value}, $branchcode_1, 'Got the expected value from check_branch_code with unexpected branch code');
491
492
    # Given ... a known branch code. Relies on database sample data
493
    my $branchcode_2        = 'FFL';
494
    my $borrowerline_2      = 'some, line,'.$branchcode_2;
495
    my $line_number_2       = 80;
496
    my @missing_criticals_2 = ();
497
498
    # When ...
499
    $patrons_import->check_branch_code($branchcode_2, $borrowerline_2, $line_number_2, \@missing_criticals_2);
500
501
    # Then ...
502
    is(scalar @missing_criticals_2, 0, 'Got the expected missing critical array size of 0 from check_branch_code');
503
};
504
505
subtest 'test_check_borrower_category' => sub {
506
    plan tests => 11;
507
508
    # Given ... no category code.
509
    my $borrowerline      = 'some, line';
510
    my $line_number       = 781;
511
    my @missing_criticals = ();
512
513
    # When ...
514
    $patrons_import->check_borrower_category(undef, $borrowerline, $line_number, \@missing_criticals);
515
516
    # Then ...
517
    is(scalar @missing_criticals, 1, 'Got the expected missing critical array size of 1 from check_branch_code with no category code');
518
519
    is($missing_criticals[0]->{key}, 'categorycode', 'Got the expected categorycode key from check_branch_code with no category code');
520
    is($missing_criticals[0]->{line}, $line_number, 'Got the expected line number from check_branch_code with no category code');
521
    is($missing_criticals[0]->{lineraw}, $borrowerline, 'Got the expected lineraw value from check_branch_code with no category code');
522
523
    # Given ... unknown category code
524
    my $categorycode_1      = 'unexpected';
525
    my $borrowerline_1      = 'some, line, line, '.$categorycode_1;
526
    my $line_number_1       = 791;
527
    my @missing_criticals_1 = ();
528
529
    # When ...
530
    $patrons_import->check_borrower_category($categorycode_1, $borrowerline_1, $line_number_1, \@missing_criticals_1);
531
532
    # Then ...
533
    is(scalar @missing_criticals_1, 1, 'Got the expected missing critical array size of 1 from check_branch_code with unexpected category code');
534
535
    is($missing_criticals_1[0]->{category_map}, 1, 'Got the expected 1 category_map from check_branch_code with unexpected category code');
536
    is($missing_criticals_1[0]->{key}, 'categorycode', 'Got the expected branchcode key from check_branch_code with unexpected category code');
537
    is($missing_criticals_1[0]->{line}, $line_number_1, 'Got the expected line number from check_branch_code with unexpected category code');
538
    is($missing_criticals_1[0]->{lineraw}, $borrowerline_1, 'Got the expected lineraw value from check_branch_code with unexpected category code');
539
    is($missing_criticals_1[0]->{value}, $categorycode_1, 'Got the expected value from check_branch_code with unexpected category code');
540
541
    # Given ... a known category code. Relies on database sample data.
542
    my $categorycode_2      = 'T';
543
    my $borrowerline_2      = 'some, line,'.$categorycode_2;
544
    my $line_number_2       = 801;
545
    my @missing_criticals_2 = ();
546
547
    # When ...
548
    $patrons_import->check_borrower_category($categorycode_2, $borrowerline_2, $line_number_2, \@missing_criticals_2);
549
550
    # Then ...
551
    is(scalar @missing_criticals_2, 0, 'Got the expected missing critical array size of 0 from check_branch_code');
552
};
553
554
subtest 'test_format_dates' => sub {
555
    plan tests => 22;
556
557
    # Given ... no borrower data.
558
    my $borrowerline      = 'another line';
559
    my $line_number       = 987;
560
    my @missing_criticals = ();
561
    my %borrower;
562
    my $params = {borrower => \%borrower, lineraw => $borrowerline, line => $line_number, missing_criticals => \@missing_criticals, };
563
564
    # When ...
565
    $patrons_import->format_dates($params);
566
567
    # Then ...
568
    ok( not(%borrower), 'Got the expected no borrower from format_dates with no dates');
569
    is(scalar @missing_criticals, 0, 'Got the expected missing critical array size of 0 from format_dates with no dates');
570
571
    # Given ... some good dates
572
    my @missing_criticals_1 = ();
573
    my $dateofbirth_1  = '2016-05-03';
574
    my $dateenrolled_1 = '2016-05-04';
575
    my $dateexpiry_1   = '2016-05-06';
576
    my $borrower_1     = { dateofbirth => $dateofbirth_1, dateenrolled => $dateenrolled_1, dateexpiry => $dateexpiry_1, };
577
    my $params_1       = {borrower => $borrower_1, lineraw => $borrowerline, line => $line_number, missing_criticals => \@missing_criticals_1, };
578
579
    # When ...
580
    $patrons_import->format_dates($params_1);
581
582
    # Then ...
583
    is($borrower_1->{dateofbirth}, $dateofbirth_1, 'Got the expected date of birth from format_dates with good dates');
584
    is($borrower_1->{dateenrolled}, $dateenrolled_1, 'Got the expected date of birth from format_dates with good dates');
585
    is($borrower_1->{dateexpiry}, $dateexpiry_1, 'Got the expected date of birth from format_dates with good dates');
586
    is(scalar @missing_criticals_1, 0, 'Got the expected missing critical array size of 0 from check_branch_code with good dates');
587
588
    # Given ... some very bad dates
589
    my @missing_criticals_2 = ();
590
    my $dateofbirth_2  = '03-2016-05';
591
    my $dateenrolled_2 = '04-2016-05';
592
    my $dateexpiry_2   = '06-2016-05';
593
    my $borrower_2     = { dateofbirth => $dateofbirth_2, dateenrolled => $dateenrolled_2, dateexpiry => $dateexpiry_2, };
594
    my $params_2       = {borrower => $borrower_2, lineraw => $borrowerline, line => $line_number, missing_criticals => \@missing_criticals_2, };
595
596
    # When ...
597
    $patrons_import->format_dates($params_2);
598
599
    # Then ...
600
    is($borrower_2->{dateofbirth}, '', 'Got the expected empty date of birth from format_dates with bad dates');
601
    is($borrower_2->{dateenrolled}, '', 'Got the expected emptydate of birth from format_dates with bad dates');
602
    is($borrower_2->{dateexpiry}, '', 'Got the expected empty date of birth from format_dates with bad dates');
603
604
    is(scalar @missing_criticals_2, 3, 'Got the expected missing critical array size of 3 from check_branch_code with bad dates');
605
    is($missing_criticals_2[0]->{bad_date}, 1, 'Got the expected first bad date flag from check_branch_code with bad dates');
606
    is($missing_criticals_2[0]->{key}, 'dateofbirth', 'Got the expected dateofbirth key from check_branch_code with bad dates');
607
    is($missing_criticals_2[0]->{line}, $line_number, 'Got the expected first line from check_branch_code with bad dates');
608
    is($missing_criticals_2[0]->{lineraw}, $borrowerline, 'Got the expected first lineraw from check_branch_code with bad dates');
609
610
    is($missing_criticals_2[1]->{bad_date}, 1, 'Got the expected second bad date flag from check_branch_code with bad dates');
611
    is($missing_criticals_2[1]->{key}, 'dateenrolled', 'Got the expected dateenrolled key from check_branch_code with bad dates');
612
    is($missing_criticals_2[1]->{line}, $line_number, 'Got the expected second line from check_branch_code with bad dates');
613
    is($missing_criticals_2[1]->{lineraw}, $borrowerline, 'Got the expected second lineraw from check_branch_code with bad dates');
614
615
    is($missing_criticals_2[2]->{bad_date}, 1, 'Got the expected third bad date flag from check_branch_code with bad dates');
616
    is($missing_criticals_2[2]->{key}, 'dateexpiry', 'Got the expected dateexpiry key from check_branch_code with bad dates');
617
    is($missing_criticals_2[2]->{line}, $line_number, 'Got the expected third line from check_branch_code with bad dates');
618
    is($missing_criticals_2[2]->{lineraw}, $borrowerline, 'Got the expected third lineraw from check_branch_code with bad dates');
619
};
620
621
# ###### Test utility ###########
622
sub make_csv {
623
    my ($temp_dir, @lines) = @_;
624
625
    my ($fh, $filename) = tempfile( DIR => $temp_dir) or die $!;
626
    print $fh $_."\r\n" foreach @lines;
627
    close $fh or die $!;
628
629
    return $filename;
630
}
631
632
1;
(-)a/tools/import_borrowers.pl (-287 / +53 lines)
Lines 34-50 Link Here
34
# dates should be in the format you have set up Koha to expect
34
# dates should be in the format you have set up Koha to expect
35
# branchcode and categorycode need to be valid
35
# branchcode and categorycode need to be valid
36
36
37
use strict;
37
use Modern::Perl;
38
use warnings;
39
38
40
use C4::Auth;
39
use C4::Auth;
41
use C4::Output;
40
use C4::Output;
42
use C4::Context;
43
use C4::Members;
44
use C4::Members::Attributes qw(:all);
45
use C4::Members::AttributeTypes;
46
use C4::Members::Messaging;
47
use C4::Reports::Guided;
48
use C4::Templates;
41
use C4::Templates;
49
use Koha::Patron::Debarments;
42
use Koha::Patron::Debarments;
50
use Koha::Patrons;
43
use Koha::Patrons;
Lines 54-87 use Koha::Libraries; Link Here
54
use Koha::Patron::Categories;
47
use Koha::Patron::Categories;
55
use Koha::List::Patron;
48
use Koha::List::Patron;
56
49
50
use Koha::Patrons::Import;
51
my $Import = Koha::Patrons::Import->new();
52
57
use Text::CSV;
53
use Text::CSV;
54
58
# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals:
55
# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals:
59
# ė
56
# ė
60
# č
57
# č
61
58
62
use CGI qw ( -utf8 );
59
use CGI qw ( -utf8 );
63
60
64
my (@errors, @feedback);
61
my ( @errors, @feedback );
65
my $extended = C4::Context->preference('ExtendedPatronAttributes');
62
my $extended = C4::Context->preference('ExtendedPatronAttributes');
66
my $set_messaging_prefs = C4::Context->preference('EnhancedMessagingPreferences');
63
67
my @columnkeys = Koha::Patrons->columns();
64
my @columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } Koha::Patrons->columns();
68
@columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } @columnkeys;
65
push( @columnkeys, 'patron_attributes' ) if $extended;
69
if ($extended) {
70
    push @columnkeys, 'patron_attributes';
71
}
72
66
73
my $input = CGI->new();
67
my $input = CGI->new();
74
our $csv  = Text::CSV->new({binary => 1});  # binary needed for non-ASCII Unicode
68
75
#push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend}; #XXX
69
#push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend}; #XXX
76
70
77
my ( $template, $loggedinuser, $cookie ) = get_template_and_user({
71
my ( $template, $loggedinuser, $cookie ) = get_template_and_user(
72
    {
78
        template_name   => "tools/import_borrowers.tt",
73
        template_name   => "tools/import_borrowers.tt",
79
        query           => $input,
74
        query           => $input,
80
        type            => "intranet",
75
        type            => "intranet",
81
        authnotrequired => 0,
76
        authnotrequired => 0,
82
        flagsrequired   => { tools => 'import_patrons' },
77
        flagsrequired   => { tools => 'import_patrons' },
83
        debug           => 1,
78
        debug           => 1,
84
});
79
    }
80
);
85
81
86
# get the patron categories and pass them to the template
82
# get the patron categories and pass them to the template
87
my @patron_categories = Koha::Patron::Categories->search_limited({}, {order_by => ['description']});
83
my @patron_categories = Koha::Patron::Categories->search_limited({}, {order_by => ['description']});
Lines 90-110 my $columns = C4::Templates::GetColumnDefs( $input )->{borrowers}; Link Here
90
$columns = [ grep { $_->{field} ne 'borrowernumber' ? $_ : () } @$columns ];
86
$columns = [ grep { $_->{field} ne 'borrowernumber' ? $_ : () } @$columns ];
91
$template->param( borrower_fields => $columns );
87
$template->param( borrower_fields => $columns );
92
88
93
if ($input->param('sample')) {
89
if ( $input->param('sample') ) {
90
    our $csv = Text::CSV->new( { binary => 1 } );    # binary needed for non-ASCII Unicode
94
    print $input->header(
91
    print $input->header(
95
        -type       => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ?
92
        -type       => 'application/vnd.sun.xml.calc',    # 'application/vnd.ms-excel' ?
96
        -attachment => 'patron_import.csv',
93
        -attachment => 'patron_import.csv',
97
    );
94
    );
98
    $csv->combine(@columnkeys);
95
    $csv->combine(@columnkeys);
99
    print $csv->string, "\n";
96
    print $csv->string, "\n";
100
    exit 0;
97
    exit 0;
101
}
98
}
99
102
my $uploadborrowers = $input->param('uploadborrowers');
100
my $uploadborrowers = $input->param('uploadborrowers');
103
my $matchpoint      = $input->param('matchpoint');
101
my $matchpoint      = $input->param('matchpoint');
104
if ($matchpoint) {
102
if ($matchpoint) {
105
    $matchpoint =~ s/^patron_attribute_//;
103
    $matchpoint =~ s/^patron_attribute_//;
106
}
104
}
107
my $overwrite_cardnumber = $input->param('overwrite_cardnumber');
108
105
109
#create a patronlist
106
#create a patronlist
110
my $createpatronlist = $input->param('createpatronlist') || 0;
107
my $createpatronlist = $input->param('createpatronlist') || 0;
Lines 121-407 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
121
            token  => scalar $input->param('csrf_token'),
118
            token  => scalar $input->param('csrf_token'),
122
        });
119
        });
123
120
124
    push @feedback, {feedback=>1, name=>'filename', value=>$uploadborrowers, filename=>$uploadborrowers};
121
    my $handle   = $input->upload('uploadborrowers');
125
    my $handle = $input->upload('uploadborrowers');
126
    my $uploadinfo = $input->uploadInfo($uploadborrowers);
127
    foreach (keys %$uploadinfo) {
128
        push @feedback, {feedback=>1, name=>$_, value=>$uploadinfo->{$_}, $_=>$uploadinfo->{$_}};
129
    }
130
131
    my $imported    = 0;
132
    my @imported_borrowers;
133
    my $alreadyindb = 0;
134
    my $overwritten = 0;
135
    my $invalid     = 0;
136
    my $matchpoint_attr_type; 
137
    my %defaults = $input->Vars;
122
    my %defaults = $input->Vars;
138
123
139
    # use header line to construct key to column map
124
    my $return = $Import->import_patrons(
140
    my $borrowerline = <$handle>;
125
        {
141
    my $status = $csv->parse($borrowerline);
126
            file                         => $handle,
142
    ($status) or push @errors, {badheader=>1,line=>$., lineraw=>$borrowerline};
127
            defaults                     => \%defaults,
143
    my @csvcolumns = $csv->fields();
128
            matchpoint                   => $matchpoint,
144
    my %csvkeycol;
129
            overwrite_cardnumber         => $input->param('overwrite_cardnumber'),
145
    my $col = 0;
130
            preserve_extended_attributes => $input->param('ext_preserve') || 0,
146
    foreach my $keycol (@csvcolumns) {
147
    	# columnkeys don't contain whitespace, but some stupid tools add it
148
    	$keycol =~ s/ +//g;
149
        $csvkeycol{$keycol} = $col++;
150
    }
151
    #warn($borrowerline);
152
    my $ext_preserve = $input->param('ext_preserve') || 0;
153
    if ($extended) {
154
        $matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint);
155
    }
156
157
    push @feedback, {feedback=>1, name=>'headerrow', value=>join(', ', @csvcolumns)};
158
    my $today = output_pref;
159
    my @criticals = qw(surname branchcode categorycode);    # there probably should be others
160
    my @bad_dates;  # I've had a few.
161
    LINE: while ( my $borrowerline = <$handle> ) {
162
        my %borrower;
163
        my @missing_criticals;
164
        my $patron_attributes;
165
        my $status  = $csv->parse($borrowerline);
166
        my @columns = $csv->fields();
167
        if (! $status) {
168
            push @missing_criticals, {badparse=>1, line=>$., lineraw=>$borrowerline};
169
        } elsif (@columns == @columnkeys) {
170
            @borrower{@columnkeys} = @columns;
171
            # MJR: try to fill blanks gracefully by using default values
172
            foreach my $key (@columnkeys) {
173
                if ($borrower{$key} !~ /\S/) {
174
                    $borrower{$key} = $defaults{$key};
175
                }
176
            } 
177
        } else {
178
            # MJR: try to recover gracefully by using default values
179
            foreach my $key (@columnkeys) {
180
            	if (defined($csvkeycol{$key}) and $columns[$csvkeycol{$key}] =~ /\S/) { 
181
            	    $borrower{$key} = $columns[$csvkeycol{$key}];
182
            	} elsif ( $defaults{$key} ) {
183
            	    $borrower{$key} = $defaults{$key};
184
            	} elsif ( scalar grep {$key eq $_} @criticals ) {
185
            	    # a critical field is undefined
186
            	    push @missing_criticals, {key=>$key, line=>$., lineraw=>$borrowerline};
187
            	} else {
188
            		$borrower{$key} = '';
189
            	}
190
            }
191
        }
192
        #warn join(':',%borrower);
193
        if ($borrower{categorycode}) {
194
            push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline, value=>$borrower{categorycode}, category_map=>1}
195
                unless Koha::Patron::Categories->find($borrower{categorycode});
196
        } else {
197
            push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline};
198
        }
199
        if ($borrower{branchcode}) {
200
            push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline, value=>$borrower{branchcode}, branch_map=>1}
201
                unless Koha::Libraries->find($borrower{branchcode});
202
        } else {
203
            push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline};
204
        }
205
        if (@missing_criticals) {
206
            foreach (@missing_criticals) {
207
                $_->{borrowernumber} = $borrower{borrowernumber} || 'UNDEF';
208
                $_->{surname}        = $borrower{surname} || 'UNDEF';
209
            }
210
            $invalid++;
211
            (25 > scalar @errors) and push @errors, {missing_criticals=>\@missing_criticals};
212
            # The first 25 errors are enough.  Keeping track of 30,000+ would destroy performance.
213
            next LINE;
214
        }
215
        if ($extended) {
216
            my $attr_str = $borrower{patron_attributes};
217
            $attr_str =~ s/\xe2\x80\x9c/"/g; # fixup double quotes in case we are passed smart quotes
218
            $attr_str =~ s/\xe2\x80\x9d/"/g;
219
            push @feedback, {feedback=>1, name=>'attribute string', value=>$attr_str, filename=>$uploadborrowers};
220
            delete $borrower{patron_attributes};    # not really a field in borrowers, so we don't want to pass it to ModMember.
221
            $patron_attributes = extended_attributes_code_value_arrayref($attr_str); 
222
        }
223
	# Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it.
224
        foreach (qw(dateofbirth dateenrolled dateexpiry)) {
225
            my $tempdate = $borrower{$_} or next;
226
            $tempdate = eval { output_pref( { dt => dt_from_string( $tempdate ), dateonly => 1, dateformat => 'iso' } ); };
227
            if ($tempdate) {
228
                $borrower{$_} = $tempdate;
229
            } else {
230
                $borrower{$_} = '';
231
                push @missing_criticals, {key=>$_, line=>$. , lineraw=>$borrowerline, bad_date=>1};
232
            }
233
        }
234
        $borrower{dateenrolled} ||= $today;
235
        $borrower{dateexpiry}   ||= Koha::Patron::Categories->find( $borrower{categorycode} )->get_expiry_date( $borrower{dateenrolled} );
236
        my $borrowernumber;
237
        my $member;
238
        if ( ($matchpoint eq 'cardnumber') && ($borrower{'cardnumber'}) ) {
239
            $member = Koha::Patrons->find( { cardnumber => $borrower{'cardnumber'} } );
240
        } elsif ( ($matchpoint eq 'userid') && ($borrower{'userid'}) ) {
241
            $member = Koha::Patrons->find( { userid => $borrower{'userid'} } );
242
        } elsif ($extended) {
243
            if (defined($matchpoint_attr_type)) {
244
                foreach my $attr (@$patron_attributes) {
245
                    if ($attr->{code} eq $matchpoint and $attr->{value} ne '') {
246
                        my @borrowernumbers = $matchpoint_attr_type->get_patrons($attr->{value});
247
                        $borrowernumber = $borrowernumbers[0] if scalar(@borrowernumbers) == 1;
248
                        last;
249
                    }
250
                }
251
            }
252
        }
253
254
        if ($member) {
255
            $member = $member->unblessed;
256
            $borrowernumber = $member->{'borrowernumber'};
257
        } else {
258
            $member = {};
259
        }
260
261
        if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) {
262
            push @errors, {
263
                invalid_cardnumber => 1,
264
                borrowernumber => $borrowernumber,
265
                cardnumber => $borrower{cardnumber}
266
            };
267
            $invalid++;
268
            next;
269
        }
131
        }
132
    );
270
133
271
        if ($borrowernumber) {
134
    my $feedback    = $return->{feedback};
272
            # borrower exists
135
    my $errors      = $return->{errors};
273
            unless ($overwrite_cardnumber) {
136
    my $imported    = $return->{imported};
274
                $alreadyindb++;
137
    my $overwritten = $return->{overwritten};
275
                $template->param('lastalreadyindb'=>$borrower{'surname'}.' / '.$borrowernumber);
138
    my $alreadyindb = $return->{already_in_db};
276
                next LINE;
139
    my $invalid     = $return->{invalid};
277
            }
278
            $borrower{'borrowernumber'} = $borrowernumber;
279
            for my $col (keys %borrower) {
280
                # use values from extant patron unless our csv file includes this column or we provided a default.
281
                # FIXME : You cannot update a field with a  perl-evaluated false value using the defaults.
282
283
                # The password is always encrypted, skip it!
284
                next if $col eq 'password';
285
286
                unless(exists($csvkeycol{$col}) || $defaults{$col}) {
287
                    $borrower{$col} = $member->{$col} if($member->{$col}) ;
288
                }
289
            }
290
291
            # Check if the userid provided does not exist yet
292
            if (  exists $borrower{userid}
293
                     and $borrower{userid}
294
                 and not Check_Userid( $borrower{userid}, $borrower{borrowernumber} ) ) {
295
                push @errors, { duplicate_userid => 1, userid => $borrower{userid} };
296
                $invalid++;
297
                next LINE;
298
            }
299
300
            unless (ModMember(%borrower)) {
301
                $invalid++;
302
                # until we have better error trapping, we have no way of knowing why ModMember errored out...
303
                push @errors, {unknown_error => 1};
304
                $template->param('lastinvalid'=>$borrower{'surname'}.' / '.$borrowernumber);
305
                next LINE;
306
            }
307
308
            # Don't add a new restriction if the existing 'combined' restriction matches this one
309
            if ( $borrower{debarred} && ( ( $borrower{debarred} ne $member->{debarred} ) || ( $borrower{debarredcomment} ne $member->{debarredcomment} ) ) ) {
310
                # Check to see if this debarment already exists
311
                my $debarrments = GetDebarments(
312
                    {
313
                        borrowernumber => $borrowernumber,
314
                        expiration     => $borrower{debarred},
315
                        comment        => $borrower{debarredcomment}
316
                    }
317
                );
318
                # If it doesn't, then add it!
319
                unless (@$debarrments) {
320
                    AddDebarment(
321
                        {
322
                            borrowernumber => $borrowernumber,
323
                            expiration     => $borrower{debarred},
324
                            comment        => $borrower{debarredcomment}
325
                        }
326
                    );
327
                }
328
            }
329
330
            if ($extended) {
331
                if ($ext_preserve) {
332
                    my $old_attributes = GetBorrowerAttributes($borrowernumber);
333
                    $patron_attributes = extended_attributes_merge($old_attributes, $patron_attributes);  #TODO: expose repeatable options in template
334
                }
335
                push @errors, {unknown_error => 1} unless SetBorrowerAttributes($borrower{'borrowernumber'}, $patron_attributes, 'no_branch_limit' );
336
            }
337
            $overwritten++;
338
            $template->param('lastoverwritten'=>$borrower{'surname'}.' / '.$borrowernumber);
339
        } else {
340
            # FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either.
341
            # At least this is closer to AddMember than in members/memberentry.pl
342
            if (!$borrower{'cardnumber'}) {
343
                $borrower{'cardnumber'} = fixup_cardnumber(undef);
344
            }
345
            if ($borrowernumber = AddMember(%borrower)) {
346
347
                if ( $borrower{debarred} ) {
348
                    AddDebarment(
349
                        {
350
                            borrowernumber => $borrowernumber,
351
                            expiration     => $borrower{debarred},
352
                            comment        => $borrower{debarredcomment}
353
                        }
354
                    );
355
                }
356
357
                if ($extended) {
358
                    SetBorrowerAttributes($borrowernumber, $patron_attributes);
359
                }
360
361
                if ($set_messaging_prefs) {
362
                    C4::Members::Messaging::SetMessagingPreferencesFromDefaults({ borrowernumber => $borrowernumber,
363
                                                                                  categorycode => $borrower{categorycode} });
364
                }
365
140
366
                $imported++;
141
    my $uploadinfo = $input->uploadInfo($uploadborrowers);
367
                $template->param('lastimported'=>$borrower{'surname'}.' / '.$borrowernumber);
142
    foreach ( keys %$uploadinfo ) {
368
                push @imported_borrowers, $borrowernumber; #for patronlist
143
        push @$feedback, { feedback => 1, name => $_, value => $uploadinfo->{$_}, $_ => $uploadinfo->{$_} };
369
            } else {
370
                $invalid++;
371
                push @errors, {unknown_error => 1};
372
                $template->param('lastinvalid'=>$borrower{'surname'}.' / AddMember');
373
            }
374
        }
375
    }
144
    }
376
145
377
    if ( $imported && $createpatronlist ) {
146
    push @$feedback, { feedback => 1, name => 'filename', value => $uploadborrowers, filename => $uploadborrowers };
378
        my $patronlist = AddPatronList({ name => $patronlistname });
379
        AddPatronsToList({ list => $patronlist, borrowernumbers => \@imported_borrowers });
380
        $template->param('patronlistname' => $patronlistname);
381
    }
382
147
383
    (@errors  ) and $template->param(  ERRORS=>\@errors  );
384
    (@feedback) and $template->param(FEEDBACK=>\@feedback);
385
    $template->param(
148
    $template->param(
386
        'uploadborrowers' => 1,
149
        uploadborrowers => 1,
387
        'imported'        => $imported,
150
        errors          => $errors,
388
        'overwritten'     => $overwritten,
151
        feedback        => $feedback,
389
        'alreadyindb'     => $alreadyindb,
152
        imported        => $imported,
390
        'invalid'         => $invalid,
153
        overwritten     => $overwritten,
391
        'total'           => $imported + $alreadyindb + $invalid + $overwritten,
154
        alreadyindb     => $alreadyindb,
155
        invalid         => $invalid,
156
        total           => $imported + $alreadyindb + $invalid + $overwritten,
392
    );
157
    );
393
158
394
} else {
159
}
160
else {
395
    if ($extended) {
161
    if ($extended) {
396
        my @matchpoints = ();
162
        my @matchpoints = ();
397
        my @attr_types = C4::Members::AttributeTypes::GetAttributeTypes(undef, 1);
163
        my @attr_types = C4::Members::AttributeTypes::GetAttributeTypes( undef, 1 );
398
        foreach my $type (@attr_types) {
164
        foreach my $type (@attr_types) {
399
            my $attr_type = C4::Members::AttributeTypes->fetch($type->{code});
165
            my $attr_type = C4::Members::AttributeTypes->fetch( $type->{code} );
400
            if ($attr_type->unique_id()) {
166
            if ( $attr_type->unique_id() ) {
401
            push @matchpoints, { code =>  "patron_attribute_" . $attr_type->code(), description => $attr_type->description() };
167
                push @matchpoints,
168
                  { code => "patron_attribute_" . $attr_type->code(), description => $attr_type->description() };
402
            }
169
            }
403
        }
170
        }
404
        $template->param(matchpoints => \@matchpoints);
171
        $template->param( matchpoints => \@matchpoints );
405
    }
172
    }
406
173
407
    $template->param(
174
    $template->param(
408
- 

Return to bug 12598