View | Details | Raw Unified | Return to bug 12598
Collapse All | Expand All

(-)a/tools/import_borrowers.pl (-68 / +177 lines)
Lines 2-7 Link Here
2
2
3
# Copyright 2007 Liblime
3
# Copyright 2007 Liblime
4
# Parts copyright 2010 BibLibre
4
# Parts copyright 2010 BibLibre
5
# Parts copyright 2014 ByWater Solutions
5
#
6
#
6
# This file is part of Koha.
7
# This file is part of Koha.
7
#
8
#
Lines 34-41 Link Here
34
# dates should be in the format you have set up Koha to expect
35
# dates should be in the format you have set up Koha to expect
35
# branchcode and categorycode need to be valid
36
# branchcode and categorycode need to be valid
36
37
37
use strict;
38
use Modern::Perl;
38
use warnings;
39
39
40
use C4::Auth;
40
use C4::Auth;
41
use C4::Output;
41
use C4::Output;
Lines 48-60 use C4::Members::AttributeTypes; Link Here
48
use C4::Members::Messaging;
48
use C4::Members::Messaging;
49
use Koha::Borrower::Debarments;
49
use Koha::Borrower::Debarments;
50
50
51
use CGI;
52
use Getopt::Long;
51
use Text::CSV;
53
use Text::CSV;
52
# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals:
54
# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals:
53
# ė
55
# ė
54
# č
56
# č
55
57
56
use CGI;
58
# Checks if the script is called from commandline
57
# use encoding 'utf8';    # don't do this
59
my $is_cli = not defined $ENV{GATEWAY_INTERFACE};
58
60
59
my (@errors, @feedback);
61
my (@errors, @feedback);
60
my $extended = C4::Context->preference('ExtendedPatronAttributes');
62
my $extended = C4::Context->preference('ExtendedPatronAttributes');
Lines 67-123 my $columnkeystpl = [ map { {'key' => $_} } grep {$_ ne 'borrowernumber' } @col Link Here
67
69
68
my $input = CGI->new();
70
my $input = CGI->new();
69
our $csv  = Text::CSV->new({binary => 1});  # binary needed for non-ASCII Unicode
71
our $csv  = Text::CSV->new({binary => 1});  # binary needed for non-ASCII Unicode
70
#push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend}; #XXX
72
71
73
my ( $template, $loggedinuser, $cookie );
72
my ( $template, $loggedinuser, $cookie ) = get_template_and_user({
74
unless ($is_cli) {
73
        template_name   => "tools/import_borrowers.tt",
75
    ( $template, $loggedinuser, $cookie ) = get_template_and_user(
74
        query           => $input,
76
        {
75
        type            => "intranet",
77
            template_name   => "tools/import_borrowers.tt",
76
        authnotrequired => 0,
78
            query           => $input,
77
        flagsrequired   => { tools => 'import_patrons' },
79
            type            => "intranet",
78
        debug           => 1,
80
            authnotrequired => 0,
79
});
81
            flagsrequired   => { tools => 'import_patrons' },
80
82
            debug           => 1,
81
$template->param(columnkeys => $columnkeystpl);
83
        }
82
84
    );
83
if ($input->param('sample')) {
85
84
    print $input->header(
86
    $template->param( columnkeys => $columnkeystpl );
85
        -type       => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ?
87
86
        -attachment => 'patron_import.csv',
88
    if ( $input->param('sample') ) {
89
        print $input->header(
90
            -type =>
91
              'application/vnd.sun.xml.calc',    # 'application/vnd.ms-excel' ?
92
            -attachment => 'patron_import.csv',
93
        );
94
        $csv->combine(@columnkeys);
95
        print $csv->string, "\n";
96
        exit 1;
97
    }
98
99
    $template->param(
100
        SCRIPT_NAME              => $ENV{'SCRIPT_NAME'},
101
        ExtendedPatronAttributes => $extended,
87
    );
102
    );
88
    $csv->combine(@columnkeys);
89
    print $csv->string, "\n";
90
    exit 1;
91
}
92
my $uploadborrowers = $input->param('uploadborrowers');
93
my $matchpoint      = $input->param('matchpoint');
94
if ($matchpoint) {
95
    $matchpoint =~ s/^patron_attribute_//;
96
}
103
}
97
my $overwrite_cardnumber = $input->param('overwrite_cardnumber');
98
104
99
$template->param( SCRIPT_NAME => $ENV{'SCRIPT_NAME'} );
105
my $uploadborrowers;
106
my $matchpoint;
107
my $overwrite_cardnumber;
108
my %defaults;
109
my $ext_preserve = 0;
110
my $verbose;
111
112
if ($is_cli) {
113
    my $help;
114
    GetOptions(
115
        'c|csv=s'                       => \$uploadborrowers,
116
        'm|matchpoint=s'                => \$matchpoint,
117
        'd|default=s'                   => \%defaults,
118
        'o|overwrite'                   => \$overwrite_cardnumber,
119
        'p|preserve-extended-atributes' => \$ext_preserve,
120
        'v|verbose'                     => \$verbose,
121
        'h|help|?'                      => \$help,
122
    );
100
123
101
($extended) and $template->param(ExtendedPatronAttributes => 1);
124
    print_help() if ( $help || !$uploadborrowers || !$matchpoint );
125
}
126
else {
127
    $uploadborrowers = $input->param('uploadborrowers');
128
    $matchpoint      = $input->param('matchpoint');
129
    if ($matchpoint) {
130
        $matchpoint =~ s/^patron_attribute_//;
131
    }
132
    $overwrite_cardnumber = $input->param('overwrite_cardnumber');
133
    %defaults             = $input->Vars;
134
    $ext_preserve         = $input->param('ext_preserve') || 0;
135
}
102
136
103
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
137
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
104
    push @feedback, {feedback=>1, name=>'filename', value=>$uploadborrowers, filename=>$uploadborrowers};
138
    push @feedback, {feedback=>1, name=>'filename', value=>$uploadborrowers, filename=>$uploadborrowers};
105
    my $handle = $input->upload('uploadborrowers');
139
106
    my $uploadinfo = $input->uploadInfo($uploadborrowers);
140
    my $handle;
107
    foreach (keys %$uploadinfo) {
141
    if ($is_cli) {
108
        push @feedback, {feedback=>1, name=>$_, value=>$uploadinfo->{$_}, $_=>$uploadinfo->{$_}};
142
        open( $handle, "<", $uploadborrowers ) or die $!;
143
    }
144
    else {
145
        $handle = $uploadborrowers;
146
147
        my $uploadinfo = $input->uploadInfo($uploadborrowers);
148
        foreach ( keys %$uploadinfo ) {
149
            push @feedback,
150
              {
151
                feedback => 1,
152
                name     => $_,
153
                value    => $uploadinfo->{$_},
154
                $_       => $uploadinfo->{$_}
155
              };
156
        }
109
    }
157
    }
158
110
    my $imported    = 0;
159
    my $imported    = 0;
111
    my $alreadyindb = 0;
160
    my $alreadyindb = 0;
112
    my $overwritten = 0;
161
    my $overwritten = 0;
113
    my $invalid     = 0;
162
    my $invalid     = 0;
114
    my $matchpoint_attr_type; 
163
    my $matchpoint_attr_type; 
115
    my %defaults = $input->Vars;
116
164
117
    # use header line to construct key to column map
165
    # use header line to construct key to column map
118
    my $borrowerline = <$handle>;
166
    my $borrowerline = <$handle>;
119
    my $status = $csv->parse($borrowerline);
167
    my $status = $csv->parse($borrowerline);
120
    ($status) or push @errors, {badheader=>1,line=>$., lineraw=>$borrowerline};
168
    ($status) or push @errors, {badheader=>1,line=>$., lineraw=>$borrowerline};
169
    print "WARNING: CSV header line is incomplete: $borrowerline\n" if $is_cli;
121
    my @csvcolumns = $csv->fields();
170
    my @csvcolumns = $csv->fields();
122
    my %csvkeycol;
171
    my %csvkeycol;
123
    my $col = 0;
172
    my $col = 0;
Lines 126-133 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
126
    	$keycol =~ s/ +//g;
175
    	$keycol =~ s/ +//g;
127
        $csvkeycol{$keycol} = $col++;
176
        $csvkeycol{$keycol} = $col++;
128
    }
177
    }
129
    #warn($borrowerline);
130
    my $ext_preserve = $input->param('ext_preserve') || 0;
131
    if ($extended) {
178
    if ($extended) {
132
        $matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint);
179
        $matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint);
133
    }
180
    }
Lines 146-151 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
146
        my @columns = $csv->fields();
193
        my @columns = $csv->fields();
147
        if (! $status) {
194
        if (! $status) {
148
            push @missing_criticals, {badparse=>1, line=>$., lineraw=>$borrowerline};
195
            push @missing_criticals, {badparse=>1, line=>$., lineraw=>$borrowerline};
196
            print "ERROR: Unable to parse line $.: $borrowerline" if $is_cli;
149
        } elsif (@columns == @columnkeys) {
197
        } elsif (@columns == @columnkeys) {
150
            @borrower{@columnkeys} = @columns;
198
            @borrower{@columnkeys} = @columns;
151
            # MJR: try to fill blanks gracefully by using default values
199
            # MJR: try to fill blanks gracefully by using default values
Lines 164-169 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
164
            	} elsif ( scalar grep {$key eq $_} @criticals ) {
212
            	} elsif ( scalar grep {$key eq $_} @criticals ) {
165
            	    # a critical field is undefined
213
            	    # a critical field is undefined
166
            	    push @missing_criticals, {key=>$key, line=>$., lineraw=>$borrowerline};
214
            	    push @missing_criticals, {key=>$key, line=>$., lineraw=>$borrowerline};
215
                    print "ERROR: missing critical column data '$key' for line $.: $borrowerline\n" if $is_cli;
167
            	} else {
216
            	} else {
168
            		$borrower{$key} = '';
217
            		$borrower{$key} = '';
169
            	}
218
            	}
Lines 171-186 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
171
        }
220
        }
172
        #warn join(':',%borrower);
221
        #warn join(':',%borrower);
173
        if ($borrower{categorycode}) {
222
        if ($borrower{categorycode}) {
174
            push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline, value=>$borrower{categorycode}, category_map=>1}
223
            unless ( GetBorrowercategory( $borrower{categorycode} ) ) {
175
                unless GetBorrowercategory($borrower{categorycode});
224
                push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline, value=>$borrower{categorycode}, category_map=>1};
225
                print "ERROR: invalid categorycode for line $.: $borrowerline\n" if $is_cli;
226
            }
176
        } else {
227
        } else {
177
            push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline};
228
            push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline};
229
            print "ERROR: missing categorycode for line $.: $borrowerline\n" if $is_cli;
178
        }
230
        }
179
        if ($borrower{branchcode}) {
231
        if ($borrower{branchcode}) {
180
            push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline, value=>$borrower{branchcode}, branch_map=>1}
232
            unless ( GetBranchName( $borrower{branchcode} ) ) {
181
                unless GetBranchName($borrower{branchcode});
233
                push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline, value=>$borrower{branchcode}, branch_map=>1};
234
                print "ERROR: invalid branchcode for line $.: $borrowerline\n" if $is_cli;
235
            }
182
        } else {
236
        } else {
183
            push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline};
237
            push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline};
238
            print "ERROR: missing branchcode for line $.: $borrowerline\n" if $is_cli;
184
        }
239
        }
185
        if (@missing_criticals) {
240
        if (@missing_criticals) {
186
            foreach (@missing_criticals) {
241
            foreach (@missing_criticals) {
Lines 234-244 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
234
        }
289
        }
235
290
236
        if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) {
291
        if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) {
292
            $borrowernumber ||= q{};
237
            push @errors, {
293
            push @errors, {
238
                invalid_cardnumber => 1,
294
                invalid_cardnumber => 1,
239
                borrowernumber => $borrowernumber,
295
                borrowernumber => $borrowernumber,
240
                cardnumber => $borrower{cardnumber}
296
                cardnumber => $borrower{cardnumber}
241
            };
297
            };
298
            print "ERROR: invalid cardnumber $borrower{cardnumber} for borrowernumber $borrowernumber\n" if $is_cli;
242
            $invalid++;
299
            $invalid++;
243
            next;
300
            next;
244
        }
301
        }
Lines 248-254 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
248
            # borrower exists
305
            # borrower exists
249
            unless ($overwrite_cardnumber) {
306
            unless ($overwrite_cardnumber) {
250
                $alreadyindb++;
307
                $alreadyindb++;
251
                $template->param('lastalreadyindb'=>$borrower{'surname'}.' / '.$borrowernumber);
308
                if ( $is_cli ) {
309
                    print "$borrower{'surname'} / $borrowernumber alredy in database.\n" if $verbose;
310
                } else {
311
                    $template->param( 'lastalreadyindb' => $borrower{'surname'} . ' / ' . $borrowernumber );
312
                }
252
                next LINE;
313
                next LINE;
253
            }
314
            }
254
            $borrower{'borrowernumber'} = $borrowernumber;
315
            $borrower{'borrowernumber'} = $borrowernumber;
Lines 267-273 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
267
                $invalid++;
328
                $invalid++;
268
                # untill we have better error trapping, we have no way of knowing why ModMember errored out...
329
                # untill we have better error trapping, we have no way of knowing why ModMember errored out...
269
                push @errors, {unknown_error => 1};
330
                push @errors, {unknown_error => 1};
270
                $template->param('lastinvalid'=>$borrower{'surname'}.' / '.$borrowernumber);
331
                if ( $is_cli ) {
332
                    print "Failure to update $borrower{'surname'} / $borrowernumber\n" if $verbose;
333
                } else {
334
                    $template->param('lastinvalid'=>$borrower{'surname'}.' / '.$borrowernumber);
335
                }
271
                next LINE;
336
                next LINE;
272
            }
337
            }
273
            if ( $borrower{debarred} ) {
338
            if ( $borrower{debarred} ) {
Lines 298-304 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
298
                push @errors, {unknown_error => 1} unless SetBorrowerAttributes($borrower{'borrowernumber'}, $patron_attributes);
363
                push @errors, {unknown_error => 1} unless SetBorrowerAttributes($borrower{'borrowernumber'}, $patron_attributes);
299
            }
364
            }
300
            $overwritten++;
365
            $overwritten++;
301
            $template->param('lastoverwritten'=>$borrower{'surname'}.' / '.$borrowernumber);
366
367
            if ( $is_cli ) {
368
                print "Overwriting $borrower{'surname'} / $borrowernumber with new data\n";
369
            } else {
370
                $template->param( 'lastoverwritten' => $borrower{'surname'} . ' / ' . $borrowernumber );
371
            }
302
        } else {
372
        } else {
303
            # FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either.
373
            # FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either.
304
            # At least this is closer to AddMember than in members/memberentry.pl
374
            # At least this is closer to AddMember than in members/memberentry.pl
Lines 327-364 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
327
                }
397
                }
328
398
329
                $imported++;
399
                $imported++;
330
                $template->param('lastimported'=>$borrower{'surname'}.' / '.$borrowernumber);
400
401
                if ($is_cli) {
402
                    print "Imported new patron $borrower{'surname'} / $borrowernumber\n" if $verbose;
403
                } else {
404
                    $template->param( 'lastimported' => $borrower{'surname'} . ' / ' . $borrowernumber );
405
                }
331
            } else {
406
            } else {
332
                $invalid++;
407
                $invalid++;
333
                push @errors, {unknown_error => 1};
408
                push @errors, {unknown_error => 1};
334
                $template->param('lastinvalid'=>$borrower{'surname'}.' / AddMember');
409
                if ( $is_cli ) {
410
                    print "Failure to import $borrower{'surname'}\n" if $verbose;
411
                } else {
412
                    $template->param( 'lastinvalid' => $borrower{'surname'} . ' / AddMember');
413
                }
335
            }
414
            }
336
        }
415
        }
337
    }
416
    }
338
    (@errors  ) and $template->param(  ERRORS=>\@errors  );
417
339
    (@feedback) and $template->param(FEEDBACK=>\@feedback);
418
    if ($is_cli) {
340
    $template->param(
419
        if ($verbose) {
341
        'uploadborrowers' => 1,
420
            my $total = $imported + $alreadyindb + $invalid + $overwritten;
342
        'imported'        => $imported,
421
            print "\nImport complete:\n";
343
        'overwritten'     => $overwritten,
422
            print "Imported:    $imported\n";
344
        'alreadyindb'     => $alreadyindb,
423
            print "Overwritten: $overwritten\n";
345
        'invalid'         => $invalid,
424
            print "Skipped:     $alreadyindb\n";
346
        'total'           => $imported + $alreadyindb + $invalid + $overwritten,
425
            print "Invalid:     $invalid\n";
347
    );
426
            print "Total:       $total\n\n";
427
        }
428
    } else {
429
        (@errors)   and $template->param( ERRORS   => \@errors );
430
        (@feedback) and $template->param( FEEDBACK => \@feedback );
431
        $template->param(
432
            'uploadborrowers' => 1,
433
            'imported'        => $imported,
434
            'overwritten'     => $overwritten,
435
            'alreadyindb'     => $alreadyindb,
436
            'invalid'         => $invalid,
437
            'total' => $imported + $alreadyindb + $invalid + $overwritten,
438
        );
439
    }
348
440
349
} else {
441
} else {
350
    if ($extended) {
442
    unless ( $is_cli ) {
351
        my @matchpoints = ();
443
        if ($extended) {
352
        my @attr_types = C4::Members::AttributeTypes::GetAttributeTypes(undef, 1);
444
            my @matchpoints = ();
353
        foreach my $type (@attr_types) {
445
            my @attr_types = C4::Members::AttributeTypes::GetAttributeTypes(undef, 1);
354
            my $attr_type = C4::Members::AttributeTypes->fetch($type->{code});
446
            foreach my $type (@attr_types) {
355
            if ($attr_type->unique_id()) {
447
                my $attr_type = C4::Members::AttributeTypes->fetch( $type->{code} );
356
            push @matchpoints, { code =>  "patron_attribute_" . $attr_type->code(), description => $attr_type->description() };
448
                if ( $attr_type->unique_id() ) {
449
                    push @matchpoints,
450
                      {
451
                        code        => "patron_attribute_" . $attr_type->code(),
452
                        description => $attr_type->description()
453
                      };
454
                }
357
            }
455
            }
456
            $template->param(matchpoints => \@matchpoints);
358
        }
457
        }
359
        $template->param(matchpoints => \@matchpoints);
360
    }
458
    }
361
}
459
}
362
460
363
output_html_with_http_headers $input, $cookie, $template->output;
461
output_html_with_http_headers( $input, $cookie, $template->output ) unless $is_cli;
364
462
365
- 
463
sub print_help {
464
    print <<_USAGE_;
465
import_borrowers.pl -c /path/to/borrowers.csv -m cardnumber
466
    -c --csv                            Path to the CSV file of patrons  to import
467
    -m --matchpoint                     Field on which to match incoming patrons to existing patrons
468
    -d --default                        Set defaults to patron fields, repeatable e.g. --default branchcode=MPL --default categorycode=PT
469
    -p --preserve-extended-atributes    Retain extended patron attributes for existing patrons being overwritten
470
    -o --overwrite                      Overwrite existing patrons with new data if a match is found
471
    -v --verbose                        Be verbose
472
_USAGE_
473
    exit;
474
}

Return to bug 12598