View | Details | Raw Unified | Return to bug 5633
Collapse All | Expand All

(-)a/tools/import_borrowers.pl (-38 / +141 lines)
Lines 46-52 use C4::Members; Link Here
46
use C4::Members::Attributes qw(:all);
46
use C4::Members::Attributes qw(:all);
47
use C4::Members::AttributeTypes;
47
use C4::Members::AttributeTypes;
48
use C4::Members::Messaging;
48
use C4::Members::Messaging;
49
49
use Date::Calc qw(Today_and_Now);
50
use Getopt::Long;
50
use Text::CSV;
51
use Text::CSV;
51
# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals:
52
# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals:
52
# ė
53
# ė
Lines 55-60 use Text::CSV; Link Here
55
use CGI;
56
use CGI;
56
# use encoding 'utf8';    # don't do this
57
# use encoding 'utf8';    # don't do this
57
58
59
my $input = CGI->new();
60
61
# Checks if the script is called from commandline
62
my $commandline = 0;
63
my $uploadborrowers;
64
my $matchpoint;
65
my $overwrite_cardnumber;
66
my $ext_preserve;
67
my $file;
68
my $cl_defaults;
69
my $help;
70
if (scalar @ARGV > 0) { 
71
    # Getting parameters
72
    GetOptions ('matchpoint=s' => \$matchpoint, 'overwrite' => \$overwrite_cardnumber, 'preserve_attributes' => \$ext_preserve, 'file=s' => \$file, 'help|?' => \$help);
73
    $commandline = 1;
74
75
    if ($help) {
76
	print "\nimport_borrowers.pl [--matchpoint=matchpoint] [--overwrite] [--preserve_attributes] --file=csvtoimport.csv\n\n";
77
	print " * matchpoint is either 'cardnumber' or like 'patron_attribute_' + patron attribute code (example: patron_attribute_EXTERNALID)\n";
78
	print " * Default values can be specified in import_borrowers.yaml (keys must be column names from the borrowers table)\n\n";
79
	exit;
80
    } 
81
82
    # Default parameters values : 
83
    $matchpoint ||= "cardnumber";
84
    $overwrite_cardnumber ||= 0;
85
    $ext_preserve ||= 0;
86
87
    # Default values
88
    $cl_defaults = YAML::LoadFile('import_borrowers.yaml') if (-e 'import_borrowers.yaml');
89
90
    open($uploadborrowers, '<', $file) or die("Unable to open $file");
91
} else {
92
    $uploadborrowers      = $input->param('uploadborrowers');
93
    $matchpoint           = $input->param('matchpoint');
94
    $overwrite_cardnumber = $input->param('overwrite_cardnumber');
95
    $ext_preserve         = $input->param('ext_preserve') || 0;
96
}
97
58
my (@errors, @feedback);
98
my (@errors, @feedback);
59
my $extended = C4::Context->preference('ExtendedPatronAttributes');
99
my $extended = C4::Context->preference('ExtendedPatronAttributes');
60
my $set_messaging_prefs = C4::Context->preference('EnhancedMessagingPreferences');
100
my $set_messaging_prefs = C4::Context->preference('EnhancedMessagingPreferences');
Lines 64-117 if ($extended) { Link Here
64
}
104
}
65
my $columnkeystpl = [ map { {'key' => $_} }  grep {$_ ne 'borrowernumber' && $_ ne 'cardnumber'} @columnkeys ];  # ref. to array of hashrefs.
105
my $columnkeystpl = [ map { {'key' => $_} }  grep {$_ ne 'borrowernumber' && $_ ne 'cardnumber'} @columnkeys ];  # ref. to array of hashrefs.
66
106
67
my $input = CGI->new();
68
our $csv  = Text::CSV->new({binary => 1});  # binary needed for non-ASCII Unicode
107
our $csv  = Text::CSV->new({binary => 1});  # binary needed for non-ASCII Unicode
69
# push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend};
108
# push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend};
70
109
71
my ( $template, $loggedinuser, $cookie ) = get_template_and_user({
110
my ( $template, $loggedinuser, $cookie ) = get_template_and_user({
72
        template_name   => "tools/import_borrowers.tmpl",
111
	    template_name   => "tools/import_borrowers.tmpl",
73
        query           => $input,
112
	    query           => $input,
74
        type            => "intranet",
113
	    type            => "intranet",
75
        authnotrequired => 0,
114
	    authnotrequired => $commandline,
76
        flagsrequired   => { tools => 'import_patrons' },
115
	    flagsrequired   => { tools => 'import_patrons' },
77
        debug           => 1,
116
	    debug           => 1,
78
});
117
    });
79
118
80
$template->param(columnkeys => $columnkeystpl);
119
if (!$commandline) {
81
120
    $template->param(columnkeys => $columnkeystpl);
82
if ($input->param('sample')) {
121
    $template->param( SCRIPT_NAME => $ENV{'SCRIPT_NAME'} );
83
    print $input->header(
122
    ($extended) and $template->param(ExtendedPatronAttributes => 1);
84
        -type       => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ?
123
85
        -attachment => 'patron_import.csv',
124
    if ($input->param('sample')) {
86
    );
125
	print $input->header(
87
    $csv->combine(@columnkeys);
126
	    -type       => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ?
88
    print $csv->string, "\n";
127
	    -attachment => 'patron_import.csv',
89
    exit 1;
128
	);
129
	$csv->combine(@columnkeys);
130
	print $csv->string, "\n";
131
	exit 1;
132
    }
90
}
133
}
91
my $uploadborrowers = $input->param('uploadborrowers');
134
92
my $matchpoint      = $input->param('matchpoint');
93
if ($matchpoint) {
135
if ($matchpoint) {
94
    $matchpoint =~ s/^patron_attribute_//;
136
    $matchpoint =~ s/^patron_attribute_//;
95
}
137
}
96
my $overwrite_cardnumber = $input->param('overwrite_cardnumber');
97
138
98
$template->param( SCRIPT_NAME => $ENV{'SCRIPT_NAME'} );
99
139
100
($extended) and $template->param(ExtendedPatronAttributes => 1);
101
140
102
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
141
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
103
    push @feedback, {feedback=>1, name=>'filename', value=>$uploadborrowers, filename=>$uploadborrowers};
142
    push @feedback, {feedback=>1, name=>'filename', value=>$uploadborrowers, filename=>$uploadborrowers};
104
    my $handle = $input->upload('uploadborrowers');
143
    my $handle = ($commandline == 1) ? $uploadborrowers : $input->upload('uploadborrowers');
105
    my $uploadinfo = $input->uploadInfo($uploadborrowers);
144
    my $uploadinfo = $input->uploadInfo($uploadborrowers);
106
    foreach (keys %$uploadinfo) {
145
    foreach (keys %$uploadinfo) {
107
        push @feedback, {feedback=>1, name=>$_, value=>$uploadinfo->{$_}, $_=>$uploadinfo->{$_}};
146
        push @feedback, {feedback=>1, name=>$_, value=>$uploadinfo->{$_}, $_=>$uploadinfo->{$_}};
108
    }
147
    }
109
    my $imported    = 0;
148
    my $imported    = 0;
110
    my $alreadyindb = 0;
149
    my $alreadyindb = 0;
150
    my $lastalreadyindb;
111
    my $overwritten = 0;
151
    my $overwritten = 0;
112
    my $invalid     = 0;
152
    my $invalid     = 0;
153
    my $lastinvalid;
113
    my $matchpoint_attr_type; 
154
    my $matchpoint_attr_type; 
114
    my %defaults = $input->Vars;
155
    my %defaults = ($commandline) ? %$cl_defaults : $input->Vars;
115
156
116
    # use header line to construct key to column map
157
    # use header line to construct key to column map
117
    my $borrowerline = <$handle>;
158
    my $borrowerline = <$handle>;
Lines 126-132 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
126
        $csvkeycol{$keycol} = $col++;
167
        $csvkeycol{$keycol} = $col++;
127
    }
168
    }
128
    #warn($borrowerline);
169
    #warn($borrowerline);
129
    my $ext_preserve = $input->param('ext_preserve') || 0;
130
    if ($extended) {
170
    if ($extended) {
131
        $matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint);
171
        $matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint);
132
    }
172
    }
Lines 187-193 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
187
                $_->{surname}        = $borrower{surname} || 'UNDEF';
227
                $_->{surname}        = $borrower{surname} || 'UNDEF';
188
            }
228
            }
189
            $invalid++;
229
            $invalid++;
190
            (25 > scalar @errors) and push @errors, {missing_criticals=>\@missing_criticals};
230
            ($commandline or 25 > scalar @errors) and push @errors, {missing_criticals=>\@missing_criticals};
191
            # The first 25 errors are enough.  Keeping track of 30,000+ would destroy performance.
231
            # The first 25 errors are enough.  Keeping track of 30,000+ would destroy performance.
192
            next LINE;
232
            next LINE;
193
        }
233
        }
Lines 233-239 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
233
            # borrower exists
273
            # borrower exists
234
            unless ($overwrite_cardnumber) {
274
            unless ($overwrite_cardnumber) {
235
                $alreadyindb++;
275
                $alreadyindb++;
236
                $template->param('lastalreadyindb'=>$borrower{'surname'}.' / '.$borrowernumber);
276
                $template->param('lastalreadyindb'=>$borrower{'surname'}.' / '.$borrowernumber) if (!$commandline);
277
                $lastalreadyindb = $borrower{'surname'}.' / '.$borrowernumber;
237
                next LINE;
278
                next LINE;
238
            }
279
            }
239
            $borrower{'borrowernumber'} = $borrowernumber;
280
            $borrower{'borrowernumber'} = $borrowernumber;
Lines 246-252 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
246
            }
287
            }
247
            unless (ModMember(%borrower)) {
288
            unless (ModMember(%borrower)) {
248
                $invalid++;
289
                $invalid++;
249
                $template->param('lastinvalid'=>$borrower{'surname'}.' / '.$borrowernumber);
290
                $template->param('lastinvalid'=>$borrower{'surname'}.' / '.$borrowernumber) if (!$commandline);
291
		$lastinvalid = $borrower{'surname'}.' / '.$borrowernumber;
250
                next LINE;
292
                next LINE;
251
            }
293
            }
252
            if ($extended) {
294
            if ($extended) {
Lines 257-263 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
257
                SetBorrowerAttributes($borrower{'borrowernumber'}, $patron_attributes);
299
                SetBorrowerAttributes($borrower{'borrowernumber'}, $patron_attributes);
258
            }
300
            }
259
            $overwritten++;
301
            $overwritten++;
260
            $template->param('lastoverwritten'=>$borrower{'surname'}.' / '.$borrowernumber);
302
            $template->param('lastoverwritten'=>$borrower{'surname'}.' / '.$borrowernumber) if (!$commandline);
261
        } else {
303
        } else {
262
            # FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either.
304
            # FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either.
263
            # At least this is closer to AddMember than in members/memberentry.pl
305
            # At least this is closer to AddMember than in members/memberentry.pl
Lines 273-287 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
273
                                                                                  categorycode => $borrower{categorycode} });
315
                                                                                  categorycode => $borrower{categorycode} });
274
                }
316
                }
275
                $imported++;
317
                $imported++;
276
                $template->param('lastimported'=>$borrower{'surname'}.' / '.$borrowernumber);
318
                $template->param('lastimported'=>$borrower{'surname'}.' / '.$borrowernumber) if (!$commandline);
277
            } else {
319
            } else {
278
                $invalid++;
320
                $invalid++;
279
                $template->param('lastinvalid'=>$borrower{'surname'}.' / AddMember');
321
                $template->param('lastinvalid'=>$borrower{'surname'}.' / AddMember') if (!$commandline);
322
                $lastinvalid = $borrower{'surname'}.' / AddMember';
280
            }
323
            }
281
        }
324
        }
282
    }
325
    }
283
    (@errors  ) and $template->param(  ERRORS=>\@errors  );
326
    (@errors  ) and $template->param(  ERRORS=>\@errors  ) if (!$commandline);
284
    (@feedback) and $template->param(FEEDBACK=>\@feedback);
327
    (@feedback) and $template->param(FEEDBACK=>\@feedback) if (!$commandline);
285
    $template->param(
328
    $template->param(
286
        'uploadborrowers' => 1,
329
        'uploadborrowers' => 1,
287
        'imported'        => $imported,
330
        'imported'        => $imported,
Lines 289-295 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
289
        'alreadyindb'     => $alreadyindb,
332
        'alreadyindb'     => $alreadyindb,
290
        'invalid'         => $invalid,
333
        'invalid'         => $invalid,
291
        'total'           => $imported + $alreadyindb + $invalid + $overwritten,
334
        'total'           => $imported + $alreadyindb + $invalid + $overwritten,
292
    );
335
    ) if (!$commandline);
336
    if ($commandline) {
337
338
	my $total = $imported + $alreadyindb + $invalid + $overwritten;
339
	my $output;
340
341
	my $timestamp = C4::Dates->new()->output . " " . POSIX::strftime("%H:%M:%S",localtime);
342
	$output .= "Timestamp : $timestamp\n";
343
	$output .= "Import results\n";
344
	$output .= "$imported imported records\n";
345
	$output .= "$overwritten overwritten records\n";
346
	$output .= "$alreadyindb not imported because already in borrowers table and overwrite disabled\n"; 
347
	$output .= "(last was $lastalreadyindb)\n" if ($lastalreadyindb);
348
	$output .= "$invalid not imported because they are not in the expected format\n"; 
349
	$output .= "(last was $lastinvalid)\n" if ($lastinvalid);
350
	$output .= "$total records parsed\n";
351
352
353
	$output .= "\nError analysis\n";
354
	foreach my $hash (@errors) {
355
	   $output .= "Header row could not be parsed" if ($hash->{'badheader'});
356
	   foreach my $array ($hash->{'missing_criticals'}) {
357
	       foreach (@$array) {
358
		    $output .= "Line $_->{'line'}: ";
359
		    if ($hash->{'badparse'}) {
360
			$output .= "could not be parsed!";
361
		    } elsif ($hash->{'bad_date'}) { 
362
			$output .= "has $_->{'key'} in unrecognized format: $_->{'value'} ";
363
		    } else {
364
			$output .= "Critical field $_->{'key'}: ";
365
			if ($_->{'branch_map'} || $_->{'category_map'}) {
366
			    $output .= "has unrecognized value: $_->{'value'}";
367
			} else {
368
			    $output .= " missing";
369
			}
370
			$output .= " (borrowernumber: $_->{'borrowernumber'}; surname: $_->{'surname'})";
371
		    }
372
		    $output .= "\n". $_->{'lineraw'} . "\n";
373
		}
374
	   }
375
	}
376
377
    # Write log file
378
    my $logfile = "/var/log/koha/reports/import_borrowers.log";
379
    if (open (FH, ">>$logfile")) {
380
	print FH $output;
381
	close(FH);
382
    } else {
383
	$output .= "Unable to write to log file : $logfile\n";
384
    }
385
386
387
    # Send email with log
388
     my $mail = MIME::Lite->new(
389
                To      => C4::Context->preference('KohaAdminEmailAddress'),
390
                Subject => "Import borrowers log email",
391
                Type    => 'text/plain',
392
                Data    => $output
393
            );
394
    $mail->send() or print "Unable to send log email";
395
    }
293
396
294
} else {
397
} else {
295
    if ($extended) {
398
    if ($extended) {
Lines 305-309 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
305
    }
408
    }
306
}
409
}
307
410
308
output_html_with_http_headers $input, $cookie, $template->output;
411
output_html_with_http_headers $input, $cookie, $template->output if (!$commandline);
309
412
(-)a/tools/import_borrowers.yaml (-1 / +3 lines)
Line 0 Link Here
0
- 
1
# Default values for import_borrowers.pl used in command line
2
column: value
3
othercolumn: value

Return to bug 5633