|
Lines 46-52
use C4::Members;
Link Here
|
| 46 |
use C4::Members::Attributes qw(:all); |
46 |
use C4::Members::Attributes qw(:all); |
| 47 |
use C4::Members::AttributeTypes; |
47 |
use C4::Members::AttributeTypes; |
| 48 |
use C4::Members::Messaging; |
48 |
use C4::Members::Messaging; |
| 49 |
|
49 |
use Date::Calc qw(Today_and_Now); |
|
|
50 |
use Getopt::Long; |
| 51 |
use File::Temp; |
| 50 |
use Text::CSV; |
52 |
use Text::CSV; |
| 51 |
# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals: |
53 |
# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals: |
| 52 |
# ė |
54 |
# ė |
|
Lines 69-92
our $csv = Text::CSV->new({binary => 1}); # binary needed for non-ASCII Unicod
Link Here
|
| 69 |
#push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend}; #XXX |
71 |
#push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend}; #XXX |
| 70 |
|
72 |
|
| 71 |
my ( $template, $loggedinuser, $cookie ) = get_template_and_user({ |
73 |
my ( $template, $loggedinuser, $cookie ) = get_template_and_user({ |
| 72 |
template_name => "tools/import_borrowers.tmpl", |
74 |
template_name => "tools/import_borrowers.tmpl", |
| 73 |
query => $input, |
75 |
query => $input, |
| 74 |
type => "intranet", |
76 |
type => "intranet", |
| 75 |
authnotrequired => 0, |
77 |
authnotrequired => $commandline, |
| 76 |
flagsrequired => { tools => 'import_patrons' }, |
78 |
flagsrequired => { tools => 'import_patrons' }, |
| 77 |
debug => 1, |
79 |
debug => 1, |
| 78 |
}); |
80 |
}); |
|
|
81 |
|
| 82 |
if (!$commandline) { |
| 83 |
$template->param(columnkeys => $columnkeystpl); |
| 84 |
$template->param( SCRIPT_NAME => $ENV{'SCRIPT_NAME'} ); |
| 85 |
($extended) and $template->param(ExtendedPatronAttributes => 1); |
| 79 |
|
86 |
|
| 80 |
$template->param(columnkeys => $columnkeystpl); |
87 |
if ($input->param('sample')) { |
|
|
88 |
print $input->header( |
| 89 |
-type => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ? |
| 90 |
-attachment => 'patron_import.csv', |
| 91 |
); |
| 92 |
$csv->combine(@columnkeys); |
| 93 |
print $csv->string, "\n"; |
| 94 |
exit 1; |
| 95 |
} |
| 81 |
|
96 |
|
| 82 |
if ($input->param('sample')) { |
97 |
if ($input->param('report')) { |
| 83 |
print $input->header( |
98 |
open (FH, $input->param('errors_filename')); |
| 84 |
-type => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ? |
99 |
print $input->header( |
| 85 |
-attachment => 'patron_import.csv', |
100 |
-type => 'text/plain', |
| 86 |
); |
101 |
-attachment => 'import_borrowers_report.txt' |
| 87 |
$csv->combine(@columnkeys); |
102 |
); |
| 88 |
print $csv->string, "\n"; |
103 |
print <FH>; |
| 89 |
exit 1; |
104 |
close FH; |
|
|
105 |
#TODO : We surely want to check that is it really a temp file that we are unlinking |
| 106 |
unlink $input->param('errors_filename'); |
| 107 |
exit 1; |
| 108 |
} |
| 90 |
} |
109 |
} |
| 91 |
my $uploadborrowers = $input->param('uploadborrowers'); |
110 |
my $uploadborrowers = $input->param('uploadborrowers'); |
| 92 |
my $matchpoint = $input->param('matchpoint'); |
111 |
my $matchpoint = $input->param('matchpoint'); |
|
Lines 187-194
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
| 187 |
$_->{surname} = $borrower{surname} || 'UNDEF'; |
206 |
$_->{surname} = $borrower{surname} || 'UNDEF'; |
| 188 |
} |
207 |
} |
| 189 |
$invalid++; |
208 |
$invalid++; |
| 190 |
(25 > scalar @errors) and push @errors, {missing_criticals=>\@missing_criticals}; |
209 |
push @errors, {missing_criticals=>\@missing_criticals}; |
| 191 |
# The first 25 errors are enough. Keeping track of 30,000+ would destroy performance. |
|
|
| 192 |
next LINE; |
210 |
next LINE; |
| 193 |
} |
211 |
} |
| 194 |
if ($extended) { |
212 |
if ($extended) { |
|
Lines 295-301
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
| 295 |
'alreadyindb' => $alreadyindb, |
313 |
'alreadyindb' => $alreadyindb, |
| 296 |
'invalid' => $invalid, |
314 |
'invalid' => $invalid, |
| 297 |
'total' => $imported + $alreadyindb + $invalid + $overwritten, |
315 |
'total' => $imported + $alreadyindb + $invalid + $overwritten, |
| 298 |
); |
316 |
) if (!$commandline); |
|
|
317 |
|
| 318 |
if (scalar(@errors) > 25 or $commandline) { |
| 319 |
|
| 320 |
my $total = $imported + $alreadyindb + $invalid + $overwritten; |
| 321 |
my $output; |
| 322 |
|
| 323 |
my $timestamp = C4::Dates->new()->output . " " . POSIX::strftime("%H:%M:%S",localtime); |
| 324 |
$output .= "Timestamp : $timestamp\n"; |
| 325 |
$output .= "Import results\n"; |
| 326 |
$output .= "$imported imported records\n"; |
| 327 |
$output .= "$overwritten overwritten records\n"; |
| 328 |
$output .= "$alreadyindb not imported because already in borrowers table and overwrite disabled\n"; |
| 329 |
$output .= "(last was $lastalreadyindb)\n" if ($lastalreadyindb); |
| 330 |
$output .= "$invalid not imported because they are not in the expected format\n"; |
| 331 |
$output .= "(last was $lastinvalid)\n" if ($lastinvalid); |
| 332 |
$output .= "$total records parsed\n"; |
| 333 |
|
| 334 |
|
| 335 |
$output .= "\nError analysis\n"; |
| 336 |
foreach my $hash (@errors) { |
| 337 |
$output .= "Header row could not be parsed" if ($hash->{'badheader'}); |
| 338 |
foreach my $array ($hash->{'missing_criticals'}) { |
| 339 |
foreach (@$array) { |
| 340 |
$output .= "Line $_->{'line'}: "; |
| 341 |
if ($hash->{'badparse'}) { |
| 342 |
$output .= "could not be parsed!"; |
| 343 |
} elsif ($hash->{'bad_date'}) { |
| 344 |
$output .= "has $_->{'key'} in unrecognized format: $_->{'value'} "; |
| 345 |
} else { |
| 346 |
$output .= "Critical field $_->{'key'}: "; |
| 347 |
if ($_->{'branch_map'} || $_->{'category_map'}) { |
| 348 |
$output .= "has unrecognized value: $_->{'value'}"; |
| 349 |
} else { |
| 350 |
$output .= " missing"; |
| 351 |
} |
| 352 |
$output .= " (borrowernumber: $_->{'borrowernumber'}; surname: $_->{'surname'})"; |
| 353 |
} |
| 354 |
$output .= "\n"; |
| 355 |
$output .= $_->{'lineraw'} . "\n" if ($commandline); |
| 356 |
} |
| 357 |
} |
| 358 |
} |
| 359 |
|
| 360 |
if (scalar(@errors) > 25 && !$commandline) { |
| 361 |
my $tmpf = File::Temp->new(UNLINK => 0); |
| 362 |
print $tmpf $output; |
| 363 |
$template->param(download_errors => 1, errors_filename => $tmpf->filename); |
| 364 |
close $tmpf; |
| 365 |
} |
| 366 |
|
| 367 |
if ($commandline) { |
| 368 |
# Write log file |
| 369 |
my $logfile = "/var/log/koha/reports/import_borrowers.log"; |
| 370 |
if (open (FH, ">>$logfile")) { |
| 371 |
print FH $output; |
| 372 |
close(FH); |
| 373 |
} else { |
| 374 |
$output .= "Unable to write to log file : $logfile\n"; |
| 375 |
} |
| 376 |
|
| 377 |
|
| 378 |
# Send email with log |
| 379 |
my $mail = MIME::Lite->new( |
| 380 |
To => C4::Context->preference('KohaAdminEmailAddress'), |
| 381 |
Subject => "Import borrowers log email", |
| 382 |
Type => 'text/plain', |
| 383 |
Data => $output |
| 384 |
); |
| 385 |
$mail->send() or print "Unable to send log email"; |
| 386 |
} |
| 387 |
} |
| 299 |
|
388 |
|
| 300 |
} else { |
389 |
} else { |
| 301 |
if ($extended) { |
390 |
if ($extended) { |
| 302 |
- |
|
|