|
Lines 45-51
use C4::Members;
Link Here
|
| 45 |
use C4::Members::Attributes qw(:all); |
45 |
use C4::Members::Attributes qw(:all); |
| 46 |
use C4::Members::AttributeTypes; |
46 |
use C4::Members::AttributeTypes; |
| 47 |
use C4::Members::Messaging; |
47 |
use C4::Members::Messaging; |
| 48 |
|
48 |
use Date::Calc qw(Today_and_Now); |
|
|
49 |
use Getopt::Long; |
| 50 |
use File::Temp; |
| 49 |
use Text::CSV; |
51 |
use Text::CSV; |
| 50 |
# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals: |
52 |
# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals: |
| 51 |
# ė |
53 |
# ė |
|
Lines 68-91
our $csv = Text::CSV->new({binary => 1}); # binary needed for non-ASCII Unicod
Link Here
|
| 68 |
# push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend}; |
70 |
# push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend}; |
| 69 |
|
71 |
|
| 70 |
my ( $template, $loggedinuser, $cookie ) = get_template_and_user({ |
72 |
my ( $template, $loggedinuser, $cookie ) = get_template_and_user({ |
| 71 |
template_name => "tools/import_borrowers.tmpl", |
73 |
template_name => "tools/import_borrowers.tmpl", |
| 72 |
query => $input, |
74 |
query => $input, |
| 73 |
type => "intranet", |
75 |
type => "intranet", |
| 74 |
authnotrequired => 0, |
76 |
authnotrequired => $commandline, |
| 75 |
flagsrequired => { tools => 'import_patrons' }, |
77 |
flagsrequired => { tools => 'import_patrons' }, |
| 76 |
debug => 1, |
78 |
debug => 1, |
| 77 |
}); |
79 |
}); |
|
|
80 |
|
| 81 |
if (!$commandline) { |
| 82 |
$template->param(columnkeys => $columnkeystpl); |
| 83 |
$template->param( SCRIPT_NAME => $ENV{'SCRIPT_NAME'} ); |
| 84 |
($extended) and $template->param(ExtendedPatronAttributes => 1); |
| 78 |
|
85 |
|
| 79 |
$template->param(columnkeys => $columnkeystpl); |
86 |
if ($input->param('sample')) { |
|
|
87 |
print $input->header( |
| 88 |
-type => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ? |
| 89 |
-attachment => 'patron_import.csv', |
| 90 |
); |
| 91 |
$csv->combine(@columnkeys); |
| 92 |
print $csv->string, "\n"; |
| 93 |
exit 1; |
| 94 |
} |
| 80 |
|
95 |
|
| 81 |
if ($input->param('sample')) { |
96 |
if ($input->param('report')) { |
| 82 |
print $input->header( |
97 |
open (FH, $input->param('errors_filename')); |
| 83 |
-type => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ? |
98 |
print $input->header( |
| 84 |
-attachment => 'patron_import.csv', |
99 |
-type => 'text/plain', |
| 85 |
); |
100 |
-attachment => 'import_borrowers_report.txt' |
| 86 |
$csv->combine(@columnkeys); |
101 |
); |
| 87 |
print $csv->string, "\n"; |
102 |
print <FH>; |
| 88 |
exit 1; |
103 |
close FH; |
|
|
104 |
#TODO : We surely want to check that is it really a temp file that we are unlinking |
| 105 |
unlink $input->param('errors_filename'); |
| 106 |
exit 1; |
| 107 |
} |
| 89 |
} |
108 |
} |
| 90 |
my $uploadborrowers = $input->param('uploadborrowers'); |
109 |
my $uploadborrowers = $input->param('uploadborrowers'); |
| 91 |
my $matchpoint = $input->param('matchpoint'); |
110 |
my $matchpoint = $input->param('matchpoint'); |
|
Lines 186-193
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
| 186 |
$_->{surname} = $borrower{surname} || 'UNDEF'; |
205 |
$_->{surname} = $borrower{surname} || 'UNDEF'; |
| 187 |
} |
206 |
} |
| 188 |
$invalid++; |
207 |
$invalid++; |
| 189 |
(25 > scalar @errors) and push @errors, {missing_criticals=>\@missing_criticals}; |
208 |
push @errors, {missing_criticals=>\@missing_criticals}; |
| 190 |
# The first 25 errors are enough. Keeping track of 30,000+ would destroy performance. |
|
|
| 191 |
next LINE; |
209 |
next LINE; |
| 192 |
} |
210 |
} |
| 193 |
if ($extended) { |
211 |
if ($extended) { |
|
Lines 288-294
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
| 288 |
'alreadyindb' => $alreadyindb, |
306 |
'alreadyindb' => $alreadyindb, |
| 289 |
'invalid' => $invalid, |
307 |
'invalid' => $invalid, |
| 290 |
'total' => $imported + $alreadyindb + $invalid + $overwritten, |
308 |
'total' => $imported + $alreadyindb + $invalid + $overwritten, |
| 291 |
); |
309 |
) if (!$commandline); |
|
|
310 |
|
| 311 |
if (scalar(@errors) > 25 or $commandline) { |
| 312 |
|
| 313 |
my $total = $imported + $alreadyindb + $invalid + $overwritten; |
| 314 |
my $output; |
| 315 |
|
| 316 |
my $timestamp = C4::Dates->new()->output . " " . POSIX::strftime("%H:%M:%S",localtime); |
| 317 |
$output .= "Timestamp : $timestamp\n"; |
| 318 |
$output .= "Import results\n"; |
| 319 |
$output .= "$imported imported records\n"; |
| 320 |
$output .= "$overwritten overwritten records\n"; |
| 321 |
$output .= "$alreadyindb not imported because already in borrowers table and overwrite disabled\n"; |
| 322 |
$output .= "(last was $lastalreadyindb)\n" if ($lastalreadyindb); |
| 323 |
$output .= "$invalid not imported because they are not in the expected format\n"; |
| 324 |
$output .= "(last was $lastinvalid)\n" if ($lastinvalid); |
| 325 |
$output .= "$total records parsed\n"; |
| 326 |
|
| 327 |
|
| 328 |
$output .= "\nError analysis\n"; |
| 329 |
foreach my $hash (@errors) { |
| 330 |
$output .= "Header row could not be parsed" if ($hash->{'badheader'}); |
| 331 |
foreach my $array ($hash->{'missing_criticals'}) { |
| 332 |
foreach (@$array) { |
| 333 |
$output .= "Line $_->{'line'}: "; |
| 334 |
if ($hash->{'badparse'}) { |
| 335 |
$output .= "could not be parsed!"; |
| 336 |
} elsif ($hash->{'bad_date'}) { |
| 337 |
$output .= "has $_->{'key'} in unrecognized format: $_->{'value'} "; |
| 338 |
} else { |
| 339 |
$output .= "Critical field $_->{'key'}: "; |
| 340 |
if ($_->{'branch_map'} || $_->{'category_map'}) { |
| 341 |
$output .= "has unrecognized value: $_->{'value'}"; |
| 342 |
} else { |
| 343 |
$output .= " missing"; |
| 344 |
} |
| 345 |
$output .= " (borrowernumber: $_->{'borrowernumber'}; surname: $_->{'surname'})"; |
| 346 |
} |
| 347 |
$output .= "\n"; |
| 348 |
$output .= $_->{'lineraw'} . "\n" if ($commandline); |
| 349 |
} |
| 350 |
} |
| 351 |
} |
| 352 |
|
| 353 |
if (scalar(@errors) > 25 && !$commandline) { |
| 354 |
my $tmpf = File::Temp->new(UNLINK => 0); |
| 355 |
print $tmpf $output; |
| 356 |
$template->param(download_errors => 1, errors_filename => $tmpf->filename); |
| 357 |
close $tmpf; |
| 358 |
} |
| 359 |
|
| 360 |
if ($commandline) { |
| 361 |
# Write log file |
| 362 |
my $logfile = "/var/log/koha/reports/import_borrowers.log"; |
| 363 |
if (open (FH, ">>$logfile")) { |
| 364 |
print FH $output; |
| 365 |
close(FH); |
| 366 |
} else { |
| 367 |
$output .= "Unable to write to log file : $logfile\n"; |
| 368 |
} |
| 369 |
|
| 370 |
|
| 371 |
# Send email with log |
| 372 |
my $mail = MIME::Lite->new( |
| 373 |
To => C4::Context->preference('KohaAdminEmailAddress'), |
| 374 |
Subject => "Import borrowers log email", |
| 375 |
Type => 'text/plain', |
| 376 |
Data => $output |
| 377 |
); |
| 378 |
$mail->send() or print "Unable to send log email"; |
| 379 |
} |
| 380 |
} |
| 292 |
|
381 |
|
| 293 |
} else { |
382 |
} else { |
| 294 |
if ($extended) { |
383 |
if ($extended) { |
| 295 |
- |
|
|