View | Details | Raw Unified | Return to bug 5771
Collapse All | Expand All

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tmpl (+8 lines)
Lines 51-56 Link Here
51
  <br /><br />
51
  <br /><br />
52
    <div>
52
    <div>
53
    <h5>Error analysis:</h5>
53
    <h5>Error analysis:</h5>
54
    <!-- TMPL_IF NAME="download_errors" -->
55
    <form method="post">
56
	<input type="hidden" name="errors_filename" value="<!-- TMPL_VAR NAME="errors_filename" -->" />
57
	<input type="hidden" name="report" value="report" />
58
	<input type="submit" value="Download report" />
59
    </form>
60
    <!-- TMPL_ELSE -->
54
    <ul>
61
    <ul>
55
    <!-- TMPL_LOOP NAME="ERRORS" -->
62
    <!-- TMPL_LOOP NAME="ERRORS" -->
56
        <!-- TMPL_IF NAME="badheader" --><li>Header row could not be parsed</li><!-- /TMPL_IF -->
63
        <!-- TMPL_IF NAME="badheader" --><li>Header row could not be parsed</li><!-- /TMPL_IF -->
Lines 74-79 Link Here
74
        <!-- /TMPL_LOOP -->
81
        <!-- /TMPL_LOOP -->
75
    <!-- /TMPL_LOOP -->
82
    <!-- /TMPL_LOOP -->
76
    </ul>
83
    </ul>
84
    <!-- /TMPL_IF -->
77
    </div>
85
    </div>
78
  <!-- /TMPL_IF -->
86
  <!-- /TMPL_IF -->
79
<!-- TMPL_ELSE -->
87
<!-- TMPL_ELSE -->
(-)a/tools/import_borrowers.pl (-21 / +109 lines)
Lines 45-51 use C4::Members; Link Here
45
use C4::Members::Attributes qw(:all);
45
use C4::Members::Attributes qw(:all);
46
use C4::Members::AttributeTypes;
46
use C4::Members::AttributeTypes;
47
use C4::Members::Messaging;
47
use C4::Members::Messaging;
48
48
use Date::Calc qw(Today_and_Now);
49
use Getopt::Long;
50
use File::Temp;
49
use Text::CSV;
51
use Text::CSV;
50
# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals:
52
# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals:
51
# ė
53
# ė
Lines 68-91 our $csv = Text::CSV->new({binary => 1}); # binary needed for non-ASCII Unicod Link Here
68
# push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend};
70
# push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend};
69
71
70
my ( $template, $loggedinuser, $cookie ) = get_template_and_user({
72
my ( $template, $loggedinuser, $cookie ) = get_template_and_user({
71
        template_name   => "tools/import_borrowers.tmpl",
73
	    template_name   => "tools/import_borrowers.tmpl",
72
        query           => $input,
74
	    query           => $input,
73
        type            => "intranet",
75
	    type            => "intranet",
74
        authnotrequired => 0,
76
	    authnotrequired => $commandline,
75
        flagsrequired   => { tools => 'import_patrons' },
77
	    flagsrequired   => { tools => 'import_patrons' },
76
        debug           => 1,
78
	    debug           => 1,
77
});
79
    });
80
81
if (!$commandline) {
82
    $template->param(columnkeys => $columnkeystpl);
83
    $template->param( SCRIPT_NAME => $ENV{'SCRIPT_NAME'} );
84
    ($extended) and $template->param(ExtendedPatronAttributes => 1);
78
85
79
$template->param(columnkeys => $columnkeystpl);
86
    if ($input->param('sample')) {
87
	print $input->header(
88
	    -type       => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ?
89
	    -attachment => 'patron_import.csv',
90
	);
91
	$csv->combine(@columnkeys);
92
	print $csv->string, "\n";
93
	exit 1;
94
    }
80
95
81
if ($input->param('sample')) {
96
    if ($input->param('report')) {
82
    print $input->header(
97
	open (FH, $input->param('errors_filename'));
83
        -type       => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ?
98
	print $input->header(
84
        -attachment => 'patron_import.csv',
99
	    -type => 'text/plain',
85
    );
100
	    -attachment => 'import_borrowers_report.txt'
86
    $csv->combine(@columnkeys);
101
	);
87
    print $csv->string, "\n";
102
	print <FH>;
88
    exit 1;
103
	close FH;
104
	#TODO : We surely want to check that is it really a temp file that we are unlinking
105
	unlink $input->param('errors_filename');
106
	exit 1;
107
    }
89
}
108
}
90
my $uploadborrowers = $input->param('uploadborrowers');
109
my $uploadborrowers = $input->param('uploadborrowers');
91
my $matchpoint      = $input->param('matchpoint');
110
my $matchpoint      = $input->param('matchpoint');
Lines 186-193 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
186
                $_->{surname}        = $borrower{surname} || 'UNDEF';
205
                $_->{surname}        = $borrower{surname} || 'UNDEF';
187
            }
206
            }
188
            $invalid++;
207
            $invalid++;
189
            (25 > scalar @errors) and push @errors, {missing_criticals=>\@missing_criticals};
208
            push @errors, {missing_criticals=>\@missing_criticals};
190
            # The first 25 errors are enough.  Keeping track of 30,000+ would destroy performance.
191
            next LINE;
209
            next LINE;
192
        }
210
        }
193
        if ($extended) {
211
        if ($extended) {
Lines 288-294 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
288
        'alreadyindb'     => $alreadyindb,
306
        'alreadyindb'     => $alreadyindb,
289
        'invalid'         => $invalid,
307
        'invalid'         => $invalid,
290
        'total'           => $imported + $alreadyindb + $invalid + $overwritten,
308
        'total'           => $imported + $alreadyindb + $invalid + $overwritten,
291
    );
309
    ) if (!$commandline);
310
311
    if (scalar(@errors) > 25 or $commandline) {
312
313
	my $total = $imported + $alreadyindb + $invalid + $overwritten;
314
	my $output;
315
316
	my $timestamp = C4::Dates->new()->output . " " . POSIX::strftime("%H:%M:%S",localtime);
317
	$output .= "Timestamp : $timestamp\n";
318
	$output .= "Import results\n";
319
	$output .= "$imported imported records\n";
320
	$output .= "$overwritten overwritten records\n";
321
	$output .= "$alreadyindb not imported because already in borrowers table and overwrite disabled\n"; 
322
	$output .= "(last was $lastalreadyindb)\n" if ($lastalreadyindb);
323
	$output .= "$invalid not imported because they are not in the expected format\n"; 
324
	$output .= "(last was $lastinvalid)\n" if ($lastinvalid);
325
	$output .= "$total records parsed\n";
326
327
328
	$output .= "\nError analysis\n";
329
	foreach my $hash (@errors) {
330
	   $output .= "Header row could not be parsed" if ($hash->{'badheader'});
331
	   foreach my $array ($hash->{'missing_criticals'}) {
332
	       foreach (@$array) {
333
		    $output .= "Line $_->{'line'}: ";
334
		    if ($hash->{'badparse'}) {
335
			$output .= "could not be parsed!";
336
		    } elsif ($hash->{'bad_date'}) { 
337
			$output .= "has $_->{'key'} in unrecognized format: $_->{'value'} ";
338
		    } else {
339
			$output .= "Critical field $_->{'key'}: ";
340
			if ($_->{'branch_map'} || $_->{'category_map'}) {
341
			    $output .= "has unrecognized value: $_->{'value'}";
342
			} else {
343
			    $output .= " missing";
344
			}
345
			$output .= " (borrowernumber: $_->{'borrowernumber'}; surname: $_->{'surname'})";
346
		    }
347
		    $output .= "\n";
348
		    $output .= $_->{'lineraw'} . "\n" if ($commandline);
349
		}
350
	   }
351
	}
352
353
    if (scalar(@errors) > 25 && !$commandline) {
354
	my $tmpf = File::Temp->new(UNLINK => 0);
355
	print $tmpf $output;
356
	$template->param(download_errors => 1, errors_filename => $tmpf->filename);
357
	close $tmpf;
358
    }
359
360
    if ($commandline) {
361
	# Write log file
362
	my $logfile = "/var/log/koha/reports/import_borrowers.log";
363
	if (open (FH, ">>$logfile")) {
364
	    print FH $output;
365
	    close(FH);
366
	} else {
367
	    $output .= "Unable to write to log file : $logfile\n";
368
	}
369
370
371
	# Send email with log
372
	 my $mail = MIME::Lite->new(
373
		    To      => C4::Context->preference('KohaAdminEmailAddress'),
374
		    Subject => "Import borrowers log email",
375
		    Type    => 'text/plain',
376
		    Data    => $output
377
		);
378
	$mail->send() or print "Unable to send log email";
379
    }
380
   }
292
381
293
} else {
382
} else {
294
    if ($extended) {
383
    if ($extended) {
295
- 

Return to bug 5771