Lines 1-7
Link Here
|
1 |
#!/usr/bin/perl |
1 |
#!/usr/bin/perl |
2 |
|
2 |
|
3 |
# Copyright 2007 Liblime |
3 |
# Copyright 2007 Liblime |
4 |
# Parts copyright 2010 BibLibre |
4 |
# Parts copyright 2013 BibLibre |
5 |
# |
5 |
# |
6 |
# This file is part of Koha. |
6 |
# This file is part of Koha. |
7 |
# |
7 |
# |
Lines 34-41
Link Here
|
34 |
# dates should be in the format you have set up Koha to expect |
34 |
# dates should be in the format you have set up Koha to expect |
35 |
# branchcode and categorycode need to be valid |
35 |
# branchcode and categorycode need to be valid |
36 |
|
36 |
|
37 |
use strict; |
37 |
use Modern::Perl; |
38 |
use warnings; |
|
|
39 |
|
38 |
|
40 |
use C4::Auth; |
39 |
use C4::Auth; |
41 |
use C4::Output; |
40 |
use C4::Output; |
Lines 46-53
use C4::Members;
Link Here
|
46 |
use C4::Members::Attributes qw(:all); |
45 |
use C4::Members::Attributes qw(:all); |
47 |
use C4::Members::AttributeTypes; |
46 |
use C4::Members::AttributeTypes; |
48 |
use C4::Members::Messaging; |
47 |
use C4::Members::Messaging; |
49 |
|
48 |
use Koha::DateUtils; |
|
|
49 |
use Date::Calc qw(Today_and_Now); |
50 |
use Getopt::Long; |
51 |
use File::Temp; |
50 |
use Text::CSV; |
52 |
use Text::CSV; |
|
|
53 |
|
51 |
# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals: |
54 |
# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals: |
52 |
# ė |
55 |
# ė |
53 |
# č |
56 |
# č |
Lines 75-84
my ( $template, $loggedinuser, $cookie ) = get_template_and_user({
Link Here
|
75 |
authnotrequired => 0, |
78 |
authnotrequired => 0, |
76 |
flagsrequired => { tools => 'import_patrons' }, |
79 |
flagsrequired => { tools => 'import_patrons' }, |
77 |
debug => 1, |
80 |
debug => 1, |
78 |
}); |
81 |
}); |
79 |
|
82 |
|
80 |
$template->param(columnkeys => $columnkeystpl); |
83 |
$template->param(columnkeys => $columnkeystpl); |
81 |
|
|
|
82 |
if ($input->param('sample')) { |
84 |
if ($input->param('sample')) { |
83 |
print $input->header( |
85 |
print $input->header( |
84 |
-type => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ? |
86 |
-type => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ? |
Lines 88-93
if ($input->param('sample')) {
Link Here
|
88 |
print $csv->string, "\n"; |
90 |
print $csv->string, "\n"; |
89 |
exit 1; |
91 |
exit 1; |
90 |
} |
92 |
} |
|
|
93 |
|
94 |
if ($input->param('report')) { |
95 |
print $input->header( |
96 |
-type => 'text/plain', |
97 |
-attachment => 'import_borrowers_report.txt' |
98 |
); |
99 |
my $filename = $input->param('errors_filename'); |
100 |
if ( -f $filename ) { |
101 |
open my $fh, '<', $filename; |
102 |
print <$fh>; |
103 |
close $fh; |
104 |
|
105 |
unlink $filename; |
106 |
exit 1; |
107 |
} |
108 |
} |
109 |
|
91 |
my $uploadborrowers = $input->param('uploadborrowers'); |
110 |
my $uploadborrowers = $input->param('uploadborrowers'); |
92 |
my $matchpoint = $input->param('matchpoint'); |
111 |
my $matchpoint = $input->param('matchpoint'); |
93 |
if ($matchpoint) { |
112 |
if ($matchpoint) { |
Lines 110-116
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
110 |
my $alreadyindb = 0; |
129 |
my $alreadyindb = 0; |
111 |
my $overwritten = 0; |
130 |
my $overwritten = 0; |
112 |
my $invalid = 0; |
131 |
my $invalid = 0; |
113 |
my $matchpoint_attr_type; |
132 |
my $matchpoint_attr_type; |
114 |
my %defaults = $input->Vars; |
133 |
my %defaults = $input->Vars; |
115 |
|
134 |
|
116 |
# use header line to construct key to column map |
135 |
# use header line to construct key to column map |
Lines 121-131
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
121 |
my %csvkeycol; |
140 |
my %csvkeycol; |
122 |
my $col = 0; |
141 |
my $col = 0; |
123 |
foreach my $keycol (@csvcolumns) { |
142 |
foreach my $keycol (@csvcolumns) { |
124 |
# columnkeys don't contain whitespace, but some stupid tools add it |
143 |
# columnkeys don't contain whitespace, but some stupid tools add it |
125 |
$keycol =~ s/ +//g; |
144 |
$keycol =~ s/ +//g; |
126 |
$csvkeycol{$keycol} = $col++; |
145 |
$csvkeycol{$keycol} = $col++; |
127 |
} |
146 |
} |
128 |
#warn($borrowerline); |
147 |
|
129 |
my $ext_preserve = $input->param('ext_preserve') || 0; |
148 |
my $ext_preserve = $input->param('ext_preserve') || 0; |
130 |
if ($extended) { |
149 |
if ($extended) { |
131 |
$matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint); |
150 |
$matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint); |
Lines 137-142
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
137 |
my @bad_dates; # I've had a few. |
156 |
my @bad_dates; # I've had a few. |
138 |
my $date_re = C4::Dates->new->regexp('syspref'); |
157 |
my $date_re = C4::Dates->new->regexp('syspref'); |
139 |
my $iso_re = C4::Dates->new->regexp('iso'); |
158 |
my $iso_re = C4::Dates->new->regexp('iso'); |
|
|
159 |
my $lastalreadyindb; |
160 |
my $lastinvalid; |
140 |
LINE: while ( my $borrowerline = <$handle> ) { |
161 |
LINE: while ( my $borrowerline = <$handle> ) { |
141 |
my %borrower; |
162 |
my %borrower; |
142 |
my @missing_criticals; |
163 |
my @missing_criticals; |
Lines 152-174
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
152 |
if ($borrower{$key} !~ /\S/) { |
173 |
if ($borrower{$key} !~ /\S/) { |
153 |
$borrower{$key} = $defaults{$key}; |
174 |
$borrower{$key} = $defaults{$key}; |
154 |
} |
175 |
} |
155 |
} |
176 |
} |
156 |
} else { |
177 |
} else { |
157 |
# MJR: try to recover gracefully by using default values |
178 |
# MJR: try to recover gracefully by using default values |
158 |
foreach my $key (@columnkeys) { |
179 |
foreach my $key (@columnkeys) { |
159 |
if (defined($csvkeycol{$key}) and $columns[$csvkeycol{$key}] =~ /\S/) { |
180 |
if (defined($csvkeycol{$key}) and $columns[$csvkeycol{$key}] =~ /\S/) { |
160 |
$borrower{$key} = $columns[$csvkeycol{$key}]; |
181 |
$borrower{$key} = $columns[$csvkeycol{$key}]; |
161 |
} elsif ( $defaults{$key} ) { |
182 |
} elsif ( $defaults{$key} ) { |
162 |
$borrower{$key} = $defaults{$key}; |
183 |
$borrower{$key} = $defaults{$key}; |
163 |
} elsif ( scalar grep {$key eq $_} @criticals ) { |
184 |
} elsif ( scalar grep {$key eq $_} @criticals ) { |
164 |
# a critical field is undefined |
185 |
# a critical field is undefined |
165 |
push @missing_criticals, {key=>$key, line=>$., lineraw=>$borrowerline}; |
186 |
push @missing_criticals, {key=>$key, line=>$., lineraw=>$borrowerline}; |
166 |
} else { |
187 |
} else { |
167 |
$borrower{$key} = ''; |
188 |
$borrower{$key} = ''; |
168 |
} |
189 |
} |
169 |
} |
190 |
} |
170 |
} |
191 |
} |
171 |
#warn join(':',%borrower); |
192 |
|
172 |
if ($borrower{categorycode}) { |
193 |
if ($borrower{categorycode}) { |
173 |
push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline, value=>$borrower{categorycode}, category_map=>1} |
194 |
push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline, value=>$borrower{categorycode}, category_map=>1} |
174 |
unless GetBorrowercategory($borrower{categorycode}); |
195 |
unless GetBorrowercategory($borrower{categorycode}); |
Lines 187-194
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
187 |
$_->{surname} = $borrower{surname} || 'UNDEF'; |
208 |
$_->{surname} = $borrower{surname} || 'UNDEF'; |
188 |
} |
209 |
} |
189 |
$invalid++; |
210 |
$invalid++; |
190 |
(25 > scalar @errors) and push @errors, {missing_criticals=>\@missing_criticals}; |
211 |
push @errors, {missing_criticals=>\@missing_criticals}; |
191 |
# The first 25 errors are enough. Keeping track of 30,000+ would destroy performance. |
|
|
192 |
next LINE; |
212 |
next LINE; |
193 |
} |
213 |
} |
194 |
if ($extended) { |
214 |
if ($extended) { |
Lines 197-205
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
197 |
$attr_str =~ s/\xe2\x80\x9d/"/g; |
217 |
$attr_str =~ s/\xe2\x80\x9d/"/g; |
198 |
push @feedback, {feedback=>1, name=>'attribute string', value=>$attr_str, filename=>$uploadborrowers}; |
218 |
push @feedback, {feedback=>1, name=>'attribute string', value=>$attr_str, filename=>$uploadborrowers}; |
199 |
delete $borrower{patron_attributes}; # not really a field in borrowers, so we don't want to pass it to ModMember. |
219 |
delete $borrower{patron_attributes}; # not really a field in borrowers, so we don't want to pass it to ModMember. |
200 |
$patron_attributes = extended_attributes_code_value_arrayref($attr_str); |
220 |
$patron_attributes = extended_attributes_code_value_arrayref($attr_str); |
201 |
} |
221 |
} |
202 |
# Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it. |
222 |
# Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it. |
203 |
foreach (qw(dateofbirth dateenrolled dateexpiry)) { |
223 |
foreach (qw(dateofbirth dateenrolled dateexpiry)) { |
204 |
my $tempdate = $borrower{$_} or next; |
224 |
my $tempdate = $borrower{$_} or next; |
205 |
if ($tempdate =~ /$date_re/) { |
225 |
if ($tempdate =~ /$date_re/) { |
Lines 211-218
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
211 |
push @missing_criticals, {key=>$_, line=>$. , lineraw=>$borrowerline, bad_date=>1}; |
231 |
push @missing_criticals, {key=>$_, line=>$. , lineraw=>$borrowerline, bad_date=>1}; |
212 |
} |
232 |
} |
213 |
} |
233 |
} |
214 |
$borrower{dateenrolled} = $today_iso unless $borrower{dateenrolled}; |
234 |
$borrower{dateenrolled} = $today_iso unless $borrower{dateenrolled}; |
215 |
$borrower{dateexpiry} = GetExpiryDate($borrower{categorycode},$borrower{dateenrolled}) unless $borrower{dateexpiry}; |
235 |
$borrower{dateexpiry} = GetExpiryDate($borrower{categorycode},$borrower{dateenrolled}) unless $borrower{dateexpiry}; |
216 |
my $borrowernumber; |
236 |
my $borrowernumber; |
217 |
my $member; |
237 |
my $member; |
218 |
if ( ($matchpoint eq 'cardnumber') && ($borrower{'cardnumber'}) ) { |
238 |
if ( ($matchpoint eq 'cardnumber') && ($borrower{'cardnumber'}) ) { |
Lines 242-253
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
242 |
next; |
262 |
next; |
243 |
} |
263 |
} |
244 |
|
264 |
|
245 |
|
|
|
246 |
if ($borrowernumber) { |
265 |
if ($borrowernumber) { |
247 |
# borrower exists |
266 |
# borrower exists |
248 |
unless ($overwrite_cardnumber) { |
267 |
unless ($overwrite_cardnumber) { |
249 |
$alreadyindb++; |
268 |
$alreadyindb++; |
250 |
$template->param('lastalreadyindb'=>$borrower{'surname'}.' / '.$borrowernumber); |
269 |
$lastalreadyindb = $borrower{'surname'}.' / '.$borrowernumber; |
|
|
270 |
$template->param('lastalreadyindb'=>$lastalreadyindb); |
251 |
next LINE; |
271 |
next LINE; |
252 |
} |
272 |
} |
253 |
$borrower{'borrowernumber'} = $borrowernumber; |
273 |
$borrower{'borrowernumber'} = $borrowernumber; |
Lines 266-272
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
266 |
$invalid++; |
286 |
$invalid++; |
267 |
# untill we have better error trapping, we have no way of knowing why ModMember errored out... |
287 |
# untill we have better error trapping, we have no way of knowing why ModMember errored out... |
268 |
push @errors, {unknown_error => 1}; |
288 |
push @errors, {unknown_error => 1}; |
269 |
$template->param('lastinvalid'=>$borrower{'surname'}.' / '.$borrowernumber); |
289 |
$lastinvalid = $borrower{'surname'}.' / '.$borrowernumber; |
|
|
290 |
$template->param('lastinvalid'=>$lastinvalid); |
270 |
next LINE; |
291 |
next LINE; |
271 |
} |
292 |
} |
272 |
if ($extended) { |
293 |
if ($extended) { |
Lines 312-317
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
312 |
'total' => $imported + $alreadyindb + $invalid + $overwritten, |
333 |
'total' => $imported + $alreadyindb + $invalid + $overwritten, |
313 |
); |
334 |
); |
314 |
|
335 |
|
|
|
336 |
my $max_errors = 25; |
337 |
if (scalar(@errors) > $max_errors ) { |
338 |
|
339 |
my $total = $imported + $alreadyindb + $invalid + $overwritten; |
340 |
my $output; |
341 |
|
342 |
my $timestamp = output_pref( dt_from_string ); |
343 |
$output .= "Timestamp : $timestamp\n"; |
344 |
$output .= "Import results\n"; |
345 |
$output .= "$imported imported records\n"; |
346 |
$output .= "$overwritten overwritten records\n"; |
347 |
$output .= "$alreadyindb not imported because already in borrowers table and overwrite disabled\n"; |
348 |
$output .= "(last was $lastalreadyindb)\n" if ($lastalreadyindb); |
349 |
$output .= "$invalid not imported because they are not in the expected format\n"; |
350 |
$output .= "(last was $lastinvalid)\n" if ($lastinvalid); |
351 |
$output .= "$total records parsed\n"; |
352 |
|
353 |
|
354 |
$output .= "\nError analysis\n"; |
355 |
foreach my $error (@errors) { |
356 |
$output .= "Header row could not be parsed" if ($error->{'badheader'}); |
357 |
foreach my $array ($error->{'missing_criticals'}) { |
358 |
foreach (@$array) { |
359 |
$output .= "Line $_->{'line'}: "; |
360 |
if ($error->{'badparse'}) { |
361 |
$output .= "could not be parsed!"; |
362 |
} elsif ($error->{'bad_date'}) { |
363 |
$output .= "has $_->{'key'} in unrecognized format: $_->{'value'} "; |
364 |
} else { |
365 |
$output .= "Critical field $_->{'key'}: "; |
366 |
if ($_->{'branch_map'} || $_->{'category_map'}) { |
367 |
$output .= "has unrecognized value: $_->{'value'}"; |
368 |
} else { |
369 |
$output .= " missing"; |
370 |
} |
371 |
$output .= " (borrowernumber: $_->{'borrowernumber'}; surname: $_->{'surname'})"; |
372 |
} |
373 |
$output .= "\n"; |
374 |
} |
375 |
} |
376 |
} |
377 |
|
378 |
my $tmpf = File::Temp->new(UNLINK => 0); |
379 |
print $tmpf $output; |
380 |
$template->param( |
381 |
download_errors => 1, |
382 |
errors_filename => $tmpf->filename |
383 |
); |
384 |
close $tmpf; |
385 |
|
386 |
} |
387 |
|
315 |
} else { |
388 |
} else { |
316 |
if ($extended) { |
389 |
if ($extended) { |
317 |
my @matchpoints = (); |
390 |
my @matchpoints = (); |
Lines 327-330
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
327 |
} |
400 |
} |
328 |
|
401 |
|
329 |
output_html_with_http_headers $input, $cookie, $template->output; |
402 |
output_html_with_http_headers $input, $cookie, $template->output; |
330 |
|
|
|
331 |
- |