Lines 2-7
Link Here
|
2 |
|
2 |
|
3 |
# Copyright 2007 Liblime |
3 |
# Copyright 2007 Liblime |
4 |
# Parts copyright 2010 BibLibre |
4 |
# Parts copyright 2010 BibLibre |
|
|
5 |
# Parts copyright 2014 ByWater Solutions |
5 |
# |
6 |
# |
6 |
# This file is part of Koha. |
7 |
# This file is part of Koha. |
7 |
# |
8 |
# |
Lines 34-41
Link Here
|
34 |
# dates should be in the format you have set up Koha to expect |
35 |
# dates should be in the format you have set up Koha to expect |
35 |
# branchcode and categorycode need to be valid |
36 |
# branchcode and categorycode need to be valid |
36 |
|
37 |
|
37 |
use strict; |
38 |
use Modern::Perl; |
38 |
use warnings; |
|
|
39 |
|
39 |
|
40 |
use C4::Auth; |
40 |
use C4::Auth; |
41 |
use C4::Output; |
41 |
use C4::Output; |
Lines 48-60
use C4::Members::AttributeTypes;
Link Here
|
48 |
use C4::Members::Messaging; |
48 |
use C4::Members::Messaging; |
49 |
use Koha::Borrower::Debarments; |
49 |
use Koha::Borrower::Debarments; |
50 |
|
50 |
|
|
|
51 |
use CGI; |
52 |
use Getopt::Long; |
51 |
use Text::CSV; |
53 |
use Text::CSV; |
52 |
# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals: |
54 |
# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals: |
53 |
# ė |
55 |
# ė |
54 |
# č |
56 |
# č |
55 |
|
57 |
|
56 |
use CGI; |
58 |
# Checks if the script is called from commandline |
57 |
# use encoding 'utf8'; # don't do this |
59 |
my $is_cli = not defined $ENV{GATEWAY_INTERFACE}; |
58 |
|
60 |
|
59 |
my (@errors, @feedback); |
61 |
my (@errors, @feedback); |
60 |
my $extended = C4::Context->preference('ExtendedPatronAttributes'); |
62 |
my $extended = C4::Context->preference('ExtendedPatronAttributes'); |
Lines 67-123
my $columnkeystpl = [ map { {'key' => $_} } grep {$_ ne 'borrowernumber' } @col
Link Here
|
67 |
|
69 |
|
68 |
my $input = CGI->new(); |
70 |
my $input = CGI->new(); |
69 |
our $csv = Text::CSV->new({binary => 1}); # binary needed for non-ASCII Unicode |
71 |
our $csv = Text::CSV->new({binary => 1}); # binary needed for non-ASCII Unicode |
70 |
#push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend}; #XXX |
72 |
|
71 |
|
73 |
my ( $template, $loggedinuser, $cookie ); |
72 |
my ( $template, $loggedinuser, $cookie ) = get_template_and_user({ |
74 |
unless ($is_cli) { |
73 |
template_name => "tools/import_borrowers.tmpl", |
75 |
( $template, $loggedinuser, $cookie ) = get_template_and_user( |
74 |
query => $input, |
76 |
{ |
75 |
type => "intranet", |
77 |
template_name => "tools/import_borrowers.tmpl", |
76 |
authnotrequired => 0, |
78 |
query => $input, |
77 |
flagsrequired => { tools => 'import_patrons' }, |
79 |
type => "intranet", |
78 |
debug => 1, |
80 |
authnotrequired => 0, |
79 |
}); |
81 |
flagsrequired => { tools => 'import_patrons' }, |
80 |
|
82 |
debug => 1, |
81 |
$template->param(columnkeys => $columnkeystpl); |
83 |
} |
82 |
|
84 |
); |
83 |
if ($input->param('sample')) { |
85 |
|
84 |
print $input->header( |
86 |
$template->param( columnkeys => $columnkeystpl ); |
85 |
-type => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ? |
87 |
|
86 |
-attachment => 'patron_import.csv', |
88 |
if ( $input->param('sample') ) { |
|
|
89 |
print $input->header( |
90 |
-type => |
91 |
'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ? |
92 |
-attachment => 'patron_import.csv', |
93 |
); |
94 |
$csv->combine(@columnkeys); |
95 |
print $csv->string, "\n"; |
96 |
exit 1; |
97 |
} |
98 |
|
99 |
$template->param( |
100 |
SCRIPT_NAME => $ENV{'SCRIPT_NAME'}, |
101 |
ExtendedPatronAttributes => $extended, |
87 |
); |
102 |
); |
88 |
$csv->combine(@columnkeys); |
|
|
89 |
print $csv->string, "\n"; |
90 |
exit 1; |
91 |
} |
92 |
my $uploadborrowers = $input->param('uploadborrowers'); |
93 |
my $matchpoint = $input->param('matchpoint'); |
94 |
if ($matchpoint) { |
95 |
$matchpoint =~ s/^patron_attribute_//; |
96 |
} |
103 |
} |
97 |
my $overwrite_cardnumber = $input->param('overwrite_cardnumber'); |
|
|
98 |
|
104 |
|
99 |
$template->param( SCRIPT_NAME => $ENV{'SCRIPT_NAME'} ); |
105 |
my $uploadborrowers; |
|
|
106 |
my $matchpoint; |
107 |
my $overwrite_cardnumber; |
108 |
my %defaults; |
109 |
my $ext_preserve = 0; |
110 |
my $verbose; |
111 |
|
112 |
if ($is_cli) { |
113 |
my $help; |
114 |
GetOptions( |
115 |
'c|csv=s' => \$uploadborrowers, |
116 |
'm|matchpoint=s' => \$matchpoint, |
117 |
'd|default=s' => \%defaults, |
118 |
'o|overwrite' => \$overwrite_cardnumber, |
119 |
'p|preserve-extended-atributes' => \$ext_preserve, |
120 |
'v|verbose' => \$verbose, |
121 |
'h|help|?' => \$help, |
122 |
); |
100 |
|
123 |
|
101 |
($extended) and $template->param(ExtendedPatronAttributes => 1); |
124 |
print_help() if ( $help || !$uploadborrowers || !$matchpoint ); |
|
|
125 |
} |
126 |
else { |
127 |
$uploadborrowers = $input->param('uploadborrowers'); |
128 |
$matchpoint = $input->param('matchpoint'); |
129 |
if ($matchpoint) { |
130 |
$matchpoint =~ s/^patron_attribute_//; |
131 |
} |
132 |
$overwrite_cardnumber = $input->param('overwrite_cardnumber'); |
133 |
%defaults = $input->Vars; |
134 |
$ext_preserve = $input->param('ext_preserve') || 0; |
135 |
} |
102 |
|
136 |
|
103 |
if ( $uploadborrowers && length($uploadborrowers) > 0 ) { |
137 |
if ( $uploadborrowers && length($uploadborrowers) > 0 ) { |
104 |
push @feedback, {feedback=>1, name=>'filename', value=>$uploadborrowers, filename=>$uploadborrowers}; |
138 |
push @feedback, {feedback=>1, name=>'filename', value=>$uploadborrowers, filename=>$uploadborrowers}; |
105 |
my $handle = $input->upload('uploadborrowers'); |
139 |
|
106 |
my $uploadinfo = $input->uploadInfo($uploadborrowers); |
140 |
my $handle; |
107 |
foreach (keys %$uploadinfo) { |
141 |
if ($is_cli) { |
108 |
push @feedback, {feedback=>1, name=>$_, value=>$uploadinfo->{$_}, $_=>$uploadinfo->{$_}}; |
142 |
open $handle, $uploadborrowers or die $!; |
|
|
143 |
} |
144 |
else { |
145 |
$handle = $uploadborrowers; |
146 |
|
147 |
my $uploadinfo = $input->uploadInfo($uploadborrowers); |
148 |
foreach ( keys %$uploadinfo ) { |
149 |
push @feedback, |
150 |
{ |
151 |
feedback => 1, |
152 |
name => $_, |
153 |
value => $uploadinfo->{$_}, |
154 |
$_ => $uploadinfo->{$_} |
155 |
}; |
156 |
} |
109 |
} |
157 |
} |
|
|
158 |
|
110 |
my $imported = 0; |
159 |
my $imported = 0; |
111 |
my $alreadyindb = 0; |
160 |
my $alreadyindb = 0; |
112 |
my $overwritten = 0; |
161 |
my $overwritten = 0; |
113 |
my $invalid = 0; |
162 |
my $invalid = 0; |
114 |
my $matchpoint_attr_type; |
163 |
my $matchpoint_attr_type; |
115 |
my %defaults = $input->Vars; |
|
|
116 |
|
164 |
|
117 |
# use header line to construct key to column map |
165 |
# use header line to construct key to column map |
118 |
my $borrowerline = <$handle>; |
166 |
my $borrowerline = <$handle>; |
119 |
my $status = $csv->parse($borrowerline); |
167 |
my $status = $csv->parse($borrowerline); |
120 |
($status) or push @errors, {badheader=>1,line=>$., lineraw=>$borrowerline}; |
168 |
($status) or push @errors, {badheader=>1,line=>$., lineraw=>$borrowerline}; |
|
|
169 |
print "WARNING: CSV header line is incomplete: $borrowerline\n" if $is_cli; |
121 |
my @csvcolumns = $csv->fields(); |
170 |
my @csvcolumns = $csv->fields(); |
122 |
my %csvkeycol; |
171 |
my %csvkeycol; |
123 |
my $col = 0; |
172 |
my $col = 0; |
Lines 126-133
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
126 |
$keycol =~ s/ +//g; |
175 |
$keycol =~ s/ +//g; |
127 |
$csvkeycol{$keycol} = $col++; |
176 |
$csvkeycol{$keycol} = $col++; |
128 |
} |
177 |
} |
129 |
#warn($borrowerline); |
|
|
130 |
my $ext_preserve = $input->param('ext_preserve') || 0; |
131 |
if ($extended) { |
178 |
if ($extended) { |
132 |
$matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint); |
179 |
$matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint); |
133 |
} |
180 |
} |
Lines 146-151
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
146 |
my @columns = $csv->fields(); |
193 |
my @columns = $csv->fields(); |
147 |
if (! $status) { |
194 |
if (! $status) { |
148 |
push @missing_criticals, {badparse=>1, line=>$., lineraw=>$borrowerline}; |
195 |
push @missing_criticals, {badparse=>1, line=>$., lineraw=>$borrowerline}; |
|
|
196 |
print "ERROR: Unable to parse line $.: $borrowerline" if $is_cli; |
149 |
} elsif (@columns == @columnkeys) { |
197 |
} elsif (@columns == @columnkeys) { |
150 |
@borrower{@columnkeys} = @columns; |
198 |
@borrower{@columnkeys} = @columns; |
151 |
# MJR: try to fill blanks gracefully by using default values |
199 |
# MJR: try to fill blanks gracefully by using default values |
Lines 164-169
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
164 |
} elsif ( scalar grep {$key eq $_} @criticals ) { |
212 |
} elsif ( scalar grep {$key eq $_} @criticals ) { |
165 |
# a critical field is undefined |
213 |
# a critical field is undefined |
166 |
push @missing_criticals, {key=>$key, line=>$., lineraw=>$borrowerline}; |
214 |
push @missing_criticals, {key=>$key, line=>$., lineraw=>$borrowerline}; |
|
|
215 |
print "ERROR: missing critical column data '$key' for line $.: $borrowerline\n" if $is_cli; |
167 |
} else { |
216 |
} else { |
168 |
$borrower{$key} = ''; |
217 |
$borrower{$key} = ''; |
169 |
} |
218 |
} |
Lines 171-186
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
171 |
} |
220 |
} |
172 |
#warn join(':',%borrower); |
221 |
#warn join(':',%borrower); |
173 |
if ($borrower{categorycode}) { |
222 |
if ($borrower{categorycode}) { |
174 |
push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline, value=>$borrower{categorycode}, category_map=>1} |
223 |
unless ( GetBorrowercategory( $borrower{categorycode} ) ) { |
175 |
unless GetBorrowercategory($borrower{categorycode}); |
224 |
push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline, value=>$borrower{categorycode}, category_map=>1}; |
|
|
225 |
print "ERROR: invalid categorycode for line $.: $borrowerline\n" if $is_cli; |
226 |
} |
176 |
} else { |
227 |
} else { |
177 |
push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline}; |
228 |
push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline}; |
|
|
229 |
print "ERROR: missing categorycode for line $.: $borrowerline\n" if $is_cli; |
178 |
} |
230 |
} |
179 |
if ($borrower{branchcode}) { |
231 |
if ($borrower{branchcode}) { |
180 |
push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline, value=>$borrower{branchcode}, branch_map=>1} |
232 |
unless ( GetBranchName( $borrower{branchcode} ) ) { |
181 |
unless GetBranchName($borrower{branchcode}); |
233 |
push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline, value=>$borrower{branchcode}, branch_map=>1}; |
|
|
234 |
print "ERROR: invalid branchcode for line $.: $borrowerline\n" if $is_cli; |
235 |
} |
182 |
} else { |
236 |
} else { |
183 |
push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline}; |
237 |
push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline}; |
|
|
238 |
print "ERROR: missing branchcode for line $.: $borrowerline\n" if $is_cli; |
184 |
} |
239 |
} |
185 |
if (@missing_criticals) { |
240 |
if (@missing_criticals) { |
186 |
foreach (@missing_criticals) { |
241 |
foreach (@missing_criticals) { |
Lines 234-244
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
234 |
} |
289 |
} |
235 |
|
290 |
|
236 |
if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) { |
291 |
if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) { |
|
|
292 |
$borrowernumber ||= q{}; |
237 |
push @errors, { |
293 |
push @errors, { |
238 |
invalid_cardnumber => 1, |
294 |
invalid_cardnumber => 1, |
239 |
borrowernumber => $borrowernumber, |
295 |
borrowernumber => $borrowernumber, |
240 |
cardnumber => $borrower{cardnumber} |
296 |
cardnumber => $borrower{cardnumber} |
241 |
}; |
297 |
}; |
|
|
298 |
print "ERROR: invalid cardnumber $borrower{cardnumber} for borrowernumber $borrowernumber\n" if $is_cli; |
242 |
$invalid++; |
299 |
$invalid++; |
243 |
next; |
300 |
next; |
244 |
} |
301 |
} |
Lines 248-254
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
248 |
# borrower exists |
305 |
# borrower exists |
249 |
unless ($overwrite_cardnumber) { |
306 |
unless ($overwrite_cardnumber) { |
250 |
$alreadyindb++; |
307 |
$alreadyindb++; |
251 |
$template->param('lastalreadyindb'=>$borrower{'surname'}.' / '.$borrowernumber); |
308 |
if ( $is_cli ) { |
|
|
309 |
print "$borrower{'surname'} / $borrowernumber alredy in database.\n" if $verbose; |
310 |
} else { |
311 |
$template->param( 'lastalreadyindb' => $borrower{'surname'} . ' / ' . $borrowernumber ); |
312 |
} |
252 |
next LINE; |
313 |
next LINE; |
253 |
} |
314 |
} |
254 |
$borrower{'borrowernumber'} = $borrowernumber; |
315 |
$borrower{'borrowernumber'} = $borrowernumber; |
Lines 267-273
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
267 |
$invalid++; |
328 |
$invalid++; |
268 |
# untill we have better error trapping, we have no way of knowing why ModMember errored out... |
329 |
# untill we have better error trapping, we have no way of knowing why ModMember errored out... |
269 |
push @errors, {unknown_error => 1}; |
330 |
push @errors, {unknown_error => 1}; |
270 |
$template->param('lastinvalid'=>$borrower{'surname'}.' / '.$borrowernumber); |
331 |
if ( $is_cli ) { |
|
|
332 |
print "Failure to update $borrower{'surname'} / $borrowernumber\n" if $verbose; |
333 |
} else { |
334 |
$template->param('lastinvalid'=>$borrower{'surname'}.' / '.$borrowernumber); |
335 |
} |
271 |
next LINE; |
336 |
next LINE; |
272 |
} |
337 |
} |
273 |
if ( $borrower{debarred} ) { |
338 |
if ( $borrower{debarred} ) { |
Lines 298-304
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
298 |
push @errors, {unknown_error => 1} unless SetBorrowerAttributes($borrower{'borrowernumber'}, $patron_attributes); |
363 |
push @errors, {unknown_error => 1} unless SetBorrowerAttributes($borrower{'borrowernumber'}, $patron_attributes); |
299 |
} |
364 |
} |
300 |
$overwritten++; |
365 |
$overwritten++; |
301 |
$template->param('lastoverwritten'=>$borrower{'surname'}.' / '.$borrowernumber); |
366 |
|
|
|
367 |
if ( $is_cli ) { |
368 |
print "Overwriting $borrower{'surname'} / $borrowernumber with new data\n"; |
369 |
} else { |
370 |
$template->param( 'lastoverwritten' => $borrower{'surname'} . ' / ' . $borrowernumber ); |
371 |
} |
302 |
} else { |
372 |
} else { |
303 |
# FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either. |
373 |
# FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either. |
304 |
# At least this is closer to AddMember than in members/memberentry.pl |
374 |
# At least this is closer to AddMember than in members/memberentry.pl |
Lines 327-364
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
327 |
} |
397 |
} |
328 |
|
398 |
|
329 |
$imported++; |
399 |
$imported++; |
330 |
$template->param('lastimported'=>$borrower{'surname'}.' / '.$borrowernumber); |
400 |
|
|
|
401 |
if ($is_cli) { |
402 |
print "Imported new patron $borrower{'surname'} / $borrowernumber\n" if $verbose; |
403 |
} else { |
404 |
$template->param( 'lastimported' => $borrower{'surname'} . ' / ' . $borrowernumber ); |
405 |
} |
331 |
} else { |
406 |
} else { |
332 |
$invalid++; |
407 |
$invalid++; |
333 |
push @errors, {unknown_error => 1}; |
408 |
push @errors, {unknown_error => 1}; |
334 |
$template->param('lastinvalid'=>$borrower{'surname'}.' / AddMember'); |
409 |
if ( $is_cli ) { |
|
|
410 |
print "Failure to import $borrower{'surname'}\n" if $verbose; |
411 |
} else { |
412 |
$template->param( 'lastinvalid' => $borrower{'surname'} . ' / AddMember'); |
413 |
} |
335 |
} |
414 |
} |
336 |
} |
415 |
} |
337 |
} |
416 |
} |
338 |
(@errors ) and $template->param( ERRORS=>\@errors ); |
417 |
|
339 |
(@feedback) and $template->param(FEEDBACK=>\@feedback); |
418 |
if ($is_cli) { |
340 |
$template->param( |
419 |
if ($verbose) { |
341 |
'uploadborrowers' => 1, |
420 |
my $total = $imported + $alreadyindb + $invalid + $overwritten; |
342 |
'imported' => $imported, |
421 |
print "\nImport complete:\n"; |
343 |
'overwritten' => $overwritten, |
422 |
print "Imported: $imported\n"; |
344 |
'alreadyindb' => $alreadyindb, |
423 |
print "Overwritten: $overwritten\n"; |
345 |
'invalid' => $invalid, |
424 |
print "Skipped: $alreadyindb\n"; |
346 |
'total' => $imported + $alreadyindb + $invalid + $overwritten, |
425 |
print "Invalid: $invalid\n"; |
347 |
); |
426 |
print "Total: $total\n\n"; |
|
|
427 |
} |
428 |
} else { |
429 |
(@errors) and $template->param( ERRORS => \@errors ); |
430 |
(@feedback) and $template->param( FEEDBACK => \@feedback ); |
431 |
$template->param( |
432 |
'uploadborrowers' => 1, |
433 |
'imported' => $imported, |
434 |
'overwritten' => $overwritten, |
435 |
'alreadyindb' => $alreadyindb, |
436 |
'invalid' => $invalid, |
437 |
'total' => $imported + $alreadyindb + $invalid + $overwritten, |
438 |
); |
439 |
} |
348 |
|
440 |
|
349 |
} else { |
441 |
} else { |
350 |
if ($extended) { |
442 |
unless ( $is_cli ) { |
351 |
my @matchpoints = (); |
443 |
if ($extended) { |
352 |
my @attr_types = C4::Members::AttributeTypes::GetAttributeTypes(undef, 1); |
444 |
my @matchpoints = (); |
353 |
foreach my $type (@attr_types) { |
445 |
my @attr_types = C4::Members::AttributeTypes::GetAttributeTypes(undef, 1); |
354 |
my $attr_type = C4::Members::AttributeTypes->fetch($type->{code}); |
446 |
foreach my $type (@attr_types) { |
355 |
if ($attr_type->unique_id()) { |
447 |
my $attr_type = C4::Members::AttributeTypes->fetch( $type->{code} ); |
356 |
push @matchpoints, { code => "patron_attribute_" . $attr_type->code(), description => $attr_type->description() }; |
448 |
if ( $attr_type->unique_id() ) { |
|
|
449 |
push @matchpoints, |
450 |
{ |
451 |
code => "patron_attribute_" . $attr_type->code(), |
452 |
description => $attr_type->description() |
453 |
}; |
454 |
} |
357 |
} |
455 |
} |
|
|
456 |
$template->param(matchpoints => \@matchpoints); |
358 |
} |
457 |
} |
359 |
$template->param(matchpoints => \@matchpoints); |
|
|
360 |
} |
458 |
} |
361 |
} |
459 |
} |
362 |
|
460 |
|
363 |
output_html_with_http_headers $input, $cookie, $template->output; |
461 |
output_html_with_http_headers( $input, $cookie, $template->output ) unless $is_cli; |
364 |
|
462 |
|
365 |
- |
463 |
sub print_help { |
|
|
464 |
print <<_USAGE_; |
465 |
import_borrowers.pl -c /path/to/borrowers.csv -m cardnumber |
466 |
-c --csv Path to the CSV file of patrons to import |
467 |
-m --matchpoint Field on which to match incoming patrons to existing patrons |
468 |
-d --default Set defaults to patron fields, repeatable e.g. --default branchcode=MPL --default categorycode=PT |
469 |
-p --preserve-extended-atributes Retain extended patron attributes for existing patrons being overwritten |
470 |
-o --overwrite Overwrite existing patrons with new data if a match is found |
471 |
-v --verbose Be verbose |
472 |
_USAGE_ |
473 |
exit; |
474 |
} |