|
Lines 1-7
Link Here
|
| 1 |
#!/usr/bin/perl |
1 |
#!/usr/bin/perl |
| 2 |
|
2 |
|
| 3 |
# Copyright 2007 Liblime |
3 |
# Copyright 2007 Liblime |
| 4 |
# Parts copyright 2010 BibLibre |
4 |
# Parts copyright 2013 BibLibre |
| 5 |
# |
5 |
# |
| 6 |
# This file is part of Koha. |
6 |
# This file is part of Koha. |
| 7 |
# |
7 |
# |
|
Lines 34-41
Link Here
|
| 34 |
# dates should be in the format you have set up Koha to expect |
34 |
# dates should be in the format you have set up Koha to expect |
| 35 |
# branchcode and categorycode need to be valid |
35 |
# branchcode and categorycode need to be valid |
| 36 |
|
36 |
|
| 37 |
use strict; |
37 |
use Modern::Perl; |
| 38 |
use warnings; |
|
|
| 39 |
|
38 |
|
| 40 |
use C4::Auth; |
39 |
use C4::Auth; |
| 41 |
use C4::Output; |
40 |
use C4::Output; |
|
Lines 46-53
use C4::Members;
Link Here
|
| 46 |
use C4::Members::Attributes qw(:all); |
45 |
use C4::Members::Attributes qw(:all); |
| 47 |
use C4::Members::AttributeTypes; |
46 |
use C4::Members::AttributeTypes; |
| 48 |
use C4::Members::Messaging; |
47 |
use C4::Members::Messaging; |
| 49 |
|
48 |
use Koha::DateUtils; |
|
|
49 |
use Date::Calc qw(Today_and_Now); |
| 50 |
use Getopt::Long; |
| 51 |
use File::Temp; |
| 50 |
use Text::CSV; |
52 |
use Text::CSV; |
|
|
53 |
|
| 51 |
# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals: |
54 |
# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals: |
| 52 |
# ė |
55 |
# ė |
| 53 |
# č |
56 |
# č |
|
Lines 75-84
my ( $template, $loggedinuser, $cookie ) = get_template_and_user({
Link Here
|
| 75 |
authnotrequired => 0, |
78 |
authnotrequired => 0, |
| 76 |
flagsrequired => { tools => 'import_patrons' }, |
79 |
flagsrequired => { tools => 'import_patrons' }, |
| 77 |
debug => 1, |
80 |
debug => 1, |
| 78 |
}); |
81 |
}); |
| 79 |
|
82 |
|
| 80 |
$template->param(columnkeys => $columnkeystpl); |
83 |
$template->param(columnkeys => $columnkeystpl); |
| 81 |
|
|
|
| 82 |
if ($input->param('sample')) { |
84 |
if ($input->param('sample')) { |
| 83 |
print $input->header( |
85 |
print $input->header( |
| 84 |
-type => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ? |
86 |
-type => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ? |
|
Lines 88-93
if ($input->param('sample')) {
Link Here
|
| 88 |
print $csv->string, "\n"; |
90 |
print $csv->string, "\n"; |
| 89 |
exit 1; |
91 |
exit 1; |
| 90 |
} |
92 |
} |
|
|
93 |
|
| 94 |
if ($input->param('report')) { |
| 95 |
print $input->header( |
| 96 |
-type => 'text/plain', |
| 97 |
-attachment => 'import_borrowers_report.txt' |
| 98 |
); |
| 99 |
my $filename = $input->param('errors_filename'); |
| 100 |
if ( -f $filename ) { |
| 101 |
open my $fh, '<', $filename; |
| 102 |
print <$fh>; |
| 103 |
close $fh; |
| 104 |
|
| 105 |
unlink $filename; |
| 106 |
exit 1; |
| 107 |
} |
| 108 |
} |
| 109 |
|
| 91 |
my $uploadborrowers = $input->param('uploadborrowers'); |
110 |
my $uploadborrowers = $input->param('uploadborrowers'); |
| 92 |
my $matchpoint = $input->param('matchpoint'); |
111 |
my $matchpoint = $input->param('matchpoint'); |
| 93 |
if ($matchpoint) { |
112 |
if ($matchpoint) { |
|
Lines 110-116
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
| 110 |
my $alreadyindb = 0; |
129 |
my $alreadyindb = 0; |
| 111 |
my $overwritten = 0; |
130 |
my $overwritten = 0; |
| 112 |
my $invalid = 0; |
131 |
my $invalid = 0; |
| 113 |
my $matchpoint_attr_type; |
132 |
my $matchpoint_attr_type; |
| 114 |
my %defaults = $input->Vars; |
133 |
my %defaults = $input->Vars; |
| 115 |
|
134 |
|
| 116 |
# use header line to construct key to column map |
135 |
# use header line to construct key to column map |
|
Lines 121-131
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
| 121 |
my %csvkeycol; |
140 |
my %csvkeycol; |
| 122 |
my $col = 0; |
141 |
my $col = 0; |
| 123 |
foreach my $keycol (@csvcolumns) { |
142 |
foreach my $keycol (@csvcolumns) { |
| 124 |
# columnkeys don't contain whitespace, but some stupid tools add it |
143 |
# columnkeys don't contain whitespace, but some stupid tools add it |
| 125 |
$keycol =~ s/ +//g; |
144 |
$keycol =~ s/ +//g; |
| 126 |
$csvkeycol{$keycol} = $col++; |
145 |
$csvkeycol{$keycol} = $col++; |
| 127 |
} |
146 |
} |
| 128 |
#warn($borrowerline); |
147 |
|
| 129 |
my $ext_preserve = $input->param('ext_preserve') || 0; |
148 |
my $ext_preserve = $input->param('ext_preserve') || 0; |
| 130 |
if ($extended) { |
149 |
if ($extended) { |
| 131 |
$matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint); |
150 |
$matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint); |
|
Lines 137-142
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
| 137 |
my @bad_dates; # I've had a few. |
156 |
my @bad_dates; # I've had a few. |
| 138 |
my $date_re = C4::Dates->new->regexp('syspref'); |
157 |
my $date_re = C4::Dates->new->regexp('syspref'); |
| 139 |
my $iso_re = C4::Dates->new->regexp('iso'); |
158 |
my $iso_re = C4::Dates->new->regexp('iso'); |
|
|
159 |
my $lastalreadyindb; |
| 160 |
my $lastinvalid; |
| 140 |
LINE: while ( my $borrowerline = <$handle> ) { |
161 |
LINE: while ( my $borrowerline = <$handle> ) { |
| 141 |
my %borrower; |
162 |
my %borrower; |
| 142 |
my @missing_criticals; |
163 |
my @missing_criticals; |
|
Lines 152-174
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
| 152 |
if ($borrower{$key} !~ /\S/) { |
173 |
if ($borrower{$key} !~ /\S/) { |
| 153 |
$borrower{$key} = $defaults{$key}; |
174 |
$borrower{$key} = $defaults{$key}; |
| 154 |
} |
175 |
} |
| 155 |
} |
176 |
} |
| 156 |
} else { |
177 |
} else { |
| 157 |
# MJR: try to recover gracefully by using default values |
178 |
# MJR: try to recover gracefully by using default values |
| 158 |
foreach my $key (@columnkeys) { |
179 |
foreach my $key (@columnkeys) { |
| 159 |
if (defined($csvkeycol{$key}) and $columns[$csvkeycol{$key}] =~ /\S/) { |
180 |
if (defined($csvkeycol{$key}) and $columns[$csvkeycol{$key}] =~ /\S/) { |
| 160 |
$borrower{$key} = $columns[$csvkeycol{$key}]; |
181 |
$borrower{$key} = $columns[$csvkeycol{$key}]; |
| 161 |
} elsif ( $defaults{$key} ) { |
182 |
} elsif ( $defaults{$key} ) { |
| 162 |
$borrower{$key} = $defaults{$key}; |
183 |
$borrower{$key} = $defaults{$key}; |
| 163 |
} elsif ( scalar grep {$key eq $_} @criticals ) { |
184 |
} elsif ( scalar grep {$key eq $_} @criticals ) { |
| 164 |
# a critical field is undefined |
185 |
# a critical field is undefined |
| 165 |
push @missing_criticals, {key=>$key, line=>$., lineraw=>$borrowerline}; |
186 |
push @missing_criticals, {key=>$key, line=>$., lineraw=>$borrowerline}; |
| 166 |
} else { |
187 |
} else { |
| 167 |
$borrower{$key} = ''; |
188 |
$borrower{$key} = ''; |
| 168 |
} |
189 |
} |
| 169 |
} |
190 |
} |
| 170 |
} |
191 |
} |
| 171 |
#warn join(':',%borrower); |
192 |
|
| 172 |
if ($borrower{categorycode}) { |
193 |
if ($borrower{categorycode}) { |
| 173 |
push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline, value=>$borrower{categorycode}, category_map=>1} |
194 |
push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline, value=>$borrower{categorycode}, category_map=>1} |
| 174 |
unless GetBorrowercategory($borrower{categorycode}); |
195 |
unless GetBorrowercategory($borrower{categorycode}); |
|
Lines 187-194
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
| 187 |
$_->{surname} = $borrower{surname} || 'UNDEF'; |
208 |
$_->{surname} = $borrower{surname} || 'UNDEF'; |
| 188 |
} |
209 |
} |
| 189 |
$invalid++; |
210 |
$invalid++; |
| 190 |
(25 > scalar @errors) and push @errors, {missing_criticals=>\@missing_criticals}; |
211 |
push @errors, {missing_criticals=>\@missing_criticals}; |
| 191 |
# The first 25 errors are enough. Keeping track of 30,000+ would destroy performance. |
|
|
| 192 |
next LINE; |
212 |
next LINE; |
| 193 |
} |
213 |
} |
| 194 |
if ($extended) { |
214 |
if ($extended) { |
|
Lines 197-205
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
| 197 |
$attr_str =~ s/\xe2\x80\x9d/"/g; |
217 |
$attr_str =~ s/\xe2\x80\x9d/"/g; |
| 198 |
push @feedback, {feedback=>1, name=>'attribute string', value=>$attr_str, filename=>$uploadborrowers}; |
218 |
push @feedback, {feedback=>1, name=>'attribute string', value=>$attr_str, filename=>$uploadborrowers}; |
| 199 |
delete $borrower{patron_attributes}; # not really a field in borrowers, so we don't want to pass it to ModMember. |
219 |
delete $borrower{patron_attributes}; # not really a field in borrowers, so we don't want to pass it to ModMember. |
| 200 |
$patron_attributes = extended_attributes_code_value_arrayref($attr_str); |
220 |
$patron_attributes = extended_attributes_code_value_arrayref($attr_str); |
| 201 |
} |
221 |
} |
| 202 |
# Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it. |
222 |
# Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it. |
| 203 |
foreach (qw(dateofbirth dateenrolled dateexpiry)) { |
223 |
foreach (qw(dateofbirth dateenrolled dateexpiry)) { |
| 204 |
my $tempdate = $borrower{$_} or next; |
224 |
my $tempdate = $borrower{$_} or next; |
| 205 |
if ($tempdate =~ /$date_re/) { |
225 |
if ($tempdate =~ /$date_re/) { |
|
Lines 211-218
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
| 211 |
push @missing_criticals, {key=>$_, line=>$. , lineraw=>$borrowerline, bad_date=>1}; |
231 |
push @missing_criticals, {key=>$_, line=>$. , lineraw=>$borrowerline, bad_date=>1}; |
| 212 |
} |
232 |
} |
| 213 |
} |
233 |
} |
| 214 |
$borrower{dateenrolled} = $today_iso unless $borrower{dateenrolled}; |
234 |
$borrower{dateenrolled} = $today_iso unless $borrower{dateenrolled}; |
| 215 |
$borrower{dateexpiry} = GetExpiryDate($borrower{categorycode},$borrower{dateenrolled}) unless $borrower{dateexpiry}; |
235 |
$borrower{dateexpiry} = GetExpiryDate($borrower{categorycode},$borrower{dateenrolled}) unless $borrower{dateexpiry}; |
| 216 |
my $borrowernumber; |
236 |
my $borrowernumber; |
| 217 |
my $member; |
237 |
my $member; |
| 218 |
if ( ($matchpoint eq 'cardnumber') && ($borrower{'cardnumber'}) ) { |
238 |
if ( ($matchpoint eq 'cardnumber') && ($borrower{'cardnumber'}) ) { |
|
Lines 231-242
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
| 231 |
} |
251 |
} |
| 232 |
} |
252 |
} |
| 233 |
} |
253 |
} |
| 234 |
|
254 |
|
| 235 |
if ($borrowernumber) { |
255 |
if ($borrowernumber) { |
| 236 |
# borrower exists |
256 |
# borrower exists |
| 237 |
unless ($overwrite_cardnumber) { |
257 |
unless ($overwrite_cardnumber) { |
| 238 |
$alreadyindb++; |
258 |
$alreadyindb++; |
| 239 |
$template->param('lastalreadyindb'=>$borrower{'surname'}.' / '.$borrowernumber); |
259 |
$lastalreadyindb = $borrower{'surname'}.' / '.$borrowernumber; |
|
|
260 |
$template->param('lastalreadyindb'=>$lastalreadyindb); |
| 240 |
next LINE; |
261 |
next LINE; |
| 241 |
} |
262 |
} |
| 242 |
$borrower{'borrowernumber'} = $borrowernumber; |
263 |
$borrower{'borrowernumber'} = $borrowernumber; |
|
Lines 255-261
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
| 255 |
$invalid++; |
276 |
$invalid++; |
| 256 |
# untill we have better error trapping, we have no way of knowing why ModMember errored out... |
277 |
# untill we have better error trapping, we have no way of knowing why ModMember errored out... |
| 257 |
push @errors, {unknown_error => 1}; |
278 |
push @errors, {unknown_error => 1}; |
| 258 |
$template->param('lastinvalid'=>$borrower{'surname'}.' / '.$borrowernumber); |
279 |
$lastinvalid = $borrower{'surname'}.' / '.$borrowernumber; |
|
|
280 |
$template->param('lastinvalid'=>$lastinvalid); |
| 259 |
next LINE; |
281 |
next LINE; |
| 260 |
} |
282 |
} |
| 261 |
if ($extended) { |
283 |
if ($extended) { |
|
Lines 301-306
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
| 301 |
'total' => $imported + $alreadyindb + $invalid + $overwritten, |
323 |
'total' => $imported + $alreadyindb + $invalid + $overwritten, |
| 302 |
); |
324 |
); |
| 303 |
|
325 |
|
|
|
326 |
my $max_errors = 25; |
| 327 |
if (scalar(@errors) > $max_errors ) { |
| 328 |
|
| 329 |
my $total = $imported + $alreadyindb + $invalid + $overwritten; |
| 330 |
my $output; |
| 331 |
|
| 332 |
my $timestamp = output_pref( dt_from_string ); |
| 333 |
$output .= "Timestamp : $timestamp\n"; |
| 334 |
$output .= "Import results\n"; |
| 335 |
$output .= "$imported imported records\n"; |
| 336 |
$output .= "$overwritten overwritten records\n"; |
| 337 |
$output .= "$alreadyindb not imported because already in borrowers table and overwrite disabled\n"; |
| 338 |
$output .= "(last was $lastalreadyindb)\n" if ($lastalreadyindb); |
| 339 |
$output .= "$invalid not imported because they are not in the expected format\n"; |
| 340 |
$output .= "(last was $lastinvalid)\n" if ($lastinvalid); |
| 341 |
$output .= "$total records parsed\n"; |
| 342 |
|
| 343 |
|
| 344 |
$output .= "\nError analysis\n"; |
| 345 |
foreach my $error (@errors) { |
| 346 |
$output .= "Header row could not be parsed" if ($error->{'badheader'}); |
| 347 |
foreach my $array ($error->{'missing_criticals'}) { |
| 348 |
foreach (@$array) { |
| 349 |
$output .= "Line $_->{'line'}: "; |
| 350 |
if ($error->{'badparse'}) { |
| 351 |
$output .= "could not be parsed!"; |
| 352 |
} elsif ($error->{'bad_date'}) { |
| 353 |
$output .= "has $_->{'key'} in unrecognized format: $_->{'value'} "; |
| 354 |
} else { |
| 355 |
$output .= "Critical field $_->{'key'}: "; |
| 356 |
if ($_->{'branch_map'} || $_->{'category_map'}) { |
| 357 |
$output .= "has unrecognized value: $_->{'value'}"; |
| 358 |
} else { |
| 359 |
$output .= " missing"; |
| 360 |
} |
| 361 |
$output .= " (borrowernumber: $_->{'borrowernumber'}; surname: $_->{'surname'})"; |
| 362 |
} |
| 363 |
$output .= "\n"; |
| 364 |
} |
| 365 |
} |
| 366 |
} |
| 367 |
|
| 368 |
my $tmpf = File::Temp->new(UNLINK => 0); |
| 369 |
print $tmpf $output; |
| 370 |
$template->param( |
| 371 |
download_errors => 1, |
| 372 |
errors_filename => $tmpf->filename |
| 373 |
); |
| 374 |
close $tmpf; |
| 375 |
|
| 376 |
} |
| 377 |
|
| 304 |
} else { |
378 |
} else { |
| 305 |
if ($extended) { |
379 |
if ($extended) { |
| 306 |
my @matchpoints = (); |
380 |
my @matchpoints = (); |
|
Lines 316-319
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
Link Here
|
| 316 |
} |
390 |
} |
| 317 |
|
391 |
|
| 318 |
output_html_with_http_headers $input, $cookie, $template->output; |
392 |
output_html_with_http_headers $input, $cookie, $template->output; |
| 319 |
|
|
|
| 320 |
- |