From 45a8a62dc8f1732cee6bbf26c835b46086925caf Mon Sep 17 00:00:00 2001 From: Kyle M Hall Date: Thu, 17 Jul 2014 14:21:33 -0400 Subject: [PATCH] Bug 12598 - Allow tools/import_borrowers.pl to be used from the command line. Many libraries have the need to automate the importing of patrons into Koha from external sources. It would be nice if we could use Koha's own import_borrowers.pl to do that from the command line, in the same way as tools/export.pl can be used from both the staff interface and the command line. Test Plan: 1) Apply this patch 2) Test importing patrons from staff interface, no changes should be noted. 3) Test importing patrons from command line, options are availble with --help. --- tools/import_borrowers.pl | 244 ++++++++++++++++++++++++++++++++------------ 1 files changed, 177 insertions(+), 67 deletions(-) diff --git a/tools/import_borrowers.pl b/tools/import_borrowers.pl index 3623684..8c03b6b 100755 --- a/tools/import_borrowers.pl +++ b/tools/import_borrowers.pl @@ -2,6 +2,7 @@ # Copyright 2007 Liblime # Parts copyright 2010 BibLibre +# Parts copyright 2014 ByWater Solutions # # This file is part of Koha. # @@ -34,8 +35,7 @@ # dates should be in the format you have set up Koha to expect # branchcode and categorycode need to be valid -use strict; -use warnings; +use Modern::Perl; use C4::Auth; use C4::Output; @@ -48,13 +48,15 @@ use C4::Members::AttributeTypes; use C4::Members::Messaging; use Koha::Borrower::Debarments; +use CGI; +use Getopt::Long; use Text::CSV; # Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals: # ė # č -use CGI; -# use encoding 'utf8'; # don't do this +# Checks if the script is called from commandline +my $is_cli = not defined $ENV{GATEWAY_INTERFACE}; my (@errors, @feedback); my $extended = C4::Context->preference('ExtendedPatronAttributes'); @@ -67,57 +69,104 @@ my $columnkeystpl = [ map { {'key' => $_} } grep {$_ ne 'borrowernumber' } @col my $input = CGI->new(); our $csv = Text::CSV->new({binary => 1}); # binary needed for non-ASCII Unicode -#push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend}; #XXX - -my ( $template, $loggedinuser, $cookie ) = get_template_and_user({ - template_name => "tools/import_borrowers.tmpl", - query => $input, - type => "intranet", - authnotrequired => 0, - flagsrequired => { tools => 'import_patrons' }, - debug => 1, -}); - -$template->param(columnkeys => $columnkeystpl); - -if ($input->param('sample')) { - print $input->header( - -type => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ? - -attachment => 'patron_import.csv', + +my ( $template, $loggedinuser, $cookie ); +unless ($is_cli) { + ( $template, $loggedinuser, $cookie ) = get_template_and_user( + { + template_name => "tools/import_borrowers.tmpl", + query => $input, + type => "intranet", + authnotrequired => 0, + flagsrequired => { tools => 'import_patrons' }, + debug => 1, + } + ); + + $template->param( columnkeys => $columnkeystpl ); + + if ( $input->param('sample') ) { + print $input->header( + -type => + 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ? + -attachment => 'patron_import.csv', + ); + $csv->combine(@columnkeys); + print $csv->string, "\n"; + exit 1; + } + + $template->param( + SCRIPT_NAME => $ENV{'SCRIPT_NAME'}, + ExtendedPatronAttributes => $extended, ); - $csv->combine(@columnkeys); - print $csv->string, "\n"; - exit 1; -} -my $uploadborrowers = $input->param('uploadborrowers'); -my $matchpoint = $input->param('matchpoint'); -if ($matchpoint) { - $matchpoint =~ s/^patron_attribute_//; } -my $overwrite_cardnumber = $input->param('overwrite_cardnumber'); -$template->param( SCRIPT_NAME => $ENV{'SCRIPT_NAME'} ); +my $uploadborrowers; +my $matchpoint; +my $overwrite_cardnumber; +my %defaults; +my $ext_preserve = 0; +my $verbose; + +if ($is_cli) { + my $help; + GetOptions( + 'c|csv=s' => \$uploadborrowers, + 'm|matchpoint=s' => \$matchpoint, + 'd|default=s' => \%defaults, + 'o|overwrite' => \$overwrite_cardnumber, + 'p|preserve-extended-atributes' => \$ext_preserve, + 'v|verbose' => \$verbose, + 'h|help|?' => \$help, + ); -($extended) and $template->param(ExtendedPatronAttributes => 1); + print_help() if ( $help || !$uploadborrowers || !$matchpoint ); +} +else { + $uploadborrowers = $input->param('uploadborrowers'); + $matchpoint = $input->param('matchpoint'); + if ($matchpoint) { + $matchpoint =~ s/^patron_attribute_//; + } + $overwrite_cardnumber = $input->param('overwrite_cardnumber'); + %defaults = $input->Vars; + $ext_preserve = $input->param('ext_preserve') || 0; +} if ( $uploadborrowers && length($uploadborrowers) > 0 ) { push @feedback, {feedback=>1, name=>'filename', value=>$uploadborrowers, filename=>$uploadborrowers}; - my $handle = $input->upload('uploadborrowers'); - my $uploadinfo = $input->uploadInfo($uploadborrowers); - foreach (keys %$uploadinfo) { - push @feedback, {feedback=>1, name=>$_, value=>$uploadinfo->{$_}, $_=>$uploadinfo->{$_}}; + + my $handle; + if ($is_cli) { + open $handle, $uploadborrowers or die $!; + } + else { + $handle = $uploadborrowers; + + my $uploadinfo = $input->uploadInfo($uploadborrowers); + foreach ( keys %$uploadinfo ) { + push @feedback, + { + feedback => 1, + name => $_, + value => $uploadinfo->{$_}, + $_ => $uploadinfo->{$_} + }; + } } + my $imported = 0; my $alreadyindb = 0; my $overwritten = 0; my $invalid = 0; my $matchpoint_attr_type; - my %defaults = $input->Vars; # use header line to construct key to column map my $borrowerline = <$handle>; my $status = $csv->parse($borrowerline); ($status) or push @errors, {badheader=>1,line=>$., lineraw=>$borrowerline}; + print "WARNING: CSV header line is incomplete: $borrowerline\n" if $is_cli; my @csvcolumns = $csv->fields(); my %csvkeycol; my $col = 0; @@ -126,8 +175,6 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) { $keycol =~ s/ +//g; $csvkeycol{$keycol} = $col++; } - #warn($borrowerline); - my $ext_preserve = $input->param('ext_preserve') || 0; if ($extended) { $matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint); } @@ -146,6 +193,7 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) { my @columns = $csv->fields(); if (! $status) { push @missing_criticals, {badparse=>1, line=>$., lineraw=>$borrowerline}; + print "ERROR: Unable to parse line $.: $borrowerline" if $is_cli; } elsif (@columns == @columnkeys) { @borrower{@columnkeys} = @columns; # MJR: try to fill blanks gracefully by using default values @@ -164,6 +212,7 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) { } elsif ( scalar grep {$key eq $_} @criticals ) { # a critical field is undefined push @missing_criticals, {key=>$key, line=>$., lineraw=>$borrowerline}; + print "ERROR: missing critical column data '$key' for line $.: $borrowerline\n" if $is_cli; } else { $borrower{$key} = ''; } @@ -171,16 +220,22 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) { } #warn join(':',%borrower); if ($borrower{categorycode}) { - push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline, value=>$borrower{categorycode}, category_map=>1} - unless GetBorrowercategory($borrower{categorycode}); + unless ( GetBorrowercategory( $borrower{categorycode} ) ) { + push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline, value=>$borrower{categorycode}, category_map=>1}; + print "ERROR: invalid categorycode for line $.: $borrowerline\n" if $is_cli; + } } else { push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline}; + print "ERROR: missing categorycode for line $.: $borrowerline\n" if $is_cli; } if ($borrower{branchcode}) { - push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline, value=>$borrower{branchcode}, branch_map=>1} - unless GetBranchName($borrower{branchcode}); + unless ( GetBranchName( $borrower{branchcode} ) ) { + push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline, value=>$borrower{branchcode}, branch_map=>1}; + print "ERROR: invalid branchcode for line $.: $borrowerline\n" if $is_cli; + } } else { push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline}; + print "ERROR: missing branchcode for line $.: $borrowerline\n" if $is_cli; } if (@missing_criticals) { foreach (@missing_criticals) { @@ -234,11 +289,13 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) { } if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) { + $borrowernumber ||= q{}; push @errors, { invalid_cardnumber => 1, borrowernumber => $borrowernumber, cardnumber => $borrower{cardnumber} }; + print "ERROR: invalid cardnumber $borrower{cardnumber} for borrowernumber $borrowernumber\n" if $is_cli; $invalid++; next; } @@ -248,7 +305,11 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) { # borrower exists unless ($overwrite_cardnumber) { $alreadyindb++; - $template->param('lastalreadyindb'=>$borrower{'surname'}.' / '.$borrowernumber); + if ( $is_cli ) { + print "$borrower{'surname'} / $borrowernumber alredy in database.\n" if $verbose; + } else { + $template->param( 'lastalreadyindb' => $borrower{'surname'} . ' / ' . $borrowernumber ); + } next LINE; } $borrower{'borrowernumber'} = $borrowernumber; @@ -267,7 +328,11 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) { $invalid++; # untill we have better error trapping, we have no way of knowing why ModMember errored out... push @errors, {unknown_error => 1}; - $template->param('lastinvalid'=>$borrower{'surname'}.' / '.$borrowernumber); + if ( $is_cli ) { + print "Failure to update $borrower{'surname'} / $borrowernumber\n" if $verbose; + } else { + $template->param('lastinvalid'=>$borrower{'surname'}.' / '.$borrowernumber); + } next LINE; } if ( $borrower{debarred} ) { @@ -298,7 +363,12 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) { push @errors, {unknown_error => 1} unless SetBorrowerAttributes($borrower{'borrowernumber'}, $patron_attributes); } $overwritten++; - $template->param('lastoverwritten'=>$borrower{'surname'}.' / '.$borrowernumber); + + if ( $is_cli ) { + print "Overwriting $borrower{'surname'} / $borrowernumber with new data\n"; + } else { + $template->param( 'lastoverwritten' => $borrower{'surname'} . ' / ' . $borrowernumber ); + } } else { # FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either. # At least this is closer to AddMember than in members/memberentry.pl @@ -327,38 +397,78 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) { } $imported++; - $template->param('lastimported'=>$borrower{'surname'}.' / '.$borrowernumber); + + if ($is_cli) { + print "Imported new patron $borrower{'surname'} / $borrowernumber\n" if $verbose; + } else { + $template->param( 'lastimported' => $borrower{'surname'} . ' / ' . $borrowernumber ); + } } else { $invalid++; push @errors, {unknown_error => 1}; - $template->param('lastinvalid'=>$borrower{'surname'}.' / AddMember'); + if ( $is_cli ) { + print "Failure to import $borrower{'surname'}\n" if $verbose; + } else { + $template->param( 'lastinvalid' => $borrower{'surname'} . ' / AddMember'); + } } } } - (@errors ) and $template->param( ERRORS=>\@errors ); - (@feedback) and $template->param(FEEDBACK=>\@feedback); - $template->param( - 'uploadborrowers' => 1, - 'imported' => $imported, - 'overwritten' => $overwritten, - 'alreadyindb' => $alreadyindb, - 'invalid' => $invalid, - 'total' => $imported + $alreadyindb + $invalid + $overwritten, - ); + + if ($is_cli) { + if ($verbose) { + my $total = $imported + $alreadyindb + $invalid + $overwritten; + print "\nImport complete:\n"; + print "Imported: $imported\n"; + print "Overwritten: $overwritten\n"; + print "Skipped: $alreadyindb\n"; + print "Invalid: $invalid\n"; + print "Total: $total\n\n"; + } + } else { + (@errors) and $template->param( ERRORS => \@errors ); + (@feedback) and $template->param( FEEDBACK => \@feedback ); + $template->param( + 'uploadborrowers' => 1, + 'imported' => $imported, + 'overwritten' => $overwritten, + 'alreadyindb' => $alreadyindb, + 'invalid' => $invalid, + 'total' => $imported + $alreadyindb + $invalid + $overwritten, + ); + } } else { - if ($extended) { - my @matchpoints = (); - my @attr_types = C4::Members::AttributeTypes::GetAttributeTypes(undef, 1); - foreach my $type (@attr_types) { - my $attr_type = C4::Members::AttributeTypes->fetch($type->{code}); - if ($attr_type->unique_id()) { - push @matchpoints, { code => "patron_attribute_" . $attr_type->code(), description => $attr_type->description() }; + unless ( $is_cli ) { + if ($extended) { + my @matchpoints = (); + my @attr_types = C4::Members::AttributeTypes::GetAttributeTypes(undef, 1); + foreach my $type (@attr_types) { + my $attr_type = C4::Members::AttributeTypes->fetch( $type->{code} ); + if ( $attr_type->unique_id() ) { + push @matchpoints, + { + code => "patron_attribute_" . $attr_type->code(), + description => $attr_type->description() + }; + } } + $template->param(matchpoints => \@matchpoints); } - $template->param(matchpoints => \@matchpoints); } } -output_html_with_http_headers $input, $cookie, $template->output; +output_html_with_http_headers( $input, $cookie, $template->output ) unless $is_cli; +sub print_help { + print <<_USAGE_; +import_borrowers.pl -c /path/to/borrowers.csv -m cardnumber + -c --csv Path to the CSV file of patrons to import + -m --matchpoint Field on which to match incoming patrons to existing patrons + -d --default Set defaults to patron fields, repeatable e.g. --default branchcode=MPL --default categorycode=PT + -p --preserve-extended-atributes Retain extended patron attributes for existing patrons being overwritten + -o --overwrite Overwrite existing patrons with new data if a match is found + -v --verbose Be verbose +_USAGE_ + exit; +} -- 1.7.2.5