Attachment 29808 Details for Bug 12598 – Bug 12598 - Allow tools/import_borrowers.pl to be used from the command line.

[patch] Bug 12598 - Allow tools/import_borrowers.pl to be used from the command line.

Bug-12598---Allow-toolsimportborrowerspl-to-be-use.patch (text/plain), 16.66 KB, created by Kyle M Hall (khall) on 2014-07-17 18:31:07 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Kyle M Hall (khall)

Created: 2014-07-17 18:31:07 UTC

Size: 16.66 KB

patch

obsolete

>From 50b0bb76d609ec1c9d01f33db48d31e1f047847f Mon Sep 17 00:00:00 2001
>From: Kyle M Hall <kyle@bywatersolutions.com>
>Date: Thu, 17 Jul 2014 14:30:35 -0400
>Subject: [PATCH] Bug 12598 - Allow tools/import_borrowers.pl to be used from the command line.
>
>Many libraries have the need to automate the importing of patrons into
>Koha from external sources. It would be nice if we could use Koha's own
>import_borrowers.pl to do that from the command line, in the same way as
>tools/export.pl can be used from both the staff interface and the
>command line.
>
>Test Plan:
>1) Apply this patch
>2) Test importing patrons from staff interface,
>   no changes should be noted.
>3) Test importing patrons from command line,
>   options are availble with --help.
>---
> tools/import_borrowers.pl |  244 ++++++++++++++++++++++++++++++++------------
> 1 files changed, 177 insertions(+), 67 deletions(-)
>
>diff --git a/tools/import_borrowers.pl b/tools/import_borrowers.pl
>index abbef70..cf4205d 100755
>--- a/tools/import_borrowers.pl
>+++ b/tools/import_borrowers.pl
>@@ -2,6 +2,7 @@
> 
> # Copyright 2007 Liblime
> # Parts copyright 2010 BibLibre
>+# Parts copyright 2014 ByWater Solutions
> #
> # This file is part of Koha.
> #
>@@ -34,8 +35,7 @@
> # dates should be in the format you have set up Koha to expect
> # branchcode and categorycode need to be valid
> 
>-use strict;
>-use warnings;
>+use Modern::Perl;
> 
> use C4::Auth;
> use C4::Output;
>@@ -48,13 +48,15 @@ use C4::Members::AttributeTypes;
> use C4::Members::Messaging;
> use Koha::Borrower::Debarments;
> 
>+use CGI;
>+use Getopt::Long;
> use Text::CSV;
> # Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals:
> # Ä
> # Ä
> 
>-use CGI;
>-# use encoding 'utf8';    # don't do this
>+# Checks if the script is called from commandline
>+my $is_cli = not defined $ENV{GATEWAY_INTERFACE};
> 
> my (@errors, @feedback);
> my $extended = C4::Context->preference('ExtendedPatronAttributes');
>@@ -67,57 +69,104 @@ my $columnkeystpl = [ map { {'key' => $_} }  grep {$_ ne 'borrowernumber' } @col
> 
> my $input = CGI->new();
> our $csv  = Text::CSV->new({binary => 1});  # binary needed for non-ASCII Unicode
>-#push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend}; #XXX
>-
>-my ( $template, $loggedinuser, $cookie ) = get_template_and_user({
>-        template_name   => "tools/import_borrowers.tt",
>-        query           => $input,
>-        type            => "intranet",
>-        authnotrequired => 0,
>-        flagsrequired   => { tools => 'import_patrons' },
>-        debug           => 1,
>-});
>-
>-$template->param(columnkeys => $columnkeystpl);
>-
>-if ($input->param('sample')) {
>-    print $input->header(
>-        -type       => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ?
>-        -attachment => 'patron_import.csv',
>+
>+my ( $template, $loggedinuser, $cookie );
>+unless ($is_cli) {
>+    ( $template, $loggedinuser, $cookie ) = get_template_and_user(
>+        {
>+            template_name   => "tools/import_borrowers.tt",
>+            query           => $input,
>+            type            => "intranet",
>+            authnotrequired => 0,
>+            flagsrequired   => { tools => 'import_patrons' },
>+            debug           => 1,
>+        }
>+    );
>+
>+    $template->param( columnkeys => $columnkeystpl );
>+
>+    if ( $input->param('sample') ) {
>+        print $input->header(
>+            -type =>
>+              'application/vnd.sun.xml.calc',    # 'application/vnd.ms-excel' ?
>+            -attachment => 'patron_import.csv',
>+        );
>+        $csv->combine(@columnkeys);
>+        print $csv->string, "\n";
>+        exit 1;
>+    }
>+
>+    $template->param(
>+        SCRIPT_NAME              => $ENV{'SCRIPT_NAME'},
>+        ExtendedPatronAttributes => $extended,
>     );
>-    $csv->combine(@columnkeys);
>-    print $csv->string, "\n";
>-    exit 1;
>-}
>-my $uploadborrowers = $input->param('uploadborrowers');
>-my $matchpoint      = $input->param('matchpoint');
>-if ($matchpoint) {
>-    $matchpoint =~ s/^patron_attribute_//;
> }
>-my $overwrite_cardnumber = $input->param('overwrite_cardnumber');
> 
>-$template->param( SCRIPT_NAME => $ENV{'SCRIPT_NAME'} );
>+my $uploadborrowers;
>+my $matchpoint;
>+my $overwrite_cardnumber;
>+my %defaults;
>+my $ext_preserve = 0;
>+my $verbose;
>+
>+if ($is_cli) {
>+    my $help;
>+    GetOptions(
>+        'c|csv=s'                       => \$uploadborrowers,
>+        'm|matchpoint=s'                => \$matchpoint,
>+        'd|default=s'                   => \%defaults,
>+        'o|overwrite'                   => \$overwrite_cardnumber,
>+        'p|preserve-extended-atributes' => \$ext_preserve,
>+        'v|verbose'                     => \$verbose,
>+        'h|help|?'                      => \$help,
>+    );
> 
>-($extended) and $template->param(ExtendedPatronAttributes => 1);
>+    print_help() if ( $help || !$uploadborrowers || !$matchpoint );
>+}
>+else {
>+    $uploadborrowers = $input->param('uploadborrowers');
>+    $matchpoint      = $input->param('matchpoint');
>+    if ($matchpoint) {
>+        $matchpoint =~ s/^patron_attribute_//;
>+    }
>+    $overwrite_cardnumber = $input->param('overwrite_cardnumber');
>+    %defaults             = $input->Vars;
>+    $ext_preserve         = $input->param('ext_preserve') || 0;
>+}
> 
> if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
>     push @feedback, {feedback=>1, name=>'filename', value=>$uploadborrowers, filename=>$uploadborrowers};
>-    my $handle = $input->upload('uploadborrowers');
>-    my $uploadinfo = $input->uploadInfo($uploadborrowers);
>-    foreach (keys %$uploadinfo) {
>-        push @feedback, {feedback=>1, name=>$_, value=>$uploadinfo->{$_}, $_=>$uploadinfo->{$_}};
>+
>+    my $handle;
>+    if ($is_cli) {
>+        open $handle, $uploadborrowers or die $!;
>+    }
>+    else {
>+        $handle = $uploadborrowers;
>+
>+        my $uploadinfo = $input->uploadInfo($uploadborrowers);
>+        foreach ( keys %$uploadinfo ) {
>+            push @feedback,
>+              {
>+                feedback => 1,
>+                name     => $_,
>+                value    => $uploadinfo->{$_},
>+                $_       => $uploadinfo->{$_}
>+              };
>+        }
>     }
>+
>     my $imported    = 0;
>     my $alreadyindb = 0;
>     my $overwritten = 0;
>     my $invalid     = 0;
>     my $matchpoint_attr_type; 
>-    my %defaults = $input->Vars;
> 
>     # use header line to construct key to column map
>     my $borrowerline = <$handle>;
>     my $status = $csv->parse($borrowerline);
>     ($status) or push @errors, {badheader=>1,line=>$., lineraw=>$borrowerline};
>+    print "WARNING: CSV header line is incomplete: $borrowerline\n" if $is_cli;
>     my @csvcolumns = $csv->fields();
>     my %csvkeycol;
>     my $col = 0;
>@@ -126,8 +175,6 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
>     	$keycol =~ s/ +//g;
>         $csvkeycol{$keycol} = $col++;
>     }
>-    #warn($borrowerline);
>-    my $ext_preserve = $input->param('ext_preserve') || 0;
>     if ($extended) {
>         $matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint);
>     }
>@@ -146,6 +193,7 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
>         my @columns = $csv->fields();
>         if (! $status) {
>             push @missing_criticals, {badparse=>1, line=>$., lineraw=>$borrowerline};
>+            print "ERROR: Unable to parse line $.: $borrowerline" if $is_cli;
>         } elsif (@columns == @columnkeys) {
>             @borrower{@columnkeys} = @columns;
>             # MJR: try to fill blanks gracefully by using default values
>@@ -164,6 +212,7 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
>             	} elsif ( scalar grep {$key eq $_} @criticals ) {
>             	    # a critical field is undefined
>             	    push @missing_criticals, {key=>$key, line=>$., lineraw=>$borrowerline};
>+                    print "ERROR: missing critical column data '$key' for line $.: $borrowerline\n" if $is_cli;
>             	} else {
>             		$borrower{$key} = '';
>             	}
>@@ -171,16 +220,22 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
>         }
>         #warn join(':',%borrower);
>         if ($borrower{categorycode}) {
>-            push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline, value=>$borrower{categorycode}, category_map=>1}
>-                unless GetBorrowercategory($borrower{categorycode});
>+            unless ( GetBorrowercategory( $borrower{categorycode} ) ) {
>+                push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline, value=>$borrower{categorycode}, category_map=>1};
>+                print "ERROR: invalid categorycode for line $.: $borrowerline\n" if $is_cli;
>+            }
>         } else {
>             push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline};
>+            print "ERROR: missing categorycode for line $.: $borrowerline\n" if $is_cli;
>         }
>         if ($borrower{branchcode}) {
>-            push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline, value=>$borrower{branchcode}, branch_map=>1}
>-                unless GetBranchName($borrower{branchcode});
>+            unless ( GetBranchName( $borrower{branchcode} ) ) {
>+                push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline, value=>$borrower{branchcode}, branch_map=>1};
>+                print "ERROR: invalid branchcode for line $.: $borrowerline\n" if $is_cli;
>+            }
>         } else {
>             push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline};
>+            print "ERROR: missing branchcode for line $.: $borrowerline\n" if $is_cli;
>         }
>         if (@missing_criticals) {
>             foreach (@missing_criticals) {
>@@ -234,11 +289,13 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
>         }
> 
>         if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) {
>+            $borrowernumber ||= q{};
>             push @errors, {
>                 invalid_cardnumber => 1,
>                 borrowernumber => $borrowernumber,
>                 cardnumber => $borrower{cardnumber}
>             };
>+            print "ERROR: invalid cardnumber $borrower{cardnumber} for borrowernumber $borrowernumber\n" if $is_cli;
>             $invalid++;
>             next;
>         }
>@@ -248,7 +305,11 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
>             # borrower exists
>             unless ($overwrite_cardnumber) {
>                 $alreadyindb++;
>-                $template->param('lastalreadyindb'=>$borrower{'surname'}.' / '.$borrowernumber);
>+                if ( $is_cli ) {
>+                    print "$borrower{'surname'} / $borrowernumber alredy in database.\n" if $verbose;
>+                } else {
>+                    $template->param( 'lastalreadyindb' => $borrower{'surname'} . ' / ' . $borrowernumber );
>+                }
>                 next LINE;
>             }
>             $borrower{'borrowernumber'} = $borrowernumber;
>@@ -267,7 +328,11 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
>                 $invalid++;
>                 # untill we have better error trapping, we have no way of knowing why ModMember errored out...
>                 push @errors, {unknown_error => 1};
>-                $template->param('lastinvalid'=>$borrower{'surname'}.' / '.$borrowernumber);
>+                if ( $is_cli ) {
>+                    print "Failure to update $borrower{'surname'} / $borrowernumber\n" if $verbose;
>+                } else {
>+                    $template->param('lastinvalid'=>$borrower{'surname'}.' / '.$borrowernumber);
>+                }
>                 next LINE;
>             }
>             if ( $borrower{debarred} ) {
>@@ -298,7 +363,12 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
>                 push @errors, {unknown_error => 1} unless SetBorrowerAttributes($borrower{'borrowernumber'}, $patron_attributes);
>             }
>             $overwritten++;
>-            $template->param('lastoverwritten'=>$borrower{'surname'}.' / '.$borrowernumber);
>+
>+            if ( $is_cli ) {
>+                print "Overwriting $borrower{'surname'} / $borrowernumber with new data\n";
>+            } else {
>+                $template->param( 'lastoverwritten' => $borrower{'surname'} . ' / ' . $borrowernumber );
>+            }
>         } else {
>             # FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either.
>             # At least this is closer to AddMember than in members/memberentry.pl
>@@ -327,38 +397,78 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
>                 }
> 
>                 $imported++;
>-                $template->param('lastimported'=>$borrower{'surname'}.' / '.$borrowernumber);
>+
>+                if ($is_cli) {
>+                    print "Imported new patron $borrower{'surname'} / $borrowernumber\n" if $verbose;
>+                } else {
>+                    $template->param( 'lastimported' => $borrower{'surname'} . ' / ' . $borrowernumber );
>+                }
>             } else {
>                 $invalid++;
>                 push @errors, {unknown_error => 1};
>-                $template->param('lastinvalid'=>$borrower{'surname'}.' / AddMember');
>+                if ( $is_cli ) {
>+                    print "Failure to import $borrower{'surname'}\n" if $verbose;
>+                } else {
>+                    $template->param( 'lastinvalid' => $borrower{'surname'} . ' / AddMember');
>+                }
>             }
>         }
>     }
>-    (@errors  ) and $template->param(  ERRORS=>\@errors  );
>-    (@feedback) and $template->param(FEEDBACK=>\@feedback);
>-    $template->param(
>-        'uploadborrowers' => 1,
>-        'imported'        => $imported,
>-        'overwritten'     => $overwritten,
>-        'alreadyindb'     => $alreadyindb,
>-        'invalid'         => $invalid,
>-        'total'           => $imported + $alreadyindb + $invalid + $overwritten,
>-    );
>+
>+    if ($is_cli) {
>+        if ($verbose) {
>+            my $total = $imported + $alreadyindb + $invalid + $overwritten;
>+            print "\nImport complete:\n";
>+            print "Imported:    $imported\n";
>+            print "Overwritten: $overwritten\n";
>+            print "Skipped:     $alreadyindb\n";
>+            print "Invalid:     $invalid\n";
>+            print "Total:       $total\n\n";
>+        }
>+    } else {
>+        (@errors)   and $template->param( ERRORS   => \@errors );
>+        (@feedback) and $template->param( FEEDBACK => \@feedback );
>+        $template->param(
>+            'uploadborrowers' => 1,
>+            'imported'        => $imported,
>+            'overwritten'     => $overwritten,
>+            'alreadyindb'     => $alreadyindb,
>+            'invalid'         => $invalid,
>+            'total' => $imported + $alreadyindb + $invalid + $overwritten,
>+        );
>+    }
> 
> } else {
>-    if ($extended) {
>-        my @matchpoints = ();
>-        my @attr_types = C4::Members::AttributeTypes::GetAttributeTypes(undef, 1);
>-        foreach my $type (@attr_types) {
>-            my $attr_type = C4::Members::AttributeTypes->fetch($type->{code});
>-            if ($attr_type->unique_id()) {
>-            push @matchpoints, { code =>  "patron_attribute_" . $attr_type->code(), description => $attr_type->description() };
>+    unless ( $is_cli ) {
>+        if ($extended) {
>+            my @matchpoints = ();
>+            my @attr_types = C4::Members::AttributeTypes::GetAttributeTypes(undef, 1);
>+            foreach my $type (@attr_types) {
>+                my $attr_type = C4::Members::AttributeTypes->fetch( $type->{code} );
>+                if ( $attr_type->unique_id() ) {
>+                    push @matchpoints,
>+                      {
>+                        code        => "patron_attribute_" . $attr_type->code(),
>+                        description => $attr_type->description()
>+                      };
>+                }
>             }
>+            $template->param(matchpoints => \@matchpoints);
>         }
>-        $template->param(matchpoints => \@matchpoints);
>     }
> }
> 
>-output_html_with_http_headers $input, $cookie, $template->output;
>+output_html_with_http_headers( $input, $cookie, $template->output ) unless $is_cli;
> 
>+sub print_help {
>+    print <<_USAGE_;
>+import_borrowers.pl -c /path/to/borrowers.csv -m cardnumber
>+    -c --csv                            Path to the CSV file of patrons  to import
>+    -m --matchpoint                     Field on which to match incoming patrons to existing patrons
>+    -d --default                        Set defaults to patron fields, repeatable e.g. --default branchcode=MPL --default categorycode=PT
>+    -p --preserve-extended-atributes    Retain extended patron attributes for existing patrons being overwritten
>+    -o --overwrite                      Overwrite existing patrons with new data if a match is found
>+    -v --verbose                        Be verbose
>+_USAGE_
>+    exit;
>+}
>-- 
>1.7.2.5

Actions: View | Diff | Splinter Review

Attachments on bug 12598: 29807 | 29808 | 29809 | 29939 | 30755 | 30780 | 35557 | 35558 | 35559 | 35560 | 35848 | 35849 | 35850 | 35851 | 35964 | 36684 | 36685 | 36687 | 36688 | 36690 | 36692 | 36693 | 36694 | 36695 | 36696 | 39124 | 39125 | 39126 | 39127 | 39128 | 41790 | 41791 | 41792 | 41793 | 41794 | 44065 | 44066 | 44067 | 44068 | 44069 | 48753 | 48754 | 48755 | 48756 | 48757 | 50693 | 50694 | 50695 | 50696 | 50697 | 50698 | 50699 | 50700 | 50704 | 51122 | 51357 | 51358 | 51359 | 51360 | 51361 | 51362 | 51363 | 51364 | 51365 | 51366 | 51367 | 51368 | 51369 | 51370 | 51371 | 51372 | 52323 | 52324 | 52325 | 52326 | 52327 | 52328 | 52329 | 52330 | 52331 | 52332 | 52333 | 52540 | 64437 | 64438 | 64855 | 65214 | 65215 | 65216 | 65217 | 65218 | 65819 | 65820 | 65821 | 65822 | 65823 | 65824 | 66353 | 66354 | 66355 | 66356 | 66357 | 66358 | 66359 | 66575 | 66576 | 66577 | 66578 | 66579 | 66580 | 66581 | 66582 | 67276 | 67277 | 70265 | 70266 | 71755 | 71756 | 71757 | 71758 | 71759 | 71760 | 71761 | 71762 | 71763 | 71764 | 71765 | 71766