From 391ed07068de0541b3086d48f3db7aa941de5a01 Mon Sep 17 00:00:00 2001 From: Kyle M Hall Date: Tue, 27 Jan 2015 10:14:02 -0500 Subject: [PATCH] Bug 12598 - Move importing code to a subroutine Signed-off-by: Benjamin Rokseth Signed-off-by: Kyle M Hall --- Koha/Patrons/Import.pm | 370 ++++++++++++++++++ .../prog/en/modules/tools/import_borrowers.tt | 426 ++++++++++++--------- tools/import_borrowers.pl | 356 ++--------------- 3 files changed, 649 insertions(+), 503 deletions(-) create mode 100644 Koha/Patrons/Import.pm diff --git a/Koha/Patrons/Import.pm b/Koha/Patrons/Import.pm new file mode 100644 index 0000000..1041262 --- /dev/null +++ b/Koha/Patrons/Import.pm @@ -0,0 +1,370 @@ +package Koha::Patrons::Import; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; + +use Carp; +use Text::CSV; + +use C4::Members; +use C4::Branch; + +sub import_patrons { + my ($params) = @_; + + my $handle = $params->{file}; + my $matchpoint = $params->{matchpoint}; + my $defaults = $params->{defaults}; + my $ext_preserve = $params->{preserve_extended_attributes}; + my $overwrite_cardnumber = $params->{overwrite_cardnumber}; + + carp("No file handle passed in!") && return unless $handle; + + my $extended = C4::Context->preference('ExtendedPatronAttributes'); + my $set_messaging_prefs = C4::Context->preference('EnhancedMessagingPreferences'); + + my @columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } C4::Members::columns(); + push( @columnkeys, 'patron_attributes' ) if $extended; + + our $csv = Text::CSV->new( { binary => 1 } ); # binary needed for non-ASCII Unicode + + my @feedback; + my @errors; + + my $imported = 0; + my $alreadyindb = 0; + my $overwritten = 0; + my $invalid = 0; + my $matchpoint_attr_type; + + # use header line to construct key to column map + my $borrowerline = <$handle>; + my $status = $csv->parse($borrowerline); + ($status) or push @errors, { badheader => 1, line => $., lineraw => $borrowerline }; + my @csvcolumns = $csv->fields(); + my %csvkeycol; + my $col = 0; + foreach my $keycol (@csvcolumns) { + + # columnkeys don't contain whitespace, but some stupid tools add it + $keycol =~ s/ +//g; + $csvkeycol{$keycol} = $col++; + } + + #warn($borrowerline); + if ($extended) { + $matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint); + } + + push @feedback, { feedback => 1, name => 'headerrow', value => join( ', ', @csvcolumns ) }; + my $today_iso = C4::Dates->new()->output('iso'); + my @criticals = qw(surname branchcode categorycode); # there probably should be others + my @bad_dates; # I've had a few. + my $date_re = C4::Dates->new->regexp('syspref'); + my $iso_re = C4::Dates->new->regexp('iso'); + LINE: while ( my $borrowerline = <$handle> ) { + my %borrower; + my @missing_criticals; + my $patron_attributes; + my $status = $csv->parse($borrowerline); + my @columns = $csv->fields(); + if ( !$status ) { + push @missing_criticals, { badparse => 1, line => $., lineraw => $borrowerline }; + } + elsif ( @columns == @columnkeys ) { + @borrower{@columnkeys} = @columns; + + # MJR: try to fill blanks gracefully by using default values + foreach my $key (@columnkeys) { + if ( $borrower{$key} !~ /\S/ ) { + $borrower{$key} = $defaults->{$key}; + } + } + } + else { + # MJR: try to recover gracefully by using default values + foreach my $key (@columnkeys) { + if ( defined( $csvkeycol{$key} ) and $columns[ $csvkeycol{$key} ] =~ /\S/ ) { + $borrower{$key} = $columns[ $csvkeycol{$key} ]; + } + elsif ( $defaults->{$key} ) { + $borrower{$key} = $defaults->{$key}; + } + elsif ( scalar grep { $key eq $_ } @criticals ) { + + # a critical field is undefined + push @missing_criticals, { key => $key, line => $., lineraw => $borrowerline }; + } + else { + $borrower{$key} = ''; + } + } + } + + #warn join(':',%borrower); + if ( $borrower{categorycode} ) { + push @missing_criticals, + { + key => 'categorycode', + line => $., + lineraw => $borrowerline, + value => $borrower{categorycode}, + category_map => 1 + } + unless GetBorrowercategory( $borrower{categorycode} ); + } + else { + push @missing_criticals, { key => 'categorycode', line => $., lineraw => $borrowerline }; + } + if ( $borrower{branchcode} ) { + push @missing_criticals, + { + key => 'branchcode', + line => $., + lineraw => $borrowerline, + value => $borrower{branchcode}, + branch_map => 1 + } + unless GetBranchName( $borrower{branchcode} ); + } + else { + push @missing_criticals, { key => 'branchcode', line => $., lineraw => $borrowerline }; + } + if (@missing_criticals) { + foreach (@missing_criticals) { + $_->{borrowernumber} = $borrower{borrowernumber} || 'UNDEF'; + $_->{surname} = $borrower{surname} || 'UNDEF'; + } + $invalid++; + ( 25 > scalar @errors ) and push @errors, { missing_criticals => \@missing_criticals }; + + # The first 25 errors are enough. Keeping track of 30,000+ would destroy performance. + next LINE; + } + if ($extended) { + my $attr_str = $borrower{patron_attributes}; + $attr_str =~ s/\xe2\x80\x9c/"/g; # fixup double quotes in case we are passed smart quotes + $attr_str =~ s/\xe2\x80\x9d/"/g; + push @feedback, { feedback => 1, name => 'attribute string', value => $attr_str }; + delete $borrower{patron_attributes} + ; # not really a field in borrowers, so we don't want to pass it to ModMember. + $patron_attributes = extended_attributes_code_value_arrayref($attr_str); + } + + # Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it. + foreach (qw(dateofbirth dateenrolled dateexpiry)) { + my $tempdate = $borrower{$_} or next; + if ( $tempdate =~ /$date_re/ ) { + $borrower{$_} = format_date_in_iso($tempdate); + } + elsif ( $tempdate =~ /$iso_re/ ) { + $borrower{$_} = $tempdate; + } + else { + $borrower{$_} = ''; + push @missing_criticals, { key => $_, line => $., lineraw => $borrowerline, bad_date => 1 }; + } + } + $borrower{dateenrolled} = $today_iso unless $borrower{dateenrolled}; + $borrower{dateexpiry} = GetExpiryDate( $borrower{categorycode}, $borrower{dateenrolled} ) + unless $borrower{dateexpiry}; + my $borrowernumber; + my $member; + if ( ( $matchpoint eq 'cardnumber' ) && ( $borrower{'cardnumber'} ) ) { + $member = GetMember( 'cardnumber' => $borrower{'cardnumber'} ); + if ($member) { + $borrowernumber = $member->{'borrowernumber'}; + } + } + elsif ($extended) { + if ( defined($matchpoint_attr_type) ) { + foreach my $attr (@$patron_attributes) { + if ( $attr->{code} eq $matchpoint and $attr->{value} ne '' ) { + my @borrowernumbers = $matchpoint_attr_type->get_patrons( $attr->{value} ); + $borrowernumber = $borrowernumbers[0] if scalar(@borrowernumbers) == 1; + last; + } + } + } + } + + if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) { + push @errors, + { + invalid_cardnumber => 1, + borrowernumber => $borrowernumber, + cardnumber => $borrower{cardnumber} + }; + $invalid++; + next; + } + + if ($borrowernumber) { + + # borrower exists + unless ($overwrite_cardnumber) { + $alreadyindb++; + push( + @feedback, + { + already_in_db => 1, + value => $borrower{'surname'} . ' / ' . $borrowernumber + } + ); + next LINE; + } + $borrower{'borrowernumber'} = $borrowernumber; + for my $col ( keys %borrower ) { + + # use values from extant patron unless our csv file includes this column or we provided a default. + # FIXME : You cannot update a field with a perl-evaluated false value using the defaults. + + # The password is always encrypted, skip it! + next if $col eq 'password'; + + unless ( exists( $csvkeycol{$col} ) || $defaults->{$col} ) { + $borrower{$col} = $member->{$col} if ( $member->{$col} ); + } + } + unless ( ModMember(%borrower) ) { + $invalid++; + + push( + @errors, + { + name => 'lastinvalid', + value => $borrower{'surname'} . ' / ' . $borrowernumber + } + ); + next LINE; + } + if ( $borrower{debarred} ) { + + # Check to see if this debarment already exists + my $debarrments = GetDebarments( + { + borrowernumber => $borrowernumber, + expiration => $borrower{debarred}, + comment => $borrower{debarredcomment} + } + ); + + # If it doesn't, then add it! + unless (@$debarrments) { + AddDebarment( + { + borrowernumber => $borrowernumber, + expiration => $borrower{debarred}, + comment => $borrower{debarredcomment} + } + ); + } + } + if ($extended) { + if ($ext_preserve) { + my $old_attributes = GetBorrowerAttributes($borrowernumber); + $patron_attributes = extended_attributes_merge( $old_attributes, $patron_attributes ); + } + push @errors, { unknown_error => 1 } + unless SetBorrowerAttributes( $borrower{'borrowernumber'}, $patron_attributes ); + } + $overwritten++; + push( + @feedback, + { + feedback => 1, + name => 'lastoverwritten', + value => $borrower{'surname'} . ' / ' . $borrowernumber + } + ); + } + else { + # FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either. + # At least this is closer to AddMember than in members/memberentry.pl + if ( !$borrower{'cardnumber'} ) { + $borrower{'cardnumber'} = fixup_cardnumber(undef); + } + if ( $borrowernumber = AddMember(%borrower) ) { + + if ( $borrower{debarred} ) { + AddDebarment( + { + borrowernumber => $borrowernumber, + expiration => $borrower{debarred}, + comment => $borrower{debarredcomment} + } + ); + } + + if ($extended) { + SetBorrowerAttributes( $borrowernumber, $patron_attributes ); + } + + if ($set_messaging_prefs) { + C4::Members::Messaging::SetMessagingPreferencesFromDefaults( + { + borrowernumber => $borrowernumber, + categorycode => $borrower{categorycode} + } + ); + } + + $imported++; + push( + @feedback, + { + feedback => 1, + name => 'lastimported', + value => $borrower{'surname'} . ' / ' . $borrowernumber + } + ); + } + else { + $invalid++; + push @errors, { unknown_error => 1 }; + push( + @errors, + { + name => 'lastinvalid', + value => $borrower{'surname'} . ' / AddMember', + } + ); + } + } + } + + return { + feedback => \@feedback, + errors => \@errors, + imported => $imported, + overwritten => $overwritten, + already_in_db => $alreadyindb, + invalid => $invalid, + }; +} + +END { } # module clean-up code here (global destructor) + +1; + +__END__ + +=head1 AUTHOR + +Koha Team + +=cut diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tt index d9f25db..d894adc 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tt @@ -33,203 +33,255 @@

Import patrons

[% IF ( uploadborrowers ) %] -
Import results :
-
    -
  • [% imported %] imported records [% IF ( lastimported ) %](last was [% lastimported %])[% END %]
    • -
  • [% overwritten %] overwritten [% IF ( lastoverwritten ) %](last was [% lastoverwritten %])[% END %]
    • -
  • [% alreadyindb %] not imported because already in borrowers table and overwrite disabled [% IF ( lastalreadyindb ) %](last was [% lastalreadyindb %])[% END %]
    • -
  • [% invalid %] not imported because they are not in the expected format [% IF ( lastinvalid ) %](last was [% lastinvalid %])[% END %]
    • -
  • [% total %] records parsed
    • -
  • Back to Tools
    • -
- [% IF ( FEEDBACK ) %] -

-
-
Feedback:
- -
- [% END %] - [% IF ( ERRORS ) %] -

-
-
Error analysis:
+
Import results :
    - [% FOREACH ERROR IN ERRORS %] - [% IF ( ERROR.badheader ) %]
  • Header row could not be parsed
    • [% END %] - [% FOREACH missing_critical IN ERROR.missing_criticals %] -
  • - Line [% missing_critical.line %] - [% IF ( missing_critical.badparse ) %] - could not be parsed! - [% ELSIF ( missing_critical.bad_date ) %] - has "[% missing_critical.key %]" in unrecognized format: "[% missing_critical.value %]" - [% ELSE %] - Critical field "[% missing_critical.key %]" - [% IF ( missing_critical.branch_map ) %]has unrecognized value "[% missing_critical.value %]" - [% ELSIF ( missing_critical.category_map ) %]has unrecognized value "[% missing_critical.value %]" - [% ELSE %]missing +
  • [% imported %] imported records [% IF ( lastimported ) %](last was [% lastimported %])[% END %]
    • +
  • [% overwritten %] overwritten [% IF ( lastoverwritten ) %](last was [% lastoverwritten %])[% END %]
    • +
  • [% alreadyindb %] not imported because already in borrowers table and overwrite disabled [% IF ( lastalreadyindb ) %](last was [% lastalreadyindb %])[% END %]
    • +
  • [% invalid %] not imported because they are not in the expected format [% IF ( lastinvalid ) %](last was [% lastinvalid %])[% END %]
    • +
  • [% total %] records parsed
    • +
  • Back to Tools
    • +
+ + [% IF ( feedback ) %] +

+ +
+
Feedback:
+ +
+ [% END %] + + [% IF ( errors ) %] +

+ +
+
Error analysis:
+
    + [% FOREACH e IN errors %] + [% IF ( e.badheader ) %]
  • Header row could not be parsed
    • [% END %] + + [% FOREACH missing_critical IN e.missing_criticals %] +
  • + Line [% missing_critical.line %] + + [% IF ( missing_critical.badparse ) %] + could not be parsed! + [% ELSIF ( missing_critical.bad_date ) %] + has "[% missing_critical.key %]" in unrecognized format: "[% missing_critical.value %]" + [% ELSE %] + Critical field "[% missing_critical.key %]" + + [% IF ( missing_critical.branch_map ) %] + has unrecognized value "[% missing_critical.value %]" + [% ELSIF ( missing_critical.category_map ) %] + has unrecognized value "[% missing_critical.value %]" + [% ELSE %] + missing + [% END %] + + (borrowernumber: [% missing_critical.borrowernumber %]; surname: [% missing_critical.surname %]). + [% END %] + +
    + [% missing_critical.lineraw %] +
    • + [% END %] + + [% IF e.invalid_cardnumber %] +
  • + Cardnumber [% e.cardnumber %] is not a valid cardnumber + [% IF e.borrowernumber %] (for patron with borrowernumber [% e.borrowernumber %])[% END %] +
    • + [% END %] + [% IF e.duplicate_userid %] +
  • + Userid [% e.userid %] is already used by another patron. +
    • + [% END %] [% END %] - (borrowernumber: [% missing_critical.borrowernumber %]; surname: [% missing_critical.surname %]). - [% END %] -
    [% missing_critical.lineraw %] - - [% END %] - [% IF ERROR.invalid_cardnumber %] -
  • - Cardnumber [% ERROR.cardnumber %] is not a valid cardnumber - [% IF ERROR.borrowernumber %] (for patron with borrowernumber [% ERROR.borrowernumber %])[% END %] -
    • - [% END %] - [% IF ERROR.duplicate_userid %] -
  • - Userid [% ERROR.userid %] is already used by another patron. -
    • - [% END %] +
+
[% END %] - -
- [% END %] [% ELSE %] -
    -
  • Select a file to import into the borrowers table.
    • -
  • If a cardnumber exists in the table, you can choose whether to ignore the new one or overwrite the old one.
    • -
-
-
-Import into the borrowers table -
    -
  1. - - -
    2. -
-
- Field to use for record matching -
    -
  1. - +
    2. +
+
+ +
+ Field to use for record matching +
    +
  1. + +
    2. +
+
+ +
+ Default values + +
    + [% FOREACH borrower_field IN borrower_fields %] + + [% SWITCH borrower_field.field %] + [% CASE 'branchcode' %] +
  1. + + + [% borrower_field.field %] +
    2. + [% CASE 'categorycode' %] +
  2. + + + [% borrower_field.field %] +
    3. + [% CASE %] +
  3. + + + [% borrower_field.field %] +
    4. [% END %] - - -
-
-
-Default values -
    -[% FOREACH borrower_field IN borrower_fields %] - [% SWITCH borrower_field.field %] - [% CASE 'branchcode' %] -
  1. - - [% borrower_field.field %] -
    2. - [% CASE 'categorycode' %] -
  2. - - + patron_attributes +
    3. + [% END %] + +
+
+ +
+ If matching record is already in the borrowers table: + +
    +
  1. + +
    2. + +
  2. + +
    3. +
+
+ + [% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %] +
+ Patron attributes + +
    +
  1. + +
    2. + +
  2. + +
    3. +
+
[% END %] - [% borrower_field.field %] - - [% CASE %] -
  • - - [% borrower_field.field %] -
    • - [% END %] -[% END %] -[% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %] -
  • - - - patron_attributes -
    • + +
    +
    [% END %] - -
    - If matching record is already in the borrowers table: -
    1. - + +
    + +
    +

    Notes:

    +
      +
    • The first line in the file must be a header row defining which columns you are supplying in the import file.
      • + +
    • Download a starter CSV file with all the columns here. Values are comma-separated.
      • + +
    • + OR choose which fields you want to supply from the following list: +
        +
      • + [% FOREACH columnkey IN borrower_fields %]'[% columnkey.field %]', [% END %] +
        • +
      • -
    • - + + [% IF ( ExtendedPatronAttributes ) %] +
    • + If loading patron attributes, the 'patron_attributes' field should contain a comma-separated list of attribute types and values. The attribute type code and a colon should precede each value. For example: INSTID:12345,LANG:fr or STARTDATE:January 1 2010,TRACK:Day. If an input record has more than one attribute, the fields should either be entered as an unquoted string (previous examples), or with each field wrapped in separate double quotes and delimited by a comma: "STARTDATE:January 1, 2010","TRACK:Day". The second syntax would be required if the data might have a comma in it, like a date string. +
      • + [% END %] + +
    • + The fields 'branchcode' and 'categorycode' are required and must match valid entries in your database.
      • - - - [% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %] -
      - Patron attributes -
      1. - + +
      2. + 'password' should be stored in plaintext, and will be converted to a Bcrypt hash (if your passwords are already encrypted, talk to your system administrator about options).
        3. -
      3. - + +
      4. + Date formats should match your system preference, and must be zero-padded, e.g. '01/02/2008'. Alternatively, +you can supply dates in ISO format (e.g., '2010-10-28').
        5. -
      -
      - [% END %] -
      - -[% END %] +
    +
    -
    -

    Notes:

    -
      -
    • Header: The first line in the file must be a header row defining which columns you are supplying in the import file.
      • -
    • Separator: Values are comma-separated.
      • -
    • Starter CSV: Koha provides a starter CSV with all the columns. - -
      • -
    • Field list: Alternatively, you can create your own CSV and choose which fields you want to supply from the following list: -
      • - [% FOREACH columnkey IN borrower_fields %]'[% columnkey.field %]', [% END %] -
      -
      • -[% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %] -
    • Extended patron attributes: If loading patron attributes, the 'patron_attributes' field should contain a comma-separated list of attribute types and values. The attribute type code and a colon should precede each value. -
      • Example 1: INSTID:12345,LANG:fr
      • Example 2: STARTDATE:January 1 2010,TRACK:Day
      -If an input record has more than one attribute, the fields should either be entered as an unquoted string (previous examples), or with each field wrapped in separate double quotes and delimited by a comma: -
      • Example 3: "STARTDATE:January 1, 2010","TRACK:Day"
      -The second syntax would be required if the data might have a comma in it, like a date string.
      • -[% END %] -
    • Required fields: The fields 'branchcode' and 'categorycode' are required and must match valid entries in your database.
      • -
    • Password: Values for the field 'password' should be stored in plaintext, and will be converted to a Bcrypt hash (if your passwords are already encrypted, talk to your system administrator about options).
      • -
    • Date formats: Date values should match your system preference, and must be zero-padded. -
      • Example: '01/02/2008'
      -Alternatively, you can supply dates in ISO format. -
      • Example: '2010-10-28'
      -
      • -
    - -
    - - - -
    -[% INCLUDE 'tools-menu.inc' %] -
    - + + + + +
    + [% INCLUDE 'tools-menu.inc' %] +
    + + [% INCLUDE 'intranet-bottom.inc' %] diff --git a/tools/import_borrowers.pl b/tools/import_borrowers.pl index b6b5312..41b776c 100755 --- a/tools/import_borrowers.pl +++ b/tools/import_borrowers.pl @@ -34,22 +34,19 @@ # dates should be in the format you have set up Koha to expect # branchcode and categorycode need to be valid -use strict; -use warnings; +use Modern::Perl; use C4::Auth; use C4::Output; -use C4::Context; -use C4::Branch qw/GetBranchesLoop GetBranchName/; -use C4::Members; -use C4::Members::Attributes qw(:all); -use C4::Members::AttributeTypes; -use C4::Members::Messaging; -use C4::Reports::Guided; use C4::Templates; use Koha::Patron::Debarments; use Koha::DateUtils; +use C4::Branch; +use C4::Members; + +use Koha::Patrons::Import qw(import_patrons); + use Text::CSV; # Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals: @@ -61,16 +58,12 @@ use CGI qw ( -utf8 ); # use encoding 'utf8'; # don't do this my ( @errors, @feedback ); -my $extended = C4::Context->preference('ExtendedPatronAttributes'); -my $set_messaging_prefs = C4::Context->preference('EnhancedMessagingPreferences'); -my @columnkeys = C4::Members::columns(); -@columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } @columnkeys; -if ($extended) { - push @columnkeys, 'patron_attributes'; -} +my $extended = C4::Context->preference('ExtendedPatronAttributes'); + +my @columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } C4::Members::columns(); +push( @columnkeys, 'patron_attributes' ) if $extended; my $input = CGI->new(); -our $csv = Text::CSV->new( { binary => 1 } ); # binary needed for non-ASCII Unicode #push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend}; #XXX @@ -97,6 +90,7 @@ $columns = [ grep { $_->{field} ne 'borrowernumber' ? $_ : () } @$columns ]; $template->param( borrower_fields => $columns ); if ( $input->param('sample') ) { + our $csv = Text::CSV->new( { binary => 1 } ); # binary needed for non-ASCII Unicode print $input->header( -type => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ? -attachment => 'patron_import.csv', @@ -105,322 +99,52 @@ if ( $input->param('sample') ) { print $csv->string, "\n"; exit 0; } + my $uploadborrowers = $input->param('uploadborrowers'); my $matchpoint = $input->param('matchpoint'); if ($matchpoint) { $matchpoint =~ s/^patron_attribute_//; } -my $overwrite_cardnumber = $input->param('overwrite_cardnumber'); $template->param( SCRIPT_NAME => '/cgi-bin/koha/tools/import_borrowers.pl' ); if ( $uploadborrowers && length($uploadborrowers) > 0 ) { - push @feedback, { feedback => 1, name => 'filename', value => $uploadborrowers, filename => $uploadborrowers }; - my $handle = $input->upload('uploadborrowers'); - my $uploadinfo = $input->uploadInfo($uploadborrowers); - foreach ( keys %$uploadinfo ) { - push @feedback, { feedback => 1, name => $_, value => $uploadinfo->{$_}, $_ => $uploadinfo->{$_} }; - } - my $imported = 0; - my $alreadyindb = 0; - my $overwritten = 0; - my $invalid = 0; - my $matchpoint_attr_type; + my $handle = $input->upload('uploadborrowers'); my %defaults = $input->Vars; - # use header line to construct key to column map - my $borrowerline = <$handle>; - my $status = $csv->parse($borrowerline); - ($status) or push @errors, { badheader => 1, line => $., lineraw => $borrowerline }; - my @csvcolumns = $csv->fields(); - my %csvkeycol; - my $col = 0; - foreach my $keycol (@csvcolumns) { - - # columnkeys don't contain whitespace, but some stupid tools add it - $keycol =~ s/ +//g; - $csvkeycol{$keycol} = $col++; - } - - #warn($borrowerline); - my $ext_preserve = $input->param('ext_preserve') || 0; - if ($extended) { - $matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint); - } - - push @feedback, { feedback => 1, name => 'headerrow', value => join( ', ', @csvcolumns ) }; - my $today_iso = output_pref( { dt => dt_from_string, dateonly => 1, dateformat => 'iso' } ); - my @criticals = qw(surname branchcode categorycode); # there probably should be others - my @bad_dates; # I've had a few. - LINE: while ( my $borrowerline = <$handle> ) { - my %borrower; - my @missing_criticals; - my $patron_attributes; - my $status = $csv->parse($borrowerline); - my @columns = $csv->fields(); - if ( !$status ) { - push @missing_criticals, { badparse => 1, line => $., lineraw => $borrowerline }; - } - elsif ( @columns == @columnkeys ) { - @borrower{@columnkeys} = @columns; - - # MJR: try to fill blanks gracefully by using default values - foreach my $key (@columnkeys) { - if ( $borrower{$key} !~ /\S/ ) { - $borrower{$key} = $defaults{$key}; - } - } - } - else { - # MJR: try to recover gracefully by using default values - foreach my $key (@columnkeys) { - if ( defined( $csvkeycol{$key} ) and $columns[ $csvkeycol{$key} ] =~ /\S/ ) { - $borrower{$key} = $columns[ $csvkeycol{$key} ]; - } - elsif ( $defaults{$key} ) { - $borrower{$key} = $defaults{$key}; - } - elsif ( scalar grep { $key eq $_ } @criticals ) { - - # a critical field is undefined - push @missing_criticals, { key => $key, line => $., lineraw => $borrowerline }; - } - else { - $borrower{$key} = ''; - } - } - } - - #warn join(':',%borrower); - if ( $borrower{categorycode} ) { - push @missing_criticals, - { - key => 'categorycode', - line => $., - lineraw => $borrowerline, - value => $borrower{categorycode}, - category_map => 1 - } - unless GetBorrowercategory( $borrower{categorycode} ); - } - else { - push @missing_criticals, { key => 'categorycode', line => $., lineraw => $borrowerline }; - } - if ( $borrower{branchcode} ) { - push @missing_criticals, - { - key => 'branchcode', - line => $., - lineraw => $borrowerline, - value => $borrower{branchcode}, - branch_map => 1 - } - unless GetBranchName( $borrower{branchcode} ); - } - else { - push @missing_criticals, { key => 'branchcode', line => $., lineraw => $borrowerline }; + my $return = Koha::Patrons::Import::import_patrons( + { + file => $handle, + defaults => \%defaults, + matchpoint => $matchpoint, + overwrite_cardnumber => $input->param('overwrite_cardnumber'), + preserve_extended_attributes => $input->param('ext_preserve') || 0, } - if (@missing_criticals) { - foreach (@missing_criticals) { - $_->{borrowernumber} = $borrower{borrowernumber} || 'UNDEF'; - $_->{surname} = $borrower{surname} || 'UNDEF'; - } - $invalid++; - ( 25 > scalar @errors ) and push @errors, { missing_criticals => \@missing_criticals }; - - # The first 25 errors are enough. Keeping track of 30,000+ would destroy performance. - next LINE; - } - if ($extended) { - my $attr_str = $borrower{patron_attributes}; - $attr_str =~ s/\xe2\x80\x9c/"/g; # fixup double quotes in case we are passed smart quotes - $attr_str =~ s/\xe2\x80\x9d/"/g; - push @feedback, - { feedback => 1, name => 'attribute string', value => $attr_str, filename => $uploadborrowers }; - delete $borrower{patron_attributes} - ; # not really a field in borrowers, so we don't want to pass it to ModMember. - $patron_attributes = extended_attributes_code_value_arrayref($attr_str); - } - - # Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it. - foreach (qw(dateofbirth dateenrolled dateexpiry)) { - my $tempdate = $borrower{$_} or next; - $tempdate = - eval { output_pref( { dt => dt_from_string($tempdate), dateonly => 1, dateformat => 'iso' } ); }; - if ($tempdate) { - $borrower{$_} = $tempdate; - } - else { - $borrower{$_} = ''; - push @missing_criticals, { key => $_, line => $., lineraw => $borrowerline, bad_date => 1 }; - } - } - $borrower{dateenrolled} = $today_iso unless $borrower{dateenrolled}; - $borrower{dateexpiry} = GetExpiryDate( $borrower{categorycode}, $borrower{dateenrolled} ) - unless $borrower{dateexpiry}; - my $borrowernumber; - my $member; - if ( ( $matchpoint eq 'cardnumber' ) && ( $borrower{'cardnumber'} ) ) { - $member = GetMember( 'cardnumber' => $borrower{'cardnumber'} ); - if ($member) { - $borrowernumber = $member->{'borrowernumber'}; - } - } - elsif ( ( $matchpoint eq 'userid' ) && ( $borrower{'userid'} ) ) { - $member = GetMember( 'userid' => $borrower{'userid'} ); - if ($member) { - $borrowernumber = $member->{'borrowernumber'}; - } - } - elsif ($extended) { - if ( defined($matchpoint_attr_type) ) { - foreach my $attr (@$patron_attributes) { - if ( $attr->{code} eq $matchpoint and $attr->{value} ne '' ) { - my @borrowernumbers = $matchpoint_attr_type->get_patrons( $attr->{value} ); - $borrowernumber = $borrowernumbers[0] if scalar(@borrowernumbers) == 1; - last; - } - } - } - } - - if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) { - push @errors, - { - invalid_cardnumber => 1, - borrowernumber => $borrowernumber, - cardnumber => $borrower{cardnumber} - }; - $invalid++; - next; - } - - if ($borrowernumber) { - - # borrower exists - unless ($overwrite_cardnumber) { - $alreadyindb++; - $template->param( 'lastalreadyindb' => $borrower{'surname'} . ' / ' . $borrowernumber ); - next LINE; - } - $borrower{'borrowernumber'} = $borrowernumber; - for my $col ( keys %borrower ) { - - # use values from extant patron unless our csv file includes this column or we provided a default. - # FIXME : You cannot update a field with a perl-evaluated false value using the defaults. - - # The password is always encrypted, skip it! - next if $col eq 'password'; - - unless ( exists( $csvkeycol{$col} ) || $defaults{$col} ) { - $borrower{$col} = $member->{$col} if ( $member->{$col} ); - } - } - - # Check if the userid provided does not exist yet - if ( exists $borrower{userid} - and $borrower{userid} - and not Check_Userid( $borrower{userid}, $borrower{borrowernumber} ) ) { - push @errors, { duplicate_userid => 1, userid => $borrower{userid} }; - $invalid++; - next LINE; - } - - unless (ModMember(%borrower)) { - $invalid++; - - # until we have better error trapping, we have no way of knowing why ModMember errored out... - push @errors, { unknown_error => 1 }; - $template->param( 'lastinvalid' => $borrower{'surname'} . ' / ' . $borrowernumber ); - next LINE; - } - - # Don't add a new restriction if the existing 'combined' restriction matches this one - if ( $borrower{debarred} && ( ( $borrower{debarred} ne $member->{debarred} ) || ( $borrower{debarredcomment} ne $member->{debarredcomment} ) ) ) { - # Check to see if this debarment already exists - my $debarrments = GetDebarments( - { - borrowernumber => $borrowernumber, - expiration => $borrower{debarred}, - comment => $borrower{debarredcomment} - } - ); - - # If it doesn't, then add it! - unless (@$debarrments) { - AddDebarment( - { - borrowernumber => $borrowernumber, - expiration => $borrower{debarred}, - comment => $borrower{debarredcomment} - } - ); - } - } - - if ($extended) { - if ($ext_preserve) { - my $old_attributes = GetBorrowerAttributes($borrowernumber); - $patron_attributes = extended_attributes_merge( $old_attributes, $patron_attributes ) - ; #TODO: expose repeatable options in template - } - - push @errors, { unknown_error => 1 } - unless SetBorrowerAttributes( $borrower{'borrowernumber'}, $patron_attributes ); - } - $overwritten++; - $template->param( 'lastoverwritten' => $borrower{'surname'} . ' / ' . $borrowernumber ); - } - else { - # FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either. - # At least this is closer to AddMember than in members/memberentry.pl - if ( !$borrower{'cardnumber'} ) { - $borrower{'cardnumber'} = fixup_cardnumber(undef); - } - if ( $borrowernumber = AddMember(%borrower) ) { + ); - if ( $borrower{debarred} ) { - AddDebarment( - { - borrowernumber => $borrowernumber, - expiration => $borrower{debarred}, - comment => $borrower{debarredcomment} - } - ); - } + my $feedback = $return->{feedback}; + my $errors = $return->{errors}; + my $imported = $return->{imported}; + my $overwritten = $return->{overwritten}; + my $alreadyindb = $return->{already_in_db}; + my $invalid = $return->{invalid}; - if ($extended) { - SetBorrowerAttributes( $borrowernumber, $patron_attributes ); - } + my $uploadinfo = $input->uploadInfo($uploadborrowers); + foreach ( keys %$uploadinfo ) { + push @$feedback, { feedback => 1, name => $_, value => $uploadinfo->{$_}, $_ => $uploadinfo->{$_} }; + } - if ($set_messaging_prefs) { - C4::Members::Messaging::SetMessagingPreferencesFromDefaults( - { - borrowernumber => $borrowernumber, - categorycode => $borrower{categorycode} - } - ); - } + push @$feedback, { feedback => 1, name => 'filename', value => $uploadborrowers, filename => $uploadborrowers }; - $imported++; - $template->param( 'lastimported' => $borrower{'surname'} . ' / ' . $borrowernumber ); - } - else { - $invalid++; - push @errors, { unknown_error => 1 }; - $template->param( 'lastinvalid' => $borrower{'surname'} . ' / AddMember' ); - } - } - } - (@errors) and $template->param( ERRORS => \@errors ); - (@feedback) and $template->param( FEEDBACK => \@feedback ); $template->param( - 'uploadborrowers' => 1, - 'imported' => $imported, - 'overwritten' => $overwritten, - 'alreadyindb' => $alreadyindb, - 'invalid' => $invalid, - 'total' => $imported + $alreadyindb + $invalid + $overwritten, + uploadborrowers => 1, + errors => $errors, + feedback => $feedback, + imported => $imported, + overwritten => $overwritten, + alreadyindb => $alreadyindb, + invalid => $invalid, + total => $imported + $alreadyindb + $invalid + $overwritten, ); } -- 2.1.4