From 83530a6e8f18dd980a1b29524055fba45f696f8b Mon Sep 17 00:00:00 2001 From: Florent Mara Date: Tue, 3 May 2016 16:19:38 +1200 Subject: [PATCH] Bug 12598 - Refactoring Koha::Patrons::Import includes bug fixed for critical date types and header column parsing Signed-off-by: Kyle M Hall --- Koha/Patrons/Import.pm | 159 +++++++++++++-------- t/db_dependent/Koha/Patrons/Import.t | 259 +++++++++++++++++++++++++++++------ 2 files changed, 320 insertions(+), 98 deletions(-) diff --git a/Koha/Patrons/Import.pm b/Koha/Patrons/Import.pm index 018f609..e53bbac 100644 --- a/Koha/Patrons/Import.pm +++ b/Koha/Patrons/Import.pm @@ -25,6 +25,7 @@ use Text::CSV; use C4::Members; use C4::Branch; use C4::Members::Attributes qw(:all); +use C4::Members::AttributeTypes; use Koha::DateUtils; @@ -52,17 +53,11 @@ Further pod documentation needed here. =cut -has 'today_iso' => ( - is => 'ro', - lazy => 1, - default => sub { output_pref( { dt => dt_from_string(), dateonly => 1, dateformat => 'iso' } ); }, - ); +has 'today_iso' => ( is => 'ro', lazy => 1, + default => sub { output_pref( { dt => dt_from_string(), dateonly => 1, dateformat => 'iso' } ); }, ); -has 'text_csv' => ( - is => 'ro', - lazy => 1, - default => sub { Text::CSV->new( { binary => 1 } ), }, - ); +has 'text_csv' => ( is => 'rw', lazy => 1, + default => sub { Text::CSV->new( { binary => 1, } ); }, ); sub import_patrons { my ($self, $params) = @_; @@ -77,7 +72,7 @@ sub import_patrons { my $extended = C4::Context->preference('ExtendedPatronAttributes'); my $set_messaging_prefs = C4::Context->preference('EnhancedMessagingPreferences'); - my @columnkeys = set_column_keys($extended); + my @columnkeys = $self->set_column_keys($extended); my @feedback; my @errors; @@ -85,28 +80,14 @@ sub import_patrons { my $alreadyindb = 0; my $overwritten = 0; my $invalid = 0; - my $matchpoint_attr_type; + my $matchpoint_attr_type = $self->set_attribute_types({ extended => $extended, matchpoint => $matchpoint, }); - # use header line to construct key to column map - my $borrowerline = <$handle>; - my $status = $self->text_csv->parse($borrowerline); - ($status) or push @errors, { badheader => 1, line => $., lineraw => $borrowerline }; - - my @csvcolumns = $self->text_csv->fields(); + # Use header line to construct key to column map my %csvkeycol; - my $col = 0; - foreach my $keycol (@csvcolumns) { - - # columnkeys don't contain whitespace, but some stupid tools add it - $keycol =~ s/ +//g; - $csvkeycol{$keycol} = $col++; - } - - if ($extended) { - $matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint); - } + my $borrowerline = <$handle>; + my @csvcolumns = $self->prepare_columns({headerrow => $borrowerline, keycol => \%csvkeycol, errors => \@errors, }); + push(@feedback, { feedback => 1, name => 'headerrow', value => join( ', ', @csvcolumns ) }); - push @feedback, { feedback => 1, name => 'headerrow', value => join( ', ', @csvcolumns ) }; my @criticals = qw( surname ); # there probably should be others - rm branchcode && categorycode LINE: while ( my $borrowerline = <$handle> ) { my $line_number = $.; @@ -116,7 +97,7 @@ sub import_patrons { my $status = $self->text_csv->parse($borrowerline); my @columns = $self->text_csv->fields(); if ( !$status ) { - push @missing_criticals, { badparse => 1, line => $., lineraw => $borrowerline }; + push @missing_criticals, { badparse => 1, line => $line_number, lineraw => $borrowerline }; } elsif ( @columns == @columnkeys ) { @borrower{@columnkeys} = @columns; @@ -148,11 +129,15 @@ sub import_patrons { } } - # Checking if borrower category code exists and if it matches to a known category. Pushing error to missing_criticals otherwise. - check_borrower_category($borrower{categorycode}, $borrowerline, $line_number, \@missing_criticals); + # Check if borrower category code exists and if it matches to a known category. Pushing error to missing_criticals otherwise. + $self->check_borrower_category($borrower{categorycode}, $borrowerline, $line_number, \@missing_criticals); + + # Check if branch code exists and if it matches to a branch name. Pushing error to missing_criticals otherwise. + $self->check_branch_code($borrower{branchcode}, $borrowerline, $line_number, \@missing_criticals); + + # Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it. + $self->format_dates({borrower => \%borrower, lineraw => $borrowerline, line => $line_number, missing_criticals => \@missing_criticals, }); - # Checking if branch code exists and if it matches to a branch name. Pushing error to missing_criticals otherwise. - check_branch_code($borrower{branchcode}, $borrowerline, $line_number, \@missing_criticals); if (@missing_criticals) { foreach (@missing_criticals) { @@ -166,25 +151,11 @@ sub import_patrons { next LINE; } - # Setting patron attributes if extended. - my $patron_attributes = set_patron_attributes($extended, $borrower{patron_attributes}, \@feedback); - - # Not really a field in borrowers, so we don't want to pass it to ModMember. - if ($extended) { delete $borrower{patron_attributes}; } + # Set patron attributes if extended. + my $patron_attributes = $self->set_patron_attributes($extended, $borrower{patron_attributes}, \@feedback); + if( $extended ) { delete $borrower{patron_attributes}; } # Not really a field in borrowers. - # Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it. - foreach (qw(dateofbirth dateenrolled dateexpiry)) { - my $tempdate = $borrower{$_} or next; - - $tempdate = eval { output_pref( { dt => dt_from_string( $tempdate ), dateonly => 1, dateformat => 'iso' } ); }; - - if ($tempdate) { - $borrower{$_} = $tempdate; - } else { - $borrower{$_} = ''; - push @missing_criticals, { key => $_, line => $line_number, lineraw => $borrowerline, bad_date => 1 }; - } - } + # Default date enrolled and date expiry if not already set. $borrower{dateenrolled} = $self->today_iso() unless $borrower{dateenrolled}; $borrower{dateexpiry} = GetExpiryDate( $borrower{categorycode}, $borrower{dateenrolled} ) unless $borrower{dateexpiry}; @@ -370,6 +341,53 @@ sub import_patrons { }; } +=head2 prepare_columns + + my @csvcolumns = $self->prepare_columns({headerrow => $borrowerline, keycol => \%csvkeycol, errors => \@errors, }); + +Returns an array of all column key and populates a hash of colunm key positions. + +=cut + +sub prepare_columns { + my ($self, $params) = @_; + + my $status = $self->text_csv->parse($params->{headerrow}); + unless( $status ) { + push( @{$params->{errors}}, { badheader => 1, line => 1, lineraw => $params->{headerrow} }); + return; + } + + my @csvcolumns = $self->text_csv->fields(); + my $col = 0; + foreach my $keycol (@csvcolumns) { + # columnkeys don't contain whitespace, but some stupid tools add it + $keycol =~ s/ +//g; + $params->{keycol}->{$keycol} = $col++; + } + + return @csvcolumns; +} + +=head2 set_attribute_types + + my $matchpoint_attr_type = $self->set_attribute_types({ extended => $extended, matchpoint => $matchpoint, }); + +Returns an attribute type based on matchpoint parameter. + +=cut + +sub set_attribute_types { + my ($self, $params) = @_; + + my $attribute_types; + if( $params->{extended} ) { + $attribute_types = C4::Members::AttributeTypes->fetch($params->{matchpoint}); + } + + return $attribute_types; +} + =head2 set_column_keys my @columnkeys = set_column_keys($extended); @@ -379,7 +397,7 @@ Returns an array of borrowers' table columns. =cut sub set_column_keys { - my ($extended) = @_; + my ($self, $extended) = @_; my @columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } C4::Members::columns(); push( @columnkeys, 'patron_attributes' ) if $extended; @@ -396,9 +414,9 @@ Returns a reference to array of hashrefs data structure as expected by SetBorrow =cut sub set_patron_attributes { - my ($extended, $patron_attributes, $feedback) = @_; + my ($self, $extended, $patron_attributes, $feedback) = @_; - unless($extended) { return; } + unless( $extended ) { return; } unless( defined($patron_attributes) ) { return; } # Fixup double quotes in case we are passed smart quotes @@ -421,7 +439,7 @@ Pushes a 'missing_criticals' error entry if no branch code or branch code does n =cut sub check_branch_code { - my ($branchcode, $borrowerline, $line_number, $missing_criticals) = @_; + my ($self, $branchcode, $borrowerline, $line_number, $missing_criticals) = @_; # No branch code unless( $branchcode ) { @@ -446,7 +464,7 @@ Pushes a 'missing_criticals' error entry if no category code or category code do =cut sub check_borrower_category { - my ($categorycode, $borrowerline, $line_number, $missing_criticals) = @_; + my ($self, $categorycode, $borrowerline, $line_number, $missing_criticals) = @_; # No branch code unless( $categorycode ) { @@ -462,6 +480,31 @@ sub check_borrower_category { } } +=head2 format_dates + + format_dates({borrower => \%borrower, lineraw => $lineraw, line => $line_number, missing_criticals => \@missing_criticals, }); + +Pushes a 'missing_criticals' error entry for each of the 3 date types dateofbirth, dateenrolled and dateexpiry if it can not +be formatted to the chosen date format. Populates the correctly formatted date otherwise. + +=cut + +sub format_dates { + my ($self, $params) = @_; + + foreach my $date_type (qw(dateofbirth dateenrolled dateexpiry)) { + my $tempdate = $params->{borrower}->{$date_type} or next(); + my $formatted_date = eval { output_pref( { dt => dt_from_string( $tempdate ), dateonly => 1, dateformat => 'iso' } ); }; + + if ($formatted_date) { + $params->{borrower}->{$date_type} = $formatted_date; + } else { + $params->{borrower}->{$date_type} = ''; + push (@{$params->{missing_criticals}}, { key => $date_type, line => $params->{line}, lineraw => $params->{lineraw}, bad_date => 1 }); + } + } +} + 1; =head1 AUTHOR diff --git a/t/db_dependent/Koha/Patrons/Import.t b/t/db_dependent/Koha/Patrons/Import.t index 651b331..8714968 100644 --- a/t/db_dependent/Koha/Patrons/Import.t +++ b/t/db_dependent/Koha/Patrons/Import.t @@ -18,7 +18,7 @@ # along with Koha; if not, see . use Modern::Perl; -use Test::More tests => 98; +use Test::More tests => 122; use Test::MockModule; use Koha::Database; @@ -42,7 +42,14 @@ subtest 'test_methods' => sub { plan tests => 1; # Given ... we can reach the method(s) - my @methods = ('import_patrons', 'set_column_keys', 'set_patron_attributes', 'check_branch_code'); + my @methods = ('import_patrons', + 'set_attribute_types', + 'prepare_columns', + 'set_column_keys', + 'set_patron_attributes', + 'check_branch_code', + 'format_dates', + ); can_ok('Koha::Patrons::Import', @methods); }; @@ -66,8 +73,9 @@ my $result_0 = $patrons_import->import_patrons($params_0); is($result_0, undef, 'Got the expected undef from import_patrons with no file handle'); # Given ... a file handle to file with headers only. -my $csv_headers = 'cardnumber,surname,firstname,title,othernames,initials,streetnumber,streettype,address,address2,city,state,zipcode,country,email,phone,mobile,fax,dateofbirth,branchcode,categorycode,dateenrolled,dateexpiry,userid,password'; -my $csv_one_line = '1000,Nancy,Jenkins,Dr,,NJ,78,Circle,Bunting,El Paso,Henderson,Texas,79984,United States,ajenkins0@sourceforge.net,7-(388)559-6763,3-(373)151-4471,8-(509)286-4001,16/10/1965,CPL,PT,28/12/2014,01/07/2015,jjenkins0,DPQILy'; +my $csv_headers = 'cardnumber,surname,firstname,title,othernames,initials,streetnumber,streettype,address,address2,city,state,zipcode,country,email,phone,mobile,fax,dateofbirth,branchcode,categorycode,dateenrolled,dateexpiry,userid,password'; +my $res_header = 'cardnumber, surname, firstname, title, othernames, initials, streetnumber, streettype, address, address2, city, state, zipcode, country, email, phone, mobile, fax, dateofbirth, branchcode, categorycode, dateenrolled, dateexpiry, userid, password'; +my $csv_one_line = '1000,Nancy,Jenkins,Dr,,NJ,78,Circle,Bunting,El Paso,Henderson,Texas,79984,United States,ajenkins0@sourceforge.net,7-(388)559-6763,3-(373)151-4471,8-(509)286-4001,10/16/1965,CPL,PT,12/28/2014,07/01/2015,jjenkins0,DPQILy'; my $filename_1 = make_csv($temp_dir, $csv_headers, $csv_one_line); open(my $handle_1, "<", $filename_1) or die "cannot open < $filename_1: $!"; @@ -82,8 +90,7 @@ is(scalar @{$result_1->{errors}}, 0, 'Got the expected 0 size error array from i is($result_1->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons with no matchpoint defined'); is($result_1->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons with no matchpoint defined'); -is($result_1->{feedback}->[0]->{value}, 'cardnumber, surname, firstname, title, othernames, initials, streetnumber, streettype, address, address2, city, state, zipcode, country, email, phone, mobile, fax, dateofbirth, branchcode, categorycode, dateenrolled, dateexpiry, userid, password', - 'Got the expected header row value from import_patrons with no matchpoint defined'); +is($result_1->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons with no matchpoint defined'); is($result_1->{feedback}->[1]->{feedback}, 1, 'Got the expected second feedback from import_patrons with no matchpoint defined'); is($result_1->{feedback}->[1]->{name}, 'lastimported', 'Got the expected last imported name from import_patrons with no matchpoint defined'); @@ -109,8 +116,7 @@ is($result_2->{errors}->[0]->{invalid_cardnumber}, 1, 'Got the expected invalid is($result_2->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons with invalid card number'); is($result_2->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons with invalid card number'); -is($result_2->{feedback}->[0]->{value}, 'cardnumber, surname, firstname, title, othernames, initials, streetnumber, streettype, address, address2, city, state, zipcode, country, email, phone, mobile, fax, dateofbirth, branchcode, categorycode, dateenrolled, dateexpiry, userid, password', - 'Got the expected header row value from import_patrons with invalid card number'); +is($result_2->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons with invalid card number'); is($result_2->{imported}, 0, 'Got the expected 0 imported result from import_patrons with invalid card number'); is($result_2->{invalid}, 1, 'Got the expected 1 invalid result from import_patrons with invalid card number'); @@ -131,8 +137,7 @@ is($result_3->{errors}->[0]->{userid}, 'jjenkins0', 'Got the expected userid err is($result_3->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons with duplicate userid'); is($result_3->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons with duplicate userid'); -is($result_3->{feedback}->[0]->{value}, 'cardnumber, surname, firstname, title, othernames, initials, streetnumber, streettype, address, address2, city, state, zipcode, country, email, phone, mobile, fax, dateofbirth, branchcode, categorycode, dateenrolled, dateexpiry, userid, password', - 'Got the expected header row value from import_patrons with duplicate userid'); +is($result_3->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons with duplicate userid'); is($result_3->{imported}, 0, 'Got the expected 0 imported result from import_patrons with duplicate userid'); is($result_3->{invalid}, 1, 'Got the expected 1 invalid result from import_patrons with duplicate userid'); @@ -140,12 +145,15 @@ is($result_3->{overwritten}, 0, 'Got the expected 0 overwritten result from impo # Given ... a new input and mocked C4::Context my $context = Test::MockModule->new('C4::Context'); -$context->mock('preference', sub { my ($mod, $meth) = @_; if ( $meth eq 'ExtendedPatronAttributes' ) { return 1; } }); +$context->mock('preference', sub { my ($mod, $meth) = @_; + if ( $meth eq 'ExtendedPatronAttributes' ) { return 1; } + if ( $meth eq 'dateformat' ) { return 'us'; } + }); -my $new_input_line = '1001,Donna,Sullivan,Mrs,Henry,DS,59,Court,Burrows,Reading,Salt Lake City,Pennsylvania,19605,United States,hsullivan1@purevolume.com,3-(864)009-3006,7-(291)885-8423,1-(879)095-5038,19/09/1970,LPL,PT,04/03/2015,01/07/2015,hsullivan1,8j6P6Dmap'; +my $new_input_line = '1001,Donna,Sullivan,Mrs,Henry,DS,59,Court,Burrows,Reading,Salt Lake City,Pennsylvania,19605,United States,hsullivan1@purevolume.com,3-(864)009-3006,7-(291)885-8423,1-(879)095-5038,09/19/1970,LPL,PT,03/04/2015,07/01/2015,hsullivan1,8j6P6Dmap'; my $filename_4 = make_csv($temp_dir, $csv_headers, $new_input_line); open(my $handle_4, "<", $filename_4) or die "cannot open < $filename_4: $!"; -my $params_4 = { file => $handle_4, matchpoint => 'cardnumber', }; +my $params_4 = { file => $handle_4, matchpoint => 'SHOW_BCODE', }; # When ... my $result_4 = $patrons_import->import_patrons($params_4); @@ -156,8 +164,7 @@ is(scalar @{$result_4->{errors}}, 0, 'Got the expected 0 size error array from i is($result_4->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons with extended user'); is($result_4->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons with extended user'); -is($result_4->{feedback}->[0]->{value}, 'cardnumber, surname, firstname, title, othernames, initials, streetnumber, streettype, address, address2, city, state, zipcode, country, email, phone, mobile, fax, dateofbirth, branchcode, categorycode, dateenrolled, dateexpiry, userid, password', - 'Got the expected header row value from import_patrons with extended user'); +is($result_4->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons with extended user'); is($result_4->{feedback}->[1]->{feedback}, 1, 'Got the expected second feedback from import_patrons with extended user'); is($result_4->{feedback}->[1]->{name}, 'attribute string', 'Got the expected attribute string from import_patrons with extended user'); @@ -174,9 +181,9 @@ is($result_4->{overwritten}, 0, 'Got the expected 0 overwritten result from impo $context->unmock('preference'); # Given ... 3 new inputs. One with no branch code, one with unexpected branch code. -my $input_no_branch = '1002,Johnny,Reynolds,Mr,Patricia,JR,12,Hill,Kennedy,Saint Louis,Colorado Springs,Missouri,63131,United States,preynolds2@washington.edu,7-(925)314-9514,0-(315)973-8956,4-(510)556-2323,18/09/1967,,PT,07/05/2015,01/07/2015,preynolds2,K3HiDzl'; -my $input_good_branch = '1003,Linda,Richardson,Mr,Kimberly,LR,90,Place,Bayside,Atlanta,Erie,Georgia,31190,United States,krichardson3@pcworld.com,8-(035)185-0387,4-(796)518-3676,3-(644)960-3789,13/04/1954,RPL,PT,06/06/2015,01/07/2015,krichardson3,P3EO0MVRPXbM'; -my $input_na_branch = '1005,Ruth,Greene,Mr,Michael,RG,3,Avenue,Grim,Peoria,Jacksonville,Illinois,61614,United States,mgreene5@seesaa.net,3-(941)565-5752,1-(483)885-8138,4-(979)577-6908,09/02/1957,ZZZ,ST,02/04/2015,01/07/2015,mgreene5,or4ORT6JH'; +my $input_no_branch = '1002,Johnny,Reynolds,Mr,Patricia,JR,12,Hill,Kennedy,Saint Louis,Colorado Springs,Missouri,63131,United States,preynolds2@washington.edu,7-(925)314-9514,0-(315)973-8956,4-(510)556-2323,09/18/1967,,PT,05/07/2015,07/01/2015,preynolds2,K3HiDzl'; +my $input_good_branch = '1003,Linda,Richardson,Mr,Kimberly,LR,90,Place,Bayside,Atlanta,Erie,Georgia,31190,United States,krichardson3@pcworld.com,8-(035)185-0387,4-(796)518-3676,3-(644)960-3789,04/13/1954,RPL,PT,06/06/2015,07/01/2015,krichardson3,P3EO0MVRPXbM'; +my $input_na_branch = '1005,Ruth,Greene,Mr,Michael,RG,3,Avenue,Grim,Peoria,Jacksonville,Illinois,61614,United States,mgreene5@seesaa.net,3-(941)565-5752,1-(483)885-8138,4-(979)577-6908,02/09/1957,ZZZ,ST,04/02/2015,07/01/2015,mgreene5,or4ORT6JH'; my $filename_5 = make_csv($temp_dir, $csv_headers, $input_no_branch, $input_good_branch, $input_na_branch); open(my $handle_5, "<", $filename_5) or die "cannot open < $filename_5: $!"; @@ -204,8 +211,7 @@ is($result_5->{errors}->[1]->{missing_criticals}->[0]->{value}, 'ZZZ', 'Got the is($result_5->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for branch tests'); is($result_5->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons for branch tests'); -is($result_5->{feedback}->[0]->{value}, 'cardnumber, surname, firstname, title, othernames, initials, streetnumber, streettype, address, address2, city, state, zipcode, country, email, phone, mobile, fax, dateofbirth, branchcode, categorycode, dateenrolled, dateexpiry, userid, password', - 'Got the expected header row value from import_patrons for branch tests'); +is($result_5->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons for branch tests'); is($result_5->{feedback}->[1]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for branch tests'); is($result_5->{feedback}->[1]->{name}, 'lastimported', 'Got the expected lastimported name from import_patrons for branch tests'); @@ -216,9 +222,9 @@ is($result_5->{invalid}, 2, 'Got the expected 2 invalid result from import patro is($result_5->{overwritten}, 0, 'Got the expected 0 overwritten result from import patrons for branch tests'); # Given ... 3 new inputs. One with no category code, one with unexpected category code. -my $input_no_category = '1006,Christina,Olson,Rev,Kimberly,CO,8,Avenue,Northridge,Lexington,Wilmington,Kentucky,40510,United States,kolson6@dropbox.com,7-(810)636-6048,1-(052)012-8984,8-(567)232-7818,26/03/1952,FFL,,07/09/2014,01/07/2015,kolson6,x5D3qGbLlptx'; -my $input_good_category = '1007,Peter,Peters,Mrs,Lawrence,PP,6,Trail,South,Oklahoma City,Topeka,Oklahoma,73135,United States,lpeters7@bandcamp.com,5-(992)205-9318,0-(732)586-9365,3-(448)146-7936,16/08/1983,PVL,T,24/03/2015,01/07/2015,lpeters7,Z19BrQ4'; -my $input_na_category = '1008,Emily,Richards,Ms,Judy,ER,73,Way,Kedzie,Fort Wayne,Phoenix,Indiana,46825,United States,jrichards8@arstechnica.com,5-(266)658-8957,3-(550)500-9107,7-(816)675-9822,09/08/1984,FFL,ZZ,09/11/2014,01/07/2015,jrichards8,D5PvU6H2R'; +my $input_no_category = '1006,Christina,Olson,Rev,Kimberly,CO,8,Avenue,Northridge,Lexington,Wilmington,Kentucky,40510,United States,kolson6@dropbox.com,7-(810)636-6048,1-(052)012-8984,8-(567)232-7818,03/26/1952,FFL,,09/07/2014,01/07/2015,kolson6,x5D3qGbLlptx'; +my $input_good_category = '1007,Peter,Peters,Mrs,Lawrence,PP,6,Trail,South,Oklahoma City,Topeka,Oklahoma,73135,United States,lpeters7@bandcamp.com,5-(992)205-9318,0-(732)586-9365,3-(448)146-7936,08/16/1983,PVL,T,03/24/2015,07/01/2015,lpeters7,Z19BrQ4'; +my $input_na_category = '1008,Emily,Richards,Ms,Judy,ER,73,Way,Kedzie,Fort Wayne,Phoenix,Indiana,46825,United States,jrichards8@arstechnica.com,5-(266)658-8957,3-(550)500-9107,7-(816)675-9822,08/09/1984,FFL,ZZ,11/09/2014,07/01/2015,jrichards8,D5PvU6H2R'; my $filename_6 = make_csv($temp_dir, $csv_headers, $input_no_category, $input_good_category, $input_na_category); open(my $handle_6, "<", $filename_6) or die "cannot open < $filename_6: $!"; @@ -246,8 +252,7 @@ is($result_6->{errors}->[1]->{missing_criticals}->[0]->{value}, 'ZZ', 'Got the e is($result_6->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for category tests'); is($result_6->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons for category tests'); -is($result_6->{feedback}->[0]->{value}, 'cardnumber, surname, firstname, title, othernames, initials, streetnumber, streettype, address, address2, city, state, zipcode, country, email, phone, mobile, fax, dateofbirth, branchcode, categorycode, dateenrolled, dateexpiry, userid, password', - 'Got the expected header row value from import_patrons for category tests'); +is($result_6->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons for category tests'); is($result_6->{feedback}->[1]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for category tests'); is($result_6->{feedback}->[1]->{name}, 'lastimported', 'Got the expected lastimported name from import_patrons for category tests'); @@ -258,27 +263,133 @@ is($result_6->{invalid}, 2, 'Got the expected 2 invalid result from import patro is($result_6->{overwritten}, 0, 'Got the expected 0 overwritten result from import patrons for category tests'); # Given ... 2 new inputs. One without dateofbirth, dateenrolled and dateexpiry values. -my $input_complete = '1009,Christina,Harris,Dr,Philip,CH,99,Street,Grayhawk,Baton Rouge,Dallas,Louisiana,70810,United States,pharris9@hp.com,9-(317)603-5513,7-(005)062-7593,8-(349)134-1627,19/06/1969,IPT,PT,09/04/2015,01/07/2015,pharris9,NcAhcvvnB'; -my $input_no_date = '1010,Ralph,Warren,Ms,Linda,RW,6,Way,Barby,Orlando,Albany,Florida,32803,United States,lwarrena@multiply.com,7-(579)753-7752,6-(847)086-7566,9-(122)729-8226,,LPL,T,,,lwarrena,tJ56RD4uV'; +my $input_complete = '1009,Christina,Harris,Dr,Philip,CH,99,Street,Grayhawk,Baton Rouge,Dallas,Louisiana,70810,United States,pharris9@hp.com,9-(317)603-5513,7-(005)062-7593,8-(349)134-1627,06/19/1969,IPT,PT,04/09/2015,07/01/2015,pharris9,NcAhcvvnB'; +my $input_no_date = '1010,Ralph,Warren,Ms,Linda,RW,6,Way,Barby,Orlando,Albany,Florida,32803,United States,lwarrena@multiply.com,7-(579)753-7752,6-(847)086-7566,9-(122)729-8226,26/01/2001,LPL,T,25/01/2001,24/01/2001,lwarrena,tJ56RD4uV'; my $filename_7 = make_csv($temp_dir, $csv_headers, $input_complete, $input_no_date); open(my $handle_7, "<", $filename_7) or die "cannot open < $filename_7: $!"; my $params_7 = { file => $handle_7, matchpoint => 'cardnumber', }; -# Need upgrade to Moo +# When ... +my $result_7 = $patrons_import->import_patrons($params_7); + +# Then ... +is($result_7->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons for dates tests'); +is(scalar @{$result_7->{errors}}, 1, 'Got the expected 1 error array size from import_patrons for dates tests'); +is(scalar @{$result_7->{errors}->[0]->{missing_criticals}}, 3, 'Got the expected 3 missing critical errors from import_patrons for dates tests'); + +is($result_7->{errors}->[0]->{missing_criticals}->[0]->{bad_date}, 1, 'Got the expected 1 bad_date error from import patrons for dates tests'); +is($result_7->{errors}->[0]->{missing_criticals}->[0]->{borrowernumber}, 'UNDEF', 'Got the expected undef borrower number error from import patrons for dates tests'); +is($result_7->{errors}->[0]->{missing_criticals}->[0]->{key}, 'dateofbirth', 'Got the expected dateofbirth key from import patrons for dates tests'); +is($result_7->{errors}->[0]->{missing_criticals}->[0]->{line}, 3, 'Got the expected 2 line number error from import patrons for dates tests'); +is($result_7->{errors}->[0]->{missing_criticals}->[0]->{lineraw}, $input_no_date."\r\n", 'Got the expected lineraw error from import patrons for dates tests'); +is($result_7->{errors}->[0]->{missing_criticals}->[0]->{surname}, 'Ralph', 'Got the expected surname error from import patrons for dates tests'); + +is($result_7->{errors}->[0]->{missing_criticals}->[1]->{key}, 'dateenrolled', 'Got the expected dateenrolled key from import patrons for dates tests'); +is($result_7->{errors}->[0]->{missing_criticals}->[2]->{key}, 'dateexpiry', 'Got the expected dateexpiry key from import patrons for dates tests'); + +is(scalar @{$result_7->{feedback}}, 2, 'Got the expected 2 feedback from import patrons for dates tests'); +is($result_7->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for dates tests'); +is($result_7->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons for dates tests'); +is($result_7->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons for dates tests'); + +is($result_7->{feedback}->[1]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for dates tests'); +is($result_7->{feedback}->[1]->{name}, 'lastimported', 'Got the expected lastimported from import_patrons for dates tests'); +like($result_7->{feedback}->[1]->{value}, qr/^Christina \/ \d+/, 'Got the expected lastimported value from import_patrons for dates tests'); + +is($result_7->{imported}, 1, 'Got the expected 1 imported result from import patrons for dates tests'); +is($result_7->{invalid}, 1, 'Got the expected 1 invalid result from import patrons for dates tests'); +is($result_7->{overwritten}, 0, 'Got the expected 0 overwritten result from import patrons for dates tests'); + +subtest 'test_prepare_columns' => sub { + plan tests => 16; + + # Given ... no header row + my $headerrow_0; + my %csvkeycol_0; + my @errors_0; + + # When ... + my @csvcolumns_0 = $patrons_import->prepare_columns({headerrow => undef, keycol => \%csvkeycol_0, errors => \@errors_0, }); + + # Then ... + is(scalar @csvcolumns_0, 0, 'Got the expected empty column array from prepare columns with no header row'); + + is(scalar @errors_0, 1, 'Got the expected 1 entry in error array from prepare columns with no header row'); + is($errors_0[0]->{badheader}, 1, 'Got the expected 1 badheader from prepare columns with no header row'); + is($errors_0[0]->{line}, 1, 'Got the expected 1 line from prepare columns with no header row'); + is($errors_0[0]->{lineraw}, undef, 'Got the expected undef lineraw from prepare columns with no header row'); + + # Given ... a good header row with plenty of whitespaces + my $headerrow_1 = 'a, b , c, , d'; + my %csvkeycol_1; + my @errors_1; + + # When ... + my @csvcolumns_1 = $patrons_import->prepare_columns({headerrow => $headerrow_1, keycol => \%csvkeycol_1, errors => \@errors_1, }); + + # Then ... + is(scalar @csvcolumns_1, 5, 'Got the expected 5 column array from prepare columns'); + is($csvcolumns_1[0], 'a', 'Got the expected a header from prepare columns'); + is($csvcolumns_1[1], 'b', 'Got the expected b header from prepare columns'); + is($csvcolumns_1[2], 'c', 'Got the expected c header from prepare columns'); + is($csvcolumns_1[3], '', 'Got the expected empty header from prepare columns'); + is($csvcolumns_1[4], 'd', 'Got the expected d header from prepare columns'); + + is($csvkeycol_1{a}, 0, 'Got the expected 0 value for key a from prepare columns hash'); + is($csvkeycol_1{b}, 1, 'Got the expected 1 value for key b from prepare columns hash'); + is($csvkeycol_1{c}, 2, 'Got the expected 2 value for key c from prepare columns hash'); + is($csvkeycol_1{''}, 3, 'Got the expected 3 value for empty string key from prepare columns hash'); + is($csvkeycol_1{d}, 4, 'Got the expected 4 value for key d from prepare columns hash'); +}; + +subtest 'test_set_column_keys' => sub { + plan tests => 5; + + # Given ... nothing at all + # When ... Then ... + my $attr_type_0 = $patrons_import->set_attribute_types(undef); + is($attr_type_0, undef, 'Got the expected undef attribute type from set attribute types with nothing'); + + # Given ... extended but not matchpoint + my $params_1 = { extended => 1, matchpoint => undef, }; + + # When ... Then ... + my $attr_type_1 = $patrons_import->set_attribute_types($params_1); + is($attr_type_1, undef, 'Got the expected undef attribute type from set attribute types with no matchpoint'); + + # Given ... extended and unexpected matchpoint + my $params_2 = { extended => 1, matchpoint => 'unexpected', }; + + # When ... Then ... + my $attr_type_2 = $patrons_import->set_attribute_types($params_2); + is($attr_type_2, undef, 'Got the expected undef attribute type from set attribute types with unexpected matchpoint'); + + # Given ... + my $code_3 = 'SHOW_BCODE'; + my $params_3 = { extended => 1, matchpoint => $code_3, }; + + # When ... + my $attr_type_3 = $patrons_import->set_attribute_types($params_3); + + # Then ... + isa_ok($attr_type_3, 'C4::Members::AttributeTypes'); + is($attr_type_3->{code}, $code_3, 'Got the expected code attribute type from set attribute types'); +}; + subtest 'test_set_column_keys' => sub { plan tests => 2; # Given ... nothing at all # When ... Then ... - my @columnkeys_0 = Koha::Patrons::Import::set_column_keys(undef); + my @columnkeys_0 = $patrons_import->set_column_keys(undef); is(scalar @columnkeys_0, 66, 'Got the expected array size from set column keys with undef extended'); # Given ... extended. my $extended = 1; # When ... Then ... - my @columnkeys_1 = Koha::Patrons::Import::set_column_keys($extended); + my @columnkeys_1 = $patrons_import->set_column_keys($extended); is(scalar @columnkeys_1, 67, 'Got the expected array size from set column keys with extended'); }; @@ -287,14 +398,14 @@ subtest 'test_set_patron_attributes' => sub { # Given ... nothing at all # When ... Then ... - my $result_0 = Koha::Patrons::Import::set_patron_attributes(undef, undef, undef); + my $result_0 = $patrons_import->set_patron_attributes(undef, undef, undef); is($result_0, undef, 'Got the expected undef from set patron attributes with nothing'); # Given ... not extended. my $extended_1 = 0; # When ... Then ... - my $result_1 = Koha::Patrons::Import::set_patron_attributes($extended_1, undef, undef); + my $result_1 = $patrons_import->set_patron_attributes($extended_1, undef, undef); is($result_1, undef, 'Got the expected undef from set patron attributes with not extended'); # Given ... NO patrons attributes @@ -303,7 +414,7 @@ subtest 'test_set_patron_attributes' => sub { my @feedback_2; # When ... - my $result_2 = Koha::Patrons::Import::set_patron_attributes($extended_2, $patron_attributes_2, \@feedback_2); + my $result_2 = $patrons_import->set_patron_attributes($extended_2, $patron_attributes_2, \@feedback_2); # Then ... is($result_2, undef, 'Got the expected undef from set patron attributes with no patrons attributes'); @@ -314,7 +425,7 @@ subtest 'test_set_patron_attributes' => sub { my @feedback_3; # When ... - my $result_3 = Koha::Patrons::Import::set_patron_attributes($extended_2, $patron_attributes_3, \@feedback_3); + my $result_3 = $patrons_import->set_patron_attributes($extended_2, $patron_attributes_3, \@feedback_3); # Then ... ok($result_3, 'Got some data back from set patron attributes'); @@ -339,7 +450,7 @@ subtest 'test_check_branch_code' => sub { my @missing_criticals = (); # When ... - Koha::Patrons::Import::check_branch_code(undef, $borrowerline, $line_number, \@missing_criticals); + $patrons_import->check_branch_code(undef, $borrowerline, $line_number, \@missing_criticals); # Then ... is(scalar @missing_criticals, 1, 'Got the expected missing critical array size of 1 from check_branch_code with no branch code'); @@ -355,7 +466,7 @@ subtest 'test_check_branch_code' => sub { my @missing_criticals_1 = (); # When ... - Koha::Patrons::Import::check_branch_code($branchcode_1, $borrowerline_1, $line_number_1, \@missing_criticals_1); + $patrons_import->check_branch_code($branchcode_1, $borrowerline_1, $line_number_1, \@missing_criticals_1); # Then ... is(scalar @missing_criticals_1, 1, 'Got the expected missing critical array size of 1 from check_branch_code with unexpected branch code'); @@ -373,7 +484,7 @@ subtest 'test_check_branch_code' => sub { my @missing_criticals_2 = (); # When ... - Koha::Patrons::Import::check_branch_code($branchcode_2, $borrowerline_2, $line_number_2, \@missing_criticals_2); + $patrons_import->check_branch_code($branchcode_2, $borrowerline_2, $line_number_2, \@missing_criticals_2); # Then ... is(scalar @missing_criticals_2, 0, 'Got the expected missing critical array size of 0 from check_branch_code'); @@ -388,7 +499,7 @@ subtest 'test_check_borrower_category' => sub { my @missing_criticals = (); # When ... - Koha::Patrons::Import::check_borrower_category(undef, $borrowerline, $line_number, \@missing_criticals); + $patrons_import->check_borrower_category(undef, $borrowerline, $line_number, \@missing_criticals); # Then ... is(scalar @missing_criticals, 1, 'Got the expected missing critical array size of 1 from check_branch_code with no category code'); @@ -404,7 +515,7 @@ subtest 'test_check_borrower_category' => sub { my @missing_criticals_1 = (); # When ... - Koha::Patrons::Import::check_borrower_category($categorycode_1, $borrowerline_1, $line_number_1, \@missing_criticals_1); + $patrons_import->check_borrower_category($categorycode_1, $borrowerline_1, $line_number_1, \@missing_criticals_1); # Then ... is(scalar @missing_criticals_1, 1, 'Got the expected missing critical array size of 1 from check_branch_code with unexpected category code'); @@ -422,11 +533,79 @@ subtest 'test_check_borrower_category' => sub { my @missing_criticals_2 = (); # When ... - Koha::Patrons::Import::check_borrower_category($categorycode_2, $borrowerline_2, $line_number_2, \@missing_criticals_2); + $patrons_import->check_borrower_category($categorycode_2, $borrowerline_2, $line_number_2, \@missing_criticals_2); # Then ... is(scalar @missing_criticals_2, 0, 'Got the expected missing critical array size of 0 from check_branch_code'); }; + +subtest 'test_format_dates' => sub { + plan tests => 22; + + # Given ... no borrower data. + my $borrowerline = 'another line'; + my $line_number = 987; + my @missing_criticals = (); + my %borrower; + my $params = {borrower => \%borrower, lineraw => $borrowerline, line => $line_number, missing_criticals => \@missing_criticals, }; + + # When ... + $patrons_import->format_dates($params); + + # Then ... + ok( not(%borrower), 'Got the expected no borrower from format_dates with no dates'); + is(scalar @missing_criticals, 0, 'Got the expected missing critical array size of 0 from format_dates with no dates'); + + # Given ... some good dates + my @missing_criticals_1 = (); + my $dateofbirth_1 = '2016-05-03'; + my $dateenrolled_1 = '2016-05-04'; + my $dateexpiry_1 = '2016-05-06'; + my $borrower_1 = { dateofbirth => $dateofbirth_1, dateenrolled => $dateenrolled_1, dateexpiry => $dateexpiry_1, }; + my $params_1 = {borrower => $borrower_1, lineraw => $borrowerline, line => $line_number, missing_criticals => \@missing_criticals_1, }; + + # When ... + $patrons_import->format_dates($params_1); + + # Then ... + is($borrower_1->{dateofbirth}, $dateofbirth_1, 'Got the expected date of birth from format_dates with good dates'); + is($borrower_1->{dateenrolled}, $dateenrolled_1, 'Got the expected date of birth from format_dates with good dates'); + is($borrower_1->{dateexpiry}, $dateexpiry_1, 'Got the expected date of birth from format_dates with good dates'); + is(scalar @missing_criticals_1, 0, 'Got the expected missing critical array size of 0 from check_branch_code with good dates'); + + # Given ... some very bad dates + my @missing_criticals_2 = (); + my $dateofbirth_2 = '03-2016-05'; + my $dateenrolled_2 = '04-2016-05'; + my $dateexpiry_2 = '06-2016-05'; + my $borrower_2 = { dateofbirth => $dateofbirth_2, dateenrolled => $dateenrolled_2, dateexpiry => $dateexpiry_2, }; + my $params_2 = {borrower => $borrower_2, lineraw => $borrowerline, line => $line_number, missing_criticals => \@missing_criticals_2, }; + + # When ... + $patrons_import->format_dates($params_2); + + # Then ... + is($borrower_2->{dateofbirth}, '', 'Got the expected empty date of birth from format_dates with bad dates'); + is($borrower_2->{dateenrolled}, '', 'Got the expected emptydate of birth from format_dates with bad dates'); + is($borrower_2->{dateexpiry}, '', 'Got the expected empty date of birth from format_dates with bad dates'); + + is(scalar @missing_criticals_2, 3, 'Got the expected missing critical array size of 3 from check_branch_code with bad dates'); + is($missing_criticals_2[0]->{bad_date}, 1, 'Got the expected first bad date flag from check_branch_code with bad dates'); + is($missing_criticals_2[0]->{key}, 'dateofbirth', 'Got the expected dateofbirth key from check_branch_code with bad dates'); + is($missing_criticals_2[0]->{line}, $line_number, 'Got the expected first line from check_branch_code with bad dates'); + is($missing_criticals_2[0]->{lineraw}, $borrowerline, 'Got the expected first lineraw from check_branch_code with bad dates'); + + is($missing_criticals_2[1]->{bad_date}, 1, 'Got the expected second bad date flag from check_branch_code with bad dates'); + is($missing_criticals_2[1]->{key}, 'dateenrolled', 'Got the expected dateenrolled key from check_branch_code with bad dates'); + is($missing_criticals_2[1]->{line}, $line_number, 'Got the expected second line from check_branch_code with bad dates'); + is($missing_criticals_2[1]->{lineraw}, $borrowerline, 'Got the expected second lineraw from check_branch_code with bad dates'); + + is($missing_criticals_2[2]->{bad_date}, 1, 'Got the expected third bad date flag from check_branch_code with bad dates'); + is($missing_criticals_2[2]->{key}, 'dateexpiry', 'Got the expected dateexpiry key from check_branch_code with bad dates'); + is($missing_criticals_2[2]->{line}, $line_number, 'Got the expected third line from check_branch_code with bad dates'); + is($missing_criticals_2[2]->{lineraw}, $borrowerline, 'Got the expected third lineraw from check_branch_code with bad dates'); +}; + # ###### Test utility ########### sub make_csv { my ($temp_dir, @lines) = @_; -- 2.1.4