Attachment #119227 for bug #27883

View | Details | Raw Unified | Return to bug 27883
Collapse All | Expand All

Lines 70-75 sub import_patrons { Link Here

(-)a/Koha/Patrons/Import.pm (-1 / +9 lines)
70		70
71	my $matchpoint = $params->{matchpoint};	71	my $matchpoint = $params->{matchpoint};
72	my $defaults = $params->{defaults};	72	my $defaults = $params->{defaults};
		73	my $preserve_fields = $params->{preserve_fields};
73	my $ext_preserve = $params->{preserve_extended_attributes};	74	my $ext_preserve = $params->{preserve_extended_attributes};
74	my $overwrite_cardnumber = $params->{overwrite_cardnumber};	75	my $overwrite_cardnumber = $params->{overwrite_cardnumber};
75	my $overwrite_passwords = $params->{overwrite_passwords};	76	my $overwrite_passwords = $params->{overwrite_passwords};
Lines 241-247 sub import_patrons { Link Here
241	if exists $borrower{$field} and $borrower{$field} eq "";	242	if exists $borrower{$field} and $borrower{$field} eq "";
242	}	243	}
243		244
244	if ($borrowernumber) {	245	if ($borrowernumber) {
245		246
246	# borrower exists	247	# borrower exists
247	unless ($overwrite_cardnumber) {	248	unless ($overwrite_cardnumber) {
Lines 256-261 sub import_patrons { Link Here
256	next LINE;	257	next LINE;
257	}	258	}
258	$borrower{'borrowernumber'} = $borrowernumber;	259	$borrower{'borrowernumber'} = $borrowernumber;
		260
		261	if ( $preserve_fields ) {
		262	for my $field ( @$preserve_fields ) {
		263	delete $borrower{$field};
		264	}
		265	}
		266
259	for my $col ( keys %borrower ) {	267	for my $col ( keys %borrower ) {
260		268
261	# use values from extant patron unless our csv file includes this column or we provided a default.	269	# use values from extant patron unless our csv file includes this column or we provided a default.





            </ol>
        </fieldset>
        <fieldset class="rows">
            <legend>
                <a href="#" class="expand_preserves"><i class="fa fa-plus-square"></i> Preserve existing values</a>
                <a href="#" class="expand_preserves" style="display:none;"><i class="fa fa-minus-square"></i> Hide preserve value fields</a>
            </legend>

            <ol class="preserve_values" style="display:none;">
                <div class="hint">Selected fields will be preserved from original patron record when overwriting existing patron.</div>
                [% FOREACH borrower_field IN borrower_fields %]
                        <li>
                            <label class="description" for="[% borrower_field.field | html %]">[% borrower_field.description | html %]: </label>
                            <input name="preserve_existing" id="preserve_existing_[%  borrower_field.field | html %]" value="[%  borrower_field.field | html %]" type="checkbox">
                            <span class="field_hint">[% borrower_field.field | html %]</span>
                        </li>
                [% END %]
            </ol>
        </fieldset>

        <fieldset class="rows">
            <legend>If matching record is already in the borrowers table:</legend>

                $(".default_values").toggle();
                $(".expand_defaults").toggle();
            });

            $(".expand_preserves").click(function(e){
                e.preventDefault();
                $(".preserve_values").toggle();
                $(".expand_preserves").toggle();
            });
        });

        $("#overwrite_cardnumberno").click(function(){




my $confirm;
my $verbose      = 0;
my $help;
my @columns_to_remove;

GetOptions(
    'c|confirm'                      => \$confirm,
    'f|file=s'                       => \$csv_file,
    'm|matchpoint=s'                 => \$matchpoint,
    'd|default=s'                    => \%defaults,
    'o|overwrite'                    => \$overwrite_cardnumber,
    'op|overwrite_passwords'         => \$overwrite_passwords,
    'p|preserve-extended-attributes' => \$ext_preserve,
    'pf|preserve-field'              => \@preserve_fields,
    'v|verbose+'                     => \$verbose,
    'h|help|?'                       => \$help,
) or pod2usage(2);

pod2usage(1) if $help;

        overwrite_cardnumber         => $overwrite_cardnumber,
        overwrite_passwords          => $overwrite_passwords,
        preserve_extended_attributes => $ext_preserve,
        preserve_fields              => \@preserve_fields,
        dry_run                      => !$confirm,
    }
);


=head1 SYNOPSIS

import_patrons.pl --file /path/to/patrons.csv --matchpoint cardnumber --confirm [--default branchcode=MPL] [--overwrite] [--preserve_field <column>] [--preserve-extended-attributes] [--verbose]

=head1 OPTIONS



Set defaults to patron fields, repeatable e.g. --default branchcode=MPL --default categorycode=PT

=item B<-k|--preserve-field>

Prevent specified patron fields for existing patrons from being overwritten

=item B<-o|--overwrite>

Overwrite existing patrons with new data if a match is found




# along with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;
use Test::More tests => 172;
use Test::Warn;
use Encode qw( encode_utf8 );
use utf8;

my $res_header   = 'cardnumber, surname, firstname, title, othernames, initials, streetnumber, streettype, address, address2, city, state, zipcode, country, email, phone, mobile, fax, dateofbirth, branchcode, categorycode, dateenrolled, dateexpiry, userid, password';
my $csv_one_line = '1000,Nancy,Jenkins,Dr,,NJ,78,Circle,Bunting,El Paso,Henderson,Texas,79984,United States,ajenkins0@sourceforge.net,7-(388)559-6763,3-(373)151-4471,8-(509)286-4001,10/16/1965,CPL,PT,12/28/2014,07/01/2015,jjenkins0,DPQILy';
my $csv_one_line_a = '1001,Nancy,Jenkins,Dr,,NJ,78,Circle,Bunting,El Paso,Henderson,Texas,79984,United States,ajenkins0@sourceforge.net,7-(388)559-6763,3-(373)151-4471,8-(509)286-4001,10/16/1965,CPL,PT,12/28/2014,07/01/2015,jjenkins0,DPQILy';
my $csv_one_line_b = '1000,Nancy2,Jenkins2,Dr,,NJ,78,Circle,Bunting,El Paso,Henderson,Texas,79984,United States,ajenkins0@sourceforge.net,7-(388)559-6763,3-(373)151-4471,8-(509)286-4001,10/16/1965,CPL,PT,12/28/2014,07/01/2015,jjenkins0,DPQILy';

my $filename_1 = make_csv($temp_dir, $csv_headers, $csv_one_line);
open(my $handle_1, "<", $filename_1) or die "cannot open < $filename_1: $!";

is($result_3a->{invalid}, 0, 'Got the expected 0 invalid result from import_patrons');
is($result_3a->{overwritten}, 1, 'Got the expected 1 overwritten result from import_patrons that matched');

# Given ... valid file handle, good matchpoint that matches should overwrite when set, surname is protected from
# overwrite but firstname is not
my $filename_3c = make_csv($temp_dir, $csv_headers, $csv_one_line_b);
open(my $handle_3c, "<", $filename_3c) or die "cannot open < $filename_3: $!";
my $params_3c = { file => $handle_3c, matchpoint => 'cardnumber', overwrite_cardnumber => 1, preserve_fields => [ 'firstname' ] };

# When ...
my $result_3c;
warning_is { $result_3c = $patrons_import->import_patrons($params_3c) }
    undef,
    "No warning raised by import_patrons";

# Then ...
is($result_3c->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons when matched and overwrite set');
is($result_3c->{errors}->[0]->{duplicate_userid}, undef, 'No duplicate userid error from import patrons with duplicate userid (it is our own)');
is($result_3c->{errors}->[0]->{userid}, undef, 'No duplicate userid error from import patrons with duplicate userid (it is our own)');

is($result_3c->{feedback}->[0]->{feedback}, 1, 'Got 1 expected feedback from import_patrons that matched and overwritten');
is($result_3c->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons with duplicate userid');
is($result_3c->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons with duplicate userid');

is($result_3c->{imported}, 0, 'Got the expected 0 imported result from import_patrons');
is($result_3c->{invalid}, 0, 'Got the expected 0 invalid result from import_patrons');
is($result_3c->{overwritten}, 1, 'Got the expected 1 overwritten result from import_patrons that matched');

my $patron_3c = Koha::Patrons->find({ cardnumber => '1000' });
is( $patron_3c->surname, "Nancy2", "Surname field is preserved from original" );
is( $patron_3c->firstname, "Jenkins", "Firstname field is overwritten" );

# Given ... valid file handle, good matchpoint that does not match and conflicting userid.
my $filename_3b = make_csv($temp_dir, $csv_headers, $csv_one_line_a);
open(my $handle_3b, "<", $filename_3b) or die "cannot open < $filename_3: $!";

Lines 94-99 if ( $input->param('sample') ) { Link Here

(-)a/tools/import_borrowers.pl (-3 / +5 lines)
94	exit 0;	94	exit 0;
95	}	95	}
96		96
		97	my @preserve_fields = $input->param('preserve_existing');
		98
97	my $uploadborrowers = $input->param('uploadborrowers');	99	my $uploadborrowers = $input->param('uploadborrowers');
98	my $matchpoint = $input->param('matchpoint');	100	my $matchpoint = $input->param('matchpoint');
99	if ($matchpoint) {	101	if ($matchpoint) {
Lines 123-131 if ( $uploadborrowers && length($uploadborrowers) > 0 ) { Link Here
123	file => $handle,	125	file => $handle,
124	defaults => \%defaults,	126	defaults => \%defaults,
125	matchpoint => $matchpoint,	127	matchpoint => $matchpoint,
126	overwrite_cardnumber => scalar $input->param('overwrite_cardnumber'),	128	overwrite_cardnumber => scalar $input->param( 'overwrite_cardnumber' ),
127	overwrite_passwords => $overwrite_passwords,	129	overwrite_passwords => $overwrite_passwords,
128	preserve_extended_attributes => scalar $input->param('ext_preserve') \|\| 0,	130	preserve_extended_attributes => scalar $input->param( 'ext_preserve' ) \|\| 0,
		131	preserve_fields => \@preserve_fields,
129	}	132	}
130	);	133	);
131		134
132	-

Return to bug 27883

Lines 259-264 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tt (+23 lines)
259		259
260	</ol>	260	</ol>
261	</fieldset>	261	</fieldset>
		262	<fieldset class="rows">
		263	<legend>
		264	<a href="#" class="expand_preserves"><i class="fa fa-plus-square"></i> Preserve existing values</a>
		265	<a href="#" class="expand_preserves" style="display:none;"><i class="fa fa-minus-square"></i> Hide preserve value fields</a>
		266	</legend>
		267
		268	<ol class="preserve_values" style="display:none;">
		269	<div class="hint">Selected fields will be preserved from original patron record when overwriting existing patron.</div>
		270	[% FOREACH borrower_field IN borrower_fields %]
		271	<li>
		272	<label class="description" for="[% borrower_field.field \| html %]">[% borrower_field.description \| html %]: </label>
		273	<input name="preserve_existing" id="preserve_existing_[% borrower_field.field \| html %]" value="[% borrower_field.field \| html %]" type="checkbox">
		274	<span class="field_hint">[% borrower_field.field \| html %]</span>
		275	</li>
		276	[% END %]
		277	</ol>
		278	</fieldset>
262		279
263	<fieldset class="rows">	280	<fieldset class="rows">
264	<legend>If matching record is already in the borrowers table:</legend>	281	<legend>If matching record is already in the borrowers table:</legend>
Lines 368-373 you can supply dates in ISO format (e.g., '2010-10-28'). Link Here
368	$(".default_values").toggle();	385	$(".default_values").toggle();
369	$(".expand_defaults").toggle();	386	$(".expand_defaults").toggle();
370	});	387	});
		388
		389	$(".expand_preserves").click(function(e){
		390	e.preventDefault();
		391	$(".preserve_values").toggle();
		392	$(".expand_preserves").toggle();
		393	});
371	});	394	});
372		395
373	$("#overwrite_cardnumberno").click(function(){	396	$("#overwrite_cardnumberno").click(function(){

Lines 35-51 my $ext_preserve = 0; Link Here

(-)a/misc/import_patrons.pl (-9 / +16 lines)
35	my $confirm;	35	my $confirm;
36	my $verbose = 0;	36	my $verbose = 0;
37	my $help;	37	my $help;
		38	my @columns_to_remove;
38		39
39	GetOptions(	40	GetOptions(
40	'c\|confirm' => \$confirm,	41	'c\|confirm' => \$confirm,
41	'f\|file=s' => \$csv_file,	42	'f\|file=s' => \$csv_file,
42	'm\|matchpoint=s' => \$matchpoint,	43	'm\|matchpoint=s' => \$matchpoint,
43	'd\|default=s' => \%defaults,	44	'd\|default=s' => \%defaults,
44	'o\|overwrite' => \$overwrite_cardnumber,	45	'o\|overwrite' => \$overwrite_cardnumber,
45	'op\|overwrite_passwords' => \$overwrite_passwords,	46	'op\|overwrite_passwords' => \$overwrite_passwords,
46	'p\|preserve-extended-attributes' => \$ext_preserve,	47	'p\|preserve-extended-attributes' => \$ext_preserve,
47	'v\|verbose+' => \$verbose,	48	'pf\|preserve-field' => \@preserve_fields,
48	'h\|help\|?' => \$help,	49	'v\|verbose+' => \$verbose,
		50	'h\|help\|?' => \$help,
49	) or pod2usage(2);	51	) or pod2usage(2);
50		52
51	pod2usage(1) if $help;	53	pod2usage(1) if $help;
Lines 65-70 my $return = $Import->import_patrons( Link Here
65	overwrite_cardnumber => $overwrite_cardnumber,	67	overwrite_cardnumber => $overwrite_cardnumber,
66	overwrite_passwords => $overwrite_passwords,	68	overwrite_passwords => $overwrite_passwords,
67	preserve_extended_attributes => $ext_preserve,	69	preserve_extended_attributes => $ext_preserve,
		70	preserve_fields => \@preserve_fields,
68	dry_run => !$confirm,	71	dry_run => !$confirm,
69	}	72	}
70	);	73	);
Lines 104-110 import_patrons.pl - CLI script to import patrons data into Koha Link Here
104		107
105	=head1 SYNOPSIS	108	=head1 SYNOPSIS
106		109
107	import_patrons.pl --file /path/to/patrons.csv --matchpoint cardnumber --confirm [--default branchcode=MPL] [--overwrite] [--preserve-extended-attributes] [--verbose]	110	import_patrons.pl --file /path/to/patrons.csv --matchpoint cardnumber --confirm [--default branchcode=MPL] [--overwrite] [--preserve_field <column>] [--preserve-extended-attributes] [--verbose]
108		111
109	=head1 OPTIONS	112	=head1 OPTIONS
110		113
Lines 130-135 Field on which to match incoming patrons to existing patrons Link Here
130		133
131	Set defaults to patron fields, repeatable e.g. --default branchcode=MPL --default categorycode=PT	134	Set defaults to patron fields, repeatable e.g. --default branchcode=MPL --default categorycode=PT
132		135
		136	=item B<-k\|--preserve-field>
		137
		138	Prevent specified patron fields for existing patrons from being overwritten
		139
133	=item B<-o\|--overwrite>	140	=item B<-o\|--overwrite>
134		141
135	Overwrite existing patrons with new data if a match is found	142	Overwrite existing patrons with new data if a match is found

Lines 18-24 Link Here

(-)a/t/db_dependent/Koha/Patrons/Import.t (-1 / +31 lines)
18	# along with Koha; if not, see <http://www.gnu.org/licenses>.	18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
19		19
20	use Modern::Perl;	20	use Modern::Perl;
21	use Test::More tests => 160;	21	use Test::More tests => 172;
22	use Test::Warn;	22	use Test::Warn;
23	use Encode qw( encode_utf8 );	23	use Encode qw( encode_utf8 );
24	use utf8;	24	use utf8;
Lines 92-97 my $csv_headers = 'cardnumber,surname,firstname,title,othernames,initials,stree Link Here
92	my $res_header = 'cardnumber, surname, firstname, title, othernames, initials, streetnumber, streettype, address, address2, city, state, zipcode, country, email, phone, mobile, fax, dateofbirth, branchcode, categorycode, dateenrolled, dateexpiry, userid, password';	92	my $res_header = 'cardnumber, surname, firstname, title, othernames, initials, streetnumber, streettype, address, address2, city, state, zipcode, country, email, phone, mobile, fax, dateofbirth, branchcode, categorycode, dateenrolled, dateexpiry, userid, password';
93	my $csv_one_line = '1000,Nancy,Jenkins,Dr,,NJ,78,Circle,Bunting,El Paso,Henderson,Texas,79984,United States,ajenkins0@sourceforge.net,7-(388)559-6763,3-(373)151-4471,8-(509)286-4001,10/16/1965,CPL,PT,12/28/2014,07/01/2015,jjenkins0,DPQILy';	93	my $csv_one_line = '1000,Nancy,Jenkins,Dr,,NJ,78,Circle,Bunting,El Paso,Henderson,Texas,79984,United States,ajenkins0@sourceforge.net,7-(388)559-6763,3-(373)151-4471,8-(509)286-4001,10/16/1965,CPL,PT,12/28/2014,07/01/2015,jjenkins0,DPQILy';
94	my $csv_one_line_a = '1001,Nancy,Jenkins,Dr,,NJ,78,Circle,Bunting,El Paso,Henderson,Texas,79984,United States,ajenkins0@sourceforge.net,7-(388)559-6763,3-(373)151-4471,8-(509)286-4001,10/16/1965,CPL,PT,12/28/2014,07/01/2015,jjenkins0,DPQILy';	94	my $csv_one_line_a = '1001,Nancy,Jenkins,Dr,,NJ,78,Circle,Bunting,El Paso,Henderson,Texas,79984,United States,ajenkins0@sourceforge.net,7-(388)559-6763,3-(373)151-4471,8-(509)286-4001,10/16/1965,CPL,PT,12/28/2014,07/01/2015,jjenkins0,DPQILy';
		95	my $csv_one_line_b = '1000,Nancy2,Jenkins2,Dr,,NJ,78,Circle,Bunting,El Paso,Henderson,Texas,79984,United States,ajenkins0@sourceforge.net,7-(388)559-6763,3-(373)151-4471,8-(509)286-4001,10/16/1965,CPL,PT,12/28/2014,07/01/2015,jjenkins0,DPQILy';
95		96
96	my $filename_1 = make_csv($temp_dir, $csv_headers, $csv_one_line);	97	my $filename_1 = make_csv($temp_dir, $csv_headers, $csv_one_line);
97	open(my $handle_1, "<", $filename_1) or die "cannot open < $filename_1: $!";	98	open(my $handle_1, "<", $filename_1) or die "cannot open < $filename_1: $!";
Lines 183-188 is($result_3a->{imported}, 0, 'Got the expected 0 imported result from import_pa Link Here
183	is($result_3a->{invalid}, 0, 'Got the expected 0 invalid result from import_patrons');	184	is($result_3a->{invalid}, 0, 'Got the expected 0 invalid result from import_patrons');
184	is($result_3a->{overwritten}, 1, 'Got the expected 1 overwritten result from import_patrons that matched');	185	is($result_3a->{overwritten}, 1, 'Got the expected 1 overwritten result from import_patrons that matched');
185		186
		187	# Given ... valid file handle, good matchpoint that matches should overwrite when set, surname is protected from
		188	# overwrite but firstname is not
		189	my $filename_3c = make_csv($temp_dir, $csv_headers, $csv_one_line_b);
		190	open(my $handle_3c, "<", $filename_3c) or die "cannot open < $filename_3: $!";
		191	my $params_3c = { file => $handle_3c, matchpoint => 'cardnumber', overwrite_cardnumber => 1, preserve_fields => [ 'firstname' ] };
		192
		193	# When ...
		194	my $result_3c;
		195	warning_is { $result_3c = $patrons_import->import_patrons($params_3c) }
		196	undef,
		197	"No warning raised by import_patrons";
		198
		199	# Then ...
		200	is($result_3c->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons when matched and overwrite set');
		201	is($result_3c->{errors}->[0]->{duplicate_userid}, undef, 'No duplicate userid error from import patrons with duplicate userid (it is our own)');
		202	is($result_3c->{errors}->[0]->{userid}, undef, 'No duplicate userid error from import patrons with duplicate userid (it is our own)');
		203
		204	is($result_3c->{feedback}->[0]->{feedback}, 1, 'Got 1 expected feedback from import_patrons that matched and overwritten');
		205	is($result_3c->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons with duplicate userid');
		206	is($result_3c->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons with duplicate userid');
		207
		208	is($result_3c->{imported}, 0, 'Got the expected 0 imported result from import_patrons');
		209	is($result_3c->{invalid}, 0, 'Got the expected 0 invalid result from import_patrons');
		210	is($result_3c->{overwritten}, 1, 'Got the expected 1 overwritten result from import_patrons that matched');
		211
		212	my $patron_3c = Koha::Patrons->find({ cardnumber => '1000' });
		213	is( $patron_3c->surname, "Nancy2", "Surname field is preserved from original" );
		214	is( $patron_3c->firstname, "Jenkins", "Firstname field is overwritten" );
		215
186	# Given ... valid file handle, good matchpoint that does not match and conflicting userid.	216	# Given ... valid file handle, good matchpoint that does not match and conflicting userid.
187	my $filename_3b = make_csv($temp_dir, $csv_headers, $csv_one_line_a);	217	my $filename_3b = make_csv($temp_dir, $csv_headers, $csv_one_line_a);
188	open(my $handle_3b, "<", $filename_3b) or die "cannot open < $filename_3: $!";	218	open(my $handle_3b, "<", $filename_3b) or die "cannot open < $filename_3: $!";