From e72f12fb17456539988e441613337897ffefc2b2 Mon Sep 17 00:00:00 2001 From: Dobrica Pavlinusic Date: Fri, 21 Feb 2014 13:56:11 +0100 Subject: [PATCH] Bug 11811 - tools/import_borrowers.pl utf-8 support for CSV Decode utf-8 characters in CSV file and perform canonical decomposition followed by canonical composition. Use case for this is XLS file converted into CSV which includes utf-8 characters which display correctly in interface but are not searchable in Koha's patrons search since browser does same decomposition as this code does. Test scenario: 1. go to Tools > Import patrons 2. create patrons using attached example files which include both variants of my surname. You will have to fill in branchcode and categorycode according to configuration of Koha instance which is used in web form since those values are not included in CSV 3. import patrons and verify then utf-8 characters are correct and that you get just one results when searching for my surname (cardnumber ITI-00000151832). You can just copy/paste my surname from example file (any of them) since browser does similar decomposition as this code. 4. apply this patch and repeat import of patrons, selecting option to overwrite existing records 5. verify that after searching for my surname you get two borrowers (cardnumbers ITI-00000151831 and ITI-00000151832) Signed-off-by: A. Sassmannshausen --- tools/import_borrowers.pl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/import_borrowers.pl b/tools/import_borrowers.pl index 5f65c97..4f2a202 100755 --- a/tools/import_borrowers.pl +++ b/tools/import_borrowers.pl @@ -48,12 +48,10 @@ use C4::Members::AttributeTypes; use C4::Members::Messaging; use Text::CSV; -# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals: -# ė -# č +use Encode; +use Unicode::Normalize; use CGI; -# use encoding 'utf8'; # don't do this my (@errors, @feedback); my $extended = C4::Context->preference('ExtendedPatronAttributes'); @@ -115,6 +113,7 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) { # use header line to construct key to column map my $borrowerline = <$handle>; + $borrowerline = NFC(decode('utf-8',$borrowerline)); my $status = $csv->parse($borrowerline); ($status) or push @errors, {badheader=>1,line=>$., lineraw=>$borrowerline}; my @csvcolumns = $csv->fields(); @@ -138,6 +137,7 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) { my $date_re = C4::Dates->new->regexp('syspref'); my $iso_re = C4::Dates->new->regexp('iso'); LINE: while ( my $borrowerline = <$handle> ) { + $borrowerline = NFC(decode('utf-8',$borrowerline)); my %borrower; my @missing_criticals; my $patron_attributes; -- 1.7.10.4