From ebdac20e6ec944d42e9548eef68066b30c2726c7 Mon Sep 17 00:00:00 2001 From: Dobrica Pavlinusic Date: Fri, 21 Feb 2014 13:56:11 +0100 Subject: [PATCH] [PASSED QA] Bug 11811 - tools/import_borrowers.pl utf-8 support for CSV Decode utf-8 characters in CSV file and perform canonical decomposition followed by canonical composition. Use case for this is XLS file converted into CSV which includes utf-8 characters which display correctly in interface but are not searchable in Koha's patrons search since browser does same decomposition as this code does. Test scenario: 1. go to Tools > Import patrons 2. create patrons using attached example files which include both variants of my surname. You will have to fill in branchcode and categorycode according to configuration of Koha instance which is used in web form since those values are not included in CSV 3. import patrons and verify then utf-8 characters are correct and that you get just one results when searching for my surname (cardnumber ITI-00000151832). You can just copy/paste my surname from example file (any of them) since browser does similar decomposition as this code. 4. apply this patch and repeat import of patrons, selecting option to overwrite existing records 5. verify that after searching for my surname you get two borrowers (cardnumbers ITI-00000151831 and ITI-00000151832) Signed-off-by: A. Sassmannshausen Signed-off-by: Katrin Fischer Passes all tests and QA script. Tested with my own file, some German umlauts and non-Latin scripts. All works as expected. --- tools/import_borrowers.pl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/import_borrowers.pl b/tools/import_borrowers.pl index 4445db5..bd63a0e 100755 --- a/tools/import_borrowers.pl +++ b/tools/import_borrowers.pl @@ -48,12 +48,10 @@ use C4::Members::AttributeTypes; use C4::Members::Messaging; use Text::CSV; -# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals: -# ė -# č +use Encode; +use Unicode::Normalize; use CGI; -# use encoding 'utf8'; # don't do this my (@errors, @feedback); my $extended = C4::Context->preference('ExtendedPatronAttributes'); @@ -115,6 +113,7 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) { # use header line to construct key to column map my $borrowerline = <$handle>; + $borrowerline = NFC(decode('utf-8',$borrowerline)); my $status = $csv->parse($borrowerline); ($status) or push @errors, {badheader=>1,line=>$., lineraw=>$borrowerline}; my @csvcolumns = $csv->fields(); @@ -138,6 +137,7 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) { my $date_re = C4::Dates->new->regexp('syspref'); my $iso_re = C4::Dates->new->regexp('iso'); LINE: while ( my $borrowerline = <$handle> ) { + $borrowerline = NFC(decode('utf-8',$borrowerline)); my %borrower; my @missing_criticals; my $patron_attributes; -- 1.8.3.2