From b8682db29d8d10296e0f277de789fa03e12b0d76 Mon Sep 17 00:00:00 2001 From: Tomas Cohen Arazi Date: Tue, 5 Nov 2013 13:59:11 -0300 Subject: [PATCH] Bug 9114: import/export routines should use UTF-8 Currently both the import_export_framework.pl script and the associated libs output data with Perl's default encoding, ISO-8859. This patch properly sets the binmode to UTF-8 for the CGI script, and makes the C4::ImportExportFramework routines apply the ":encoding(UTF-8)" layer to file descriptors. To test: Export step test - Use some non-ASCII character(s) in some field description in a chosen framework. - Export the framework at Administration > MARC frameworks - Run this to check the file is ISO-8859 encoded: $ file export_XXX.csv export_XXX.csv: ISO-8859 text, with very long lines (Note: try SQL and other output formats too. But not ODS) - Apply the patch - Export the framework again (change the name), and test encoding: $ file export_XXX_2.csv export_XXX_2.csv: UTF-8 Unicode text Import step test I assume you have two files, export_XXX.csv (ISO-8859 encoded) and export_XXX_2.csv (XXX will depend on your framework's code) - Reset your testing branch to master - Import export_XXX.csv - The string with non-ASCII chars is truncated at the first non-ASCII char's position (Note: this is the current behaviour). - Import export_XXX_2.csv - The non-ASCII chars are broken, the logs show errors on non-UNICODE chars. (Note: even thou UTF-8 is the expected encoding it is treated as ISO-8859). - Apply the patch - Import the good (UTF-8 as expected) file and check everything worked as expected. Regards To+ Sponsored-by: Universidad Nacional de Cordoba Signed-off-by: Marcel de Rooy Tested export and re-import with CSV containing Latin chars and containing Chinese chars. Signed-off-by: Kyle M Hall --- C4/ImportExportFramework.pm | 4 ++-- admin/import_export_framework.pl | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/C4/ImportExportFramework.pm b/C4/ImportExportFramework.pm index f85101d..0e6487a 100644 --- a/C4/ImportExportFramework.pm +++ b/C4/ImportExportFramework.pm @@ -606,7 +606,7 @@ sub createODS my $ok = 0; # read ods file and return as a string if (-f "$tempdir/new.ods") { - if (open ($fh, '<', "$tempdir/new.ods")) { + if (open ($fh, '<:encoding(UTF-8)', "$tempdir/new.ods")) { binmode $fh; my $buffer; while (read ($fh, $buffer, 65536)) { @@ -698,7 +698,7 @@ sub ImportFramework } } else { # They are text files, so open it to read - open($dom, '<', $filename); + open($dom, '<:encoding(UTF-8)', $filename); } if ($dom) { # For sql we execute the line diff --git a/admin/import_export_framework.pl b/admin/import_export_framework.pl index 8674ebf..3380f50 100755 --- a/admin/import_export_framework.pl +++ b/admin/import_export_framework.pl @@ -24,6 +24,8 @@ use CGI; use C4::Context; use C4::ImportExportFramework; +binmode STDOUT, ":encoding(UTF-8)"; + my $input = new CGI; my $frameworkcode = $input->param('frameworkcode') || ''; -- 1.7.2.5