From f5c3a9cff193053dcb728c144523f9a11bd0f1f9 Mon Sep 17 00:00:00 2001 From: Jonathan Druart Date: Wed, 29 Apr 2020 11:53:45 +0200 Subject: [PATCH] Bug 25305: Translation process - Open all files specifying the utf8 encoding To test: 1) Create translation files for a new language ( cd misc/translator; ./translate create xx-YY) A new language means one that isn't already in Koha, xx-YY=>something you invent. 2) Verify double encoding egrep "Aix-Marseille|Jean Prunier|periodika|Bokm" misc/translator/po/xx-YY-* check strange strings 3) Apply the patch 4) Create po files again ( cd misc/translator; rm -f po/xx-YY*; ./translate create xx-YY) 5) Verify no more double encoding egrep "Aix-Marseille|Jean Prunier|periodika|Bokm" misc/translator/po/xx-YY-* check normal string Signed-off-by: Bernardo Gonzalez Kriegel Copyied my test plan, can't find more cases, tested using something similar to http://blogs.perl.org/users/chansen/2010/10/coping-with-double-encoded-utf-8.html Signed-off-by: Didier Gautheron --- C4/TTParser.pm | 3 ++- misc/translator/xgettext.pl | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/C4/TTParser.pm b/C4/TTParser.pm index a55eccdfed..24e7b83bef 100644 --- a/C4/TTParser.pm +++ b/C4/TTParser.pm @@ -64,7 +64,8 @@ sub build_tokens{ # $self->handler(default => "default", "self, line, text, is_cdata"); # anything else $self->marked_sections(1); #treat anything inside CDATA tags as text, should really make it a C4::TmplTokenType::CDATA $self->unbroken_text(1); #make contiguous whitespace into a single token (can span multiple lines) - $self->parse_file($filename); + open(my $fh, "<:encoding(utf8)", $filename) || die "Cannot open $filename ($!)"; + $self->parse_file($fh); return $self; } diff --git a/misc/translator/xgettext.pl b/misc/translator/xgettext.pl index 89e01eed5b..35ba4d2ce5 100755 --- a/misc/translator/xgettext.pl +++ b/misc/translator/xgettext.pl @@ -255,7 +255,7 @@ EOF ############################################################################### sub convert_translation_file { - open(my $INPUT, '<', $convert_from) || die "$convert_from: $!\n"; + open(my $INPUT, '<:encoding(utf-8)', $convert_from) || die "$convert_from: $!\n"; VerboseWarnings::set_input_file_name $convert_from; while (<$INPUT>) { chomp; @@ -367,7 +367,7 @@ usage_error('You cannot specify both --convert-from and --files-from') if (defined $output && $output ne '-') { print STDERR "$0: Opening output file \"$output\"\n" if $verbose_p; - open($OUTPUT, '>:encoding(utf-8)', $output) || die "$output: $!\n"; + open($OUTPUT, '>:encoding(utf-8)', $output) || die "$output: $!\n"; } else { print STDERR "$0: Outputting to STDOUT...\n" if $verbose_p; open($OUTPUT, ">&STDOUT"); @@ -375,7 +375,7 @@ if (defined $output && $output ne '-') { if (defined $files_from) { print STDERR "$0: Opening input file list \"$files_from\"\n" if $verbose_p; - open(my $INPUT, '<', $files_from) || die "$files_from: $!\n"; + open(my $INPUT, '<:encoding(utf-8)', $files_from) || die "$files_from: $!\n"; while (<$INPUT>) { chomp; my $input = /^\//? $_: "$directory/$_"; -- 2.11.0