From a6a722ebb674ca7ba334dbfbb22e2df77488e93e Mon Sep 17 00:00:00 2001
From: Jonathan Druart <jonathan.druart@bugs.koha-community.org>
Date: Wed, 29 Apr 2020 11:53:45 +0200
Subject: [PATCH] Bug 25305: Translation process - Open all files specifying
 the utf8 encoding

To test:
1) Create translation files for a new language
( cd misc/translator; ./translate create xx-YY)
A new language means one that isn't already in Koha, xx-YY=>something you
invent.
2) Verify double encoding
egrep "Aix-Marseille|Jean Prunier|periodika|Bokm" misc/translator/po/xx-YY-*
check strange strings
3) Apply the patch
4) Create po files again
( cd misc/translator; rm -f po/xx-YY*; ./translate create xx-YY)
5) Verify no more double encoding
egrep "Aix-Marseille|Jean Prunier|periodika|Bokm" misc/translator/po/xx-YY-*
check normal string

Signed-off-by: Bernardo Gonzalez Kriegel <bgkriegel@gmail.com>
Copyied my test plan, can't find more cases, tested using
something similar to http://blogs.perl.org/users/chansen/2010/10/coping-with-double-encoded-utf-8.html
---
 C4/TTParser.pm              | 3 ++-
 misc/translator/xgettext.pl | 6 +++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/C4/TTParser.pm b/C4/TTParser.pm
index a55eccdfed..24e7b83bef 100644
--- a/C4/TTParser.pm
+++ b/C4/TTParser.pm
@@ -64,7 +64,8 @@ sub build_tokens{
 #    $self->handler(default => "default", "self, line, text, is_cdata"); # anything else
     $self->marked_sections(1); #treat anything inside CDATA tags as text, should really make it a C4::TmplTokenType::CDATA
     $self->unbroken_text(1); #make contiguous whitespace into a single token (can span multiple lines)
-    $self->parse_file($filename);
+    open(my $fh, "<:encoding(utf8)", $filename) || die "Cannot open $filename ($!)";
+    $self->parse_file($fh);
     return $self;
 }
 
diff --git a/misc/translator/xgettext.pl b/misc/translator/xgettext.pl
index 89e01eed5b..35ba4d2ce5 100755
--- a/misc/translator/xgettext.pl
+++ b/misc/translator/xgettext.pl
@@ -255,7 +255,7 @@ EOF
 ###############################################################################
 
 sub convert_translation_file {
-    open(my $INPUT, '<', $convert_from) || die "$convert_from: $!\n";
+    open(my $INPUT, '<:encoding(utf-8)', $convert_from) || die "$convert_from: $!\n";
     VerboseWarnings::set_input_file_name $convert_from;
     while (<$INPUT>) {
 	chomp;
@@ -367,7 +367,7 @@ usage_error('You cannot specify both --convert-from and --files-from')
 
 if (defined $output && $output ne '-') {
     print STDERR "$0: Opening output file \"$output\"\n" if $verbose_p;
-        open($OUTPUT, '>:encoding(utf-8)', $output) || die "$output: $!\n";
+    open($OUTPUT, '>:encoding(utf-8)', $output) || die "$output: $!\n";
 } else {
     print STDERR "$0: Outputting to STDOUT...\n" if $verbose_p;
     open($OUTPUT, ">&STDOUT");
@@ -375,7 +375,7 @@ if (defined $output && $output ne '-') {
 
 if (defined $files_from) {
     print STDERR "$0: Opening input file list \"$files_from\"\n" if $verbose_p;
-    open(my $INPUT, '<', $files_from) || die "$files_from: $!\n";
+    open(my $INPUT, '<:encoding(utf-8)', $files_from) || die "$files_from: $!\n";
     while (<$INPUT>) {
 	chomp;
 	my $input = /^\//? $_: "$directory/$_";
-- 
2.17.1