From ca429488a2d6f76a49ddc7438bde07548328abc7 Mon Sep 17 00:00:00 2001
From: Jonathan Druart <jonathan.druart@bugs.koha-community.org>
Date: Tue, 19 May 2020 10:08:53 +0200
Subject: [PATCH] Bug 25501: Encode strings in utf8 for output - translation
 process

This is a follow-up of bug 25305, all strings need to be utf8 encoded
when printed to the file.

Test plan:
Test the whole translation process and translate a language you know.
Test the translated interface for: XSLT (detail and result pages), syspref, and "normal" views
You should not get encoding errors (or warning in logs), nowhere.
---
 misc/translator/LangInstaller.pm |  5 +++--
 misc/translator/tmpl_process3.pl | 37 ++++++++++++++++----------------
 2 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/misc/translator/LangInstaller.pm b/misc/translator/LangInstaller.pm
index 7b5e23648b..79180cc64e 100644
--- a/misc/translator/LangInstaller.pm
+++ b/misc/translator/LangInstaller.pm
@@ -34,6 +34,7 @@ use File::Spec;
 use File::Temp qw( tempdir tempfile );
 use Template::Parser;
 use PPI;
+use Encode qw( encode_utf8 );
 
 
 $YAML::Syck::ImplicitTyping = 1;
@@ -830,7 +831,7 @@ sub extract_messages_from_templates {
             File::Spec->catfile($tempdir, 'koha-tmpl', 'opac-tmpl', $file);
 
         make_path(dirname($destfile));
-        open my $fh, '>', $destfile;
+        open my $fh, '>:encoding(utf-8)', $destfile;
 
         my @blocks = ($data->{BLOCK}, values %{ $data->{DEFBLOCKS} });
         foreach my $block (@blocks) {
@@ -880,7 +881,7 @@ sub extract_messages_from_templates {
                 push @required_args, shift @args if $keyword =~ /n/;
                 push @required_args, shift @args if $keyword =~ /p/;
 
-                say $fh "$keyword(" . join(', ', @required_args) . ");";
+                say $fh encode_utf8 "$keyword(" . join(', ', @required_args) . ");";
             }
 
         }
diff --git a/misc/translator/tmpl_process3.pl b/misc/translator/tmpl_process3.pl
index 68c58f0cc6..11a0ba13a1 100755
--- a/misc/translator/tmpl_process3.pl
+++ b/misc/translator/tmpl_process3.pl
@@ -22,6 +22,7 @@ use Locale::PO;
 use File::Temp qw( :POSIX );
 use TmplTokenizer;
 use VerboseWarnings qw( :warn :die );
+use Encode qw( encode_utf8 );
 
 ###############################################################################
 
@@ -127,27 +128,27 @@ sub text_replace (**) {
         print $output find_translation($t);
     } elsif ($kind eq C4::TmplTokenType::TEXT_PARAMETRIZED) {
         my $fmt = find_translation($s->form);
-        print $output TmplTokenizer::parametrize($fmt, 1, $s, sub {
+        print $output encode_utf8 TmplTokenizer::parametrize($fmt, 1, $s, sub {
         $_ = $_[0];
         my($kind, $t, $attr) = ($_->type, $_->string, $_->attributes);
         $kind == C4::TmplTokenType::TAG && %$attr?
             text_replace_tag($t, $attr): $t });
     } elsif ($kind eq C4::TmplTokenType::TAG && %$attr) {
-        print $output text_replace_tag($t, $attr);
+        print $output encode_utf8(text_replace_tag($t, $attr));
     } elsif ($s->has_js_data) {
         for my $t (@{$s->js_data}) {
         # FIXME for this whole block
         if ($t->[0]) {
-            printf $output "%s%s%s", $t->[2], find_translation $t->[3],
-                $t->[2];
+            printf $output "%s%s%s", encode_utf8($t->[2]), encode_utf8(find_translation $t->[3]),
+                encode_utf8($t->[2]);
         } else {
-            print $output $t->[1];
+            print $output encode_utf8($t->[1]);
         }
         }
     } elsif (defined $t) {
         # Quick fix to bug 4472
         $t = "<!DOCTYPE stylesheet ["  if $t =~ /DOCTYPE stylesheet/ ;
-        print $output $t;
+        print $output encode_utf8($t);
     }
     }
 }
@@ -377,14 +378,14 @@ if ($action eq 'create')  {
     # FIXME: need to check the return value
     unless (-f $str_file) {
         local(*INPUT, *OUTPUT);
-        open(INPUT, "<$tmpfile2");
-        open(OUTPUT, ">$str_file");
-        while (<INPUT>) {
-        print OUTPUT;
-        last if /^\n/s;
+        open(my $input_fh, '<:encoding(utf-8)', $tmpfile2);
+        open(my $output_fh, '>:encoding(utf-8)', $str_file);
+        while (<$input_fh>) {
+            print encode_utf8 $output_fh;
+            last if /^\n/s;
         }
-        close INPUT;
-        close OUTPUT;
+        close $input_fh;
+        close $output_fh;
     }
     $st = system("msgmerge ".($quiet?'-q':'')." -s $str_file $tmpfile2 -o - | msgattrib --no-obsolete -o $str_file");
     } else {
@@ -448,8 +449,8 @@ if ($action eq 'create')  {
     -d $out_dir || die "$out_dir: The directory does not exist\n";
 
     # Try to open the file, because Locale::PO doesn't check :-/
-    open(INPUT, "<$str_file") || die "$str_file: $!\n";
-    close INPUT;
+    open(my $input_fh, '<:encoding(utf-8)', $str_file) || die "$str_file: $!\n";
+    close $input_fh;
 
     # creates the new tmpl file using the new translation
     for my $input (@in_files) {
@@ -465,9 +466,9 @@ if ($action eq 'create')  {
             VerboseWarnings::set_input_file_name $input;
             mkdir_recursive($targetdir) unless -d $targetdir;
             print STDERR "Creating $target...\n" unless $quiet;
-            open( OUTPUT, ">$target" ) || die "$target: $!\n";
-            text_replace( $h, *OUTPUT );
-            close OUTPUT;
+            open( my $output_fh, '>encoding(utf-8)' ) || die "$target: $!\n";
+            text_replace( $h, *$output_fh);
+            close $output_fh;
         } else {
         # just copying the file
             mkdir_recursive($targetdir) unless -d $targetdir;
-- 
2.20.1