From ca429488a2d6f76a49ddc7438bde07548328abc7 Mon Sep 17 00:00:00 2001 From: Jonathan Druart Date: Tue, 19 May 2020 10:08:53 +0200 Subject: [PATCH] Bug 25501: Encode strings in utf8 for output - translation process This is a follow-up of bug 25305, all strings need to be utf8 encoded when printed to the file. Test plan: Test the whole translation process and translate a language you know. Test the translated interface for: XSLT (detail and result pages), syspref, and "normal" views You should not get encoding errors (or warning in logs), nowhere. --- misc/translator/LangInstaller.pm | 5 +++-- misc/translator/tmpl_process3.pl | 37 ++++++++++++++++---------------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/misc/translator/LangInstaller.pm b/misc/translator/LangInstaller.pm index 7b5e23648b..79180cc64e 100644 --- a/misc/translator/LangInstaller.pm +++ b/misc/translator/LangInstaller.pm @@ -34,6 +34,7 @@ use File::Spec; use File::Temp qw( tempdir tempfile ); use Template::Parser; use PPI; +use Encode qw( encode_utf8 ); $YAML::Syck::ImplicitTyping = 1; @@ -830,7 +831,7 @@ sub extract_messages_from_templates { File::Spec->catfile($tempdir, 'koha-tmpl', 'opac-tmpl', $file); make_path(dirname($destfile)); - open my $fh, '>', $destfile; + open my $fh, '>:encoding(utf-8)', $destfile; my @blocks = ($data->{BLOCK}, values %{ $data->{DEFBLOCKS} }); foreach my $block (@blocks) { @@ -880,7 +881,7 @@ sub extract_messages_from_templates { push @required_args, shift @args if $keyword =~ /n/; push @required_args, shift @args if $keyword =~ /p/; - say $fh "$keyword(" . join(', ', @required_args) . ");"; + say $fh encode_utf8 "$keyword(" . join(', ', @required_args) . ");"; } } diff --git a/misc/translator/tmpl_process3.pl b/misc/translator/tmpl_process3.pl index 68c58f0cc6..11a0ba13a1 100755 --- a/misc/translator/tmpl_process3.pl +++ b/misc/translator/tmpl_process3.pl @@ -22,6 +22,7 @@ use Locale::PO; use File::Temp qw( :POSIX ); use TmplTokenizer; use VerboseWarnings qw( :warn :die ); +use Encode qw( encode_utf8 ); ############################################################################### @@ -127,27 +128,27 @@ sub text_replace (**) { print $output find_translation($t); } elsif ($kind eq C4::TmplTokenType::TEXT_PARAMETRIZED) { my $fmt = find_translation($s->form); - print $output TmplTokenizer::parametrize($fmt, 1, $s, sub { + print $output encode_utf8 TmplTokenizer::parametrize($fmt, 1, $s, sub { $_ = $_[0]; my($kind, $t, $attr) = ($_->type, $_->string, $_->attributes); $kind == C4::TmplTokenType::TAG && %$attr? text_replace_tag($t, $attr): $t }); } elsif ($kind eq C4::TmplTokenType::TAG && %$attr) { - print $output text_replace_tag($t, $attr); + print $output encode_utf8(text_replace_tag($t, $attr)); } elsif ($s->has_js_data) { for my $t (@{$s->js_data}) { # FIXME for this whole block if ($t->[0]) { - printf $output "%s%s%s", $t->[2], find_translation $t->[3], - $t->[2]; + printf $output "%s%s%s", encode_utf8($t->[2]), encode_utf8(find_translation $t->[3]), + encode_utf8($t->[2]); } else { - print $output $t->[1]; + print $output encode_utf8($t->[1]); } } } elsif (defined $t) { # Quick fix to bug 4472 $t = "$str_file"); - while () { - print OUTPUT; - last if /^\n/s; + open(my $input_fh, '<:encoding(utf-8)', $tmpfile2); + open(my $output_fh, '>:encoding(utf-8)', $str_file); + while (<$input_fh>) { + print encode_utf8 $output_fh; + last if /^\n/s; } - close INPUT; - close OUTPUT; + close $input_fh; + close $output_fh; } $st = system("msgmerge ".($quiet?'-q':'')." -s $str_file $tmpfile2 -o - | msgattrib --no-obsolete -o $str_file"); } else { @@ -448,8 +449,8 @@ if ($action eq 'create') { -d $out_dir || die "$out_dir: The directory does not exist\n"; # Try to open the file, because Locale::PO doesn't check :-/ - open(INPUT, "<$str_file") || die "$str_file: $!\n"; - close INPUT; + open(my $input_fh, '<:encoding(utf-8)', $str_file) || die "$str_file: $!\n"; + close $input_fh; # creates the new tmpl file using the new translation for my $input (@in_files) { @@ -465,9 +466,9 @@ if ($action eq 'create') { VerboseWarnings::set_input_file_name $input; mkdir_recursive($targetdir) unless -d $targetdir; print STDERR "Creating $target...\n" unless $quiet; - open( OUTPUT, ">$target" ) || die "$target: $!\n"; - text_replace( $h, *OUTPUT ); - close OUTPUT; + open( my $output_fh, '>encoding(utf-8)' ) || die "$target: $!\n"; + text_replace( $h, *$output_fh); + close $output_fh; } else { # just copying the file mkdir_recursive($targetdir) unless -d $targetdir; -- 2.20.1