Bugzilla – Attachment 98506 Details for
Bug 24262
Translate installer data in YAML format
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 24262: Translate installer data in YAML format
Bug-24262-Translate-installer-data-in-YAML-format.patch (text/plain), 17.97 KB, created by
Bernardo Gonzalez Kriegel
on 2020-02-05 15:22:18 UTC
(
hide
)
Description:
Bug 24262: Translate installer data in YAML format
Filename:
MIME Type:
Creator:
Bernardo Gonzalez Kriegel
Created:
2020-02-05 15:22:18 UTC
Size:
17.97 KB
patch
obsolete
>From 17be288f0c3eac30ec0cc9cb1ea0be93d4c1c752 Mon Sep 17 00:00:00 2001 >From: Bernardo Gonzalez Kriegel <bgkriegel@gmail.com> >Date: Fri, 20 Dec 2019 10:53:11 -0300 >Subject: [PATCH] Bug 24262: Translate installer data in YAML format > >This patch adds the ability to: >1) Create new translation files from yaml installer files >2) Create installer directory for a given language > >It will not create a installer directory if it already exists. > >New (possible) translation files: > xx-YY-installer.po > xx-YY-installer-MARC21.po > xx-YY-installer-UNIMARC.po > >Needs Bug 13897 (for yaml files) > >NOTE: updated version adding ability to process multiline >fields, discard small ( < 2) strings, and discard strings >with pure html, TT or punctuation. > >To test: >1) Apply patches from Bug 13897 >2) Apply this patch >3) Go to misc/translation >4) Create translation files for a NEW language > $ ./translate create xx-YY > > check new file 'xx-YY-installer.po' > >5) Copy ../../installer/data/mysql/en/optional/auth_val.yml > into ../../installer/data/mysql/en/marcflavour/marc21/mandatory/ > and ../../installer/data/mysql/en/marcflavour/unimarc/mandatory/ > > remove po/xx-YY*, then repeat creation > > check new files 'xx-YY-installer.po', 'xx-YY-installer-MARC21.po' > and 'xx-YY-installer-UNIMARC.po' > > remove all new files > >6) Create for xx-YY again and try update > $ ./translate create xx-YY > edit ../../installer/data/mysql/en/optional/auth_val.yml > and change one char in one of the translatable values, > also edit 'xx-YY-installer.po', translate the same > string (in msgstr). > > Do an update > $ ./translate update xx-YY > > check in 'xx-YY-installer.po' a fuzzy value for the > translated value and the preservation of the translation > Fix the translation, or add a new one. > >7) Create an install dir for xx-YY > ./translate install xx-YY > > Check new dir '../../installer/data/mysql/xx-YY/' > Check files on it > $ tree ../../installer/data/mysql/xx-YY/ > and compare with ../../installer/data/mysql/en/ > > All installation files must be present > >8) Try a new Koha install using this language in the > usual way. > Check in authorised_values table for the translated string. > >9) Try create an install dir for an existing language > (eg. es-ES, de-DE or fr-FR ), eg. > ./translate install de-DE > > Install dir is detected and not changed, a note is printed. >--- > misc/translator/LangInstaller.pm | 290 ++++++++++++++++++++++++++++++- > 1 file changed, 282 insertions(+), 8 deletions(-) > >diff --git a/misc/translator/LangInstaller.pm b/misc/translator/LangInstaller.pm >index 3da23a84d8..f82db14ca7 100644 >--- a/misc/translator/LangInstaller.pm >+++ b/misc/translator/LangInstaller.pm >@@ -22,18 +22,20 @@ use Modern::Perl; > use C4::Context; > # WARNING: Any other tested YAML library fails to work properly in this > # script content >-use YAML::Syck qw( Dump LoadFile ); >+use YAML::Syck qw( Dump LoadFile DumpFile ); > use Locale::PO; > use FindBin qw( $Bin ); > use File::Basename; > use File::Find; > use File::Path qw( make_path ); >+use File::Copy; > use File::Slurp; > use File::Spec; >-use File::Temp qw( tempdir ); >+use File::Temp qw( tempdir tempfile ); > use Template::Parser; > use PPI; > >+ > $YAML::Syck::ImplicitTyping = 1; > > >@@ -79,12 +81,14 @@ sub new { > $self->{msgmerge} = `which msgmerge`; > $self->{msgfmt} = `which msgfmt`; > $self->{msginit} = `which msginit`; >+ $self->{msgattrib} = `which msgattrib`; > $self->{xgettext} = `which xgettext`; > $self->{sed} = `which sed`; > chomp $self->{cp}; > chomp $self->{msgmerge}; > chomp $self->{msgfmt}; > chomp $self->{msginit}; >+ chomp $self->{msgattrib}; > chomp $self->{xgettext}; > chomp $self->{sed}; > >@@ -140,16 +144,40 @@ sub new { > }; > } > >+ # EN YAML installer files >+ push @{$self->{installer}}, { >+ name => "YAML installer files", >+ dirs => [ 'installer/data/mysql/en/mandatory', >+ 'installer/data/mysql/en/optional'], >+ suffix => "-installer.po", >+ }; >+ >+ # EN MARC21 YAML installer files >+ push @{$self->{installer}}, { >+ name => "MARC21 YAML installer files", >+ dirs => [ 'installer/data/mysql/en/marcflavour/marc21/mandatory', >+ 'installer/data/mysql/en/marcflavour/marc21/optional'], >+ suffix => "-installer-MARC21.po", >+ }; >+ >+ # EN UNIMARC YAML installer files >+ push @{$self->{installer}}, { >+ name => "UNIMARC YAML installer files", >+ dirs => [ 'installer/data/mysql/en/marcflavour/unimarc/mandatory', ], >+ suffix => "-installer-UNIMARC.po", >+ }; >+ > bless $self, $class; > } > > > sub po_filename { >- my $self = shift; >+ my $self = shift; >+ my $suffix = shift; > > my $context = C4::Context->new; > my $trans_path = $Bin . '/po'; >- my $trans_file = "$trans_path/" . $self->{lang} . "-pref.po"; >+ my $trans_file = "$trans_path/" . $self->{lang} . $suffix; > return $trans_file; > } > >@@ -276,8 +304,8 @@ sub save_po { > $po->{''} ||= $default_pref_po_header; > > # Write .po entries into a file put in Koha standard po directory >- Locale::PO->save_file_fromhash( $self->po_filename, $po ); >- say "Saved in file: ", $self->po_filename if $self->{verbose}; >+ Locale::PO->save_file_fromhash( $self->po_filename("-pref.po"), $po ); >+ say "Saved in file: ", $self->po_filename("-pref.po") if $self->{verbose}; > } > > >@@ -289,7 +317,7 @@ sub get_po_merged_with_en { > my $po_current = $self->{po}; > > # Get po from previous generation >- my $po_previous = Locale::PO->load_file_ashash( $self->po_filename ); >+ my $po_previous = Locale::PO->load_file_ashash( $self->po_filename("-pref.po") ); > > for my $id ( keys %$po_current ) { > my $po = $po_previous->{Locale::PO->quote($id)}; >@@ -430,7 +458,7 @@ sub update_tmpl { > sub create_prefs { > my $self = shift; > >- if ( -e $self->po_filename ) { >+ if ( -e $self->po_filename("-pref.po") ) { > say "Preferences .po file already exists. Delete it if you want to recreate it."; > return; > } >@@ -438,6 +466,249 @@ sub create_prefs { > $self->save_po(); > } > >+sub get_po_from_target { >+ my $self = shift; >+ my $target = shift; >+ >+ my $po; >+ my $po_head = new Locale::PO; >+ $po_head->{msgid} = "\"\""; >+ $po_head->{msgstr} = "". >+ "Project-Id-Version: Koha Project - Installation files\\n" . >+ "PO-Revision-Date: YEAR-MO-DA HO:MI +ZONE\\n" . >+ "Last-Translator: FULL NAME <EMAIL\@ADDRESS>\\n" . >+ "Language-Team: Koha Translation Team\\n" . >+ "Language: ".$self->{lang}."\\n" . >+ "MIME-Version: 1.0\\n" . >+ "Content-Type: text/plain; charset=UTF-8\\n" . >+ "Content-Transfer-Encoding: 8bit\\n"; >+ >+ my @dirs = @{ $target->{dirs} }; >+ my $intradir = $self->{context}->config('intranetdir'); >+ for my $dir ( @dirs ) { # each dir >+ opendir( my $dh, "$intradir/$dir" ) or die ("Can't open $intradir/$dir"); >+ my @filelist = grep { $_ =~ m/\.yml/ } readdir($dh); # Just yaml files >+ close($dh); >+ for my $file ( @filelist ) { # each file >+ my $yaml = LoadFile( "$intradir/$dir/$file" ); >+ my @tables = @{ $yaml->{'tables'} }; >+ my $tablec; >+ for my $table ( @tables ) { # each table >+ $tablec++; >+ my $table_name = ( keys %$table )[0]; >+ my @translatable = @{ $table->{$table_name}->{translatable} }; >+ my @rows = @{ $table->{$table_name}->{rows} }; >+ my @multiline = @{ $table->{$table_name}->{'multiline'} }; # to check multiline values >+ my $rowc; >+ for my $row ( @rows ) { # each row >+ $rowc++; >+ for my $field ( @translatable ) { # each field >+ if ( @multiline and grep { $_ eq $field } @multiline ) { # multiline fields, only notices ATM >+ my $mulc; >+ foreach my $line ( @{$row->{$field}} ) { >+ $mulc++; >+ next if ( $line =~ /^(\s*<.*?>\s*$|^\s*\[.*?\]\s*|\s*)$/ ); # discard pure html, TT, empty >+ $line =~ s/(<<.*?>>|\[\%.*?\%\]|<.*?>)/\%s/g; # put placeholders >+ next if ( $line =~ /^(\s|%s|-|[[:punct:]]|\(|\))*$/ or length($line) < 2 ); # discard non strings >+ if ( not $po->{ $line } ) { >+ my $msg = new Locale::PO( >+ -msgid => $line, -msgstr => '', >+ -reference => "$dir/$file:$table_name:$tablec:row:$rowc:mul:$mulc" ); >+ $po->{ $line } = $msg; >+ } >+ } >+ } else { >+ if ( length($row->{$field}) > 1 # discard small strings >+ and not $po->{ $row->{$field} } ) { >+ my $msg = new Locale::PO( >+ -msgid => $row->{$field}, -msgstr => '', >+ -reference => "$dir/$file:$table_name:$tablec:row:$rowc" ); >+ $po->{ $row->{$field} } = $msg; >+ } >+ } >+ } >+ } >+ } >+ my $desccount; >+ for my $description ( @{ $yaml->{'description'} } ) { >+ $desccount++; >+ if ( length($description) > 1 and not $po->{ $description } ) { >+ my $msg = new Locale::PO( >+ -msgid => $description, -msgstr => '', >+ -reference => "$dir/$file:description:$desccount" ); >+ $po->{ $description } = $msg; >+ } >+ } >+ } >+ } >+ $po->{''} = $po_head if ( $po ); >+ >+ return $po; >+} >+ >+sub create_installer { >+ my $self = shift; >+ return unless ( $self->{installer} ); >+ >+ say "Create installer translation files\n" if $self->{verbose}; >+ >+ my @targets = @{ $self->{installer} }; # each installer target (common,marc21,unimarc) >+ >+ for my $target ( @targets ) { >+ if ( -e $self->po_filename( $target->{suffix} ) ) { >+ say "$self->{lang}$target->{suffix} file already exists. Delete it if you want to recreate it."; >+ return; >+ } >+ } >+ >+ for my $target ( @targets ) { >+ my $po = get_po_from_target( $self, $target ); >+ # create output file only if there is something to write >+ if ( $po ) { >+ my $po_file = $self->po_filename( $target->{suffix} ); >+ Locale::PO->save_file_fromhash( $po_file, $po ); >+ say "Saved in file: ", $po_file if $self->{verbose}; >+ } >+ } >+} >+ >+sub update_installer { >+ my $self = shift; >+ return unless ( $self->{installer} ); >+ >+ say "Update installer translation files\n" if $self->{verbose}; >+ >+ my @targets = @{ $self->{installer} }; # each installer target (common,marc21,unimarc) >+ >+ for my $target ( @targets ) { >+ return unless ( -e $self->po_filename( $target->{suffix} ) ); >+ my $po = get_po_from_target( $self, $target ); >+ # update file only if there is something to update >+ if ( $po ) { >+ my ( $fh, $po_temp ) = tempfile(); >+ binmode( $fh, ":encoding(UTF-8)" ); >+ Locale::PO->save_file_fromhash( $po_temp, $po ); >+ my $po_file = $self->po_filename( $target->{suffix} ); >+ eval { >+ my $st = system($self->{msgmerge}." ".($self->{verbose}?'':'-q'). >+ " -s $po_file $po_temp -o - | ".$self->{msgattrib}." --no-obsolete -o $po_file"); >+ }; >+ say "Updated file: ", $po_file if $self->{verbose}; >+ } >+ } >+} >+ >+sub translate_yaml { >+ my $self = shift; >+ my $target = shift; >+ my $srcyml = shift; >+ >+ my $po_file = $self->po_filename( $target->{suffix} ); >+ return $srcyml unless ( -e $po_file ); >+ >+ my $po_ref = Locale::PO->load_file_ashash( $po_file ); >+ >+ my $dstyml = LoadFile( $srcyml ); >+ >+ # translate fields in table rows >+ my @tables = @{ $dstyml->{'tables'} }; >+ for my $table ( @tables ) { # each table >+ my $table_name = ( keys %$table )[0]; >+ my @translatable = @{ $table->{$table_name}->{translatable} }; >+ my @rows = @{ $table->{$table_name}->{rows} }; >+ my @multiline = @{ $table->{$table_name}->{'multiline'} }; # to check multiline values >+ for my $row ( @rows ) { # each row >+ for my $field ( @translatable ) { # each translatable field >+ if ( @multiline and grep { $_ eq $field } @multiline ) { # multiline fields, only notices ATM >+ foreach my $line ( @{$row->{$field}} ) { >+ next if ( $line =~ /^(\s*<.*?>\s*$|^\s*\[.*?\]\s*|\s*)$/ ); # discard pure html, TT, empty >+ my @ttvar; >+ while ( $line =~ s/(<<.*?>>|\[\%.*?\%\]|<.*?>)/\%s/ ) { # put placeholders, save matches >+ my $var = $1; >+ push @ttvar, $var; >+ } >+ >+ if ( $line =~ /^(\s|%s|-|[[:punct:]]|\(|\))*$/ ) { # ignore non strings >+ while ( @ttvar ) { # restore placeholders >+ my $var = shift @ttvar; >+ $line =~ s/\%s/$var/; >+ } >+ next; >+ } else { >+ my $po = $po_ref->{"\"$line\""}; # quoted key >+ if ( $po and not defined( $po->fuzzy() ) # not fuzzy >+ and length( $po->msgid() ) > 2 # not empty msgid >+ and length( $po->msgstr() ) > 2 ) { # not empty msgstr >+ $line = $po->dequote( $po->msgstr() ); >+ } >+ while ( @ttvar ) { # restore placeholders >+ my $var = shift @ttvar; >+ $line =~ s/\%s/$var/; >+ } >+ } >+ } >+ } else { >+ my $po = $po_ref->{"\"$row->{$field}\""}; # quoted key >+ if ( $po and not defined( $po->fuzzy() ) # not fuzzy >+ and length( $po->msgid() ) > 2 # not empty msgid >+ and length( $po->msgstr() ) > 2 ) { # not empty msgstr >+ $row->{$field} = $po->dequote( $po->msgstr() ); >+ } >+ } >+ } >+ } >+ } >+ >+ # translate descriptions >+ for my $description ( @{ $dstyml->{'description'} } ) { >+ my $po = $po_ref->{"\"$description\""}; >+ if ( $po and not defined( $po->fuzzy() ) >+ and length( $po->msgid() ) > 2 >+ and length( $po->msgstr() ) > 2 ) { >+ $description = $po->dequote( $po->msgstr() ); >+ } >+ } >+ >+ return $dstyml; >+} >+ >+sub install_installer { >+ my $self = shift; >+ return unless ( $self->{installer} ); >+ >+ my $intradir = $self->{context}->config('intranetdir'); >+ my $db_scheme = $self->{context}->config('db_scheme'); >+ my $langdir = "$intradir/installer/data/$db_scheme/$self->{lang}"; >+ if ( -d $langdir ) { >+ say "$self->{lang} installer dir $langdir already exists.\nDelete it if you want to recreate it."; >+ return; >+ } >+ >+ say "Install installer files\n" if $self->{verbose}; >+ >+ for my $target ( @{ $self->{installer} } ) { >+ return unless ( -e $self->po_filename( $target->{suffix} ) ); >+ for my $dir ( @{ $target->{dirs} } ) { >+ ( my $tdir = "$dir" ) =~ s|/en/|/$self->{lang}/|; >+ make_path("$intradir/$tdir"); >+ >+ opendir( my $dh, "$intradir/$dir" ) or die ("Can't open $intradir/$dir"); >+ my @files = grep { ! /^\.+$/ } readdir($dh); >+ close($dh); >+ >+ for my $file ( @files ) { >+ if ( $file =~ /yml$/ ) { >+ my $translated_yaml = translate_yaml( $self, $target, "$intradir/$dir/$file" ); >+ open(my $fh, ">:encoding(UTF-8)", "$intradir/$tdir/$file"); >+ DumpFile( $fh, $translated_yaml ); >+ close($fh); >+ } else { >+ File::Copy::copy( "$intradir/$dir/$file", "$intradir/$tdir/$file" ); >+ } >+ } >+ } >+ } >+} > > sub create_tmpl { > my ($self, $files) = @_; >@@ -723,6 +994,7 @@ sub install { > $self->install_prefs(); > $self->install_messages(); > $self->remove_pot(); >+ $self->install_installer(); > } > > >@@ -743,6 +1015,7 @@ sub update { > $self->update_tmpl($files) unless $self->{pref_only}; > $self->update_prefs(); > $self->update_messages(); >+ $self->update_installer(); > } > $self->remove_pot(); > } >@@ -755,6 +1028,7 @@ sub create { > $self->create_prefs(); > $self->create_messages(); > $self->remove_pot(); >+ $self->create_installer(); > } > > >-- >2.17.1
<b>You cannot view the attachment while viewing its details because your browser does not support IFRAMEs. <a href="attachment.cgi?id=98506">View the attachment on a separate page</a>.</b>
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 24262
:
96538
|
96540
|
96593
|
96626
|
98506
|
99396
|
100194
|
100344
|
100345