From ec50d02c3eff746649f8761a813160c2ee9b55d3 Mon Sep 17 00:00:00 2001 From: Bernardo Gonzalez Kriegel Date: Sun, 23 Mar 2014 19:01:43 -0300 Subject: [PATCH] Bug 7939 - Separate po files for different MARC dialects [PRELIMINAR 2 - not in final form but can be tested] This patch implements separate PO files for different MARC dialects. It depends on correct filenames, i.e. it will build PO files using files with/without "unimarc|normarc|marc21" on their names. Prominent changes: A) tmpl_process3.pl three new options: exclude or include certain filenames and a new source dir that can have multiple values (I opted to add this last feature and not change current source dir in order not to brake current usage) Minor change on use of msgmerge, now there are no backup files (*~) and obsoleted translations are removed (to produce smaller files) [this can be on another bug, is not strictly related with the pourpose of this bug] B) LangInstaller.pm added definitions to create or update nn-NO-{MARCFLAVOR}.po and modification on install procedure to handle multiple target dirs C) Standarization of filenames OPAC prog po file is now xx-YY-opac-prog.po STAFF po file is now xx-YY-staff-prog.po MARC dialects po files are xx-YY-marc-{MARCFLAVOUR}.po [again, renaming not related, can be in it's own bug] To test: 1) Update po files for your preferred language, ej. nn-NO cd misc/translator perl translate update nn-NO (do it twice if there are warnings) 2) Do some copying/renaming cp po/nn-NO-i-staff-t-prog-v-3006000.po po/nn-NO-marc-UNIMARC.po cp po/nn-NO-i-staff-t-prog-v-3006000.po po/nn-NO-marc-NORMARC.po cp po/nn-NO-i-staff-t-prog-v-3006000.po po/nn-NO-marc-MARC21.po mv po/nn-NO-i-staff-t-prog-v-3006000.po po/nn-NO-staff-prog.po mv po/nn-NO-i-opac-t-prog-v-3006000.po po/nn-NO-opac-prog.po (most MARC dialect strings are on staff, so we use that as basis) 3) Apply the patch 4) Update again to fix translation files, verbose perl translate update nn-NO -v 5) Install language, verbose, verify translations perl translate install nn-NO -v 6) Create translation files rm po/nn-NO* perl translate create nn-NO we must have this list: po/nn-NO-marc-MARC21.po po/nn-NO-marc-NORMARC.po po/nn-NO-marc-UNIMARC.po po/nn-NO-opac-bootstrap.po po/nn-NO-opac-ccsr.po po/nn-NO-opac-prog.po po/nn-NO-pref.po po/nn-NO-staff-help.po po/nn-NO-staff-prog.po [staff-help could be more properly staff-prog-help, another rename] Additional tests: 7) Number of msgids 7.a) Before patch and after upgrade, extract and count msgids for i in $(ls po/nn-NO-*po); \ do msginit -i $i -o nn-old.po --no-translator --no-wrap --locale=nn_NO; \ egrep ^msgid nn-old.po >> old; \ done sort new | uniq | tee s-old | wc -l > n-old s-old: have all msgids n-old: number of msgids 7.b) After patch and after creation of new files Repeat procedure, diferent files (s-new, n-new) 7.c) Compare (diff s-old snew), they are the same (save for a strange UNIMARC char in my case) 8) Installed dirs/files 8.a) List of EN dirs/files cd koha-tmpl find | egrep "/en/" > en 8.b) List of nn-NO dirs/files. After patch and language install cd koha-tmpl find | egrep "/nn-NO/" | sed 's|/nn-NO/|/en/|' > nn 8.c) Compare (diff en nn), they are the same (save for a bogus help/nn-NO dir, without files. Working for a solution) --- misc/translator/LangInstaller.pm | 59 +++++++++++++++++++++++++++++++------- misc/translator/tmpl_process3.pl | 32 +++++++++++++++++---- 2 files changed, 74 insertions(+), 17 deletions(-) diff --git a/misc/translator/LangInstaller.pm b/misc/translator/LangInstaller.pm index 158ca22..6154174 100644 --- a/misc/translator/LangInstaller.pm +++ b/misc/translator/LangInstaller.pm @@ -75,8 +75,8 @@ sub new { # Get all available language codes opendir $fh, $self->{path_po}; - my @langs = map { ($_) =~ /(.*)-i-opac/ } - grep { $_ =~ /.*-opac-t-prog/ } readdir($fh); + my @langs = map { ($_) =~ /(.*)-pref/ } + grep { $_ =~ /.*-pref/ } readdir($fh); closedir $fh; $self->{langs} = \@langs; @@ -86,12 +86,12 @@ sub new { { name => 'OPAC prog', dir => "$opachtdocs/prog", - suffix => '-i-opac-t-prog-v-3006000.po', + suffix => '-opac-prog.po', }, { name => 'Intranet prog UI', dir => $context->config('intrahtdocs') . '/prog', - suffix => '-i-staff-t-prog-v-3006000.po', + suffix => '-staff-prog.po', }, { name => 'Intranet prog help', @@ -110,6 +110,20 @@ sub new { }; } + # MARC flavours (hardcoded) + for ( "MARC21", "UNIMARC", "NORMARC" ) { + my $dirs = $context->config('intrahtdocs') . '/prog'; + opendir $fh, $context->config('opachtdocs'); + for ( grep { not /^\.|\.\.|lib$/ } readdir($fh) ) { + $dirs .= ' ' . "$opachtdocs/$_"; + } + push @{$self->{interface}}, { + name => "$_", + dir => $dirs, + suffix => "-marc-$_.po", + }; + } + bless $self, $class; } @@ -329,29 +343,37 @@ sub install_tmpl { my ($self, $files) = @_; say "Install templates" if $self->{verbose}; for my $trans ( @{$self->{interface}} ) { + my @t_dirs = split(" ", $trans->{dir}); + for my $t_dir ( @t_dirs ) { print " Install templates '$trans->{name}'\n", - " From: $trans->{dir}/en/\n", - " To : $trans->{dir}/$self->{lang}\n", + " From: $t_dir/en/\n", + " To : $t_dir/$self->{lang}\n", " With: $self->{path_po}/$self->{lang}$trans->{suffix}\n" if $self->{verbose}; - my $trans_dir = ( $trans->{name} =~ /help/ )?"$trans->{dir}":"$trans->{dir}/en/"; - my $lang_dir = ( $trans->{name} =~ /help/ )?"$trans->{dir}":"$trans->{dir}/$self->{lang}"; + my $trans_dir = ( $trans->{name} =~ /help/ )?"$t_dir":"$t_dir/en/"; + my $lang_dir = ( $trans->{name} =~ /help/ )?"$t_dir":"$t_dir/$self->{lang}"; $lang_dir =~ s|/en/|/$self->{lang}/|; mkdir $lang_dir unless -d $lang_dir; my $excludes = ( $trans->{name} =~ /UI/ )?"-x 'help'":""; + my $match = ( $trans->{name} =~ /MARC/ )?"-m \"$trans->{name}\"":""; # for MARC translations + my $nomatch = ( $trans->{name} !~ /MARC/ )?"-n \"marc21|unimarc|normarc\"":""; # hardcoded MARC variants system "$self->{process} install " . "-i $trans_dir " . "-o $lang_dir ". - "-s $self->{path_po}/$self->{lang}$trans->{suffix} -r $excludes" . + "-s $self->{path_po}/$self->{lang}$trans->{suffix} -r " . + "$excludes " . + "$match " . + "$nomatch " . ( @$files ? ' -f ' . join ' -f ', @$files : '' ) + } } } @@ -371,11 +393,19 @@ sub update_tmpl { my $trans_dir = ( $trans->{name} =~ /help/ )?"$trans->{dir}":"$trans->{dir}/en/"; my $excludes = ( $trans->{name} =~ /UI/ )?"-x 'help'":""; + my $match = ( $trans->{name} =~ /MARC/ )?"-m \"$trans->{name}\"":""; # for MARC translations + my $dirs = ( $trans->{name} =~ /MARC/ )?"-d ".join(" -d ",split(" ",$trans->{dir})):""; # for MARC translations; + my $nomatch = ( $trans->{name} !~ /MARC/ )?"-n \"marc21|unimarc|normarc\"":""; # hardcoded MARC variants + system "$self->{process} update " . "-i $trans_dir " . - "-s $self->{path_po}/$self->{lang}$trans->{suffix} -r $excludes" . + "-s $self->{path_po}/$self->{lang}$trans->{suffix} -r " . + "$excludes " . + "$dirs " . + "$match " . + "$nomatch " . ( @$files ? ' -f ' . join ' -f ', @$files @@ -410,11 +440,18 @@ sub create_tmpl { my $trans_dir = ( $trans->{name} =~ /help/ )?"$trans->{dir}":"$trans->{dir}/en/"; my $excludes = ( $trans->{name} =~ /UI/ )?"-x 'help'":""; + my $match = ( $trans->{name} =~ /MARC/ )?"-m \"$trans->{name}\"":""; # for MARC translations + my $dirs = ( $trans->{name} =~ /MARC/ )?"-d ".join(" -d ",split(" ",$trans->{dir})):""; # for MARC translations; + my $nomatch = ( $trans->{name} !~ /MARC/ )?"-n \"marc21|unimarc|normarc\"":""; # hardcoded MARC variants system "$self->{process} create " . "-i $trans_dir " . - "-s $self->{path_po}/$self->{lang}$trans->{suffix} -r $excludes" . + "-s $self->{path_po}/$self->{lang}$trans->{suffix} -r " . + "$excludes " . + "$dirs " . + "$match " . + "$nomatch " . ( @$files ? ' -f ' . join ' -f ', @$files diff --git a/misc/translator/tmpl_process3.pl b/misc/translator/tmpl_process3.pl index e5df69a..6da5be5 100755 --- a/misc/translator/tmpl_process3.pl +++ b/misc/translator/tmpl_process3.pl @@ -3,6 +3,7 @@ # Parts copyright 2003-2004 Paul Poulain # Parts copyright 2003-2004 Jerome Vizcaino # Parts copyright 2004 Ambrose Li +# Parts copyrigth 2014 Bernardo Gonzalez Kriegel =head1 NAME @@ -24,6 +25,7 @@ use VerboseWarnings qw( :warn :die ); use vars qw( $in_dir @filenames $str_file $out_dir $quiet ); use vars qw( @excludes $exclude_regex ); +use vars qw( @dirs $match $nomatch ); use vars qw( $recursive_p ); use vars qw( $pedantic_p ); use vars qw( $href ); @@ -144,9 +146,11 @@ sub listfiles { || (defined $exclude_regex && $dirent =~ /^(?:$exclude_regex)$/)) { ; } elsif (-f $path) { - my $basename = basename $path; + my $basename = fileparse( $path ); push @it, $path if ( not @$filenames or ( grep { $path =~ /$_/ } @$filenames ) ) + and ( not defined $match or $basename =~ /($match)/i ) # files to include + and ( not defined $nomatch or $basename !~ /($nomatch)/i ) # files not to include and (!defined $type || $dirent =~ /\.(?:$type)$/) || $action eq 'install'; } elsif (-d $path && $recursive_p) { push @it, listfiles($path, $type, $action, $filenames); @@ -194,7 +198,13 @@ Create or update PO files from templates, or install translated templates. If given, only these files will be processed. -s, --str-file=FILE Specify FILE as the translation (po) file for input (install) or output (create, update) - -x, --exclude=REGEXP Exclude files matching the given REGEXP + -x, --exclude=REGEXP Exclude dirs matching the given REGEXP + -d, --dir=DIRECTORY Get or update strings from DIRECTORY. Can have + multiple values. Using this ignore -i + -m, --match=REGEXP Pipe (|) separated list. Include only files + matching the given REGEXP. + -n, --nomatch=REGEXP Pipe (|) separated list. Exclude all files + matching the given REGEXP. --help Display this help and exit -q, --quiet no output to screen (except for errors) @@ -223,6 +233,9 @@ GetOptions( 'recursive|r' => \$recursive_p, 'str-file|s=s' => \$str_file, 'exclude|x=s' => \@excludes, + 'dir|d=s' => \@dirs, + 'match|m=s' => \$match, + 'nomatch|n=s' => \$nomatch, 'quiet|q' => \$quiet, 'pedantic-warnings|pedantic' => sub { $pedantic_p = 1 }, 'help' => \&usage, @@ -261,7 +274,14 @@ for my $fn ( @filenames ) { die "You cannot specify input files and directories at the same time.\n" if -d $fn; } -@in_files = listfiles($in_dir, $type, $action, \@filenames); +if ( @dirs ) { # with any -d ignore -i + foreach ( @dirs ) { + s/\/$//; + @in_files = ( @in_files, listfiles($_, $type, $action, \@filenames)); + } +} else { + @in_files = listfiles($in_dir, $type, $action, \@filenames); +} # restores the string list from file $href = Locale::PO->load_file_ashash($str_file); @@ -348,7 +368,7 @@ if ($action eq 'create') { close INPUT; close OUTPUT; } - $st = system("msgmerge -U ".($quiet?'-q':'')." -s $str_file $tmpfile2"); + $st = system("msgmerge ".($quiet?'-q':'')." -s $str_file $tmpfile2 -o - | msgattrib --no-obsolete -o $str_file"); } else { error_normal "Text extraction failed: $xgettext: $!\n", undef; error_additional "Will not run msgmerge\n", undef; @@ -377,10 +397,10 @@ if ($action eq 'create') { if ( @filenames ) { my ($tmph3, $tmpfile3) = tmpnam(); $st = system("msgcat $str_file $tmpfile2 > $tmpfile3"); - $st = system("msgmerge -U ".($quiet?'-q':'')." -s $str_file $tmpfile3") + $st = system("msgmerge ".($quiet?'-q':'')." -s $str_file $tmpfile3 -o - | msgattrib --no-obsolete -o $str_file") unless $st; } else { - $st = system("msgmerge -U ".($quiet?'-q':'')." -s $str_file $tmpfile2"); + $st = system("msgmerge ".($quiet?'-q':'')." -s $str_file $tmpfile2 -o - | msgattrib --no-obsolete -o $str_file"); } } else { error_normal "Text extraction failed: $xgettext: $!\n", undef; -- 1.7.9.5