From 5c7499fa7b9d26cd3ff77a25c92e6e59a2eb1806 Mon Sep 17 00:00:00 2001 From: Jonathan Druart Date: Fri, 14 Mar 2014 17:09:09 +0100 Subject: [PATCH] Bug 11944: replace use of utf8 with Encode See the wiki page for the explanation. --- C4/Biblio.pm | 7 ++----- C4/Charset.pm | 7 ++++--- C4/Installer.pm | 5 +++-- C4/ItemType.pm | 5 +++-- C4/Output.pm | 2 -- C4/Search.pm | 6 +++--- Koha/SearchEngine/Zebra/Search.pm | 1 - Koha/Template/Plugin/EncodeUTF8.pm | 4 ++-- cataloguing/value_builder/macles.pl | 4 ---- labels/label-create-pdf.pl | 3 ++- reports/guided_reports.pl | 3 ++- serials/serials-edit.pl | 3 ++- t/Charset.t | 13 +++++++------ 13 files changed, 30 insertions(+), 33 deletions(-) diff --git a/C4/Biblio.pm b/C4/Biblio.pm index 330951d..094da95 100644 --- a/C4/Biblio.pm +++ b/C4/Biblio.pm @@ -23,7 +23,7 @@ use strict; use warnings; use Carp; -# use utf8; +use Encode qw( decode ); use MARC::Record; use MARC::File::USMARC; use MARC::File::XML; @@ -2337,9 +2337,6 @@ sub TransformHtmlToXml { @$values[$i] =~ s/"/"/g; @$values[$i] =~ s/'/'/g; - # if ( !utf8::is_utf8( @$values[$i] ) ) { - # utf8::decode( @$values[$i] ); - # } if ( ( @$tags[$i] ne $prevtag ) ) { $j++ unless ( @$tags[$i] eq "" ); my $indicator1 = eval { substr( @$indicator[$j], 0, 1 ) }; @@ -2474,7 +2471,7 @@ sub TransformHtmlToMarc { foreach my $param_name ( keys %$cgi_params ) { if ( $param_name =~ /^tag_/ ) { my $param_value = $cgi_params->{$param_name}; - if ( utf8::decode($param_value) ) { + if ( Encode::decode('UTF-8', $param_value) ) { $cgi_params->{$param_name} = $param_value; } diff --git a/C4/Charset.pm b/C4/Charset.pm index 179ddb5..ee875bf 100644 --- a/C4/Charset.pm +++ b/C4/Charset.pm @@ -25,6 +25,7 @@ use Text::Iconv; use C4::Context; use C4::Debug; use Unicode::Normalize; +use Encode qw( decode encode is_utf8 ); use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS); @@ -108,8 +109,8 @@ will assume that this situation occur does not very often. sub IsStringUTF8ish { my $str = shift; - return 1 if utf8::is_utf8($str); - return utf8::decode($str); + return 1 if Encode::is_utf8($str); + return Encode::decode('UTF-8', $str); } =head2 SetUTF8Flag @@ -176,7 +177,7 @@ Sample code : sub NormalizeString{ my ($string,$nfd,$transform)=@_; - utf8::decode($string) unless (utf8::is_utf8($string)); + Encode::decode('UTF-8', $string) unless (Encode::is_utf8($string)); if ($nfd){ $string= NFD($string); } diff --git a/C4/Installer.pm b/C4/Installer.pm index 2add0e8..6b1aad6 100644 --- a/C4/Installer.pm +++ b/C4/Installer.pm @@ -20,6 +20,7 @@ package C4::Installer; use strict; #use warnings; FIXME - Bug 2505 +use Encode qw( encode is_utf8 ); our $VERSION = 3.07.00.049; use C4::Context; use C4::Installer::PerlModules; @@ -137,7 +138,7 @@ sub marc_framework_sql_list { my $lines = <$fh>; $lines =~ s/\n|\r/
/g; use utf8; - utf8::encode($lines) unless ( utf8::is_utf8($lines) ); + Encode::encode('UTF-8', $lines) unless ( Encode::is_utf8($lines) ); my $mandatory = ($requirelevel =~ /(mandatory|requi|oblig|necess)/i); push @frameworklist, { @@ -215,7 +216,7 @@ sub sample_data_sql_list { my $lines = <$fh>; $lines =~ s/\n|\r/
/g; use utf8; - utf8::encode($lines) unless ( utf8::is_utf8($lines) ); + Encode::encode('UTF-8', $lines) unless ( Encode::is_utf8($lines) ); my $mandatory = ($requirelevel =~ /(mandatory|requi|oblig|necess)/i); push @frameworklist, { diff --git a/C4/ItemType.pm b/C4/ItemType.pm index ad253a7..e91617f 100644 --- a/C4/ItemType.pm +++ b/C4/ItemType.pm @@ -21,6 +21,7 @@ package C4::ItemType; use strict; use warnings; use C4::Context; +use Encode qw( encode ); our $AUTOLOAD; @@ -81,7 +82,7 @@ sub all { for ( @{$dbh->selectall_arrayref( "SELECT * FROM itemtypes ORDER BY description", { Slice => {} })} ) { - utf8::encode($_->{description}); + Encode::encode('UTF-8', $_->{description}); push @itypes, $class->new($_); } return @itypes; @@ -105,7 +106,7 @@ sub get { "SELECT * FROM itemtypes WHERE itemtype = ?", undef, $itemtype ); if ( $data->{description} ) { - utf8::encode($data->{description}); + Encode::encode('UTF-8', $data->{description}); } return $class->new($data); } diff --git a/C4/Output.pm b/C4/Output.pm index 324cf69..5dc03d7 100644 --- a/C4/Output.pm +++ b/C4/Output.pm @@ -310,8 +310,6 @@ sub output_with_http_headers { # We can't encode here, that will double encode our templates, and xslt # We need to fix the encoding as it comes out of the database, or when we pass the variables to templates - -# utf8::encode($data) if utf8::is_utf8($data); $data =~ s/\&\;amp\; /\&\; /g; print $query->header($options), $data; diff --git a/C4/Search.pm b/C4/Search.pm index 09c1951..56eca2f 100644 --- a/C4/Search.pm +++ b/C4/Search.pm @@ -36,7 +36,7 @@ use URI::Escape; use Business::ISBN; use MARC::Record; use MARC::Field; -use utf8; +use Encode qw( decode is_utf8 ); use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $DEBUG); # set the version for version checking @@ -1782,8 +1782,8 @@ sub searchResults { my @repl = $marcrecord->field($1)->subfield($2); my $subfieldvalue = $repl[$i]; - if (! utf8::is_utf8($subfieldvalue)) { - utf8::decode($subfieldvalue); + if (! Encode::is_utf8($subfieldvalue)) { + Encode::decode('UTF-8', $subfieldvalue); } $newline =~ s/\[$tag\]/$subfieldvalue/g; diff --git a/Koha/SearchEngine/Zebra/Search.pm b/Koha/SearchEngine/Zebra/Search.pm index 3b736df..b805683 100644 --- a/Koha/SearchEngine/Zebra/Search.pm +++ b/Koha/SearchEngine/Zebra/Search.pm @@ -47,7 +47,6 @@ sub search { foreach my $item (@{ $results->items }) { my $title = $item->get_value('ste_title'); - #utf8::encode($title); print "$title\n"; warn dump $title; } diff --git a/Koha/Template/Plugin/EncodeUTF8.pm b/Koha/Template/Plugin/EncodeUTF8.pm index d27ccec..871cbb4 100644 --- a/Koha/Template/Plugin/EncodeUTF8.pm +++ b/Koha/Template/Plugin/EncodeUTF8.pm @@ -21,11 +21,11 @@ use Modern::Perl; use base qw{Template::Plugin::Filter}; -use Encode qw{encode}; +use Encode qw{encode is_utf8}; sub filter { my ( $self, $value ) = @_; - return is_utf8( $value ) ? encode( 'UTF-8', $value ) : $value; + return Encode::is_utf8( $value ) ? Encode::encode( 'UTF-8', $value ) : $value; } 1; diff --git a/cataloguing/value_builder/macles.pl b/cataloguing/value_builder/macles.pl index c99508e..8f910b4 100755 --- a/cataloguing/value_builder/macles.pl +++ b/cataloguing/value_builder/macles.pl @@ -84,10 +84,6 @@ my ($input) = @_; my @innerloop; my (%numbers,%cells,@colhdr,@rowhdr,@multiplelines,@lists,$table); while (my $tab = $rq->fetchrow_hashref){ -# if (! utf8::is_utf8($tab->{lib})) { -# utf8::decode($tab->{lib}); -# } -# warn $tab->{lib}; my $number=substr($tab->{authorised_value},0,1); if ($tab->{authorised_value}=~/[0-9]XX/){ $numbers{$number}->{'hdr_tab'}=$tab->{lib}; diff --git a/labels/label-create-pdf.pl b/labels/label-create-pdf.pl index 3d7ff1f..3ef7343 100755 --- a/labels/label-create-pdf.pl +++ b/labels/label-create-pdf.pl @@ -24,6 +24,7 @@ use warnings; use CGI; use C4::Auth; use C4::Debug; +use Encode qw( decode ); use C4::Creators; use C4::Labels; @@ -90,7 +91,7 @@ sub _print_text { foreach my $text_line (@$label_text) { my $pdf_font = $pdf->Font($text_line->{'font'}); my $line = "BT /$pdf_font $text_line->{'font_size'} Tf $text_line->{'text_llx'} $text_line->{'text_lly'} Td ($text_line->{'line'}) Tj ET"; - utf8::decode($line); + Encode::decode('UTF-8', $line); $pdf->Add($line); } } diff --git a/reports/guided_reports.pl b/reports/guided_reports.pl index b9c49ad..4d41e1a 100755 --- a/reports/guided_reports.pl +++ b/reports/guided_reports.pl @@ -19,6 +19,7 @@ use CGI qw/-utf8/; +use Encode qw( decode ); use Text::CSV; use URI::Escape; use C4::Reports::Guided; @@ -853,7 +854,7 @@ sub header_cell_values { my @cols; foreach my $c (@{$sth->{NAME}}) { #FIXME apparently DBI still needs a utf8 fix for this? - utf8::decode($c); + Encode::decode('UTF-8', $c); push @cols, $c; } return @cols; diff --git a/serials/serials-edit.pl b/serials/serials-edit.pl index 8fb2e60..2f5605a 100755 --- a/serials/serials-edit.pl +++ b/serials/serials-edit.pl @@ -64,6 +64,7 @@ op can be : use strict; use warnings; use CGI; +use Encode qw( decode is_utf8 ); use C4::Auth; use C4::Dates qw/format_date format_date_in_iso/; use C4::Biblio; @@ -202,7 +203,7 @@ if ( $op and $op eq 'serialchangestatus' ) { # Convert serialseqs to UTF-8 to prevent encoding problems foreach my $seq (@serialseqs) { - utf8::decode($seq) unless utf8::is_utf8($seq); + Encode::decode('UTF-8', $seq) unless Encode::is_utf8($seq); } my $newserial; diff --git a/t/Charset.t b/t/Charset.t index 27b2a32..9f81df7 100755 --- a/t/Charset.t +++ b/t/Charset.t @@ -18,6 +18,7 @@ use Modern::Perl; use Test::More tests => 10; +use Encode qw( is_utf8 ); use MARC::Record; BEGIN { @@ -28,9 +29,9 @@ my $octets = "abc"; ok(IsStringUTF8ish($octets), "verify octets are valid UTF-8 (ASCII)"); $octets = "flamb\c3\a9"; -ok(!utf8::is_utf8($octets), "verify that string does not have Perl UTF-8 flag on"); +ok(!Encode::is_utf8($octets), "verify that string does not have Perl UTF-8 flag on"); ok(IsStringUTF8ish($octets), "verify octets are valid UTF-8 (LATIN SMALL LETTER E WITH ACUTE)"); -ok(!utf8::is_utf8($octets), "verify that IsStringUTF8ish does not magically turn Perl UTF-8 flag on"); +ok(!Encode::is_utf8($octets), "verify that IsStringUTF8ish does not magically turn Perl UTF-8 flag on"); $octets = "a\xc2" . "c"; ok(!IsStringUTF8ish($octets), "verify octets are not valid UTF-8"); @@ -45,14 +46,14 @@ $record->append_fields( MARC::Field->new('245', ' ', ' ', a => 'Rayuela'), ); # Verify our data serves its purpose -ok( !utf8::is_utf8($record->subfield('100','a')) && - !utf8::is_utf8($record->subfield('245','a')), +ok( !Encode::is_utf8($record->subfield('100','a')) && + !Encode::is_utf8($record->subfield('245','a')), 'Verify that the subfields are NOT set the UTF-8 flag yet' ); SetUTF8Flag($record); -ok( utf8::is_utf8($record->subfield('100','a')) && - utf8::is_utf8($record->subfield('245','a')), +ok( Encode::is_utf8($record->subfield('100','a')) && + Encode::is_utf8($record->subfield('245','a')), 'SetUTF8Flag sets the UTF-8 flag to all subfields' ); 1; -- 1.7.10.4