Attachment 29982 Details for Bug 11944 – Bug 11944: replace use of utf8 with Encode

[patch] Bug 11944: replace use of utf8 with Encode

Bug-11944-replace-use-of-utf8-with-Encode.patch (text/plain), 11.09 KB, created by Dobrica Pavlinusic on 2014-07-23 16:05:59 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Dobrica Pavlinusic

Created: 2014-07-23 16:05:59 UTC

Size: 11.09 KB

patch

obsolete

>From a78799847d8223d5d897ae9dd42ea16ec002d9f8 Mon Sep 17 00:00:00 2001
>From: Jonathan Druart <jonathan.druart@biblibre.com>
>Date: Fri, 14 Mar 2014 17:09:09 +0100
>Subject: [PATCH] Bug 11944: replace use of utf8 with Encode
>
>See the wiki page for the explanation.
>
>Signed-off-by: Paola Rossi <paola.rossi@cineca.it>
>Signed-off-by: Bernardo Gonzalez Kriegel <bgkriegel@gmail.com>
>Signed-off-by: Dobrica Pavlinusic <dpavlin@rot13.org>
>---
> C4/Biblio.pm                        |    7 ++-----
> C4/Charset.pm                       |    7 ++++---
> C4/Installer.pm                     |    7 +++----
> C4/ItemType.pm                      |    5 +++--
> C4/Output.pm                        |    2 --
> C4/Search.pm                        |    6 +++---
> Koha/SearchEngine/Zebra/Search.pm   |    1 -
> Koha/Template/Plugin/EncodeUTF8.pm  |    4 ++--
> cataloguing/value_builder/macles.pl |    4 ----
> reports/guided_reports.pl           |    3 ++-
> serials/serials-edit.pl             |    3 ++-
> t/Charset.t                         |   13 +++++++------
> 12 files changed, 28 insertions(+), 34 deletions(-)
>
>diff --git a/C4/Biblio.pm b/C4/Biblio.pm
>index f5903f3..04895f6 100644
>--- a/C4/Biblio.pm
>+++ b/C4/Biblio.pm
>@@ -23,7 +23,7 @@ use strict;
> use warnings;
> use Carp;
> 
>-# use utf8;
>+use Encode qw( decode );
> use MARC::Record;
> use MARC::File::USMARC;
> use MARC::File::XML;
>@@ -2350,9 +2350,6 @@ sub TransformHtmlToXml {
>         @$values[$i] =~ s/"/&quot;/g;
>         @$values[$i] =~ s/'/&apos;/g;
> 
>-        #         if ( !utf8::is_utf8( @$values[$i] ) ) {
>-        #             utf8::decode( @$values[$i] );
>-        #         }
>         if ( ( @$tags[$i] ne $prevtag ) ) {
>             $j++ unless ( @$tags[$i] eq "" );
>             my $indicator1 = eval { substr( @$indicator[$j], 0, 1 ) };
>@@ -2487,7 +2484,7 @@ sub TransformHtmlToMarc {
>     foreach my $param_name ( keys %$cgi_params ) {
>         if ( $param_name =~ /^tag_/ ) {
>             my $param_value = $cgi_params->{$param_name};
>-            if ( utf8::decode($param_value) ) {
>+            if ( $param_value = Encode::decode('UTF-8', $param_value) ) {
>                 $cgi_params->{$param_name} = $param_value;
>             }
> 
>diff --git a/C4/Charset.pm b/C4/Charset.pm
>index 8b06441..82d5ace 100644
>--- a/C4/Charset.pm
>+++ b/C4/Charset.pm
>@@ -25,6 +25,7 @@ use Text::Iconv;
> use C4::Context;
> use C4::Debug;
> use Unicode::Normalize;
>+use Encode qw( decode encode is_utf8 );
> 
> use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
> 
>@@ -110,8 +111,8 @@ will assume that this situation occur does not very often.
> sub IsStringUTF8ish {
>     my $str = shift;
> 
>-    return 1 if utf8::is_utf8($str);
>-    return utf8::decode($str);
>+    return 1 if Encode::is_utf8($str);
>+    return Encode::decode('UTF-8', $str);
> }
> 
> =head2 SetUTF8Flag
>@@ -178,7 +179,7 @@ Sample code :
> 
> sub NormalizeString{
> 	my ($string,$nfd,$transform)=@_;
>-	utf8::decode($string) unless (utf8::is_utf8($string));
>+    $string = Encode::decode('UTF-8', $string) unless (Encode::is_utf8($string));
> 	if ($nfd){
> 		$string= NFD($string);
> 	}
>diff --git a/C4/Installer.pm b/C4/Installer.pm
>index 2add0e8..3540ccf 100644
>--- a/C4/Installer.pm
>+++ b/C4/Installer.pm
>@@ -20,6 +20,7 @@ package C4::Installer;
> use strict;
> #use warnings; FIXME - Bug 2505
> 
>+use Encode qw( encode is_utf8 );
> our $VERSION = 3.07.00.049;
> use C4::Context;
> use C4::Installer::PerlModules;
>@@ -136,8 +137,7 @@ sub marc_framework_sql_list {
>             open my $fh, "<:encoding(UTF-8)", "$dir/$requirelevel/$name.txt";
>             my $lines = <$fh>;
>             $lines =~ s/\n|\r/<br \/>/g;
>-            use utf8;
>-            utf8::encode($lines) unless ( utf8::is_utf8($lines) );
>+            $lines = Encode::encode('UTF-8', $lines) unless ( Encode::is_utf8($lines) );
>             my $mandatory = ($requirelevel =~ /(mandatory|requi|oblig|necess)/i);
>             push @frameworklist,
>               {
>@@ -214,8 +214,7 @@ sub sample_data_sql_list {
>             open my $fh , "<:encoding(UTF-8)", "$dir/$requirelevel/$name.txt";
>             my $lines = <$fh>;
>             $lines =~ s/\n|\r/<br \/>/g;
>-            use utf8;
>-            utf8::encode($lines) unless ( utf8::is_utf8($lines) );
>+            $lines = Encode::encode('UTF-8', $lines) unless ( Encode::is_utf8($lines) );
>             my $mandatory = ($requirelevel =~ /(mandatory|requi|oblig|necess)/i);
>             push @frameworklist,
>               {
>diff --git a/C4/ItemType.pm b/C4/ItemType.pm
>index ad253a7..cf61d40 100644
>--- a/C4/ItemType.pm
>+++ b/C4/ItemType.pm
>@@ -21,6 +21,7 @@ package C4::ItemType;
> use strict;
> use warnings;
> use C4::Context;
>+use Encode qw( encode );
> 
> our $AUTOLOAD;
> 
>@@ -81,7 +82,7 @@ sub all {
>     for ( @{$dbh->selectall_arrayref(
>         "SELECT * FROM itemtypes ORDER BY description", { Slice => {} })} )
>     {
>-        utf8::encode($_->{description});
>+        $_->{description} = Encode::encode('UTF-8', $_->{description});
>         push @itypes, $class->new($_);
>     }
>     return @itypes;
>@@ -105,7 +106,7 @@ sub get {
>         "SELECT * FROM itemtypes WHERE itemtype = ?", undef, $itemtype
>     );
>     if ( $data->{description} ) {
>-        utf8::encode($data->{description});
>+        $data->{description} = Encode::encode('UTF-8', $data->{description});
>     }
>     return $class->new($data);
> }
>diff --git a/C4/Output.pm b/C4/Output.pm
>index 324cf69..5dc03d7 100644
>--- a/C4/Output.pm
>+++ b/C4/Output.pm
>@@ -310,8 +310,6 @@ sub output_with_http_headers {
> 
> # We can't encode here, that will double encode our templates, and xslt
> # We need to fix the encoding as it comes out of the database, or when we pass the variables to templates
>- 
>-#    utf8::encode($data) if utf8::is_utf8($data);
> 
>     $data =~ s/\&amp\;amp\; /\&amp\; /g;
>     print $query->header($options), $data;
>diff --git a/C4/Search.pm b/C4/Search.pm
>index 219de1e..7de60bf 100644
>--- a/C4/Search.pm
>+++ b/C4/Search.pm
>@@ -36,7 +36,7 @@ use URI::Escape;
> use Business::ISBN;
> use MARC::Record;
> use MARC::Field;
>-use utf8;
>+use Encode qw( decode is_utf8 );
> use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $DEBUG);
> 
> # set the version for version checking
>@@ -1800,8 +1800,8 @@ sub searchResults {
>                             my @repl = $marcrecord->field($1)->subfield($2);
>                             my $subfieldvalue = $repl[$i];
> 
>-                            if (! utf8::is_utf8($subfieldvalue)) {
>-                                utf8::decode($subfieldvalue);
>+                            if (! Encode::is_utf8($subfieldvalue)) {
>+                                $subfieldvalue = Encode::decode('UTF-8', $subfieldvalue);
>                             }
> 
>                              $newline =~ s/\[$tag\]/$subfieldvalue/g;
>diff --git a/Koha/SearchEngine/Zebra/Search.pm b/Koha/SearchEngine/Zebra/Search.pm
>index 3b736df..b805683 100644
>--- a/Koha/SearchEngine/Zebra/Search.pm
>+++ b/Koha/SearchEngine/Zebra/Search.pm
>@@ -47,7 +47,6 @@ sub search {
> 
>     foreach my $item (@{ $results->items }) {
>         my $title = $item->get_value('ste_title');
>-        #utf8::encode($title);
>         print "$title\n";
>                 warn dump $title;
>     }
>diff --git a/Koha/Template/Plugin/EncodeUTF8.pm b/Koha/Template/Plugin/EncodeUTF8.pm
>index d27ccec..871cbb4 100644
>--- a/Koha/Template/Plugin/EncodeUTF8.pm
>+++ b/Koha/Template/Plugin/EncodeUTF8.pm
>@@ -21,11 +21,11 @@ use Modern::Perl;
> 
> use base qw{Template::Plugin::Filter};
> 
>-use Encode qw{encode};
>+use Encode qw{encode is_utf8};
> 
> sub filter {
>     my ( $self, $value ) = @_;
>-    return is_utf8( $value ) ? encode( 'UTF-8', $value ) : $value;
>+    return Encode::is_utf8( $value ) ? Encode::encode( 'UTF-8', $value ) : $value;
> }
> 
> 1;
>diff --git a/cataloguing/value_builder/macles.pl b/cataloguing/value_builder/macles.pl
>index 6644e65..26aea40 100755
>--- a/cataloguing/value_builder/macles.pl
>+++ b/cataloguing/value_builder/macles.pl
>@@ -84,10 +84,6 @@ my ($input) = @_;
>     my @innerloop;
>     my (%numbers,%cells,@colhdr,@rowhdr,@multiplelines,@lists,$table);
>     while (my $tab = $rq->fetchrow_hashref){
>-#       if (! utf8::is_utf8($tab->{lib})) {
>-#         utf8::decode($tab->{lib});
>-#       }
>-#       warn $tab->{lib};
>       my $number=substr($tab->{authorised_value},0,1);
>       if ($tab->{authorised_value}=~/[0-9]XX/){
>         $numbers{$number}->{'hdr_tab'}=$tab->{lib};
>diff --git a/reports/guided_reports.pl b/reports/guided_reports.pl
>index b6cb822..388b90c 100755
>--- a/reports/guided_reports.pl
>+++ b/reports/guided_reports.pl
>@@ -20,6 +20,7 @@
> use Modern::Perl;
> use CGI qw/-utf8/;
> use Text::CSV::Encoded;
>+use Encode qw( decode );
> use URI::Escape;
> use File::Temp;
> use File::Basename qw( dirname );
>@@ -903,7 +904,7 @@ sub header_cell_values {
>     foreach my $c (@{$sth->{NAME}}) {
>         # TODO in Bug 11944
>         #FIXME apparently DBI still needs a utf8 fix for this?
>-        utf8::decode($c);
>+        $c = Encode::decode('UTF-8', $c);
>         push @cols, $c;
>     }
>     return @cols;
>diff --git a/serials/serials-edit.pl b/serials/serials-edit.pl
>index 2eeed10..54c8ccd 100755
>--- a/serials/serials-edit.pl
>+++ b/serials/serials-edit.pl
>@@ -64,6 +64,7 @@ op can be :
> use strict;
> use warnings;
> use CGI;
>+use Encode qw( decode is_utf8 );
> use C4::Auth;
> use C4::Dates qw/format_date format_date_in_iso/;
> use C4::Biblio;
>@@ -202,7 +203,7 @@ if ( $op and $op eq 'serialchangestatus' ) {
> 
>     # Convert serialseqs to UTF-8 to prevent encoding problems
>     foreach my $seq (@serialseqs) {
>-        utf8::decode($seq) unless utf8::is_utf8($seq);
>+        $seq = Encode::decode('UTF-8', $seq) unless Encode::is_utf8($seq);
>     }
> 
>     my $newserial;
>diff --git a/t/Charset.t b/t/Charset.t
>index 27b2a32..9f81df7 100755
>--- a/t/Charset.t
>+++ b/t/Charset.t
>@@ -18,6 +18,7 @@
> use Modern::Perl;
> 
> use Test::More tests => 10;
>+use Encode qw( is_utf8 );
> use MARC::Record;
> 
> BEGIN {
>@@ -28,9 +29,9 @@ my $octets = "abc";
> ok(IsStringUTF8ish($octets), "verify octets are valid UTF-8 (ASCII)");
> 
> $octets = "flamb\c3\a9";
>-ok(!utf8::is_utf8($octets), "verify that string does not have Perl UTF-8 flag on");
>+ok(!Encode::is_utf8($octets), "verify that string does not have Perl UTF-8 flag on");
> ok(IsStringUTF8ish($octets), "verify octets are valid UTF-8 (LATIN SMALL LETTER E WITH ACUTE)");
>-ok(!utf8::is_utf8($octets), "verify that IsStringUTF8ish does not magically turn Perl UTF-8 flag on");
>+ok(!Encode::is_utf8($octets), "verify that IsStringUTF8ish does not magically turn Perl UTF-8 flag on");
> 
> $octets = "a\xc2" . "c";
> ok(!IsStringUTF8ish($octets), "verify octets are not valid UTF-8");
>@@ -45,14 +46,14 @@ $record->append_fields(
>     MARC::Field->new('245', ' ', ' ', a => 'Rayuela'),
> );
> # Verify our data serves its purpose
>-ok( !utf8::is_utf8($record->subfield('100','a')) &&
>-    !utf8::is_utf8($record->subfield('245','a')),
>+ok( !Encode::is_utf8($record->subfield('100','a')) &&
>+    !Encode::is_utf8($record->subfield('245','a')),
>     'Verify that the subfields are NOT set the UTF-8 flag yet' );
> 
> SetUTF8Flag($record);
> 
>-ok( utf8::is_utf8($record->subfield('100','a')) &&
>-    utf8::is_utf8($record->subfield('245','a')),
>+ok( Encode::is_utf8($record->subfield('100','a')) &&
>+    Encode::is_utf8($record->subfield('245','a')),
>     'SetUTF8Flag sets the UTF-8 flag to all subfields' );
> 
> 1;
>-- 
>1.7.2.5

Actions: View | Diff | Splinter Review

Attachments on bug 11944: 26364 | 26365 | 26366 | 26367 | 26368 | 26369 | 26370 | 26371 | 26372 | 26407 | 26408 | 26409 | 26737 | 26738 | 26739 | 26740 | 26741 | 26742 | 26743 | 26744 | 26745 | 26746 | 26747 | 26748 | 26794 | 26857 | 26858 | 26859 | 26860 | 26861 | 26862 | 26863 | 26864 | 26865 | 26866 | 26867 | 26868 | 26869 | 26870 | 26871 | 26895 | 26921 | 27021 | 27080 | 27094 | 27158 | 27159 | 27161 | 27193 | 27194 | 27195 | 27196 | 27198 | 27199 | 27200 | 27201 | 27202 | 27203 | 27204 | 27205 | 27206 | 27207 | 27208 | 27209 | 27210 | 27211 | 27212 | 27213 | 27214 | 27215 | 27216 | 27585 | 27586 | 27587 | 27588 | 27589 | 27590 | 27591 | 27592 | 27593 | 27594 | 27595 | 27596 | 27597 | 27598 | 27599 | 27600 | 27601 | 27602 | 27603 | 27880 | 27881 | 27882 | 27883 | 27884 | 27885 | 27886 | 27887 | 27888 | 27889 | 27890 | 27891 | 27892 | 27893 | 27894 | 27895 | 27896 | 27897 | 27898 | 28046 | 28047 | 28048 | 28049 | 28050 | 28051 | 28052 | 28053 | 28054 | 28055 | 28056 | 28057 | 28058 | 28059 | 28060 | 28061 | 28062 | 28063 | 28064 | 28065 | 28066 | 28097 | 28245 | 28246 | 28247 | 28248 | 28249 | 28250 | 28251 | 28252 | 28253 | 28732 | 29981 | 29982 | 29983 | 29984 | 29985 | 29986 | 29987 | 29988 | 29989 | 29990 | 29991 | 29992 | 29993 | 29994 | 29995 | 29996 | 29997 | 29998 | 29999 | 30000 | 30001 | 30002 | 30003 | 30004 | 30005 | 30006 | 30012 | 30016 | 30945 | 31299 | 35223 | 35224 | 35225 | 35226 | 35227 | 35228 | 35229 | 35230 | 35231 | 35232 | 35233 | 35234 | 35235 | 35236 | 35237 | 35238 | 35239 | 35240 | 35241 | 35242 | 35243 | 35244 | 35245 | 35246 | 35247 | 35248 | 35249 | 35250 | 35251 | 35252 | 35253