View | Details | Raw Unified | Return to bug 11944
Collapse All | Expand All

(-)a/C4/Biblio.pm (-5 / +2 lines)
Lines 23-29 use strict; Link Here
23
use warnings;
23
use warnings;
24
use Carp;
24
use Carp;
25
25
26
# use utf8;
26
use Encode qw( decode );
27
use MARC::Record;
27
use MARC::Record;
28
use MARC::File::USMARC;
28
use MARC::File::USMARC;
29
use MARC::File::XML;
29
use MARC::File::XML;
Lines 2336-2344 sub TransformHtmlToXml { Link Here
2336
        @$values[$i] =~ s/"/"/g;
2336
        @$values[$i] =~ s/"/"/g;
2337
        @$values[$i] =~ s/'/'/g;
2337
        @$values[$i] =~ s/'/'/g;
2338
2338
2339
        #         if ( !utf8::is_utf8( @$values[$i] ) ) {
2340
        #             utf8::decode( @$values[$i] );
2341
        #         }
2342
        if ( ( @$tags[$i] ne $prevtag ) ) {
2339
        if ( ( @$tags[$i] ne $prevtag ) ) {
2343
            $j++ unless ( @$tags[$i] eq "" );
2340
            $j++ unless ( @$tags[$i] eq "" );
2344
            my $indicator1 = eval { substr( @$indicator[$j], 0, 1 ) };
2341
            my $indicator1 = eval { substr( @$indicator[$j], 0, 1 ) };
Lines 2473-2479 sub TransformHtmlToMarc { Link Here
2473
    foreach my $param_name ( keys %$cgi_params ) {
2470
    foreach my $param_name ( keys %$cgi_params ) {
2474
        if ( $param_name =~ /^tag_/ ) {
2471
        if ( $param_name =~ /^tag_/ ) {
2475
            my $param_value = $cgi_params->{$param_name};
2472
            my $param_value = $cgi_params->{$param_name};
2476
            if ( utf8::decode($param_value) ) {
2473
            if ( $param_value = Encode::decode('UTF-8', $param_value) ) {
2477
                $cgi_params->{$param_name} = $param_value;
2474
                $cgi_params->{$param_name} = $param_value;
2478
            }
2475
            }
2479
2476
(-)a/C4/Charset.pm (-3 / +4 lines)
Lines 25-30 use Text::Iconv; Link Here
25
use C4::Context;
25
use C4::Context;
26
use C4::Debug;
26
use C4::Debug;
27
use Unicode::Normalize;
27
use Unicode::Normalize;
28
use Encode qw( decode encode is_utf8 );
28
29
29
use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
30
use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
30
31
Lines 111-118 will assume that this situation occur does not very often. Link Here
111
sub IsStringUTF8ish {
112
sub IsStringUTF8ish {
112
    my $str = shift;
113
    my $str = shift;
113
114
114
    return 1 if utf8::is_utf8($str);
115
    return 1 if Encode::is_utf8($str);
115
    return utf8::decode($str);
116
    return Encode::decode('UTF-8', $str);
116
}
117
}
117
118
118
=head2 SetUTF8Flag
119
=head2 SetUTF8Flag
Lines 180-186 Sample code : Link Here
180
sub NormalizeString{
181
sub NormalizeString{
181
	my ($string,$nfd,$transform)=@_;
182
	my ($string,$nfd,$transform)=@_;
182
    return $string unless defined($string); # force scalar context return.
183
    return $string unless defined($string); # force scalar context return.
183
	utf8::decode($string) unless (utf8::is_utf8($string));
184
    $string = Encode::decode('UTF-8', $string) unless (Encode::is_utf8($string));
184
	if ($nfd){
185
	if ($nfd){
185
		$string= NFD($string);
186
		$string= NFD($string);
186
	}
187
	}
(-)a/C4/Installer.pm (-4 / +3 lines)
Lines 20-25 package C4::Installer; Link Here
20
use strict;
20
use strict;
21
#use warnings; FIXME - Bug 2505
21
#use warnings; FIXME - Bug 2505
22
22
23
use Encode qw( encode is_utf8 );
23
our $VERSION = 3.07.00.049;
24
our $VERSION = 3.07.00.049;
24
use C4::Context;
25
use C4::Context;
25
use C4::Installer::PerlModules;
26
use C4::Installer::PerlModules;
Lines 136-143 sub marc_framework_sql_list { Link Here
136
            open my $fh, "<:encoding(UTF-8)", "$dir/$requirelevel/$name.txt";
137
            open my $fh, "<:encoding(UTF-8)", "$dir/$requirelevel/$name.txt";
137
            my $lines = <$fh>;
138
            my $lines = <$fh>;
138
            $lines =~ s/\n|\r/<br \/>/g;
139
            $lines =~ s/\n|\r/<br \/>/g;
139
            use utf8;
140
            $lines = Encode::encode('UTF-8', $lines) unless ( Encode::is_utf8($lines) );
140
            utf8::encode($lines) unless ( utf8::is_utf8($lines) );
141
            my $mandatory = ($requirelevel =~ /(mandatory|requi|oblig|necess)/i);
141
            my $mandatory = ($requirelevel =~ /(mandatory|requi|oblig|necess)/i);
142
            push @frameworklist,
142
            push @frameworklist,
143
              {
143
              {
Lines 214-221 sub sample_data_sql_list { Link Here
214
            open my $fh , "<:encoding(UTF-8)", "$dir/$requirelevel/$name.txt";
214
            open my $fh , "<:encoding(UTF-8)", "$dir/$requirelevel/$name.txt";
215
            my $lines = <$fh>;
215
            my $lines = <$fh>;
216
            $lines =~ s/\n|\r/<br \/>/g;
216
            $lines =~ s/\n|\r/<br \/>/g;
217
            use utf8;
217
            $lines = Encode::encode('UTF-8', $lines) unless ( Encode::is_utf8($lines) );
218
            utf8::encode($lines) unless ( utf8::is_utf8($lines) );
219
            my $mandatory = ($requirelevel =~ /(mandatory|requi|oblig|necess)/i);
218
            my $mandatory = ($requirelevel =~ /(mandatory|requi|oblig|necess)/i);
220
            push @frameworklist,
219
            push @frameworklist,
221
              {
220
              {
(-)a/C4/ItemType.pm (-2 / +3 lines)
Lines 21-26 package C4::ItemType; Link Here
21
use strict;
21
use strict;
22
use warnings;
22
use warnings;
23
use C4::Context;
23
use C4::Context;
24
use Encode qw( encode );
24
25
25
our $AUTOLOAD;
26
our $AUTOLOAD;
26
27
Lines 81-87 sub all { Link Here
81
    for ( @{$dbh->selectall_arrayref(
82
    for ( @{$dbh->selectall_arrayref(
82
        "SELECT * FROM itemtypes ORDER BY description", { Slice => {} })} )
83
        "SELECT * FROM itemtypes ORDER BY description", { Slice => {} })} )
83
    {
84
    {
84
        utf8::encode($_->{description});
85
        $_->{description} = Encode::encode('UTF-8', $_->{description});
85
        push @itypes, $class->new($_);
86
        push @itypes, $class->new($_);
86
    }
87
    }
87
    return @itypes;
88
    return @itypes;
Lines 105-111 sub get { Link Here
105
        "SELECT * FROM itemtypes WHERE itemtype = ?", undef, $itemtype
106
        "SELECT * FROM itemtypes WHERE itemtype = ?", undef, $itemtype
106
    );
107
    );
107
    if ( $data->{description} ) {
108
    if ( $data->{description} ) {
108
        utf8::encode($data->{description});
109
        $data->{description} = Encode::encode('UTF-8', $data->{description});
109
    }
110
    }
110
    return $class->new($data);
111
    return $class->new($data);
111
}
112
}
(-)a/C4/Output.pm (-2 lines)
Lines 284-291 sub output_with_http_headers { Link Here
284
284
285
# We can't encode here, that will double encode our templates, and xslt
285
# We can't encode here, that will double encode our templates, and xslt
286
# We need to fix the encoding as it comes out of the database, or when we pass the variables to templates
286
# We need to fix the encoding as it comes out of the database, or when we pass the variables to templates
287
 
288
#    utf8::encode($data) if utf8::is_utf8($data);
289
287
290
    $data =~ s/\&amp\;amp\; /\&amp\; /g;
288
    $data =~ s/\&amp\;amp\; /\&amp\; /g;
291
    print $query->header($options), $data;
289
    print $query->header($options), $data;
(-)a/C4/Search.pm (-3 / +3 lines)
Lines 36-42 use URI::Escape; Link Here
36
use Business::ISBN;
36
use Business::ISBN;
37
use MARC::Record;
37
use MARC::Record;
38
use MARC::Field;
38
use MARC::Field;
39
use utf8;
39
use Encode qw( decode is_utf8 );
40
use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $DEBUG);
40
use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $DEBUG);
41
41
42
# set the version for version checking
42
# set the version for version checking
Lines 1982-1989 sub searchResults { Link Here
1982
                            my @repl = $marcrecord->field($1)->subfield($2);
1982
                            my @repl = $marcrecord->field($1)->subfield($2);
1983
                            my $subfieldvalue = $repl[$i];
1983
                            my $subfieldvalue = $repl[$i];
1984
1984
1985
                            if (! utf8::is_utf8($subfieldvalue)) {
1985
                            if (! Encode::is_utf8($subfieldvalue)) {
1986
                                utf8::decode($subfieldvalue);
1986
                                $subfieldvalue = Encode::decode('UTF-8', $subfieldvalue);
1987
                            }
1987
                            }
1988
1988
1989
                             $newline =~ s/\[$tag\]/$subfieldvalue/g;
1989
                             $newline =~ s/\[$tag\]/$subfieldvalue/g;
(-)a/Koha/Template/Plugin/EncodeUTF8.pm (-2 / +2 lines)
Lines 21-31 use Modern::Perl; Link Here
21
21
22
use base qw{Template::Plugin::Filter};
22
use base qw{Template::Plugin::Filter};
23
23
24
use Encode qw{encode};
24
use Encode qw{encode is_utf8};
25
25
26
sub filter {
26
sub filter {
27
    my ( $self, $value ) = @_;
27
    my ( $self, $value ) = @_;
28
    return is_utf8( $value ) ? encode( 'UTF-8', $value ) : $value;
28
    return Encode::is_utf8( $value ) ? Encode::encode( 'UTF-8', $value ) : $value;
29
}
29
}
30
30
31
1;
31
1;
(-)a/cataloguing/value_builder/macles.pl (-4 lines)
Lines 84-93 my ($input) = @_; Link Here
84
    my @innerloop;
84
    my @innerloop;
85
    my (%numbers,%cells,@colhdr,@rowhdr,@multiplelines,@lists,$table);
85
    my (%numbers,%cells,@colhdr,@rowhdr,@multiplelines,@lists,$table);
86
    while (my $tab = $rq->fetchrow_hashref){
86
    while (my $tab = $rq->fetchrow_hashref){
87
#       if (! utf8::is_utf8($tab->{lib})) {
88
#         utf8::decode($tab->{lib});
89
#       }
90
#       warn $tab->{lib};
91
      my $number=substr($tab->{authorised_value},0,1);
87
      my $number=substr($tab->{authorised_value},0,1);
92
      if ($tab->{authorised_value}=~/[0-9]XX/){
88
      if ($tab->{authorised_value}=~/[0-9]XX/){
93
        $numbers{$number}->{'hdr_tab'}=$tab->{lib};
89
        $numbers{$number}->{'hdr_tab'}=$tab->{lib};
(-)a/reports/guided_reports.pl (-1 / +2 lines)
Lines 20-25 Link Here
20
use Modern::Perl;
20
use Modern::Perl;
21
use CGI qw/-utf8/;
21
use CGI qw/-utf8/;
22
use Text::CSV::Encoded;
22
use Text::CSV::Encoded;
23
use Encode qw( decode );
23
use URI::Escape;
24
use URI::Escape;
24
use File::Temp;
25
use File::Temp;
25
use File::Basename qw( dirname );
26
use File::Basename qw( dirname );
Lines 915-921 sub header_cell_values { Link Here
915
    foreach my $c (@{$sth->{NAME}}) {
916
    foreach my $c (@{$sth->{NAME}}) {
916
        # TODO in Bug 11944
917
        # TODO in Bug 11944
917
        #FIXME apparently DBI still needs a utf8 fix for this?
918
        #FIXME apparently DBI still needs a utf8 fix for this?
918
        utf8::decode($c);
919
        $c = Encode::decode('UTF-8', $c);
919
        push @cols, $c;
920
        push @cols, $c;
920
    }
921
    }
921
    return @cols;
922
    return @cols;
(-)a/serials/serials-edit.pl (-1 / +2 lines)
Lines 64-69 op can be : Link Here
64
use strict;
64
use strict;
65
use warnings;
65
use warnings;
66
use CGI;
66
use CGI;
67
use Encode qw( decode is_utf8 );
67
use C4::Auth;
68
use C4::Auth;
68
use C4::Dates qw/format_date format_date_in_iso/;
69
use C4::Dates qw/format_date format_date_in_iso/;
69
use C4::Biblio;
70
use C4::Biblio;
Lines 202-208 if ( $op and $op eq 'serialchangestatus' ) { Link Here
202
203
203
    # Convert serialseqs to UTF-8 to prevent encoding problems
204
    # Convert serialseqs to UTF-8 to prevent encoding problems
204
    foreach my $seq (@serialseqs) {
205
    foreach my $seq (@serialseqs) {
205
        utf8::decode($seq) unless utf8::is_utf8($seq);
206
        $seq = Encode::decode('UTF-8', $seq) unless Encode::is_utf8($seq);
206
    }
207
    }
207
208
208
    my $newserial;
209
    my $newserial;
(-)a/t/Charset.t (-7 / +8 lines)
Lines 18-23 Link Here
18
use Modern::Perl;
18
use Modern::Perl;
19
19
20
use Test::More tests => 16;
20
use Test::More tests => 16;
21
use Encode qw( is_utf8 );
22
21
use MARC::Record;
23
use MARC::Record;
22
24
23
use utf8;
25
use utf8;
Lines 40-48 my $octets = "abc"; Link Here
40
ok(IsStringUTF8ish($octets), "verify octets are valid UTF-8 (ASCII)");
42
ok(IsStringUTF8ish($octets), "verify octets are valid UTF-8 (ASCII)");
41
43
42
$octets = "flamb\c3\a9";
44
$octets = "flamb\c3\a9";
43
ok(!utf8::is_utf8($octets), "verify that string does not have Perl UTF-8 flag on");
45
ok(!Encode::is_utf8($octets), "verify that string does not have Perl UTF-8 flag on");
44
ok(IsStringUTF8ish($octets), "verify octets are valid UTF-8 (LATIN SMALL LETTER E WITH ACUTE)");
46
ok(IsStringUTF8ish($octets), "verify octets are valid UTF-8 (LATIN SMALL LETTER E WITH ACUTE)");
45
ok(!utf8::is_utf8($octets), "verify that IsStringUTF8ish does not magically turn Perl UTF-8 flag on");
47
ok(!Encode::is_utf8($octets), "verify that IsStringUTF8ish does not magically turn Perl UTF-8 flag on");
46
48
47
$octets = "a\xc2" . "c";
49
$octets = "a\xc2" . "c";
48
ok(!IsStringUTF8ish($octets), "verify octets are not valid UTF-8");
50
ok(!IsStringUTF8ish($octets), "verify octets are not valid UTF-8");
Lines 57-70 $record->append_fields( Link Here
57
    MARC::Field->new('245', ' ', ' ', a => 'Rayuela'),
59
    MARC::Field->new('245', ' ', ' ', a => 'Rayuela'),
58
);
60
);
59
# Verify our data serves its purpose
61
# Verify our data serves its purpose
60
ok( !utf8::is_utf8($record->subfield('100','a')) &&
62
ok( !Encode::is_utf8($record->subfield('100','a')) &&
61
    !utf8::is_utf8($record->subfield('245','a')),
63
    !Encode::is_utf8($record->subfield('245','a')),
62
    'Verify that the subfields are NOT set the UTF-8 flag yet' );
64
    'Verify that the subfields are NOT set the UTF-8 flag yet' );
63
65
64
SetUTF8Flag($record);
66
SetUTF8Flag($record);
65
67
66
ok( utf8::is_utf8($record->subfield('100','a')) &&
68
ok( Encode::is_utf8($record->subfield('100','a')) &&
67
    utf8::is_utf8($record->subfield('245','a')),
69
    Encode::is_utf8($record->subfield('245','a')),
68
    'SetUTF8Flag sets the UTF-8 flag to all subfields' );
70
    'SetUTF8Flag sets the UTF-8 flag to all subfields' );
69
71
70
is( nsb_clean("Le Moyen Âge"), "Le Moyen Âge", "nsb_clean removes  and " );
72
is( nsb_clean("Le Moyen Âge"), "Le Moyen Âge", "nsb_clean removes  and " );
71
- 

Return to bug 11944