View | Details | Raw Unified | Return to bug 11944
Collapse All | Expand All

(-)a/C4/Biblio.pm (-5 / +2 lines)
Lines 23-29 use strict; Link Here
23
use warnings;
23
use warnings;
24
use Carp;
24
use Carp;
25
25
26
# use utf8;
26
use Encode qw( decode );
27
use MARC::Record;
27
use MARC::Record;
28
use MARC::File::USMARC;
28
use MARC::File::USMARC;
29
use MARC::File::XML;
29
use MARC::File::XML;
Lines 2350-2358 sub TransformHtmlToXml { Link Here
2350
        @$values[$i] =~ s/"/"/g;
2350
        @$values[$i] =~ s/"/"/g;
2351
        @$values[$i] =~ s/'/'/g;
2351
        @$values[$i] =~ s/'/'/g;
2352
2352
2353
        #         if ( !utf8::is_utf8( @$values[$i] ) ) {
2354
        #             utf8::decode( @$values[$i] );
2355
        #         }
2356
        if ( ( @$tags[$i] ne $prevtag ) ) {
2353
        if ( ( @$tags[$i] ne $prevtag ) ) {
2357
            $j++ unless ( @$tags[$i] eq "" );
2354
            $j++ unless ( @$tags[$i] eq "" );
2358
            my $indicator1 = eval { substr( @$indicator[$j], 0, 1 ) };
2355
            my $indicator1 = eval { substr( @$indicator[$j], 0, 1 ) };
Lines 2487-2493 sub TransformHtmlToMarc { Link Here
2487
    foreach my $param_name ( keys %$cgi_params ) {
2484
    foreach my $param_name ( keys %$cgi_params ) {
2488
        if ( $param_name =~ /^tag_/ ) {
2485
        if ( $param_name =~ /^tag_/ ) {
2489
            my $param_value = $cgi_params->{$param_name};
2486
            my $param_value = $cgi_params->{$param_name};
2490
            if ( utf8::decode($param_value) ) {
2487
            if ( $param_value = Encode::decode('UTF-8', $param_value) ) {
2491
                $cgi_params->{$param_name} = $param_value;
2488
                $cgi_params->{$param_name} = $param_value;
2492
            }
2489
            }
2493
2490
(-)a/C4/Charset.pm (-3 / +4 lines)
Lines 25-30 use Text::Iconv; Link Here
25
use C4::Context;
25
use C4::Context;
26
use C4::Debug;
26
use C4::Debug;
27
use Unicode::Normalize;
27
use Unicode::Normalize;
28
use Encode qw( decode encode is_utf8 );
28
29
29
use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
30
use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
30
31
Lines 110-117 will assume that this situation occur does not very often. Link Here
110
sub IsStringUTF8ish {
111
sub IsStringUTF8ish {
111
    my $str = shift;
112
    my $str = shift;
112
113
113
    return 1 if utf8::is_utf8($str);
114
    return 1 if Encode::is_utf8($str);
114
    return utf8::decode($str);
115
    return Encode::decode('UTF-8', $str);
115
}
116
}
116
117
117
=head2 SetUTF8Flag
118
=head2 SetUTF8Flag
Lines 178-184 Sample code : Link Here
178
179
179
sub NormalizeString{
180
sub NormalizeString{
180
	my ($string,$nfd,$transform)=@_;
181
	my ($string,$nfd,$transform)=@_;
181
	utf8::decode($string) unless (utf8::is_utf8($string));
182
    $string = Encode::decode('UTF-8', $string) unless (Encode::is_utf8($string));
182
	if ($nfd){
183
	if ($nfd){
183
		$string= NFD($string);
184
		$string= NFD($string);
184
	}
185
	}
(-)a/C4/Installer.pm (-4 / +3 lines)
Lines 20-25 package C4::Installer; Link Here
20
use strict;
20
use strict;
21
#use warnings; FIXME - Bug 2505
21
#use warnings; FIXME - Bug 2505
22
22
23
use Encode qw( encode is_utf8 );
23
our $VERSION = 3.07.00.049;
24
our $VERSION = 3.07.00.049;
24
use C4::Context;
25
use C4::Context;
25
use C4::Installer::PerlModules;
26
use C4::Installer::PerlModules;
Lines 136-143 sub marc_framework_sql_list { Link Here
136
            open my $fh, "<:encoding(UTF-8)", "$dir/$requirelevel/$name.txt";
137
            open my $fh, "<:encoding(UTF-8)", "$dir/$requirelevel/$name.txt";
137
            my $lines = <$fh>;
138
            my $lines = <$fh>;
138
            $lines =~ s/\n|\r/<br \/>/g;
139
            $lines =~ s/\n|\r/<br \/>/g;
139
            use utf8;
140
            $lines = Encode::encode('UTF-8', $lines) unless ( Encode::is_utf8($lines) );
140
            utf8::encode($lines) unless ( utf8::is_utf8($lines) );
141
            my $mandatory = ($requirelevel =~ /(mandatory|requi|oblig|necess)/i);
141
            my $mandatory = ($requirelevel =~ /(mandatory|requi|oblig|necess)/i);
142
            push @frameworklist,
142
            push @frameworklist,
143
              {
143
              {
Lines 214-221 sub sample_data_sql_list { Link Here
214
            open my $fh , "<:encoding(UTF-8)", "$dir/$requirelevel/$name.txt";
214
            open my $fh , "<:encoding(UTF-8)", "$dir/$requirelevel/$name.txt";
215
            my $lines = <$fh>;
215
            my $lines = <$fh>;
216
            $lines =~ s/\n|\r/<br \/>/g;
216
            $lines =~ s/\n|\r/<br \/>/g;
217
            use utf8;
217
            $lines = Encode::encode('UTF-8', $lines) unless ( Encode::is_utf8($lines) );
218
            utf8::encode($lines) unless ( utf8::is_utf8($lines) );
219
            my $mandatory = ($requirelevel =~ /(mandatory|requi|oblig|necess)/i);
218
            my $mandatory = ($requirelevel =~ /(mandatory|requi|oblig|necess)/i);
220
            push @frameworklist,
219
            push @frameworklist,
221
              {
220
              {
(-)a/C4/ItemType.pm (-2 / +3 lines)
Lines 21-26 package C4::ItemType; Link Here
21
use strict;
21
use strict;
22
use warnings;
22
use warnings;
23
use C4::Context;
23
use C4::Context;
24
use Encode qw( encode );
24
25
25
our $AUTOLOAD;
26
our $AUTOLOAD;
26
27
Lines 81-87 sub all { Link Here
81
    for ( @{$dbh->selectall_arrayref(
82
    for ( @{$dbh->selectall_arrayref(
82
        "SELECT * FROM itemtypes ORDER BY description", { Slice => {} })} )
83
        "SELECT * FROM itemtypes ORDER BY description", { Slice => {} })} )
83
    {
84
    {
84
        utf8::encode($_->{description});
85
        $_->{description} = Encode::encode('UTF-8', $_->{description});
85
        push @itypes, $class->new($_);
86
        push @itypes, $class->new($_);
86
    }
87
    }
87
    return @itypes;
88
    return @itypes;
Lines 105-111 sub get { Link Here
105
        "SELECT * FROM itemtypes WHERE itemtype = ?", undef, $itemtype
106
        "SELECT * FROM itemtypes WHERE itemtype = ?", undef, $itemtype
106
    );
107
    );
107
    if ( $data->{description} ) {
108
    if ( $data->{description} ) {
108
        utf8::encode($data->{description});
109
        $data->{description} = Encode::encode('UTF-8', $data->{description});
109
    }
110
    }
110
    return $class->new($data);
111
    return $class->new($data);
111
}
112
}
(-)a/C4/Output.pm (-2 lines)
Lines 310-317 sub output_with_http_headers { Link Here
310
310
311
# We can't encode here, that will double encode our templates, and xslt
311
# We can't encode here, that will double encode our templates, and xslt
312
# We need to fix the encoding as it comes out of the database, or when we pass the variables to templates
312
# We need to fix the encoding as it comes out of the database, or when we pass the variables to templates
313
 
314
#    utf8::encode($data) if utf8::is_utf8($data);
315
313
316
    $data =~ s/\&amp\;amp\; /\&amp\; /g;
314
    $data =~ s/\&amp\;amp\; /\&amp\; /g;
317
    print $query->header($options), $data;
315
    print $query->header($options), $data;
(-)a/C4/Search.pm (-3 / +3 lines)
Lines 36-42 use URI::Escape; Link Here
36
use Business::ISBN;
36
use Business::ISBN;
37
use MARC::Record;
37
use MARC::Record;
38
use MARC::Field;
38
use MARC::Field;
39
use utf8;
39
use Encode qw( decode is_utf8 );
40
use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $DEBUG);
40
use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $DEBUG);
41
41
42
# set the version for version checking
42
# set the version for version checking
Lines 1800-1807 sub searchResults { Link Here
1800
                            my @repl = $marcrecord->field($1)->subfield($2);
1800
                            my @repl = $marcrecord->field($1)->subfield($2);
1801
                            my $subfieldvalue = $repl[$i];
1801
                            my $subfieldvalue = $repl[$i];
1802
1802
1803
                            if (! utf8::is_utf8($subfieldvalue)) {
1803
                            if (! Encode::is_utf8($subfieldvalue)) {
1804
                                utf8::decode($subfieldvalue);
1804
                                $subfieldvalue = Encode::decode('UTF-8', $subfieldvalue);
1805
                            }
1805
                            }
1806
1806
1807
                             $newline =~ s/\[$tag\]/$subfieldvalue/g;
1807
                             $newline =~ s/\[$tag\]/$subfieldvalue/g;
(-)a/Koha/SearchEngine/Zebra/Search.pm (-1 lines)
Lines 47-53 sub search { Link Here
47
47
48
    foreach my $item (@{ $results->items }) {
48
    foreach my $item (@{ $results->items }) {
49
        my $title = $item->get_value('ste_title');
49
        my $title = $item->get_value('ste_title');
50
        #utf8::encode($title);
51
        print "$title\n";
50
        print "$title\n";
52
                warn dump $title;
51
                warn dump $title;
53
    }
52
    }
(-)a/Koha/Template/Plugin/EncodeUTF8.pm (-2 / +2 lines)
Lines 21-31 use Modern::Perl; Link Here
21
21
22
use base qw{Template::Plugin::Filter};
22
use base qw{Template::Plugin::Filter};
23
23
24
use Encode qw{encode};
24
use Encode qw{encode is_utf8};
25
25
26
sub filter {
26
sub filter {
27
    my ( $self, $value ) = @_;
27
    my ( $self, $value ) = @_;
28
    return is_utf8( $value ) ? encode( 'UTF-8', $value ) : $value;
28
    return Encode::is_utf8( $value ) ? Encode::encode( 'UTF-8', $value ) : $value;
29
}
29
}
30
30
31
1;
31
1;
(-)a/cataloguing/value_builder/macles.pl (-4 lines)
Lines 84-93 my ($input) = @_; Link Here
84
    my @innerloop;
84
    my @innerloop;
85
    my (%numbers,%cells,@colhdr,@rowhdr,@multiplelines,@lists,$table);
85
    my (%numbers,%cells,@colhdr,@rowhdr,@multiplelines,@lists,$table);
86
    while (my $tab = $rq->fetchrow_hashref){
86
    while (my $tab = $rq->fetchrow_hashref){
87
#       if (! utf8::is_utf8($tab->{lib})) {
88
#         utf8::decode($tab->{lib});
89
#       }
90
#       warn $tab->{lib};
91
      my $number=substr($tab->{authorised_value},0,1);
87
      my $number=substr($tab->{authorised_value},0,1);
92
      if ($tab->{authorised_value}=~/[0-9]XX/){
88
      if ($tab->{authorised_value}=~/[0-9]XX/){
93
        $numbers{$number}->{'hdr_tab'}=$tab->{lib};
89
        $numbers{$number}->{'hdr_tab'}=$tab->{lib};
(-)a/reports/guided_reports.pl (-1 / +2 lines)
Lines 20-25 Link Here
20
use Modern::Perl;
20
use Modern::Perl;
21
use CGI qw/-utf8/;
21
use CGI qw/-utf8/;
22
use Text::CSV::Encoded;
22
use Text::CSV::Encoded;
23
use Encode qw( decode );
23
use URI::Escape;
24
use URI::Escape;
24
use File::Temp;
25
use File::Temp;
25
use File::Basename qw( dirname );
26
use File::Basename qw( dirname );
Lines 903-909 sub header_cell_values { Link Here
903
    foreach my $c (@{$sth->{NAME}}) {
904
    foreach my $c (@{$sth->{NAME}}) {
904
        # TODO in Bug 11944
905
        # TODO in Bug 11944
905
        #FIXME apparently DBI still needs a utf8 fix for this?
906
        #FIXME apparently DBI still needs a utf8 fix for this?
906
        utf8::decode($c);
907
        $c = Encode::decode('UTF-8', $c);
907
        push @cols, $c;
908
        push @cols, $c;
908
    }
909
    }
909
    return @cols;
910
    return @cols;
(-)a/serials/serials-edit.pl (-1 / +2 lines)
Lines 64-69 op can be : Link Here
64
use strict;
64
use strict;
65
use warnings;
65
use warnings;
66
use CGI;
66
use CGI;
67
use Encode qw( decode is_utf8 );
67
use C4::Auth;
68
use C4::Auth;
68
use C4::Dates qw/format_date format_date_in_iso/;
69
use C4::Dates qw/format_date format_date_in_iso/;
69
use C4::Biblio;
70
use C4::Biblio;
Lines 202-208 if ( $op and $op eq 'serialchangestatus' ) { Link Here
202
203
203
    # Convert serialseqs to UTF-8 to prevent encoding problems
204
    # Convert serialseqs to UTF-8 to prevent encoding problems
204
    foreach my $seq (@serialseqs) {
205
    foreach my $seq (@serialseqs) {
205
        utf8::decode($seq) unless utf8::is_utf8($seq);
206
        $seq = Encode::decode('UTF-8', $seq) unless Encode::is_utf8($seq);
206
    }
207
    }
207
208
208
    my $newserial;
209
    my $newserial;
(-)a/t/Charset.t (-7 / +7 lines)
Lines 18-23 Link Here
18
use Modern::Perl;
18
use Modern::Perl;
19
19
20
use Test::More tests => 10;
20
use Test::More tests => 10;
21
use Encode qw( is_utf8 );
21
use MARC::Record;
22
use MARC::Record;
22
23
23
BEGIN {
24
BEGIN {
Lines 28-36 my $octets = "abc"; Link Here
28
ok(IsStringUTF8ish($octets), "verify octets are valid UTF-8 (ASCII)");
29
ok(IsStringUTF8ish($octets), "verify octets are valid UTF-8 (ASCII)");
29
30
30
$octets = "flamb\c3\a9";
31
$octets = "flamb\c3\a9";
31
ok(!utf8::is_utf8($octets), "verify that string does not have Perl UTF-8 flag on");
32
ok(!Encode::is_utf8($octets), "verify that string does not have Perl UTF-8 flag on");
32
ok(IsStringUTF8ish($octets), "verify octets are valid UTF-8 (LATIN SMALL LETTER E WITH ACUTE)");
33
ok(IsStringUTF8ish($octets), "verify octets are valid UTF-8 (LATIN SMALL LETTER E WITH ACUTE)");
33
ok(!utf8::is_utf8($octets), "verify that IsStringUTF8ish does not magically turn Perl UTF-8 flag on");
34
ok(!Encode::is_utf8($octets), "verify that IsStringUTF8ish does not magically turn Perl UTF-8 flag on");
34
35
35
$octets = "a\xc2" . "c";
36
$octets = "a\xc2" . "c";
36
ok(!IsStringUTF8ish($octets), "verify octets are not valid UTF-8");
37
ok(!IsStringUTF8ish($octets), "verify octets are not valid UTF-8");
Lines 45-58 $record->append_fields( Link Here
45
    MARC::Field->new('245', ' ', ' ', a => 'Rayuela'),
46
    MARC::Field->new('245', ' ', ' ', a => 'Rayuela'),
46
);
47
);
47
# Verify our data serves its purpose
48
# Verify our data serves its purpose
48
ok( !utf8::is_utf8($record->subfield('100','a')) &&
49
ok( !Encode::is_utf8($record->subfield('100','a')) &&
49
    !utf8::is_utf8($record->subfield('245','a')),
50
    !Encode::is_utf8($record->subfield('245','a')),
50
    'Verify that the subfields are NOT set the UTF-8 flag yet' );
51
    'Verify that the subfields are NOT set the UTF-8 flag yet' );
51
52
52
SetUTF8Flag($record);
53
SetUTF8Flag($record);
53
54
54
ok( utf8::is_utf8($record->subfield('100','a')) &&
55
ok( Encode::is_utf8($record->subfield('100','a')) &&
55
    utf8::is_utf8($record->subfield('245','a')),
56
    Encode::is_utf8($record->subfield('245','a')),
56
    'SetUTF8Flag sets the UTF-8 flag to all subfields' );
57
    'SetUTF8Flag sets the UTF-8 flag to all subfields' );
57
58
58
1;
59
1;
59
- 

Return to bug 11944