Attachment 116694 Details for Bug 27673 – Bug 27673: Fix encoding issues

[patch] Bug 27673: Fix encoding issues

Bug-27673-Fix-encoding-issues.patch (text/plain), 9.04 KB, created by Jonathan Druart on 2021-02-11 09:57:50 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Jonathan Druart

Created: 2021-02-11 09:57:50 UTC

Size: 9.04 KB

patch

obsolete

>From a2e8da83fe5f4ee68ee6ea9d8f5f3308d93a08a1 Mon Sep 17 00:00:00 2001
>From: Jonathan Druart <jonathan.druart@bugs.koha-community.org>
>Date: Thu, 11 Feb 2021 10:37:13 +0100
>Subject: [PATCH] Bug 27673: Fix encoding issues
>MIME-Version: 1.0
>Content-Type: text/plain; charset=UTF-8
>Content-Transfer-Encoding: 8bit
>
>There is a difference between YAML::Load and YAML::XS::Load
>From YAML::XS pod:
>"YAML::XS only deals with streams of utf8 octets"
>
>Test plan:
>We are going to test 1 occurence and QA will confirm others don't
>contain typos.
>0. Don't apply the patches
>1. Create a new itemtype with code=â¤ï¸
>2. Create a new item using this itemtype (to biblionumber=1 will work)
>3. Fill OpacHiddenItems with
> itype: [â¤ï¸]
>4. Search for "street shuffle" or any terms that will return the biblio
>Notice that the item is there (there is an error in logs)
>5. Apply the patches
>6. Repeat 4 and confirm that the item is now hidden
>---
> C4/Circulation.pm              | 3 ++-
> C4/Context.pm                  | 2 +-
> C4/Items.pm                    | 3 ++-
> C4/Record.pm                   | 3 ++-
> C4/Ris.pm                      | 3 ++-
> Koha/Filter/MARC/EmbedItems.pm | 2 +-
> about.pl                       | 3 ++-
> acqui/addorderiso2709.pl       | 5 +++--
> circ/pendingreserves.pl        | 3 ++-
> opac/opac-ISBDdetail.pl        | 3 ++-
> opac/opac-search.pl            | 3 ++-
> 11 files changed, 21 insertions(+), 12 deletions(-)
>
>diff --git a/C4/Circulation.pm b/C4/Circulation.pm
>index 0211de0280..e2298ad684 100644
>--- a/C4/Circulation.pm
>+++ b/C4/Circulation.pm
>@@ -22,6 +22,7 @@ use Modern::Perl;
> use DateTime;
> use POSIX qw( floor );
> use YAML::XS;
>+use Encode;
> 
> use Koha::DateUtils;
> use C4::Context;
>@@ -2016,7 +2017,7 @@ sub AddReturn {
>     if ($yaml) {
>         $yaml = "$yaml\n\n";  # YAML is anal on ending \n. Surplus does not hurt
>         my $rules;
>-        eval { $rules = YAML::XS::Load($yaml); };
>+        eval { $rules = YAML::XS::Load(Encode::encode_utf8($yaml)); };
>         if ($@) {
>             warn "Unable to parse UpdateNotForLoanStatusOnCheckin syspref : $@";
>         }
>diff --git a/C4/Context.pm b/C4/Context.pm
>index bf1df49336..f6406fd7ba 100644
>--- a/C4/Context.pm
>+++ b/C4/Context.pm
>@@ -445,7 +445,7 @@ the value cannot be properly decoded as YAML.
> sub yaml_preference {
>     my ( $self, $preference ) = @_;
> 
>-    my $yaml = eval { YAML::XS::Load( $self->preference( $preference ) ); };
>+    my $yaml = eval { YAML::XS::Load( Encode::encode_utf8( $self->preference( $preference ) ) ); };
>     if ($@) {
>         warn "Unable to parse $preference syspref : $@";
>         return;
>diff --git a/C4/Items.pm b/C4/Items.pm
>index 508e638149..c5da60d40f 100644
>--- a/C4/Items.pm
>+++ b/C4/Items.pm
>@@ -49,6 +49,7 @@ BEGIN {
> 
> use Carp;
> use Try::Tiny;
>+use Encode;
> use C4::Context;
> use C4::Koha;
> use C4::Biblio;
>@@ -982,7 +983,7 @@ sub GetHiddenItemnumbers {
>     $yaml = "$yaml\n\n"; # YAML is anal on ending \n. Surplus does not hurt
>     my $hidingrules;
>     eval {
>-        $hidingrules = YAML::XS::Load($yaml);
>+        $hidingrules = YAML::XS::Load(Encode::encode_utf8($yaml));
>     };
>     if ($@) {
>         warn "Unable to parse OpacHiddenItems syspref : $@";
>diff --git a/C4/Record.pm b/C4/Record.pm
>index 5616879cb6..0c3c670ef3 100644
>--- a/C4/Record.pm
>+++ b/C4/Record.pm
>@@ -30,6 +30,7 @@ use C4::Biblio; #marc2bibtex
> use C4::Koha; #marc2csv
> use C4::XSLT ();
> use YAML::XS; #marcrecords2csv
>+use Encode;
> use Template;
> use Text::CSV::Encoded; #marc2csv
> use Koha::Items;
>@@ -800,7 +801,7 @@ sub marc2bibtex {
>     my $additional_fields;
>     if ($BibtexExportAdditionalFields) {
>         $BibtexExportAdditionalFields = "$BibtexExportAdditionalFields\n\n";
>-        $additional_fields = eval { YAML::XS::Load($BibtexExportAdditionalFields); };
>+        $additional_fields = eval { YAML::XS::Load(Encode::encode_utf8($BibtexExportAdditionalFields)); };
>         if ($@) {
>             warn "Unable to parse BibtexExportAdditionalFields : $@";
>             $additional_fields = undef;
>diff --git a/C4/Ris.pm b/C4/Ris.pm
>index 3b39e96707..867030c73c 100644
>--- a/C4/Ris.pm
>+++ b/C4/Ris.pm
>@@ -64,6 +64,7 @@ use Modern::Perl;
> 
> use List::MoreUtils qw/uniq/;
> use YAML::XS;
>+use Encode;
> use vars qw(@ISA @EXPORT);
> 
> use Koha::SimpleMARC qw(read_field);
>@@ -120,7 +121,7 @@ sub marc2ris {
>     my $ris_additional_fields;
>     if ($RisExportAdditionalFields) {
>         $RisExportAdditionalFields = "$RisExportAdditionalFields\n\n";
>-        $ris_additional_fields = eval { YAML::XS::Load($RisExportAdditionalFields); };
>+        $ris_additional_fields = eval { YAML::XS::Load(Encode::encode_utf8($RisExportAdditionalFields)); };
>         if ($@) {
>             warn "Unable to parse RisExportAdditionalFields : $@";
>             $ris_additional_fields = undef;
>diff --git a/Koha/Filter/MARC/EmbedItems.pm b/Koha/Filter/MARC/EmbedItems.pm
>index c3623755ca..337a4dbbae 100644
>--- a/Koha/Filter/MARC/EmbedItems.pm
>+++ b/Koha/Filter/MARC/EmbedItems.pm
>@@ -31,7 +31,7 @@ my $biblio = Koha::Biblios->find(
> my $opachiddenitems_rules;
> eval {
>     my $yaml = C4::Context->preference('OpacHiddenItems') . "\n\n";
>-    $opachiddenitems_rules = YAML::XS::Load($yaml);
>+    $opachiddenitems_rules = YAML::XS::Load(Encode::encode_utf8($yaml));
> };
> 
> my @items  = grep { !$_->hidden_in_opac({ rules => $opachiddenitems_rules }) @{$biblio->items};
>diff --git a/about.pl b/about.pl
>index 3c9ceb908d..a8df6aa312 100755
>--- a/about.pl
>+++ b/about.pl
>@@ -34,6 +34,7 @@ use Config;
> use Search::Elasticsearch;
> use Try::Tiny;
> use YAML::XS;
>+use Encode;
> 
> use C4::Output;
> use C4::Auth;
>@@ -429,7 +430,7 @@ my @bad_yaml_prefs;
> foreach my $syspref (@yaml_prefs) {
>     my $yaml = C4::Context->preference( $syspref );
>     if ( $yaml ) {
>-        eval { YAML::XS::Load( "$yaml\n\n" ); };
>+        eval { YAML::XS::Load( Encode::encode_utf8("$yaml\n\n") ); };
>         if ($@) {
>             push @bad_yaml_prefs, $syspref;
>         }
>diff --git a/acqui/addorderiso2709.pl b/acqui/addorderiso2709.pl
>index 87973c1fca..202f74862e 100755
>--- a/acqui/addorderiso2709.pl
>+++ b/acqui/addorderiso2709.pl
>@@ -26,6 +26,7 @@ use CGI qw ( -utf8 );
> use Carp;
> use YAML::XS;
> use List::MoreUtils qw/uniq/;
>+use Encode;
> 
> use C4::Context;
> use C4::Auth;
>@@ -678,7 +679,7 @@ sub get_infos_syspref {
>     my $syspref = C4::Context->preference($syspref_name);
>     $syspref = "$syspref\n\n"; # YAML is anal on ending \n. Surplus does not hurt
>     my $yaml = eval {
>-        YAML::XS::Load($syspref);
>+        YAML::XS::Load(Encode::encode_utf8($syspref));
>     };
>     if ( $@ ) {
>         warn "Unable to parse $syspref syspref : $@";
>@@ -724,7 +725,7 @@ sub get_infos_syspref_on_item {
>     my $syspref = C4::Context->preference($syspref_name);
>     $syspref = "$syspref\n\n"; # YAML is anal on ending \n. Surplus does not hurt
>     my $yaml = eval {
>-        YAML::XS::Load($syspref);
>+        YAML::XS::Load(Encode::encode_utf8($syspref));
>     };
>     if ( $@ ) {
>         warn "Unable to parse $syspref syspref : $@";
>diff --git a/circ/pendingreserves.pl b/circ/pendingreserves.pl
>index 6da5e3a35a..3ed65938b7 100755
>--- a/circ/pendingreserves.pl
>+++ b/circ/pendingreserves.pl
>@@ -22,6 +22,7 @@ use Modern::Perl;
> use constant PULL_INTERVAL => 2;
> use List::MoreUtils qw( uniq );
> use YAML::XS;
>+use Encode;
> 
> use C4::Context;
> use C4::Output;
>@@ -110,7 +111,7 @@ if ( $op eq 'cancel_reserve' and $reserve_id ) {
>         if ( my $yaml = C4::Context->preference('UpdateItemWhenLostFromHoldList') ) {
>             $yaml = "$yaml\n\n";  # YAML is anal on ending \n. Surplus does not hurt
>             my $assignments;
>-            eval { $assignments = YAML::XS::Load($yaml); };
>+            eval { $assignments = YAML::XS::Load(Encode::encode_utf8($yaml)); };
>             if ($@) {
>                 warn "Unable to parse UpdateItemWhenLostFromHoldList syspref : $@" if $@;
>             }
>diff --git a/opac/opac-ISBDdetail.pl b/opac/opac-ISBDdetail.pl
>index f8ff0d2267..6278177e10 100755
>--- a/opac/opac-ISBDdetail.pl
>+++ b/opac/opac-ISBDdetail.pl
>@@ -41,6 +41,7 @@ the items attached to the biblio
> 
> use Modern::Perl;
> use YAML::XS;
>+use Encode;
> 
> use C4::Auth;
> use C4::Context;
>@@ -83,7 +84,7 @@ my $patron = Koha::Patrons->find($loggedinuser);
> my $opachiddenitems_rules;
> eval {
>     my $yaml = C4::Context->preference('OpacHiddenItems') . "\n\n";
>-    $opachiddenitems_rules = YAML::XS::Load($yaml);
>+    $opachiddenitems_rules = YAML::XS::Load(Encode::encode_utf8($yaml));
> };
> 
> unless ( $patron and $patron->category->override_hidden_items ) {
>diff --git a/opac/opac-search.pl b/opac/opac-search.pl
>index c04960d207..dd0d4884dc 100755
>--- a/opac/opac-search.pl
>+++ b/opac/opac-search.pl
>@@ -31,6 +31,7 @@ use C4::Context;
> use List::MoreUtils q/any/;
> use Try::Tiny;
> use YAML::XS;
>+use Encode;
> 
> use Data::Dumper; # TODO remove
> 
>@@ -246,7 +247,7 @@ my $yaml = C4::Context->preference('OpacHiddenItems');
> if ( $yaml =~ /\S/ ) {
>     $yaml = "$yaml\n\n"; # YAML expects trailing newline. Surplus does not hurt.
>     eval {
>-        $hidingrules = YAML::XS::Load($yaml);
>+        $hidingrules = YAML::XS::Load(Encode::encode_utf8($yaml));
>     };
>     if ($@) {
>         warn "Unable to parse OpacHiddenItems syspref : $@";
>-- 
>2.20.1

Actions: View | Diff | Splinter Review

Attachments on bug 27673: 116670 | 116675 | 116694 | 116695 | 116696 | 116737 | 116738 | 116739 | 116740 | 116762 | 116763 | 116764 | 116765 | 116766 | 116778 | 116835 | 116929 | 116990