From 0608c7c8f3f3920c35e847a49c90bf2c8c7cea8c Mon Sep 17 00:00:00 2001
From: Alex Arnaud <alex.arnaud@biblibre.com>
Date: Wed, 6 Jul 2011 10:32:14 +0200
Subject: [PATCH] Bug 8218 : Adding SanitizeEntity and entity_clean sub which
 replace &amp; by & in a record

Bug 8218 : adding script for maintenance to clean any &amp; in strings

Signed-off-by: Chris Cormack <chris@bigballofwax.co.nz>
---
 C4/Charset.pm                           |   50 ++++++++++--
 misc/maintenance/batchSanitizeEntity.pl |  128 +++++++++++++++++++++++++++++++
 2 files changed, 171 insertions(+), 7 deletions(-)
 create mode 100755 misc/maintenance/batchSanitizeEntity.pl

diff --git a/C4/Charset.pm b/C4/Charset.pm
index c294b36..32d7de3 100644
--- a/C4/Charset.pm
+++ b/C4/Charset.pm
@@ -33,13 +33,14 @@ BEGIN {
     require Exporter;
     @ISA    = qw(Exporter);
     @EXPORT = qw(
-        NormalizeString
-        IsStringUTF8ish
-        MarcToUTF8Record
-        SetUTF8Flag
-        SetMarcUnicodeFlag
-        StripNonXmlChars
-        nsb_clean
+      NormalizeString
+      IsStringUTF8ish
+      MarcToUTF8Record
+      SetUTF8Flag
+      SetMarcUnicodeFlag
+      StripNonXmlChars
+      nsb_clean
+      SanitizeEntity
     );
 }
 
@@ -1158,6 +1159,41 @@ sub char_decode5426 {
   return $result;
 }
 
+sub SanitizeEntity {
+    my $record = shift;
+
+    foreach my $field ($record->fields()) {
+        if ($field->is_control_field()) {
+            $field->update(entity_clean($field->data()));
+        } else {
+            my @subfields = $field->subfields();
+            my @new_subfields;
+            foreach my $subfield (@subfields) {
+                push @new_subfields, $subfield->[0] => entity_clean($subfield->[1]);
+            }
+            if (scalar(@new_subfields) > 0) {
+                my $new_field;
+                eval {
+                    $new_field = MARC::Field->new($field->tag(), $field->indicator(1), $field->indicator(2), @new_subfields);
+                };
+                if ($@) {
+                    warn "error : $@";
+                } else {
+                    $field->replace_with($new_field);
+                }
+
+            }
+        }
+    }
+    return $record;
+}
+
+sub entity_clean {
+    my $string=shift;
+    $string=~s/(&)(amp;)+/$1/g;
+    return $string;
+}
+
 1;
 
 
diff --git a/misc/maintenance/batchSanitizeEntity.pl b/misc/maintenance/batchSanitizeEntity.pl
new file mode 100755
index 0000000..d629faa
--- /dev/null
+++ b/misc/maintenance/batchSanitizeEntity.pl
@@ -0,0 +1,128 @@
+#!/usr/bin/perl
+# small script that replaces '&amp;amp;amp;etc.' by '&amp;' in a record
+
+use warnings;
+use C4::Charset;
+use C4::Context;
+use DBI;
+use C4::Biblio;
+use Getopt::Long;
+use Pod::Usage;
+
+my ($biblios,$run,$want_help);
+my $result = GetOptions(
+    'where=s'      => \$biblios,
+    '--run'        => \$run,
+    'help|h'       => \$want_help,
+    'batch|b'      => \$batch,
+);
+
+# We check if required entries are given :
+if ( not $result or $want_help or not $biblios or not $run) {
+    print_usage();
+    exit 0;
+}
+
+# We initialise some tools :
+my $count;
+my $bibliocount;
+my @bibliofile;
+my @biblionum;
+my @biblios;
+my $record;
+my $cleanrecord;
+
+# We first detect if we have a file or biblos directly entered by command line or if we want to use findAmp() sub :
+if ($biblios eq "search"){
+    @biblios = findAmp();
+}
+else {
+    if ($biblios =~ /\//) {
+        open (FILE, "$biblios") || die ("Can't open $biblios");
+        @bibliofile = <FILE>;
+        close (FILE);
+    }
+    else {
+        @biblionum = split ',', $biblios;
+    }
+    # We take the biblios
+    @biblios = @bibliofile?@bibliofile:@biblionum;
+}
+
+# We remove spaces
+my @biblio;
+foreach my $bib(@biblios) {
+    $bib =~ s/(^\s*|\s*$)//g;
+    next if $bib eq "";
+    push @biblio, $bib;
+}
+@biblios = @biblio;
+
+# We valid the input
+foreach my $biblio (@biblios) {
+    if ($biblio !~ /^\d+$/){
+        print "=============== \"$biblio\" is not a biblionumber !!! ===============\n";
+        print "=============== please verify \"$biblio\" !!! ===============\n";
+        $count += 1;
+    }
+    exit(1) if $count;
+}
+
+$bibliocount = scalar @biblios;
+my $confirm = 0;
+
+print "=============== $bibliocount Biblio(s) ready to be cleaned ===============\n";
+if (!$batch) {
+    print "=============== You are ok ? Types yes/no :\n";
+    while ($confirm == 0){
+                my $prout = <STDIN>;
+                if ($prout eq "yes\n") {
+                        $confirm = 1;
+                }
+                elsif ($prout eq "no\n") {
+                    print "=============== Ok, bye ===============\n";
+                    exit(0);
+                    }
+                else {
+                    print "======= Bad answer please types 'yes' or 'no' !!!!!!!!!!! ===============\n";
+                }
+            }
+}
+
+foreach my $biblio (@biblios){
+    print "=============== N° $biblio selected ===============\n";
+    $record = GetMarcBiblio($biblio);
+    $cleanrecord = SanitizeEntity($record);
+    my $frameworkcode = GetFrameworkCode($biblio);
+    ModBiblio($cleanrecord, $biblio, $frameworkcode);
+    print "=============== Biblio n° $biblio done ===============\n";
+}
+
+print "==============================================================\n";
+print "=============== $bibliocount Biblios processed ===============\n";
+
+sub print_usage {
+    print <<_USAGE_;
+$0: replaces '&amp;' by '&' in a record, you can either give some biblionumbers or a file with biblionumbers or
+
+Parameters:
+    -where                  use this to give biblionumbers in a string with "" (separated by coma)
+                            or an absolute path to a file containing biblionumbers (1 by row)
+                            or the command 'search' that creates an array with biblionumbers with "&amp;amp;..."
+    --run                   run the command
+    --batch                 run in batch mode
+    --help or -h            show this message.
+_USAGE_
+}
+
+sub findAmp {
+    my @bibliosearch;
+    my $dbh = C4::Context->dbh;
+    my $strsth = qq{SELECT biblionumber FROM biblioitems WHERE marcxml LIKE "%amp;amp;%"};
+    my $sth = $dbh->prepare($strsth);
+    $sth->execute();
+    while (my $bib = $sth-> fetchrow_array()){
+        push @bibliosearch, $bib;
+    }
+    return @bibliosearch;
+}
-- 
1.7.10