@@ -, +, @@ replace & by & in a record --- C4/Charset.pm | 50 ++++++++++-- misc/maintenance/batchSanitizeEntity.pl | 128 +++++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+), 7 deletions(-) create mode 100755 misc/maintenance/batchSanitizeEntity.pl --- a/C4/Charset.pm +++ a/C4/Charset.pm @@ -33,13 +33,14 @@ BEGIN { require Exporter; @ISA = qw(Exporter); @EXPORT = qw( - NormalizeString - IsStringUTF8ish - MarcToUTF8Record - SetUTF8Flag - SetMarcUnicodeFlag - StripNonXmlChars - nsb_clean + NormalizeString + IsStringUTF8ish + MarcToUTF8Record + SetUTF8Flag + SetMarcUnicodeFlag + StripNonXmlChars + nsb_clean + SanitizeEntity ); } @@ -1158,6 +1159,41 @@ sub char_decode5426 { return $result; } +sub SanitizeEntity { + my $record = shift; + + foreach my $field ($record->fields()) { + if ($field->is_control_field()) { + $field->update(entity_clean($field->data())); + } else { + my @subfields = $field->subfields(); + my @new_subfields; + foreach my $subfield (@subfields) { + push @new_subfields, $subfield->[0] => entity_clean($subfield->[1]); + } + if (scalar(@new_subfields) > 0) { + my $new_field; + eval { + $new_field = MARC::Field->new($field->tag(), $field->indicator(1), $field->indicator(2), @new_subfields); + }; + if ($@) { + warn "error : $@"; + } else { + $field->replace_with($new_field); + } + + } + } + } + return $record; +} + +sub entity_clean { + my $string=shift; + $string=~s/(&)(amp;)+/$1/g; + return $string; +} + 1; --- a/misc/maintenance/batchSanitizeEntity.pl +++ a/misc/maintenance/batchSanitizeEntity.pl @@ -0,0 +1,128 @@ +#!/usr/bin/perl +# small script that replaces '&amp;amp;etc.' by '&' in a record + +use warnings; +use C4::Charset; +use C4::Context; +use DBI; +use C4::Biblio; +use Getopt::Long; +use Pod::Usage; + +my ($biblios,$run,$want_help); +my $result = GetOptions( + 'where=s' => \$biblios, + '--run' => \$run, + 'help|h' => \$want_help, + 'batch|b' => \$batch, +); + +# We check if required entries are given : +if ( not $result or $want_help or not $biblios or not $run) { + print_usage(); + exit 0; +} + +# We initialise some tools : +my $count; +my $bibliocount; +my @bibliofile; +my @biblionum; +my @biblios; +my $record; +my $cleanrecord; + +# We first detect if we have a file or biblos directly entered by command line or if we want to use findAmp() sub : +if ($biblios eq "search"){ + @biblios = findAmp(); +} +else { + if ($biblios =~ /\//) { + open (FILE, "$biblios") || die ("Can't open $biblios"); + @bibliofile = ; + close (FILE); + } + else { + @biblionum = split ',', $biblios; + } + # We take the biblios + @biblios = @bibliofile?@bibliofile:@biblionum; +} + +# We remove spaces +my @biblio; +foreach my $bib(@biblios) { + $bib =~ s/(^\s*|\s*$)//g; + next if $bib eq ""; + push @biblio, $bib; +} +@biblios = @biblio; + +# We valid the input +foreach my $biblio (@biblios) { + if ($biblio !~ /^\d+$/){ + print "=============== \"$biblio\" is not a biblionumber !!! ===============\n"; + print "=============== please verify \"$biblio\" !!! ===============\n"; + $count += 1; + } + exit(1) if $count; +} + +$bibliocount = scalar @biblios; +my $confirm = 0; + +print "=============== $bibliocount Biblio(s) ready to be cleaned ===============\n"; +if (!$batch) { + print "=============== You are ok ? Types yes/no :\n"; + while ($confirm == 0){ + my $prout = ; + if ($prout eq "yes\n") { + $confirm = 1; + } + elsif ($prout eq "no\n") { + print "=============== Ok, bye ===============\n"; + exit(0); + } + else { + print "======= Bad answer please types 'yes' or 'no' !!!!!!!!!!! ===============\n"; + } + } +} + +foreach my $biblio (@biblios){ + print "=============== N° $biblio selected ===============\n"; + $record = GetMarcBiblio($biblio); + $cleanrecord = SanitizeEntity($record); + my $frameworkcode = GetFrameworkCode($biblio); + ModBiblio($cleanrecord, $biblio, $frameworkcode); + print "=============== Biblio n° $biblio done ===============\n"; +} + +print "==============================================================\n"; +print "=============== $bibliocount Biblios processed ===============\n"; + +sub print_usage { + print <<_USAGE_; +$0: replaces '&' by '&' in a record, you can either give some biblionumbers or a file with biblionumbers or + +Parameters: + -where use this to give biblionumbers in a string with "" (separated by coma) + or an absolute path to a file containing biblionumbers (1 by row) + or the command 'search' that creates an array with biblionumbers with "&amp;..." + --run run the command + --batch run in batch mode + --help or -h show this message. +_USAGE_ +} + +sub findAmp { + my @bibliosearch; + my $dbh = C4::Context->dbh; + my $strsth = qq{SELECT biblionumber FROM biblioitems WHERE marcxml LIKE "%amp;amp;%"}; + my $sth = $dbh->prepare($strsth); + $sth->execute(); + while (my $bib = $sth-> fetchrow_array()){ + push @bibliosearch, $bib; + } + return @bibliosearch; +} --