Bugzilla – Attachment 10588 Details for
Bug 8218
Add a script to sanitize entity where "&" character was wrongly replaced with "&"
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 8218 : Adding SanitizeEntity and entity_clean sub which replace & by & in a record
Bug-8218--Adding-SanitizeEntity-and-entityclean-su.patch (text/plain), 6.28 KB, created by
Chris Cormack
on 2012-07-01 03:28:47 UTC
(
hide
)
Description:
Bug 8218 : Adding SanitizeEntity and entity_clean sub which replace & by & in a record
Filename:
MIME Type:
Creator:
Chris Cormack
Created:
2012-07-01 03:28:47 UTC
Size:
6.28 KB
patch
obsolete
>From 0608c7c8f3f3920c35e847a49c90bf2c8c7cea8c Mon Sep 17 00:00:00 2001 >From: Alex Arnaud <alex.arnaud@biblibre.com> >Date: Wed, 6 Jul 2011 10:32:14 +0200 >Subject: [PATCH] Bug 8218 : Adding SanitizeEntity and entity_clean sub which > replace & by & in a record > >Bug 8218 : adding script for maintenance to clean any & in strings > >Signed-off-by: Chris Cormack <chris@bigballofwax.co.nz> >--- > C4/Charset.pm | 50 ++++++++++-- > misc/maintenance/batchSanitizeEntity.pl | 128 +++++++++++++++++++++++++++++++ > 2 files changed, 171 insertions(+), 7 deletions(-) > create mode 100755 misc/maintenance/batchSanitizeEntity.pl > >diff --git a/C4/Charset.pm b/C4/Charset.pm >index c294b36..32d7de3 100644 >--- a/C4/Charset.pm >+++ b/C4/Charset.pm >@@ -33,13 +33,14 @@ BEGIN { > require Exporter; > @ISA = qw(Exporter); > @EXPORT = qw( >- NormalizeString >- IsStringUTF8ish >- MarcToUTF8Record >- SetUTF8Flag >- SetMarcUnicodeFlag >- StripNonXmlChars >- nsb_clean >+ NormalizeString >+ IsStringUTF8ish >+ MarcToUTF8Record >+ SetUTF8Flag >+ SetMarcUnicodeFlag >+ StripNonXmlChars >+ nsb_clean >+ SanitizeEntity > ); > } > >@@ -1158,6 +1159,41 @@ sub char_decode5426 { > return $result; > } > >+sub SanitizeEntity { >+ my $record = shift; >+ >+ foreach my $field ($record->fields()) { >+ if ($field->is_control_field()) { >+ $field->update(entity_clean($field->data())); >+ } else { >+ my @subfields = $field->subfields(); >+ my @new_subfields; >+ foreach my $subfield (@subfields) { >+ push @new_subfields, $subfield->[0] => entity_clean($subfield->[1]); >+ } >+ if (scalar(@new_subfields) > 0) { >+ my $new_field; >+ eval { >+ $new_field = MARC::Field->new($field->tag(), $field->indicator(1), $field->indicator(2), @new_subfields); >+ }; >+ if ($@) { >+ warn "error : $@"; >+ } else { >+ $field->replace_with($new_field); >+ } >+ >+ } >+ } >+ } >+ return $record; >+} >+ >+sub entity_clean { >+ my $string=shift; >+ $string=~s/(&)(amp;)+/$1/g; >+ return $string; >+} >+ > 1; > > >diff --git a/misc/maintenance/batchSanitizeEntity.pl b/misc/maintenance/batchSanitizeEntity.pl >new file mode 100755 >index 0000000..d629faa >--- /dev/null >+++ b/misc/maintenance/batchSanitizeEntity.pl >@@ -0,0 +1,128 @@ >+#!/usr/bin/perl >+# small script that replaces '&amp;amp;etc.' by '&' in a record >+ >+use warnings; >+use C4::Charset; >+use C4::Context; >+use DBI; >+use C4::Biblio; >+use Getopt::Long; >+use Pod::Usage; >+ >+my ($biblios,$run,$want_help); >+my $result = GetOptions( >+ 'where=s' => \$biblios, >+ '--run' => \$run, >+ 'help|h' => \$want_help, >+ 'batch|b' => \$batch, >+); >+ >+# We check if required entries are given : >+if ( not $result or $want_help or not $biblios or not $run) { >+ print_usage(); >+ exit 0; >+} >+ >+# We initialise some tools : >+my $count; >+my $bibliocount; >+my @bibliofile; >+my @biblionum; >+my @biblios; >+my $record; >+my $cleanrecord; >+ >+# We first detect if we have a file or biblos directly entered by command line or if we want to use findAmp() sub : >+if ($biblios eq "search"){ >+ @biblios = findAmp(); >+} >+else { >+ if ($biblios =~ /\//) { >+ open (FILE, "$biblios") || die ("Can't open $biblios"); >+ @bibliofile = <FILE>; >+ close (FILE); >+ } >+ else { >+ @biblionum = split ',', $biblios; >+ } >+ # We take the biblios >+ @biblios = @bibliofile?@bibliofile:@biblionum; >+} >+ >+# We remove spaces >+my @biblio; >+foreach my $bib(@biblios) { >+ $bib =~ s/(^\s*|\s*$)//g; >+ next if $bib eq ""; >+ push @biblio, $bib; >+} >+@biblios = @biblio; >+ >+# We valid the input >+foreach my $biblio (@biblios) { >+ if ($biblio !~ /^\d+$/){ >+ print "=============== \"$biblio\" is not a biblionumber !!! ===============\n"; >+ print "=============== please verify \"$biblio\" !!! ===============\n"; >+ $count += 1; >+ } >+ exit(1) if $count; >+} >+ >+$bibliocount = scalar @biblios; >+my $confirm = 0; >+ >+print "=============== $bibliocount Biblio(s) ready to be cleaned ===============\n"; >+if (!$batch) { >+ print "=============== You are ok ? Types yes/no :\n"; >+ while ($confirm == 0){ >+ my $prout = <STDIN>; >+ if ($prout eq "yes\n") { >+ $confirm = 1; >+ } >+ elsif ($prout eq "no\n") { >+ print "=============== Ok, bye ===============\n"; >+ exit(0); >+ } >+ else { >+ print "======= Bad answer please types 'yes' or 'no' !!!!!!!!!!! ===============\n"; >+ } >+ } >+} >+ >+foreach my $biblio (@biblios){ >+ print "=============== N° $biblio selected ===============\n"; >+ $record = GetMarcBiblio($biblio); >+ $cleanrecord = SanitizeEntity($record); >+ my $frameworkcode = GetFrameworkCode($biblio); >+ ModBiblio($cleanrecord, $biblio, $frameworkcode); >+ print "=============== Biblio n° $biblio done ===============\n"; >+} >+ >+print "==============================================================\n"; >+print "=============== $bibliocount Biblios processed ===============\n"; >+ >+sub print_usage { >+ print <<_USAGE_; >+$0: replaces '&' by '&' in a record, you can either give some biblionumbers or a file with biblionumbers or >+ >+Parameters: >+ -where use this to give biblionumbers in a string with "" (separated by coma) >+ or an absolute path to a file containing biblionumbers (1 by row) >+ or the command 'search' that creates an array with biblionumbers with "&amp;..." >+ --run run the command >+ --batch run in batch mode >+ --help or -h show this message. >+_USAGE_ >+} >+ >+sub findAmp { >+ my @bibliosearch; >+ my $dbh = C4::Context->dbh; >+ my $strsth = qq{SELECT biblionumber FROM biblioitems WHERE marcxml LIKE "%amp;amp;%"}; >+ my $sth = $dbh->prepare($strsth); >+ $sth->execute(); >+ while (my $bib = $sth-> fetchrow_array()){ >+ push @bibliosearch, $bib; >+ } >+ return @bibliosearch; >+} >-- >1.7.10
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 8218
:
10039
|
10588
|
10589
|
11744
|
12591
|
13678
|
15826
|
16122
|
16720
|
26754
|
26755
|
29792
|
29793
|
29794
|
29795
|
29796
|
29798
|
30120
|
30121
|
33248