Bugzilla – Attachment 16720 Details for
Bug 8218
Add a script to sanitize entity where "&" character was wrongly replaced with "&"
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
proposed patch
0001-Bug-8218-A-script-for-maintenance-to-clean-any-amp.patch (text/plain), 8.79 KB, created by
Christophe Croullebois
on 2013-03-22 08:47:55 UTC
(
hide
)
Description:
proposed patch
Filename:
MIME Type:
Creator:
Christophe Croullebois
Created:
2013-03-22 08:47:55 UTC
Size:
8.79 KB
patch
obsolete
>From e5b2e47041faa8aef8c8d417fd9901f6d8306ea9 Mon Sep 17 00:00:00 2001 >From: Christophe Croullebois <christophe.croullebois@biblibre.com> >Date: Fri, 28 Sep 2012 15:23:37 +0200 >Subject: [PATCH 1/1] Bug 8218 : A script for maintenance to clean any & > >Two subs written by Alex Arnaud in C4/Charset.pm, SanitizeEntity and entity_clean >I had created two scripts for Biblibre, one to be used in cron, it reindexes all transformed records. >Sophie Meynieux brought together the two scripts, she has created a batch mode, usefull but without the reindexing >So I have rewrited it and I have integrated "Fixing copyright headers and license and description" >written by Chris Cormack and "Followup: Perltidy and QA FIX" written by Jonathan Druart >I have added a check and a message if the script does nothing on a record >and some other corrections with the help of Bernardo Gonzalez Kriegel >Thx to all >--- > C4/Charset.pm | 54 +++++++++ > misc/maintenance/batchSanitizeEntity.pl | 180 +++++++++++++++++++++++++++++++ > 2 files changed, 234 insertions(+), 0 deletions(-) > create mode 100755 misc/maintenance/batchSanitizeEntity.pl > >diff --git a/C4/Charset.pm b/C4/Charset.pm >index f8ddd63..2a886e0 100644 >--- a/C4/Charset.pm >+++ b/C4/Charset.pm >@@ -40,6 +40,7 @@ BEGIN { > SetMarcUnicodeFlag > StripNonXmlChars > nsb_clean >+ SanitizeEntity > ); > } > >@@ -419,6 +420,59 @@ sub nsb_clean { > return($string) ; > } > >+=head2 SanitizeEntity >+ >+SanitizeEntity($marcrecord); >+ >+Sanitize Entity >+A script for maintenance to clean any string with '&amp;...', replacing it by '&' >+ >+=cut >+ >+sub SanitizeEntity { >+ my $record = shift; >+ my $string; >+ my $noact; >+ foreach my $field ($record->fields()) { >+ if ($field->is_control_field()) { >+ ($string,$noact) = entity_clean($field->data()); >+ $field->update($string); >+ } else { >+ my @subfields = $field->subfields(); >+ my @new_subfields; >+ foreach my $subfield (@subfields) { >+ ($string,$noact) = entity_clean($subfield->[1]); >+ push @new_subfields, $subfield->[0] => $string; >+ } >+ if (scalar(@new_subfields) > 0) { >+ my $new_field; >+ eval { >+ $new_field = MARC::Field->new($field->tag(), $field->indicator(1), $field->indicator(2), @new_subfields); >+ }; >+ if ($@) { >+ warn "error : $@"; >+ } else { >+ $field->replace_with($new_field); >+ } >+ >+ } >+ } >+ } >+ >+ return $record,$noact; >+} >+ >+sub entity_clean { >+ my $string=shift; >+ my $oldstring = $string; >+ my $noact; >+ $string=~s/(&)(amp;)+/$1/g; >+ if ( $string eq $oldstring){ >+ $noact = 1; >+ return $oldstring, $noact; >+ } >+ return $string; >+} > > =head1 INTERNAL FUNCTIONS > >diff --git a/misc/maintenance/batchSanitizeEntity.pl b/misc/maintenance/batchSanitizeEntity.pl >new file mode 100755 >index 0000000..30f45aa >--- /dev/null >+++ b/misc/maintenance/batchSanitizeEntity.pl >@@ -0,0 +1,180 @@ >+#!/usr/bin/perl >+ >+# small script that replaces '&amp;amp;etc.' by '&' in a record >+ >+# Copyright 2012 Biblibre SARL >+# >+# This file is part of Koha. >+# >+# Koha is free software; you can redistribute it and/or modify it under the >+# terms of the GNU General Public License as published by the Free Software >+# Foundation; either version 2 of the License, or (at your option) any later >+# version. >+# >+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY >+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR >+# A PARTICULAR PURPOSE. See the GNU General Public License for more details. >+# >+# You should have received a copy of the GNU General Public License along >+# with Koha; if not, write to the Free Software Foundation, Inc., >+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. >+ >+use Modern::Perl; >+use C4::Charset; >+use C4::Context; >+use DBI; >+use C4::Biblio; >+use Getopt::Long; >+use Pod::Usage; >+ >+my ( $biblios,$run,$want_help,$batch,$cron,$reindex ); >+my $result = GetOptions( >+ 'where=s' => \$biblios, >+ 'run' => \$run, >+ 'help|h' => \$want_help, >+ 'batch|b' => \$batch, >+ 'cron|c' => \$cron, >+ 'reindex|r'=> \$reindex, >+); >+ >+# We check if required entries are given : >+if ( not $result or $want_help or not $biblios or not $run ) { >+ print_usage(); >+ exit 0; >+} >+ >+# We initialise some tools : >+my $count; >+my $bibliocount; >+my @bibliofile; >+my @biblionum; >+my @biblios; >+my $record; >+my $cleanrecord; >+my $noact; >+ >+# We first detect if we have a file or biblos directly entered by command line >+#or if we want to use findAmp() sub >+if ( $biblios eq "search" ) { >+ @biblios = findAmp(); >+} >+else { >+ if ( $biblios =~ m|/| ) { >+ open( FILE, "$biblios" ) || die("Can't open $biblios"); >+ @bibliofile = <FILE>; >+ close(FILE); >+ } >+ else { >+ @biblionum = split ',', $biblios; >+ } >+ >+ # We take the biblios >+ @biblios = @bibliofile ? @bibliofile : @biblionum; >+} >+ >+# We remove spaces >+s/(^\s*|\s*$)//g for @biblios; >+# Remove empty lines >+@biblios = grep {!/^$/} @biblios; >+ >+# We valid the input >+foreach my $biblio (@biblios) { >+ if ( $biblio !~ /^\d+$/ ) { >+ print >+"------------- \"$biblio\" is not a biblionumber !!! -------------\n"; >+ print "------------- please verify \"$biblio\" !!! -------------\n"; >+ $count++; >+ } >+ exit(1) if $count; >+} >+ >+$bibliocount = scalar @biblios; >+ >+print >+"------------- $bibliocount Biblio(s) ready to be cleaned -------------\n"; >+if ( !$batch or !$cron ) { >+ my $confirm = 0; >+ print "------------- Proceed? [yes/no] :\n"; >+ while ( $confirm == 0 ) { >+ my $prout = <STDIN>; >+ if ($prout eq "yes\n") { >+ $confirm = 1; >+ } >+ elsif ($prout eq "no\n") { >+ print "------------- Ok, bye -------------\n"; >+ exit(0); >+ } >+ else { >+ print "------------- Bad answer please types 'yes' or 'no' -------------\n"; >+ } >+ } >+} >+ >+my $num = -1; >+foreach my $biblio (@biblios) { >+ $num++; >+ print "------------- N° $biblio selected -------------\n"; >+ $record = GetMarcBiblio($biblio); >+ ($cleanrecord,$noact) = SanitizeEntity($record); >+ if ($noact==1){ >+ print "------------- Biblio n° $biblio nothing to clean -------------\n"; >+ splice(@biblios,$num,1); >+ }else { >+ my $frameworkcode = GetFrameworkCode($biblio); >+ ModBiblio( $cleanrecord, $biblio, $frameworkcode ); >+ print "------------- Biblio n° $biblio cleaning done -------------\n"; >+ $count++; >+ } >+} >+ >+print "-------------------------------------------------------------\n"; >+if ($count){ >+ print "------------- $count Biblios cleaned -------------\n"; >+}else{ >+ print "------------- no Biblios cleaned -------------\n" if ($count); >+} >+ >+if ( $cron or $reindex) { >+ print "-------------------------------------------------------------\n"; >+ print >+"--------- Now we are reindexing -b -v -where \"biblionumber=xxx\" ---------\n"; >+ print "-------------------------------------------------------------\n"; >+ $count = 0; >+ my $kohapath = C4::Context->config('intranetdir'); >+ foreach my $biblio ( @biblios ) { >+ system("$kohapath/misc/migration_tools/rebuild_zebra.pl >+ -b -v -where \"biblionumber=$biblio\""); >+ print "------------- Biblio n° $biblio re-indexing done -------------\n"; >+ $count ++; >+} >+ print "------------- $count Biblios re-indexed -------------\n"; >+} >+ >+sub print_usage { >+ print <<_USAGE_; >+$0: replaces '&' by '&' in a record, you can either give some biblionumbers or a file with biblionumbers or ask for a search >+ >+Parameters: >+ --where use this to give biblionumbers (separated by coma) in a string with "" >+ or an absolute path to a file containing biblionumbers (1 by row) >+ or the command 'search' that creates an array with biblionumbers containing "&amp;..." in biblioitems.marxml >+ --run run the command >+ --batch or -b run in batch mode >+ --cron or -c run in cron mode, it reindexes the changed records (you can redirect the output to a file) >+ --help or -h show this message. >+ --reindex reindexes the modified records >+_USAGE_ >+} >+ >+sub findAmp { >+ my @bibliosearch; >+ my $dbh = C4::Context->dbh; >+ my $strsth = >+ qq{SELECT biblionumber FROM biblioitems WHERE marcxml LIKE "%&amp;%"}; >+ my $sth = $dbh->prepare($strsth); >+ $sth->execute(); >+ while ( my $bib = $sth-> fetchrow_array() ) { >+ push @bibliosearch, $bib; >+ } >+ return @bibliosearch; >+} >-- >1.7.0.4 >
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 8218
:
10039
|
10588
|
10589
|
11744
|
12591
|
13678
|
15826
|
16122
|
16720
|
26754
|
26755
|
29792
|
29793
|
29794
|
29795
|
29796
|
29798
|
30120
|
30121
|
33248