Bugzilla – Attachment 15826 Details for
Bug 8218
Add a script to sanitize entity where "&" character was wrongly replaced with "&"
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
proposed patch
0001-Bug-8218-A-script-for-maintenance-to-clean-any-amp.patch (text/plain), 8.09 KB, created by
Christophe Croullebois
on 2013-03-02 13:33:42 UTC
(
hide
)
Description:
proposed patch
Filename:
MIME Type:
Creator:
Christophe Croullebois
Created:
2013-03-02 13:33:42 UTC
Size:
8.09 KB
patch
obsolete
>From cef6fb32e46fe2f7401ced3f078c7a80b99b7457 Mon Sep 17 00:00:00 2001 >From: Christophe Croullebois <christophe.croullebois@biblibre.com> >Date: Fri, 28 Sep 2012 15:23:37 +0200 >Subject: [PATCH 1/1] Bug 8218 : A script for maintenance to clean any & > >Two subs written by Alex Arnaud in C4/Charset.pm, SanitizeEntity and entity_clean >I had created two scripts for Biblibre, one to be used in cron, it reindexes all transformed records. >Sophie Meynieux brought together the two scripts, she has created a batch mode, usefull but without the reindexing >So I have rewrited it and I have integrated "Fixing copyright headers and license and description" >written by Chris Cormack and "Followup: Perltidy and QA FIX" written by Jonathan Druart >Thx to all >--- > C4/Charset.pm | 48 +++++++++ > misc/maintenance/batchSanitizeEntity.pl | 173 +++++++++++++++++++++++++++++++ > 2 files changed, 221 insertions(+) > create mode 100644 misc/maintenance/batchSanitizeEntity.pl > >diff --git a/C4/Charset.pm b/C4/Charset.pm >index f8ddd63..0b22e4c 100644 >--- a/C4/Charset.pm >+++ b/C4/Charset.pm >@@ -40,6 +40,7 @@ BEGIN { > SetMarcUnicodeFlag > StripNonXmlChars > nsb_clean >+ SanitizeEntity > ); > } > >@@ -1161,6 +1162,53 @@ sub char_decode5426 { > return $result; > } > >+=head2 SanitizeEntity >+ >+=over 4 >+ >+SanitizeEntity($marcrecord); >+ >+=back >+ >+Sanitize Entity >+ >+=cut >+ >+sub SanitizeEntity { >+ my $record = shift; >+ >+ foreach my $field ($record->fields()) { >+ if ($field->is_control_field()) { >+ $field->update(entity_clean($field->data())); >+ } else { >+ my @subfields = $field->subfields(); >+ my @new_subfields; >+ foreach my $subfield (@subfields) { >+ push @new_subfields, $subfield->[0] => entity_clean($subfield->[1]); >+ } >+ if (scalar(@new_subfields) > 0) { >+ my $new_field; >+ eval { >+ $new_field = MARC::Field->new($field->tag(), $field->indicator(1), $field->indicator(2), @new_subfields); >+ }; >+ if ($@) { >+ warn "error : $@"; >+ } else { >+ $field->replace_with($new_field); >+ } >+ >+ } >+ } >+ } >+ return $record; >+} >+ >+sub entity_clean { >+ my $string=shift; >+ $string=~s/(&)(amp;)+/$1/g; >+ return $string; >+} >+ > 1; > > >diff --git a/misc/maintenance/batchSanitizeEntity.pl b/misc/maintenance/batchSanitizeEntity.pl >new file mode 100644 >index 0000000..d226cea >--- /dev/null >+++ b/misc/maintenance/batchSanitizeEntity.pl >@@ -0,0 +1,173 @@ >+#!/usr/bin/perl >+# small script that replaces '&amp;amp;etc.' by '&' in a record >+ >+# Copyright 2012 Biblibre SARL >+# >+# This file is part of Koha. >+# >+# Koha is free software; you can redistribute it and/or modify it under the >+# terms of the GNU General Public License as published by the Free Software >+# Foundation; either version 2 of the License, or (at your option) any later >+# version. >+# >+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY >+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR >+# A PARTICULAR PURPOSE. See the GNU General Public License for more details. >+# >+# You should have received a copy of the GNU General Public License along >+# with Koha; if not, write to the Free Software Foundation, Inc., >+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. >+ >+use Modern::Perl; >+use C4::Charset; >+use C4::Context; >+use DBI; >+use C4::Biblio; >+use Getopt::Long; >+use Pod::Usage; >+ >+my ( $biblios,$run,$want_help,$batch,$cron ); >+my $result = GetOptions( >+ 'where=s' => \$biblios, >+ '--run' => \$run, >+ 'help|h' => \$want_help, >+ 'batch|b' => \$batch, >+ 'cron|c' => \$cron, >+); >+ >+# We check if required entries are given : >+if ( not $result or $want_help or not $biblios or not $run ) { >+ print_usage(); >+ exit 0; >+} >+ >+my $count; >+my $bibliocount; >+my @bibliofile; >+my @biblionum; >+my @biblios; >+my $record; >+my $cleanrecord; >+ >+# We first detect if we have a file or biblos directly entered by command line >+#or if we want to use findAmp() sub >+if ( $biblios eq "search" ) { >+ print "=============== Searching, please wait ...\n"; >+ @biblios = findAmp(); >+} >+else { >+ if ( $biblios =~ m|/| ) { >+ open( FILE, "$biblios" ) || die("Can't open $biblios"); >+ @bibliofile = <FILE>; >+ close(FILE); >+ } >+ else { >+ @biblionum = split ',', $biblios; >+ } >+ >+ # We take the biblios >+ @biblios = @bibliofile ? @bibliofile : @biblionum; >+} >+ >+# We remove spaces >+s/(^\s*|\s*$)//g for @biblios; >+# Remove empty lines >+@biblios = grep {!/^$/} @biblios; >+ >+# We valid the input >+foreach my $biblio (@biblios) { >+ if ( $biblio !~ /^\d+$/ ) { >+ print >+"=============== \"$biblio\" is not a biblionumber !!! ===============\n"; >+ print "=============== please verify \"$biblio\" !!! ===============\n"; >+ $count++; >+ } >+ exit(1) if $count; >+} >+ >+$bibliocount = scalar @biblios; >+ >+print >+"=============== $bibliocount Biblio(s) ready to be cleaned ===============\n"; >+ >+if ($bibliocount == 0) { >+ print "=============== nothing to clean\n"; >+ exit(0); >+} >+ >+if ( !$batch and !$cron ) { >+ my $confirm = 0; >+ print "=============== You are ok ? Types yes/no :\n"; >+ while ( $confirm == 0 ) { >+ my $prout = <STDIN>; >+ if ($prout eq "yes\n") { >+ $confirm = 1; >+ } >+ elsif ($prout eq "no\n") { >+ print "=============== Ok, bye ===============\n"; >+ exit(0); >+ } >+ else { >+ print "======= Bad answer please types 'yes' or 'no' ! ===============\n"; >+ } >+ } >+} >+ >+foreach my $biblio ( @biblios ) { >+ print "=============== N° $biblio selected ===============\n"; >+ $record = GetMarcBiblio($biblio); >+ $cleanrecord = SanitizeEntity($record); >+ my $frameworkcode = GetFrameworkCode($biblio); >+ ModBiblio( $cleanrecord, $biblio, $frameworkcode ); >+ print "=============== Biblio n° $biblio cleaning done ===============\n"; >+ $count++; >+} >+ >+print "==============================================================\n"; >+print "=============== $count Biblios cleaned ===============\n"; >+ >+if ( $cron ) { >+ print "==============================================================\n"; >+ print >+"========= Now we are reindexing -b -v -where \"biblionumber=xxx\" =========\n"; >+ print "==============================================================\n"; >+ $count = 0; >+ my $kohapath = C4::Context->config('intranetdir'); >+ foreach my $biblio ( @biblios ) { >+ system("$kohapath/misc/migration_tools/rebuild_zebra.pl -b -v -where \"biblionumber=$biblio\""); >+ print "========= Biblio n° $biblio re-indexing done =========\n"; >+ $count ++; >+} >+ print "=============== $count Biblios re-indexed ===============\n"; >+} >+ >+sub print_usage { >+ print <<_USAGE_; >+$0: replaces '&' by '&' in a record, you can give some biblionumbers >+or a file with biblionumbers >+or ask to search in the DataBase, may be long. >+WARNING : if there is a subfield with an url with '&' it will become '&' >+ >+Parameters: >+ --where use this to give biblionumbers in a string with "" (separated by coma) >+ or an absolute path to a file containing biblionumbers (1 by row) >+ or the command 'search' that creates an array with biblionumbers with "&amp;..." >+ --run run the command >+ --batch or -b run in batch mode >+ --cron or -c run in cron mode, it reindexes the changed records (you can redirect the output to a file) >+ --help or -h show this message. >+_USAGE_ >+} >+ >+sub findAmp { >+ my @bibliosearch; >+ my $dbh = C4::Context->dbh; >+ my $strsth = >+ qq{SELECT biblionumber FROM biblioitems WHERE marcxml LIKE "%amp;amp;%"}; >+ my $sth = $dbh->prepare($strsth); >+ $sth->execute(); >+ while ( my $bib = $sth-> fetchrow_array() ) { >+ push @bibliosearch, $bib; >+ } >+ return @bibliosearch; >+} >\ No newline at end of file >-- >1.7.9.5 >
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 8218
:
10039
|
10588
|
10589
|
11744
|
12591
|
13678
|
15826
|
16122
|
16720
|
26754
|
26755
|
29792
|
29793
|
29794
|
29795
|
29796
|
29798
|
30120
|
30121
|
33248