From e5b2e47041faa8aef8c8d417fd9901f6d8306ea9 Mon Sep 17 00:00:00 2001 From: Christophe Croullebois Date: Fri, 28 Sep 2012 15:23:37 +0200 Subject: [PATCH 1/1] Bug 8218 : A script for maintenance to clean any & Two subs written by Alex Arnaud in C4/Charset.pm, SanitizeEntity and entity_clean I had created two scripts for Biblibre, one to be used in cron, it reindexes all transformed records. Sophie Meynieux brought together the two scripts, she has created a batch mode, usefull but without the reindexing So I have rewrited it and I have integrated "Fixing copyright headers and license and description" written by Chris Cormack and "Followup: Perltidy and QA FIX" written by Jonathan Druart I have added a check and a message if the script does nothing on a record and some other corrections with the help of Bernardo Gonzalez Kriegel Thx to all --- C4/Charset.pm | 54 +++++++++ misc/maintenance/batchSanitizeEntity.pl | 180 +++++++++++++++++++++++++++++++ 2 files changed, 234 insertions(+), 0 deletions(-) create mode 100755 misc/maintenance/batchSanitizeEntity.pl diff --git a/C4/Charset.pm b/C4/Charset.pm index f8ddd63..2a886e0 100644 --- a/C4/Charset.pm +++ b/C4/Charset.pm @@ -40,6 +40,7 @@ BEGIN { SetMarcUnicodeFlag StripNonXmlChars nsb_clean + SanitizeEntity ); } @@ -419,6 +420,59 @@ sub nsb_clean { return($string) ; } +=head2 SanitizeEntity + +SanitizeEntity($marcrecord); + +Sanitize Entity +A script for maintenance to clean any string with '&amp;...', replacing it by '&' + +=cut + +sub SanitizeEntity { + my $record = shift; + my $string; + my $noact; + foreach my $field ($record->fields()) { + if ($field->is_control_field()) { + ($string,$noact) = entity_clean($field->data()); + $field->update($string); + } else { + my @subfields = $field->subfields(); + my @new_subfields; + foreach my $subfield (@subfields) { + ($string,$noact) = entity_clean($subfield->[1]); + push @new_subfields, $subfield->[0] => $string; + } + if (scalar(@new_subfields) > 0) { + my $new_field; + eval { + $new_field = MARC::Field->new($field->tag(), $field->indicator(1), $field->indicator(2), @new_subfields); + }; + if ($@) { + warn "error : $@"; + } else { + $field->replace_with($new_field); + } + + } + } + } + + return $record,$noact; +} + +sub entity_clean { + my $string=shift; + my $oldstring = $string; + my $noact; + $string=~s/(&)(amp;)+/$1/g; + if ( $string eq $oldstring){ + $noact = 1; + return $oldstring, $noact; + } + return $string; +} =head1 INTERNAL FUNCTIONS diff --git a/misc/maintenance/batchSanitizeEntity.pl b/misc/maintenance/batchSanitizeEntity.pl new file mode 100755 index 0000000..30f45aa --- /dev/null +++ b/misc/maintenance/batchSanitizeEntity.pl @@ -0,0 +1,180 @@ +#!/usr/bin/perl + +# small script that replaces '&amp;amp;etc.' by '&' in a record + +# Copyright 2012 Biblibre SARL +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use C4::Charset; +use C4::Context; +use DBI; +use C4::Biblio; +use Getopt::Long; +use Pod::Usage; + +my ( $biblios,$run,$want_help,$batch,$cron,$reindex ); +my $result = GetOptions( + 'where=s' => \$biblios, + 'run' => \$run, + 'help|h' => \$want_help, + 'batch|b' => \$batch, + 'cron|c' => \$cron, + 'reindex|r'=> \$reindex, +); + +# We check if required entries are given : +if ( not $result or $want_help or not $biblios or not $run ) { + print_usage(); + exit 0; +} + +# We initialise some tools : +my $count; +my $bibliocount; +my @bibliofile; +my @biblionum; +my @biblios; +my $record; +my $cleanrecord; +my $noact; + +# We first detect if we have a file or biblos directly entered by command line +#or if we want to use findAmp() sub +if ( $biblios eq "search" ) { + @biblios = findAmp(); +} +else { + if ( $biblios =~ m|/| ) { + open( FILE, "$biblios" ) || die("Can't open $biblios"); + @bibliofile = ; + close(FILE); + } + else { + @biblionum = split ',', $biblios; + } + + # We take the biblios + @biblios = @bibliofile ? @bibliofile : @biblionum; +} + +# We remove spaces +s/(^\s*|\s*$)//g for @biblios; +# Remove empty lines +@biblios = grep {!/^$/} @biblios; + +# We valid the input +foreach my $biblio (@biblios) { + if ( $biblio !~ /^\d+$/ ) { + print +"------------- \"$biblio\" is not a biblionumber !!! -------------\n"; + print "------------- please verify \"$biblio\" !!! -------------\n"; + $count++; + } + exit(1) if $count; +} + +$bibliocount = scalar @biblios; + +print +"------------- $bibliocount Biblio(s) ready to be cleaned -------------\n"; +if ( !$batch or !$cron ) { + my $confirm = 0; + print "------------- Proceed? [yes/no] :\n"; + while ( $confirm == 0 ) { + my $prout = ; + if ($prout eq "yes\n") { + $confirm = 1; + } + elsif ($prout eq "no\n") { + print "------------- Ok, bye -------------\n"; + exit(0); + } + else { + print "------------- Bad answer please types 'yes' or 'no' -------------\n"; + } + } +} + +my $num = -1; +foreach my $biblio (@biblios) { + $num++; + print "------------- N° $biblio selected -------------\n"; + $record = GetMarcBiblio($biblio); + ($cleanrecord,$noact) = SanitizeEntity($record); + if ($noact==1){ + print "------------- Biblio n° $biblio nothing to clean -------------\n"; + splice(@biblios,$num,1); + }else { + my $frameworkcode = GetFrameworkCode($biblio); + ModBiblio( $cleanrecord, $biblio, $frameworkcode ); + print "------------- Biblio n° $biblio cleaning done -------------\n"; + $count++; + } +} + +print "-------------------------------------------------------------\n"; +if ($count){ + print "------------- $count Biblios cleaned -------------\n"; +}else{ + print "------------- no Biblios cleaned -------------\n" if ($count); +} + +if ( $cron or $reindex) { + print "-------------------------------------------------------------\n"; + print +"--------- Now we are reindexing -b -v -where \"biblionumber=xxx\" ---------\n"; + print "-------------------------------------------------------------\n"; + $count = 0; + my $kohapath = C4::Context->config('intranetdir'); + foreach my $biblio ( @biblios ) { + system("$kohapath/misc/migration_tools/rebuild_zebra.pl + -b -v -where \"biblionumber=$biblio\""); + print "------------- Biblio n° $biblio re-indexing done -------------\n"; + $count ++; +} + print "------------- $count Biblios re-indexed -------------\n"; +} + +sub print_usage { + print <<_USAGE_; +$0: replaces '&' by '&' in a record, you can either give some biblionumbers or a file with biblionumbers or ask for a search + +Parameters: + --where use this to give biblionumbers (separated by coma) in a string with "" + or an absolute path to a file containing biblionumbers (1 by row) + or the command 'search' that creates an array with biblionumbers containing "&amp;..." in biblioitems.marxml + --run run the command + --batch or -b run in batch mode + --cron or -c run in cron mode, it reindexes the changed records (you can redirect the output to a file) + --help or -h show this message. + --reindex reindexes the modified records +_USAGE_ +} + +sub findAmp { + my @bibliosearch; + my $dbh = C4::Context->dbh; + my $strsth = + qq{SELECT biblionumber FROM biblioitems WHERE marcxml LIKE "%&amp;%"}; + my $sth = $dbh->prepare($strsth); + $sth->execute(); + while ( my $bib = $sth-> fetchrow_array() ) { + push @bibliosearch, $bib; + } + return @bibliosearch; +} -- 1.7.0.4