From 096c896c60d72c967af65778824bbec77fe7a31f Mon Sep 17 00:00:00 2001 From: Jonathan Druart Date: Fri, 28 Sep 2012 15:23:37 +0200 Subject: [PATCH] Bug 8218 : Add a maintenance script to sanitize biblio records This patch adds: - a new maintenance script batch_sanitize_records - a new subroutine C4::Charset::SanitizeRecord - new unit tests for the new subroutine Test plan: 1/ prove t/db_dependent/Charset.t 2/ Create a record containing "&amp;" (could be follow with as many 'amp;' as you want) in one of its fields and the same for the field linked to biblioitems.url. The url should not be sanitized, it cans contain "&". 3/ Launch the maintenance script with the -h paramters to see how to use it. 4/ Launch the script using the different parameters: --filename=FILENAME --biblionumbers='XXX' --auto-search The auto-search permits to sanitize all records containing "&amp;" in the marcxml field. Use the verbose flag for testing. Without the --confirm flag, nothing is done. 5/ Use the --confirm flag and verify in the biblioitems.marcxml field that the record has been sanitized. 6/ Try the --reindex flag to reindex records which have been modified. --- C4/Charset.pm | 55 +++++++ misc/maintenance/batch_sanitize_records.pl | 214 ++++++++++++++++++++++++++++ t/db_dependent/Charset.t | 51 +++++++ 3 files changed, 320 insertions(+) create mode 100755 misc/maintenance/batch_sanitize_records.pl create mode 100644 t/db_dependent/Charset.t diff --git a/C4/Charset.pm b/C4/Charset.pm index acad92a..e847051 100644 --- a/C4/Charset.pm +++ b/C4/Charset.pm @@ -41,6 +41,7 @@ BEGIN { SetMarcUnicodeFlag StripNonXmlChars nsb_clean + SanitizeRecord ); } @@ -423,6 +424,60 @@ sub nsb_clean { return($string) ; } +=head2 SanitizeRecord + +SanitizeRecord($marcrecord); + +Sanitize a record +This routine is called in the maintenance script misc/maintenance/batch_sanitize_records.pl. +It cleans any string with '&amp;...', replacing it by '&' + +=cut + +sub SanitizeRecord { + my ( $record, $biblionumber ) = @_; + my $string; + my $record_modified = 0; + my $frameworkcode = C4::Biblio::GetFrameworkCode( $biblionumber ); + my ( $url_field, $url_subfield ) = C4::Biblio::GetMarcFromKohaField('biblioitems.url', $frameworkcode); + foreach my $field ($record->fields()) { + if ($field->is_control_field()) { + my $value = $field->data(); + my $sanitized_value = _entity_clean($value); + $record_modified = 1 if $sanitized_value ne $value; + $field->update($sanitized_value); + } else { + my @subfields = $field->subfields(); + my @new_subfields; + foreach my $subfield (@subfields) { + next if $url_field eq $field->tag() and $url_subfield eq $subfield->[0]; + my $value = $subfield->[1]; + my $sanitized_value = _entity_clean($value); + push @new_subfields, $subfield->[0] => $sanitized_value; + $record_modified = 1 if $sanitized_value ne $value; + } + if (scalar(@new_subfields) > 0) { + my $new_field = eval { + MARC::Field->new($field->tag(), $field->indicator(1), $field->indicator(2), @new_subfields); + }; + if ($@) { + warn "error : $@"; + } else { + $field->replace_with($new_field); + } + + } + } + } + + return $record, $record_modified; +} + +sub _entity_clean { + my ( $string ) = @_; + $string =~ s/(&)(amp;)+/$1/g; + return $string; +} =head1 INTERNAL FUNCTIONS diff --git a/misc/maintenance/batch_sanitize_records.pl b/misc/maintenance/batch_sanitize_records.pl new file mode 100755 index 0000000..fead4b8 --- /dev/null +++ b/misc/maintenance/batch_sanitize_records.pl @@ -0,0 +1,214 @@ +#!/usr/bin/perl + +# This file is part of Koha. +# +# Copyright 2014 BibLibre +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + + + +use Modern::Perl; +use C4::Charset qw( SanitizeRecord ); +use C4::Context; +use DBI; +use C4::Biblio; +use Getopt::Long; +use Pod::Usage; + +my ( $help, $verbose, $confirm, $biblionumbers, $reindex, $filename, $auto_search ); +my $result = GetOptions( + 'h|help' => \$help, + 'v|verbose' => \$verbose, + 'c|confirm' => \$confirm, + 'biblionumbers:s' => \$biblionumbers, + 'reindex'=> \$reindex, + 'f|filename:s' => \$filename, + 'auto-search' => \$auto_search, +) || pod2usage(1); + +if ( $help ) { + pod2usage(0); +} + +unless ( $filename or $biblionumbers or $auto_search ) { + pod2usage( -exitval => 1, -message => qq{\n\tAt least one record number source should be provided.\n} ); +} + +if ( + $filename and $biblionumbers + or $filename and $auto_search + or $biblionumbers and $auto_search +) { + pod2usage( -exitval => 1, -message => qq{\n\tOnly one record number source should be provided.\n} ); +} + + +my @biblionumbers; +# We first detect if we have a file or biblos directly entered by command line +#or if we want to use findAmp() sub +if ( $auto_search ) { + @biblionumbers = biblios_to_sanitize(); +} +elsif ( $filename ) { + if ( -e $filename ) { + open( my $fh, $filename ) || die("Can't open $filename ($!)"); + while ( <$fh> ) { + chomp; + my $line = $_; + push @biblionumbers, split( " |,", $line ); + } + close $fh; + } else { + pod2usage( -exitval => 1, -message => qq{\n\tThis filename does not exist. Please verify the path is correct.\n} ); + } +} else { + @biblionumbers = split m|,|, $biblionumbers if $biblionumbers; +} + +# We remove spaces +s/(^\s*|\s*$)//g for @biblionumbers; +# Remove empty lines +@biblionumbers = grep {!/^$/} @biblionumbers; + +say @biblionumbers . " records to process" if $verbose; + +my @changes; +for my $biblionumber (@biblionumbers) { + print "processing record $biblionumber..." if $verbose; + unless ( $biblionumber =~ m|^\d+$| ) { + say " skipping. ERROR: Invalid biblionumber." if $verbose; + next; + } + my $record = C4::Biblio::GetMarcBiblio($biblionumber); + unless ( $record ) { + say " skipping. ERROR: Invalid record." if $verbose; + next; + } + + unless ( $confirm ) { + say " will be processed" if $verbose; + next; + } + my ($cleaned_record,$has_been_modified) = C4::Charset::SanitizeRecord($record, $biblionumber); + if ( $has_been_modified ){ + my $frameworkcode = C4::Biblio::GetFrameworkCode($record); + C4::Biblio::ModBiblio( $cleaned_record, $biblionumber, $frameworkcode ); + push @changes, $biblionumber; + say " Done!" if $verbose; + }else { + say " Nothing todo." if $verbose; + } +} + +if ( $verbose ) { + say "Total: " . @changes . " records " . ( $confirm ? "cleaned!" : "to clean." ); +} + + +if ( $reindex and $confirm and @changes ) { + say "Now, reindexing using -b -v" if $verbose; + my $kohapath = C4::Context->config('intranetdir'); + my $cmd = qq| + $kohapath/misc/migration_tools/rebuild_zebra.pl -b -v -where "biblionumber IN ( | + . join ( ',', @changes ) + . q| )" + |; + system($cmd); +} + +sub biblios_to_sanitize { + my $dbh = C4::Context->dbh; + my $query = q{ + SELECT biblionumber + FROM biblioitems + WHERE marcxml + LIKE "%&amp;%" + }; + return @{ + $dbh->selectcol_arrayref( + $query, + { Slice => {} }, + ) + }; +} + +=head1 NAME + +batch_sanitize_biblios - This script sanitize a biblio, replacing '&amp;amp;etc.' with '&' in it. + +=head1 SYNOPSIS + +batch_sanitize_biblios.pl [-h|--help] [-v|--verbose] [-c|--confirm] [--biblionumbers=BIBLIONUMBER_LIST] [-f|--filename=FILENAME] [--auto-search] [--reindex] + +Replace '&' by '&' in a record, you can either give some biblionumbers or a file with biblionumbers or ask for an auto-search + +=head1 OPTIONS + +=over + +=item B<-h|--help> + +Print a brief help message + +=item B<-v|--verbose> + +Verbose mode. + +=item B<-c|--confirm> + +This flag must be provided in order for the script to actually +sanitize records. If it is not supplied, the script will +only report on the record list to process. + +=item B<--biblionumbers=BIBLIONUMBER_LIST> + +Give a biblionumber list using this parameter. They must be separated by comma. + +=item B<-f|--filename=FILENAME> + +Give a biblionumber list using a filename. One biblionumber by line or separate them with a withespace character. + +=item B<--auto_search> + +Automatically search records containing "&" in biblioitems.marcxml or in the specified fields. + +=item B<--reindex> + +Reindex the modified records. + +=back + +=head1 AUTHOR + +Alex Arnaud +Christophe Croullebois +Jonathan Druart + +=head1 COPYRIGHT + +Copyright 2014 BibLibre + +=head1 LICENSE + +This file is part of Koha. + +Koha is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software +Foundation; either version 3 of the License, or (at your option) any later version. + +You should have received a copy of the GNU General Public License along +with Koha; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +=cut diff --git a/t/db_dependent/Charset.t b/t/db_dependent/Charset.t new file mode 100644 index 0000000..9a48aef --- /dev/null +++ b/t/db_dependent/Charset.t @@ -0,0 +1,51 @@ +use Modern::Perl; +use Test::More tests => 4; +use MARC::Record; + +use C4::Biblio qw( AddBiblio SetFieldMapping GetMarcFromKohaField ); +use C4::Context; +use C4::Charset qw( SanitizeRecord ); + +my $dbh = C4::Context->dbh; +$dbh->{RaiseError} = 1; +$dbh->{AutoCommit} = 0; + +my $frameworkcode = q||; + +$dbh->do(q| + DELETE FROM marc_subfield_structure WHERE kohafield='biblioitems.url' +|); +$dbh->do(qq| + INSERT INTO marc_subfield_structure(frameworkcode,kohafield,tagfield,tagsubfield) + VALUES ('$frameworkcode', 'biblioitems.url', '856', 'u') +|); +my ( $url_field, $url_subfield ) = C4::Biblio::GetMarcFromKohaField('biblioitems.url', $frameworkcode); + +my $title = q|My title & a word & another word|; +my $url = q|http://www.example.org/index.pl?arg1=val1&arg2=val2|; +my $record = MARC::Record->new(); +$record->append_fields( + MARC::Field->new('100', ' ', ' ', a => 'my author'), + MARC::Field->new('245', ' ', ' ', a => $title), + MARC::Field->new($url_field, ' ', ' ', $url_subfield => $url ), +); + +my ($biblionumber, $biblioitemnumber) = AddBiblio($record, $frameworkcode); +my ( $sanitized_record, $has_been_modified ) = C4::Charset::SanitizeRecord( $record, $biblionumber ); +is( $has_been_modified, 0, 'SanitizeRecord: the record has not been modified' ); +is( $url, $sanitized_record->subfield($url_field, $url_subfield), 'SanitizeRecord: the url has not been modified'); + +$title = q|My title &amp;amp; a word &amp; another word|; +$record = MARC::Record->new(); +$record->append_fields( + MARC::Field->new('100', ' ', ' ', a => 'my author'), + MARC::Field->new('245', ' ', ' ', a => $title), + MARC::Field->new($url_field, ' ', ' ', $url_subfield => $url ), +); + +($biblionumber, $biblioitemnumber) = AddBiblio($record, $frameworkcode); +( $sanitized_record, $has_been_modified ) = C4::Charset::SanitizeRecord( $record, $biblionumber ); +is( $has_been_modified, 1, 'SanitizeRecord: the record has been modified' ); +is( $url, $sanitized_record->subfield($url_field, $url_subfield), 'SanitizeRecord: the url has not been modified'); + +$dbh->rollback; -- 1.7.10.4