Bugzilla – Attachment 29795 Details for
Bug 8218
Add a script to sanitize entity where "&" character was wrongly replaced with "&"
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 8218 : Add a maintenance script to sanitize biblio records
Bug-8218--Add-a-maintenance-script-to-sanitize-bib.patch (text/plain), 12.62 KB, created by
Marcel de Rooy
on 2014-07-17 13:05:15 UTC
(
hide
)
Description:
Bug 8218 : Add a maintenance script to sanitize biblio records
Filename:
MIME Type:
Creator:
Marcel de Rooy
Created:
2014-07-17 13:05:15 UTC
Size:
12.62 KB
patch
obsolete
>From 2602be641172f18d030b07c415548fc0252ead05 Mon Sep 17 00:00:00 2001 >From: Jonathan Druart <jonathan.druart@biblibre.com> >Date: Fri, 28 Sep 2012 15:23:37 +0200 >Subject: [PATCH] Bug 8218 : Add a maintenance script to sanitize biblio > records >Content-Type: text/plain; charset=utf-8 > >This patch adds: >- a new maintenance script batch_sanitize_records >- a new subroutine C4::Charset::SanitizeRecord >- new unit tests for the new subroutine > >Test plan: >1/ prove t/db_dependent/Charset.t >2/ Create a record containing "&amp;" (could be follow with as many >'amp;' as you want) in one of its fields and the same for the field >linked to biblioitems.url. >The url should not be sanitized, it may contain "&". >3/ Launch the maintenance script with the -h parameter to see how to use >it. >4/ Launch the script using the different parameters: > --filename=FILENAME > --biblionumbers='XXX' > --auto-search > >The auto-search permits to sanitize all records containing "&amp;" in >the marcxml field. > >Use the verbose flag for testing. >Without the --confirm flag, nothing is done. > >5/ Use the --confirm flag and verify in the biblioitems.marcxml field >that the record has been sanitized. > >6/ Try the --reindex flag to reindex records which have been modified. > >Signed-off-by: Marcel de Rooy <m.de.rooy@rijksmuseum.nl> >--- > C4/Charset.pm | 64 ++++++++ > misc/maintenance/batch_sanitize_records.pl | 222 ++++++++++++++++++++++++++++ > t/db_dependent/Charset.t | 51 +++++++ > 3 files changed, 337 insertions(+), 0 deletions(-) > create mode 100755 misc/maintenance/batch_sanitize_records.pl > create mode 100644 t/db_dependent/Charset.t > >diff --git a/C4/Charset.pm b/C4/Charset.pm >index 2950a71..bef5c18 100644 >--- a/C4/Charset.pm >+++ b/C4/Charset.pm >@@ -41,6 +41,7 @@ BEGIN { > SetMarcUnicodeFlag > StripNonXmlChars > nsb_clean >+ SanitizeRecord > ); > } > >@@ -426,6 +427,69 @@ sub nsb_clean { > } > > >+=head2 SanitizeRecord >+ >+SanitizeRecord($marcrecord); >+ >+Sanitize a record >+This routine is called in the maintenance script misc/maintenance/batch_sanitize_records.pl. >+It cleans any string with '&amp;...', replacing it by '&' >+ >+=cut >+ >+sub SanitizeRecord { >+ my ( $record, $biblionumber ) = @_; >+ my $string; >+ my $record_modified = 0; >+ my $frameworkcode = C4::Biblio::GetFrameworkCode($biblionumber); >+ my ( $url_field, $url_subfield ) = >+ C4::Biblio::GetMarcFromKohaField( 'biblioitems.url', $frameworkcode ); >+ foreach my $field ( $record->fields() ) { >+ if ( $field->is_control_field() ) { >+ my $value = $field->data(); >+ my $sanitized_value = _entity_clean($value); >+ $record_modified = 1 if $sanitized_value ne $value; >+ $field->update($sanitized_value); >+ } >+ else { >+ my @subfields = $field->subfields(); >+ my @new_subfields; >+ foreach my $subfield (@subfields) { >+ next >+ if $url_field eq $field->tag() >+ and $url_subfield eq $subfield->[0]; >+ my $value = $subfield->[1]; >+ my $sanitized_value = _entity_clean($value); >+ push @new_subfields, $subfield->[0] => $sanitized_value; >+ $record_modified = 1 if $sanitized_value ne $value; >+ } >+ if ( scalar(@new_subfields) > 0 ) { >+ my $new_field = eval { >+ MARC::Field->new( >+ $field->tag(), $field->indicator(1), >+ $field->indicator(2), @new_subfields >+ ); >+ }; >+ if ($@) { >+ warn "error : $@"; >+ } >+ else { >+ $field->replace_with($new_field); >+ } >+ >+ } >+ } >+ } >+ >+ return $record, $record_modified; >+} >+ >+sub _entity_clean { >+ my ($string) = @_; >+ $string =~ s/(&)(amp;)+/$1/g; >+ return $string; >+} >+ > =head1 INTERNAL FUNCTIONS > > =head2 _default_marc21_charconv_to_utf8 >diff --git a/misc/maintenance/batch_sanitize_records.pl b/misc/maintenance/batch_sanitize_records.pl >new file mode 100755 >index 0000000..7e21df0 >--- /dev/null >+++ b/misc/maintenance/batch_sanitize_records.pl >@@ -0,0 +1,222 @@ >+#!/usr/bin/perl >+ >+# This file is part of Koha. >+# >+# Copyright 2014 BibLibre >+# >+# Koha is free software; you can redistribute it and/or modify it >+# under the terms of the GNU General Public License as published by >+# the Free Software Foundation; either version 3 of the License, or >+# (at your option) any later version. >+# >+# Koha is distributed in the hope that it will be useful, but >+# WITHOUT ANY WARRANTY; without even the implied warranty of >+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >+# GNU General Public License for more details. >+# >+# You should have received a copy of the GNU General Public License >+# along with Koha; if not, see <http://www.gnu.org/licenses>. >+ >+use Modern::Perl; >+use C4::Charset qw( SanitizeRecord ); >+use C4::Context; >+use DBI; >+use C4::Biblio; >+use Getopt::Long; >+use Pod::Usage; >+ >+my ( $help, $verbose, $confirm, $biblionumbers, $reindex, $filename, >+ $auto_search ); >+my $result = GetOptions( >+ 'h|help' => \$help, >+ 'v|verbose' => \$verbose, >+ 'c|confirm' => \$confirm, >+ 'biblionumbers:s' => \$biblionumbers, >+ 'reindex' => \$reindex, >+ 'f|filename:s' => \$filename, >+ 'auto-search' => \$auto_search, >+) || pod2usage(1); >+ >+if ($help) { >+ pod2usage(0); >+} >+ >+unless ( $filename or $biblionumbers or $auto_search ) { >+ pod2usage( >+ -exitval => 1, >+ -message => >+ qq{\n\tAt least one record number source should be provided.\n} >+ ); >+} >+ >+if ( $filename and $biblionumbers >+ or $filename and $auto_search >+ or $biblionumbers and $auto_search ) >+{ >+ pod2usage( >+ -exitval => 1, >+ -message => qq{\n\tOnly one record number source should be provided.\n} >+ ); >+} >+ >+my @biblionumbers; >+ >+# We first detect if we have a file or biblos directly entered by command line >+#or if we want to use findAmp() sub >+if ($auto_search) { >+ @biblionumbers = biblios_to_sanitize(); >+} >+elsif ($filename) { >+ if ( -e $filename ) { >+ open( my $fh, '<', $filename ) || die("Can't open $filename ($!)"); >+ while (<$fh>) { >+ chomp; >+ my $line = $_; >+ push @biblionumbers, split( " |,", $line ); >+ } >+ close $fh; >+ } >+ else { >+ pod2usage( >+ -exitval => 1, >+ -message => >+qq{\n\tThis filename does not exist. Please verify the path is correct.\n} >+ ); >+ } >+} >+else { >+ @biblionumbers = split m|,|, $biblionumbers if $biblionumbers; >+} >+ >+# We remove spaces >+s/(^\s*|\s*$)//g for @biblionumbers; >+ >+# Remove empty lines >+@biblionumbers = grep { !/^$/ } @biblionumbers; >+ >+say @biblionumbers . " records to process" if $verbose; >+ >+my @changes; >+for my $biblionumber (@biblionumbers) { >+ print "processing record $biblionumber..." if $verbose; >+ unless ( $biblionumber =~ m|^\d+$| ) { >+ say " skipping. ERROR: Invalid biblionumber." if $verbose; >+ next; >+ } >+ my $record = C4::Biblio::GetMarcBiblio($biblionumber); >+ unless ($record) { >+ say " skipping. ERROR: Invalid record." if $verbose; >+ next; >+ } >+ >+ my ( $cleaned_record, $has_been_modified ) = >+ C4::Charset::SanitizeRecord( $record, $biblionumber ); >+ if ($has_been_modified) { >+ my $frameworkcode = C4::Biblio::GetFrameworkCode($record); >+ >+ C4::Biblio::ModBiblio( $cleaned_record, $biblionumber, $frameworkcode ) >+ if $confirm; >+ push @changes, $biblionumber; >+ say " Done!" if $verbose; >+ } >+ else { >+ say " Nothing todo." if $verbose; >+ } >+} >+ >+if ($verbose) { >+ say "Total: " >+ . @changes >+ . " records " >+ . ( $confirm ? "cleaned!" : "to clean." ); >+} >+ >+if ( $reindex and $confirm and @changes ) { >+ say "Now, reindexing using -b -v" if $verbose; >+ my $kohapath = C4::Context->config('intranetdir'); >+ my $cmd = qq| >+ $kohapath/misc/migration_tools/rebuild_zebra.pl -b -v -where "biblionumber IN ( | >+ . join( ',', @changes ) . q| )" >+ |; >+ system($cmd); >+} >+ >+sub biblios_to_sanitize { >+ my $dbh = C4::Context->dbh; >+ my $query = q{ >+ SELECT biblionumber >+ FROM biblioitems >+ WHERE marcxml >+ LIKE "%&amp;%" >+ }; >+ return @{ $dbh->selectcol_arrayref( $query, { Slice => {} }, ) }; >+} >+ >+=head1 NAME >+ >+batch_sanitize_biblios - This script sanitize a biblio, replacing '&amp;amp;etc.' with '&' in it. >+ >+=head1 SYNOPSIS >+ >+batch_sanitize_biblios.pl [-h|--help] [-v|--verbose] [-c|--confirm] [--biblionumbers=BIBLIONUMBER_LIST] [-f|--filename=FILENAME] [--auto-search] [--reindex] >+ >+Replace '&' by '&' in a record, you can either give some biblionumbers or a file with biblionumbers or ask for an auto-search >+ >+=head1 OPTIONS >+ >+=over >+ >+=item B<-h|--help> >+ >+Print a brief help message >+ >+=item B<-v|--verbose> >+ >+Verbose mode. >+ >+=item B<-c|--confirm> >+ >+This flag must be provided in order for the script to actually >+sanitize records. If it is not supplied, the script will >+only report on the record list to process. >+ >+=item B<--biblionumbers=BIBLIONUMBER_LIST> >+ >+Give a biblionumber list using this parameter. They must be separated by comma. >+ >+=item B<-f|--filename=FILENAME> >+ >+Give a biblionumber list using a filename. One biblionumber by line or separate them with a withespace character. >+ >+=item B<--auto_search> >+ >+Automatically search records containing "&" in biblioitems.marcxml or in the specified fields. >+ >+=item B<--reindex> >+ >+Reindex the modified records. >+ >+=back >+ >+=head1 AUTHOR >+ >+Alex Arnaud <alex.arnaud@biblibre.com> >+Christophe Croullebois <christophe.croullebois@biblibre.com> >+Jonathan Druart <jonathan.druart@biblibre.com> >+ >+=head1 COPYRIGHT >+ >+Copyright 2014 BibLibre >+ >+=head1 LICENSE >+ >+This file is part of Koha. >+ >+Koha is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software >+Foundation; either version 3 of the License, or (at your option) any later version. >+ >+You should have received a copy of the GNU General Public License along >+with Koha; if not, write to the Free Software Foundation, Inc., >+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. >+ >+=cut >diff --git a/t/db_dependent/Charset.t b/t/db_dependent/Charset.t >new file mode 100644 >index 0000000..9a48aef >--- /dev/null >+++ b/t/db_dependent/Charset.t >@@ -0,0 +1,51 @@ >+use Modern::Perl; >+use Test::More tests => 4; >+use MARC::Record; >+ >+use C4::Biblio qw( AddBiblio SetFieldMapping GetMarcFromKohaField ); >+use C4::Context; >+use C4::Charset qw( SanitizeRecord ); >+ >+my $dbh = C4::Context->dbh; >+$dbh->{RaiseError} = 1; >+$dbh->{AutoCommit} = 0; >+ >+my $frameworkcode = q||; >+ >+$dbh->do(q| >+ DELETE FROM marc_subfield_structure WHERE kohafield='biblioitems.url' >+|); >+$dbh->do(qq| >+ INSERT INTO marc_subfield_structure(frameworkcode,kohafield,tagfield,tagsubfield) >+ VALUES ('$frameworkcode', 'biblioitems.url', '856', 'u') >+|); >+my ( $url_field, $url_subfield ) = C4::Biblio::GetMarcFromKohaField('biblioitems.url', $frameworkcode); >+ >+my $title = q|My title & a word & another word|; >+my $url = q|http://www.example.org/index.pl?arg1=val1&arg2=val2|; >+my $record = MARC::Record->new(); >+$record->append_fields( >+ MARC::Field->new('100', ' ', ' ', a => 'my author'), >+ MARC::Field->new('245', ' ', ' ', a => $title), >+ MARC::Field->new($url_field, ' ', ' ', $url_subfield => $url ), >+); >+ >+my ($biblionumber, $biblioitemnumber) = AddBiblio($record, $frameworkcode); >+my ( $sanitized_record, $has_been_modified ) = C4::Charset::SanitizeRecord( $record, $biblionumber ); >+is( $has_been_modified, 0, 'SanitizeRecord: the record has not been modified' ); >+is( $url, $sanitized_record->subfield($url_field, $url_subfield), 'SanitizeRecord: the url has not been modified'); >+ >+$title = q|My title &amp;amp; a word &amp; another word|; >+$record = MARC::Record->new(); >+$record->append_fields( >+ MARC::Field->new('100', ' ', ' ', a => 'my author'), >+ MARC::Field->new('245', ' ', ' ', a => $title), >+ MARC::Field->new($url_field, ' ', ' ', $url_subfield => $url ), >+); >+ >+($biblionumber, $biblioitemnumber) = AddBiblio($record, $frameworkcode); >+( $sanitized_record, $has_been_modified ) = C4::Charset::SanitizeRecord( $record, $biblionumber ); >+is( $has_been_modified, 1, 'SanitizeRecord: the record has been modified' ); >+is( $url, $sanitized_record->subfield($url_field, $url_subfield), 'SanitizeRecord: the url has not been modified'); >+ >+$dbh->rollback; >-- >1.7.7.6
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 8218
:
10039
|
10588
|
10589
|
11744
|
12591
|
13678
|
15826
|
16122
|
16720
|
26754
|
26755
|
29792
|
29793
|
29794
|
29795
|
29796
|
29798
|
30120
|
30121
|
33248