|
Lines 1-327
Link Here
|
| 1 |
#!/usr/bin/perl |
|
|
| 2 |
# |
| 3 |
# Copyright 2009 Liblime |
| 4 |
# |
| 5 |
# This file is part of Koha. |
| 6 |
# |
| 7 |
# Koha is free software; you can redistribute it and/or modify it |
| 8 |
# under the terms of the GNU General Public License as published by |
| 9 |
# the Free Software Foundation; either version 3 of the License, or |
| 10 |
# (at your option) any later version. |
| 11 |
# |
| 12 |
# Koha is distributed in the hope that it will be useful, but |
| 13 |
# WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 15 |
# GNU General Public License for more details. |
| 16 |
# |
| 17 |
# You should have received a copy of the GNU General Public License |
| 18 |
# along with Koha; if not, see <http://www.gnu.org/licenses>. |
| 19 |
|
| 20 |
use strict; |
| 21 |
use warnings; |
| 22 |
|
| 23 |
use MARC::Record; |
| 24 |
use MARC::File::XML; |
| 25 |
use MARC::File::USMARC; |
| 26 |
|
| 27 |
use open OUT => ':encoding(UTF-8)'; |
| 28 |
|
| 29 |
use Getopt::Long qw(:config auto_help auto_version); |
| 30 |
use Pod::Usage; |
| 31 |
|
| 32 |
use C4::Biblio; |
| 33 |
use C4::Charset; |
| 34 |
use C4::Context; |
| 35 |
use C4::Debug; |
| 36 |
|
| 37 |
BEGIN { |
| 38 |
# find Koha's Perl modules |
| 39 |
# test carefully before changing this |
| 40 |
use FindBin; |
| 41 |
eval { require "$FindBin::Bin/../kohalib.pl" }; |
| 42 |
} |
| 43 |
|
| 44 |
our $debug; |
| 45 |
|
| 46 |
## OPTIONS |
| 47 |
my $help = 0; |
| 48 |
my $man = 0; |
| 49 |
my $verbose = 0; |
| 50 |
|
| 51 |
my $limit; # undef, not zero. |
| 52 |
my $offset = 0; |
| 53 |
my $dump = 0; |
| 54 |
my $all = 0; |
| 55 |
my $summary = 1; |
| 56 |
my $lint = 0; |
| 57 |
my $fix = 0; |
| 58 |
my $filename = "/tmp/MARC21_parse_test.$$.marc"; |
| 59 |
|
| 60 |
GetOptions( |
| 61 |
'help|?' => \$help, |
| 62 |
'man' => \$man, |
| 63 |
'verbose' => \$verbose, |
| 64 |
'limit=i' => \$limit, |
| 65 |
'offset=i' => \$offset, |
| 66 |
'filename' => \$filename, |
| 67 |
'All!' => \$all, |
| 68 |
'Lint!' => \$lint, |
| 69 |
'dump!' => \$dump, |
| 70 |
'summary!' => \$summary, |
| 71 |
'fix!' => \$fix, |
| 72 |
) or pod2usage(2); |
| 73 |
pod2usage( -verbose => 2 ) if ($man); |
| 74 |
pod2usage( -verbose => 2 ) if ($help and $verbose); |
| 75 |
pod2usage(1) if $help; |
| 76 |
|
| 77 |
if ($debug) { |
| 78 |
$summary++; |
| 79 |
$verbose++; |
| 80 |
$lint++; |
| 81 |
} |
| 82 |
|
| 83 |
my $lint_object; |
| 84 |
if ($lint) { |
| 85 |
require MARC::Lint; |
| 86 |
$lint_object = new MARC::Lint; |
| 87 |
} |
| 88 |
my $marcflavour = C4::Context->preference('marcflavour') or die "No marcflavour (MARC21 or UNIMARC) set in syspref"; |
| 89 |
(uc($marcflavour) eq 'MARC21') or die "Only marcflavour MARC21, not '$marcflavour'"; |
| 90 |
|
| 91 |
# my $countq = C4::Context->dbh->prepare("SELECT COUNT(*) FROM biblioitems"); # Too SLOW on large systems |
| 92 |
# $countq->execute; $countq->fetchrow(); |
| 93 |
my $max = 999999; # arbitrary ceiling |
| 94 |
|
| 95 |
$limit or $limit = $max; # limit becomes max if unspecified |
| 96 |
|
| 97 |
if ($summary) { |
| 98 |
printf "# Examining marcxml from %s\n", ($all ? 'ALL biblioitems' : 'SELECT biblionumbers'); |
| 99 |
printf "# limit %d, offset %d:\n", $limit, $offset; |
| 100 |
printf "# MARC::Lint warnings: %s\n", ($lint ? 'ON' : 'OFF'); |
| 101 |
$verbose and print "# Using temp file: $filename\n" |
| 102 |
} |
| 103 |
|
| 104 |
MARC::File::XML->default_record_format($marcflavour) or die "FAILED MARC::File::XML->default_record_format($marcflavour)"; |
| 105 |
|
| 106 |
my $query = "SELECT * FROM biblioitems "; |
| 107 |
my $recs; |
| 108 |
if ($all) { |
| 109 |
if ($limit or $offset) { |
| 110 |
my $limit_clause = sprintf "LIMIT %d, %d", ($offset || 0), ($limit || $max); |
| 111 |
$query .= $limit_clause; |
| 112 |
} |
| 113 |
$verbose and print "# Query: $query\n"; |
| 114 |
$recs = C4::Context->dbh->prepare($query); |
| 115 |
$recs->execute(); |
| 116 |
} else { |
| 117 |
$query .= "WHERE biblionumber=?"; |
| 118 |
$verbose and print "# Query: $query\n"; |
| 119 |
$recs = C4::Context->dbh->prepare($query); |
| 120 |
# no execute, we execute per biblionumber |
| 121 |
print "# Reading biblionumbers from STDIN\n"; |
| 122 |
} |
| 123 |
|
| 124 |
sub next_row { |
| 125 |
$all and return $recs->fetchrow_hashref(); # no WHERE clause, just get it |
| 126 |
while (my $biblionumber = <>) { |
| 127 |
chomp($biblionumber); |
| 128 |
unless (defined $biblionumber) { |
| 129 |
print "Skipping blank line $.\n"; |
| 130 |
next; |
| 131 |
} |
| 132 |
unless ($biblionumber =~ s/^\s*(\d+)\s*$/$1/ and $biblionumber != 0) { |
| 133 |
print "Skipping illegal biblionumber: $biblionumber (line $.)\n"; |
| 134 |
next; |
| 135 |
} |
| 136 |
($verbose > 1) and printf("(%9d) plausible biblionumber\n", $biblionumber); |
| 137 |
$recs->execute($biblionumber); |
| 138 |
return $recs->fetchrow_hashref(); |
| 139 |
} |
| 140 |
return undef; # just in case |
| 141 |
} |
| 142 |
|
| 143 |
my $ilimit = $limit; |
| 144 |
$ilimit += $offset unless $all; # increase ilimit for offset. if $all, then offset is built into query. |
| 145 |
my $i = 0; |
| 146 |
my $found = 0; |
| 147 |
my $fixed = 0; |
| 148 |
my $fine = 0; |
| 149 |
my $failed = 0; |
| 150 |
my $warns = 0; |
| 151 |
my $printline = 0; |
| 152 |
while ( my $row = next_row() ) { |
| 153 |
++$i; |
| 154 |
unless ($all) { |
| 155 |
($i > $ilimit) and last; # controls for user-input data/files |
| 156 |
($i > $offset) or next; |
| 157 |
} |
| 158 |
my $xml = $row->{marcxml}; |
| 159 |
my $bibnum_prefix = sprintf "(%9d)", $row->{biblionumber}; |
| 160 |
# $xml now pared down to just the <leader> element |
| 161 |
$verbose and printf "# %4d of %4d: biblionumber %s\n", ++$printline, $limit, $row->{biblionumber}; |
| 162 |
my $stripped = StripNonXmlChars($xml); |
| 163 |
($stripped eq $xml) or printf "$bibnum_prefix: %d NON-XML Characters removed!!\n", (length($xml) - length($stripped)); |
| 164 |
my $record = eval { MARC::Record::new_from_xml( $stripped, 'utf8', $marcflavour ) }; |
| 165 |
if (not $record) { |
| 166 |
$found++; |
| 167 |
my $msg = $@ || ''; |
| 168 |
$verbose or $msg =~ s# at /usr/.*$##gs; # shorten common error message |
| 169 |
print "$bibnum_prefix ERROR: $msg\n"; |
| 170 |
} else { |
| 171 |
$fine++; |
| 172 |
} |
| 173 |
if ($lint) { |
| 174 |
open (FILE, ">$filename") or die "Cannot write to temp file: $filename"; |
| 175 |
print FILE $xml; |
| 176 |
close FILE; |
| 177 |
my $file = MARC::File::XML->in( $filename ); |
| 178 |
while ( my $marc = $file->next() ) { # should be only 1 |
| 179 |
# $marc->field("245") or print "pre check_record 245 check 1: FAIL\n"; use Data::Dumper; print Dumper($marc); |
| 180 |
$lint_object->check_record( $marc ); |
| 181 |
if ($lint_object->warnings) { |
| 182 |
$warns++; |
| 183 |
print join("\n", map {"$bibnum_prefix $_"} $lint_object->warnings), "\n"; |
| 184 |
} |
| 185 |
} |
| 186 |
} |
| 187 |
if ($fix and not $record) { |
| 188 |
my $record_from_blob = MARC::Record->new_from_usmarc($row->{marc}); |
| 189 |
unless ($record_from_blob) { |
| 190 |
print "$bibnum_prefix ERROR: Cannot recover from biblioitems.marc\n"; |
| 191 |
$failed++; |
| 192 |
} else { |
| 193 |
my $mod = ModBiblioMarc($record_from_blob, $row->{biblionumber}, ''); |
| 194 |
if ($mod) { |
| 195 |
$fixed++; print "$bibnum_prefix FIXED\n"; |
| 196 |
} else { |
| 197 |
$failed++; print "$bibnum_prefix FAILED from marc. Manual intervention required.\n"; |
| 198 |
} |
| 199 |
} |
| 200 |
} |
| 201 |
$dump and print $row->{marcxml}, "\n"; |
| 202 |
} |
| 203 |
|
| 204 |
(-f $filename) and unlink ($filename); # remove tempfile |
| 205 |
|
| 206 |
if ($summary) { |
| 207 |
printf "# Examining marcxml from %s\n", ($all ? 'ALL biblioitems' : 'SELECT biblionumbers'); |
| 208 |
printf "# limit %d, offset %d:\n", $limit, $offset; |
| 209 |
print "\nRESULTS (number of records)...\n"; |
| 210 |
printf " %6d -- OK \n", $fine; |
| 211 |
printf " %6d -- w/ bad marcxml \n", $found; |
| 212 |
printf " %6d -- w/ MARC::Lint warnings\n", $warns; |
| 213 |
printf " %6d -- fixed from marc \n", $fixed; |
| 214 |
printf " %6d -- failed to fix \n", $failed; |
| 215 |
} |
| 216 |
|
| 217 |
|
| 218 |
__END__ |
| 219 |
|
| 220 |
=head1 NAME |
| 221 |
|
| 222 |
MARC21_parse_test.pl - Try parsing and optionally fixing biblioitems.marcxml, report errors |
| 223 |
|
| 224 |
=head1 SYNOPSIS |
| 225 |
|
| 226 |
MARC21_parse_test.pl [ -h | -m ] [ -v ] [ -d ] [ -s ] [ -l=N ] [ -o=N ] [ -l ] [ -f ] [ -A | filename ...] |
| 227 |
|
| 228 |
Help Options: |
| 229 |
-h --help -? Brief help message |
| 230 |
-m --man Full documentation, same as --help --verbose |
| 231 |
--version Prints version info |
| 232 |
|
| 233 |
Feedback Options: |
| 234 |
-d --dump Dump MARCXML of biblioitems processed, default OFF |
| 235 |
-s --summary Print initial and closing summary of good and bad biblioitems counted, default ON |
| 236 |
-L --Lint Show any warnings from MARC::Lint, default OFF |
| 237 |
-v --verbose Increase verbosity of output, default OFF |
| 238 |
|
| 239 |
Run Options: |
| 240 |
-f --fix Replace biblioitems.marcxml from data in marc field, default OFF |
| 241 |
-A --All Use the whole biblioitems table as target set, default OFF |
| 242 |
-l --limit Number of biblioitems to display or fix |
| 243 |
-o --offset Number of biblioitems to skip (not displayed or fixed) |
| 244 |
|
| 245 |
=head1 OPTIONS |
| 246 |
|
| 247 |
=over 8 |
| 248 |
|
| 249 |
=item B<--All> |
| 250 |
|
| 251 |
Target the entire biblioitems table. |
| 252 |
Beware, on a large table B<--All> can be very costly to performance. |
| 253 |
|
| 254 |
=item B<--fix> |
| 255 |
|
| 256 |
Without this option, no changes to any records are made. With <--fix>, the script attempts to reconstruct |
| 257 |
biblioitems.marcxml from biblioitems.marc. |
| 258 |
|
| 259 |
=item B<--limit=N> |
| 260 |
|
| 261 |
Like a LIMIT statement in SQL, this constrains the number of records targeted by the script to an integer N. |
| 262 |
This applies whether the target records are determined by user input, filenames or <--All>. |
| 263 |
|
| 264 |
=item B<--offset=N> |
| 265 |
|
| 266 |
Like an OFFSET statement in SQL, this tells the script to skip N of the targetted records. |
| 267 |
The default is 0, i.e. skip none of them. |
| 268 |
|
| 269 |
=back |
| 270 |
|
| 271 |
The binary ON/OFF options can be negated like: |
| 272 |
B<--nosummary> Do not display summary. |
| 273 |
B<--nodump> Do not dump MARCXML. |
| 274 |
B<--noLint> Do not show MARC::Lint warnings. |
| 275 |
B<--nofix> Do not change any records. This is the default mode. |
| 276 |
|
| 277 |
=head1 ARGUMENTS |
| 278 |
|
| 279 |
Any number of filepath arguments can be referenced. They will be read in order and used to select the target |
| 280 |
set of biblioitems. The file format should be simply one biblionumber per line. The B<--limit> and B<--offset> |
| 281 |
options can still be used with biblionumbers specified from file. Files will be ignored under the B<--All> option. |
| 282 |
|
| 283 |
=head1 DESCRIPTION |
| 284 |
|
| 285 |
This checks for data corruption or otherwise unparsable data in biblioitems.marcxml. |
| 286 |
As the name suggests, this script is only useful for MARC21 and will die for marcflavour UNIMARC. |
| 287 |
|
| 288 |
Run MARC21_parse_test.pl the first time with no options and type in individual biblionumbers to test. |
| 289 |
Or run with B<--All> to go through the entire table. |
| 290 |
Run the script again with B<--fix> to attempt repair of the same target set. |
| 291 |
|
| 292 |
After fixing any records, you will need to rebuild your index, e.g. B<rebuild_zebra -b -r -x>. |
| 293 |
|
| 294 |
=head1 USAGE EXAMPLES |
| 295 |
|
| 296 |
B<MARC21_parse_test.pl> |
| 297 |
|
| 298 |
In the most basic form, allows you to input biblionumbers and checks them individually. |
| 299 |
|
| 300 |
B<MARC21_parse_test.pl --fix> |
| 301 |
|
| 302 |
Same thing but fixes them if they fail to parse. |
| 303 |
|
| 304 |
B<MARC21_parse_test.pl --fix --limit=15 bibnumbers1.txt> |
| 305 |
|
| 306 |
Fixes biblioitems from the first 15 biblionumbers in file bibnumbers1.txt. Multiple file arguments can be used. |
| 307 |
|
| 308 |
B<MARC21_parse_test.pl --All --limit=3 --offset=15 --nosummary --dump> |
| 309 |
|
| 310 |
Dumps MARCXML from the 16th, 17th and 18th records found in the database. |
| 311 |
|
| 312 |
B<MARC21_parse_test.pl -A -l=3 -o=15 -s=0 -d> |
| 313 |
|
| 314 |
Same thing as previous example in terse form. |
| 315 |
|
| 316 |
=head1 TODO |
| 317 |
|
| 318 |
Add more documentation for OPTIONS. |
| 319 |
|
| 320 |
Update zebra status so rebuild of index is not necessary. |
| 321 |
|
| 322 |
=head1 SEE ALSO |
| 323 |
|
| 324 |
MARC::Lint |
| 325 |
C4::Biblio |
| 326 |
|
| 327 |
=cut |
| 328 |
- |