View | Details | Raw Unified | Return to bug 10455
Collapse All | Expand All

(-)a/misc/cronjobs/MARC21_parse_test.pl (-328 lines)
Lines 1-327 Link Here
1
#!/usr/bin/perl
2
#
3
# Copyright 2009 Liblime
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
use strict;
21
use warnings;
22
23
use MARC::Record;
24
use MARC::File::XML;
25
use MARC::File::USMARC;
26
27
use open OUT => ':encoding(UTF-8)';
28
29
use Getopt::Long qw(:config auto_help auto_version);
30
use Pod::Usage;
31
32
use C4::Biblio;
33
use C4::Charset;
34
use C4::Context;
35
use C4::Debug;
36
37
BEGIN {
38
    # find Koha's Perl modules
39
    # test carefully before changing this
40
    use FindBin;
41
    eval { require "$FindBin::Bin/../kohalib.pl" };
42
}
43
44
our $debug;
45
46
## OPTIONS
47
my $help    = 0;
48
my $man     = 0;
49
my $verbose = 0;
50
51
my $limit;      # undef, not zero.
52
my $offset  = 0;
53
my $dump    = 0;
54
my $all     = 0;
55
my $summary = 1;
56
my $lint    = 0;
57
my $fix     = 0;
58
my $filename = "/tmp/MARC21_parse_test.$$.marc";
59
60
GetOptions(
61
       'help|?' => \$help,
62
          'man' => \$man,
63
      'verbose' => \$verbose,
64
      'limit=i' => \$limit,
65
     'offset=i' => \$offset,
66
     'filename' => \$filename,
67
         'All!' => \$all,
68
        'Lint!' => \$lint,
69
        'dump!' => \$dump,
70
     'summary!' => \$summary,
71
         'fix!' => \$fix,
72
) or pod2usage(2);
73
pod2usage( -verbose => 2 ) if ($man);
74
pod2usage( -verbose => 2 ) if ($help and $verbose);
75
pod2usage(1) if $help;
76
77
if ($debug) {
78
    $summary++;
79
    $verbose++;
80
    $lint++;
81
}
82
83
my $lint_object;
84
if ($lint) {
85
    require MARC::Lint;
86
    $lint_object = new MARC::Lint;
87
}
88
my $marcflavour = C4::Context->preference('marcflavour') or die "No marcflavour (MARC21 or UNIMARC) set in syspref";
89
(uc($marcflavour) eq 'MARC21') or die "Only marcflavour MARC21, not '$marcflavour'";
90
91
# my $countq = C4::Context->dbh->prepare("SELECT COUNT(*) FROM biblioitems");    # Too SLOW on large systems
92
# $countq->execute; $countq->fetchrow();
93
my $max = 999999;   # arbitrary ceiling
94
95
$limit or $limit = $max;       # limit becomes max if unspecified
96
97
if ($summary) {
98
    printf "# Examining marcxml from %s\n", ($all ? 'ALL biblioitems' : 'SELECT biblionumbers');
99
    printf "# limit %d, offset %d:\n", $limit, $offset;
100
    printf "# MARC::Lint warnings: %s\n", ($lint ? 'ON' : 'OFF');
101
    $verbose and print "# Using temp file: $filename\n"
102
}
103
104
MARC::File::XML->default_record_format($marcflavour) or die "FAILED MARC::File::XML->default_record_format($marcflavour)";
105
106
my $query = "SELECT  *  FROM biblioitems ";
107
my $recs;
108
if ($all) {
109
    if ($limit or $offset) {
110
        my $limit_clause = sprintf "LIMIT %d, %d", ($offset || 0), ($limit || $max);
111
        $query .= $limit_clause;
112
    }
113
    $verbose and print "# Query: $query\n";
114
    $recs = C4::Context->dbh->prepare($query);
115
    $recs->execute();
116
} else {
117
    $query .= "WHERE biblionumber=?";
118
    $verbose and print "# Query: $query\n";
119
    $recs = C4::Context->dbh->prepare($query);
120
    # no execute, we execute per biblionumber
121
    print "# Reading biblionumbers from STDIN\n";
122
}
123
124
sub next_row {
125
    $all and return $recs->fetchrow_hashref();  # no WHERE clause, just get it
126
    while (my $biblionumber = <>) {
127
        chomp($biblionumber);
128
        unless (defined $biblionumber) {
129
            print "Skipping blank line $.\n";
130
            next;
131
        } 
132
        unless ($biblionumber =~ s/^\s*(\d+)\s*$/$1/ and $biblionumber != 0) {
133
            print "Skipping illegal biblionumber: $biblionumber  (line $.)\n";
134
            next;
135
        }
136
        ($verbose > 1) and printf("(%9d) plausible biblionumber\n", $biblionumber);
137
        $recs->execute($biblionumber);
138
        return $recs->fetchrow_hashref();
139
    }
140
    return undef;   # just in case
141
}
142
143
my $ilimit = $limit;
144
$ilimit += $offset unless $all;    # increase ilimit for offset.  if $all, then offset is built into query.
145
my $i = 0;
146
my $found  = 0;
147
my $fixed  = 0;
148
my $fine   = 0;
149
my $failed = 0;
150
my $warns  = 0;
151
my $printline = 0;
152
while ( my $row = next_row() ) {
153
    ++$i;
154
    unless ($all) {
155
        ($i > $ilimit) and last;  # controls for user-input data/files
156
        ($i > $offset) or next;
157
    }
158
    my $xml = $row->{marcxml};
159
    my $bibnum_prefix = sprintf "(%9d)", $row->{biblionumber};
160
    # $xml now pared down to just the <leader> element
161
    $verbose and printf "# %4d of %4d: biblionumber %s\n", ++$printline, $limit, $row->{biblionumber};
162
    my $stripped = StripNonXmlChars($xml);
163
    ($stripped eq $xml) or printf "$bibnum_prefix: %d NON-XML Characters removed!!\n", (length($xml) - length($stripped));
164
    my $record = eval { MARC::Record::new_from_xml( $stripped, 'utf8', $marcflavour ) };
165
    if (not $record) {
166
        $found++;
167
    	my $msg = $@ || '';
168
        $verbose or $msg =~ s# at /usr/.*$##gs;    # shorten common error message
169
        print "$bibnum_prefix ERROR: $msg\n";
170
    } else {
171
        $fine++;
172
    }
173
    if ($lint) {
174
        open (FILE, ">$filename") or die "Cannot write to temp file: $filename";
175
        print FILE $xml;
176
        close FILE;
177
        my $file = MARC::File::XML->in( $filename );
178
        while ( my $marc = $file->next() ) {    # should be only 1
179
            # $marc->field("245") or print "pre check_record 245 check 1: FAIL\n"; use Data::Dumper;  print Dumper($marc);
180
            $lint_object->check_record( $marc );
181
            if ($lint_object->warnings) {
182
                $warns++;
183
                print join("\n", map {"$bibnum_prefix $_"} $lint_object->warnings), "\n";
184
            }
185
        }
186
    }
187
    if ($fix and not $record) {
188
        my $record_from_blob = MARC::Record->new_from_usmarc($row->{marc});
189
        unless ($record_from_blob) {
190
            print "$bibnum_prefix ERROR: Cannot recover from biblioitems.marc\n";
191
            $failed++;
192
        } else {
193
            my $mod = ModBiblioMarc($record_from_blob, $row->{biblionumber}, '');
194
            if ($mod) {
195
                $fixed++;  print "$bibnum_prefix FIXED\n";
196
            } else {
197
                $failed++; print "$bibnum_prefix FAILED from marc.  Manual intervention required.\n";
198
            }
199
        }
200
    }
201
    $dump and print $row->{marcxml}, "\n";
202
}
203
204
(-f $filename) and unlink ($filename);  # remove tempfile
205
206
if ($summary) {
207
    printf "# Examining marcxml from %s\n", ($all ? 'ALL biblioitems' : 'SELECT biblionumbers');
208
    printf "# limit %d, offset %d:\n", $limit, $offset;
209
    print "\nRESULTS (number of records)...\n";
210
    printf "  %6d -- OK              \n",  $fine;
211
    printf "  %6d -- w/ bad marcxml  \n",  $found;
212
    printf "  %6d -- w/ MARC::Lint warnings\n", $warns;
213
    printf "  %6d -- fixed from marc \n",  $fixed;
214
    printf "  %6d -- failed to fix   \n",  $failed;
215
}
216
217
218
__END__
219
220
=head1 NAME
221
222
MARC21_parse_test.pl - Try parsing and optionally fixing biblioitems.marcxml, report errors
223
224
=head1 SYNOPSIS
225
226
MARC21_parse_test.pl [ -h | -m ] [ -v ] [ -d ] [ -s ] [ -l=N ] [ -o=N ] [ -l ] [ -f ] [ -A | filename ...]
227
228
 Help Options:
229
   -h --help -?   Brief help message
230
   -m --man       Full documentation, same as --help --verbose
231
      --version   Prints version info
232
233
 Feedback Options:
234
   -d --dump      Dump MARCXML of biblioitems processed, default OFF
235
   -s --summary   Print initial and closing summary of good and bad biblioitems counted, default ON
236
   -L --Lint      Show any warnings from MARC::Lint, default OFF
237
   -v --verbose   Increase verbosity of output, default OFF
238
239
 Run Options:
240
   -f --fix       Replace biblioitems.marcxml from data in marc field, default OFF
241
   -A --All       Use the whole biblioitems table as target set, default OFF
242
   -l --limit     Number of biblioitems to display or fix
243
   -o --offset    Number of biblioitems to skip (not displayed or fixed)
244
245
=head1 OPTIONS
246
247
=over 8
248
249
=item B<--All>
250
251
Target the entire biblioitems table.
252
Beware, on a large table B<--All> can be very costly to performance.
253
254
=item B<--fix>
255
256
Without this option, no changes to any records are made.  With <--fix>, the script attempts to reconstruct
257
biblioitems.marcxml from biblioitems.marc.  
258
259
=item B<--limit=N>
260
261
Like a LIMIT statement in SQL, this constrains the number of records targeted by the script to an integer N.  
262
This applies whether the target records are determined by user input, filenames or <--All>.
263
264
=item B<--offset=N>
265
266
Like an OFFSET statement in SQL, this tells the script to skip N of the targetted records.
267
The default is 0, i.e. skip none of them.
268
269
=back
270
271
The binary ON/OFF options can be negated like:
272
   B<--nosummary>   Do not display summary.
273
   B<--nodump>      Do not dump MARCXML.
274
   B<--noLint>      Do not show MARC::Lint warnings.
275
   B<--nofix>       Do not change any records.  This is the default mode.
276
277
=head1 ARGUMENTS
278
279
Any number of filepath arguments can be referenced.  They will be read in order and used to select the target
280
set of biblioitems.  The file format should be simply one biblionumber per line.  The B<--limit> and B<--offset>
281
options can still be used with biblionumbers specified from file.  Files will be ignored under the B<--All> option.
282
283
=head1 DESCRIPTION
284
285
This checks for data corruption or otherwise unparsable data in biblioitems.marcxml.  
286
As the name suggests, this script is only useful for MARC21 and will die for marcflavour UNIMARC.
287
288
Run MARC21_parse_test.pl the first time with no options and type in individual biblionumbers to test.
289
Or run with B<--All> to go through the entire table.
290
Run the script again with B<--fix> to attempt repair of the same target set.
291
292
After fixing any records, you will need to rebuild your index, e.g. B<rebuild_zebra -b -r -x>.
293
294
=head1 USAGE EXAMPLES
295
296
B<MARC21_parse_test.pl>
297
298
In the most basic form, allows you to input biblionumbers and checks them individually.
299
300
B<MARC21_parse_test.pl --fix>
301
302
Same thing but fixes them if they fail to parse.
303
304
B<MARC21_parse_test.pl --fix --limit=15 bibnumbers1.txt>
305
306
Fixes biblioitems from the first 15 biblionumbers in file bibnumbers1.txt.  Multiple file arguments can be used.
307
308
B<MARC21_parse_test.pl --All --limit=3 --offset=15 --nosummary --dump>
309
310
Dumps MARCXML from the 16th, 17th and 18th records found in the database.
311
312
B<MARC21_parse_test.pl -A -l=3 -o=15 -s=0 -d>
313
314
Same thing as previous example in terse form.
315
316
=head1 TODO
317
318
Add more documentation for OPTIONS.
319
320
Update zebra status so rebuild of index is not necessary.
321
322
=head1 SEE ALSO
323
324
MARC::Lint
325
C4::Biblio
326
327
=cut
328
- 

Return to bug 10455