View | Details | Raw Unified | Return to bug 5166
Collapse All | Expand All

(-)a/C4/Catalog/Rebuild.pm (-1 / +426 lines)
Line 0 Link Here
0
- 
1
package C4::Catalog::Rebuild;
2
3
use strict;
4
use C4::Context;
5
use Getopt::Long;
6
use File::Temp qw/ tempdir /;
7
use File::Path;
8
use C4::Biblio;
9
use C4::AuthoritiesMarc;
10
11
# library for checking for updates in zerbaqueue
12
# can also check zebradir structure & create directories & 
13
# mandatory files if needed
14
15
    my $nosanitize = '';
16
    my $skip_export = '';
17
    my $keep_export = '';
18
    my $noxml = '';
19
    my $as_xml = '';
20
    my $process_zebraqueue = 1;
21
    my $do_not_clear_zebraqueue = '';
22
    my $verbose_logging = '';
23
    my $reset = '';
24
    my $noshadow = '';
25
    my $biblioserverdir     = C4::Context->zebraconfig('biblioserver')->{directory};
26
    my $authorityserverdir  = C4::Context->zebraconfig('authorityserver')->{directory};
27
    my $kohadir             = C4::Context->config('intranetdir');
28
    my $dbh                 = C4::Context->dbh;
29
    my ($biblionumbertagfield,$biblionumbertagsubfield) = &GetMarcFromKohaField("biblio.biblionumber","");
30
    my ($biblioitemnumbertagfield,$biblioitemnumbertagsubfield) = &GetMarcFromKohaField("biblioitems.biblioitemnumber","");
31
32
33
sub update_auth {
34
35
    # Create tmp dir
36
    my $directory = File::Temp->newdir();
37
    my $zebraidx_log_opt = " -v none,fatal,warn ";
38
39
40
    # Update authorities
41
    index_records('authority', $directory, $skip_export, "1", $as_xml,\
42
                  $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt);
43
44
    return 0;
45
46
}
47
48
sub update_auth_and_biblio {
49
50
    # Create tmp dir
51
    my $directory = File::Temp->newdir();
52
    my $zebraidx_log_opt = " -v none,fatal,warn ";
53
54
    # Update authorities
55
    index_records('authority', $directory, 0, 1, 1, 0, 0, 0, $verbose_logging, $zebraidx_log_opt);
56
57
    # Update biblios
58
    index_records('biblio', $directory, 0, 1, 1, 0, 0, 0, $verbose_logging, $zebraidx_log_opt);
59
60
    return 0;
61
}
62
63
sub index_records {
64
    my ($record_type, $directory, $skip_export, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt) = @_;
65
66
67
    my $num_records_exported = 0;
68
    my $num_records_deleted = 0;
69
70
    if ($skip_export && $verbose_logging) {
71
        print "====================\n";
72
        print "SKIPPING $record_type export\n";
73
        print "====================\n";
74
    } else {
75
        if ( $verbose_logging ) {
76
            print "====================\n";
77
            print "exporting $record_type\n";
78
            print "====================\n";
79
        }
80
        mkdir "$directory" unless (-d $directory);
81
        mkdir "$directory/$record_type" unless (-d "$directory/$record_type");
82
        if ($process_zebraqueue) {
83
            my $entries = select_zebraqueue_records($record_type, 'deleted');
84
            mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type");
85
            $num_records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_xml);
86
            mark_zebraqueue_batch_done($entries);
87
            $entries = select_zebraqueue_records($record_type, 'updated');
88
            mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type");
89
            $num_records_exported = export_marc_records_from_list($record_type, 
90
                                                                  $entries, "$directory/upd_$record_type", $as_xml, $noxml);
91
            mark_zebraqueue_batch_done($entries);
92
        } else {
93
            my $sth = select_all_records($record_type);
94
            $num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_xml, $noxml, $nosanitize);
95
            unless ($do_not_clear_zebraqueue) {
96
                mark_all_zebraqueue_done($record_type);
97
            }
98
        }
99
    }
100
    
101
    #
102
    # and reindexing everything
103
    #
104
    if ( $verbose_logging ) {
105
        print "====================\n";
106
        print "REINDEXING zebra\n";
107
        print "====================\n";
108
    }
109
	my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
110
    if ($process_zebraqueue) {
111
        do_indexing($record_type, 'delete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) 
112
            if $num_records_deleted;
113
        do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
114
            if $num_records_exported;
115
    } else {
116
        do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
117
            if ($num_records_exported or $skip_export);
118
    }
119
}
120
121
sub select_zebraqueue_records {
122
    my ($record_type, $update_type) = @_;
123
124
    my $server = ($record_type eq 'biblio') ? 'biblioserver' : 'authorityserver';
125
    my $op = ($update_type eq 'deleted') ? 'recordDelete' : 'specialUpdate';
126
127
    my $sth = $dbh->prepare("SELECT id, biblio_auth_number 
128
                             FROM zebraqueue
129
                             WHERE server = ?
130
                             AND   operation = ?
131
                             AND   done = 0
132
                             ORDER BY id DESC");
133
    $sth->execute($server, $op);
134
    my $entries = $sth->fetchall_arrayref({});
135
}
136
137
sub mark_all_zebraqueue_done {
138
    my ($record_type) = @_;
139
140
    my $server = ($record_type eq 'biblio') ? 'biblioserver' : 'authorityserver';
141
142
    my $sth = $dbh->prepare("UPDATE zebraqueue SET done = 1
143
                             WHERE server = ?
144
                             AND done = 0");
145
    $sth->execute($server);
146
    print "MARK\n";
147
}
148
149
sub mark_zebraqueue_batch_done {
150
    my ($entries) = @_;
151
152
    $dbh->{AutoCommit} = 0;
153
    my $sth = $dbh->prepare("UPDATE zebraqueue SET done = 1 WHERE id = ?");
154
    $dbh->commit();
155
    foreach my $id (map { $_->{id} } @$entries) {
156
        $sth->execute($id);
157
    }
158
    $dbh->{AutoCommit} = 1;
159
}
160
161
sub select_all_records {
162
    my $record_type = shift;
163
    return ($record_type eq 'biblio') ? select_all_biblios() : select_all_authorities();
164
}
165
166
sub select_all_authorities {
167
    my $sth = $dbh->prepare("SELECT authid FROM auth_header");
168
    $sth->execute();
169
    return $sth;
170
}
171
172
sub select_all_biblios {
173
    my $sth = $dbh->prepare("SELECT biblionumber FROM biblioitems ORDER BY biblionumber");
174
    $sth->execute();
175
    return $sth;
176
}
177
178
sub export_marc_records_from_sth {
179
    my ($record_type, $sth, $directory, $as_xml, $noxml, $nosanitize) = @_;
180
181
    my $num_exported = 0;
182
    open (OUT, ">:utf8 ", "$directory/exported_records") or die $!;
183
    my $i = 0;
184
    while (my ($record_number) = $sth->fetchrow_array) {
185
        print "." if ( $verbose_logging );
186
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
187
        if ( $nosanitize ) {
188
            my $marcxml = $record_type eq 'biblio'
189
                          ? GetXmlBiblio( $record_number )
190
                          : GetAuthorityXML( $record_number );
191
            if ( $marcxml ) {
192
                print OUT $marcxml if $marcxml;
193
                $num_exported++;
194
            }
195
            next;
196
        }
197
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
198
        if (defined $marc) {
199
            # FIXME - when more than one record is exported and $as_xml is true,
200
            # the output file is not valid XML - it's just multiple <record> elements
201
            # strung together with no single root element.  zebraidx doesn't seem
202
            # to care, though, at least if you're using the GRS-1 filter.  It does
203
            # care if you're using the DOM filter, which requires valid XML file(s).
204
            print OUT ($as_xml) ? $marc->as_xml_record() : $marc->as_usmarc();
205
            $num_exported++;
206
        }
207
    }
208
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
209
    close OUT;
210
    return $num_exported;
211
}
212
213
sub export_marc_records_from_list {
214
    my ($record_type, $entries, $directory, $as_xml, $noxml) = @_;
215
216
    my $num_exported = 0;
217
    open (OUT, ">:utf8 ", "$directory/exported_records") or die $!;
218
    my $i = 0;
219
    my %found = ();
220
    foreach my $record_number ( map { $_->{biblio_auth_number} }
221
                                grep { !$found{ $_->{biblio_auth_number} }++ }
222
                                @$entries ) {
223
        print "." if ( $verbose_logging );
224
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
225
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
226
        if (defined $marc) {
227
            # FIXME - when more than one record is exported and $as_xml is true,
228
            # the output file is not valid XML - it's just multiple <record> elements
229
            # strung together with no single root element.  zebraidx doesn't seem
230
            # to care, though, at least if you're using the GRS-1 filter.  It does
231
            # care if you're using the DOM filter, which requires valid XML file(s).
232
            print OUT ($as_xml) ? $marc->as_xml_record() : $marc->as_usmarc();
233
            $num_exported++;
234
        }
235
    }
236
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
237
    close OUT;
238
    return $num_exported;
239
}
240
241
sub generate_deleted_marc_records {
242
    my ($record_type, $entries, $directory, $as_xml) = @_;
243
244
    my $num_exported = 0;
245
    open (OUT, ">:utf8 ", "$directory/exported_records") or die $!;
246
    my $i = 0;
247
    foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) {
248
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
249
        print "." if ( $verbose_logging );
250
251
        my $marc = MARC::Record->new();
252
        if ($record_type eq 'biblio') {
253
            fix_biblio_ids($marc, $record_number, $record_number);
254
        } else {
255
            fix_authority_id($marc, $record_number);
256
        }
257
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
258
            fix_unimarc_100($marc);
259
        }
260
261
        print OUT ($as_xml) ? $marc->as_xml_record() : $marc->as_usmarc();
262
        $num_exported++;
263
    }
264
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
265
    close OUT;
266
    return $num_exported;
267
    
268
269
}
270
271
sub get_corrected_marc_record {
272
    my ($record_type, $record_number, $noxml) = @_;
273
274
    my $marc = get_raw_marc_record($record_type, $record_number, $noxml); 
275
276
    if (defined $marc) {
277
        fix_leader($marc);
278
        if ($record_type eq 'biblio') {
279
            my $succeeded = fix_biblio_ids($marc, $record_number);
280
            return unless $succeeded;
281
        } else {
282
            fix_authority_id($marc, $record_number);
283
        }
284
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
285
            fix_unimarc_100($marc);
286
        }
287
    }
288
289
    return $marc;
290
}
291
292
sub get_raw_marc_record {
293
    my ($record_type, $record_number, $noxml) = @_;
294
  
295
    my $marc; 
296
    if ($record_type eq 'biblio') {
297
        if ($noxml) {
298
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems WHERE biblionumber = ?");
299
            $fetch_sth->execute($record_number);
300
            if (my ($blob) = $fetch_sth->fetchrow_array) {
301
                $marc = MARC::Record->new_from_usmarc($blob);
302
                $fetch_sth->finish();
303
            } else {
304
                return; # failure to find a bib is not a problem -
305
                        # a delete could have been done before
306
                        # trying to process a record update
307
            }
308
        } else {
309
            eval { $marc = GetMarcBiblio($record_number); };
310
            if ($@) {
311
                # here we do warn since catching an exception
312
                # means that the bib was found but failed
313
                # to be parsed
314
                warn "error retrieving biblio $record_number";
315
                return;
316
            }
317
        }
318
    } else {
319
        eval { $marc = GetAuthority($record_number); };
320
        if ($@) {
321
            warn "error retrieving authority $record_number";
322
            return;
323
        }
324
    }
325
    return $marc;
326
}
327
328
sub fix_leader {
329
    # FIXME - this routine is suspect
330
    # It blanks the Leader/00-05 and Leader/12-16 to
331
    # force them to be recalculated correct when
332
    # the $marc->as_usmarc() or $marc->as_xml() is called.
333
    # But why is this necessary?  It would be a serious bug
334
    # in MARC::Record (definitely) and MARC::File::XML (arguably) 
335
    # if they are emitting incorrect leader values.
336
    my $marc = shift;
337
338
    my $leader = $marc->leader;
339
    substr($leader,  0, 5) = '     ';
340
    substr($leader, 10, 7) = '22     ';
341
    $marc->leader(substr($leader, 0, 24));
342
}
343
344
sub fix_biblio_ids {
345
    # FIXME - it is essential to ensure that the biblionumber is present,
346
    #         otherwise, Zebra will choke on the record.  However, this
347
    #         logic belongs in the relevant C4::Biblio APIs.
348
    my $marc = shift;
349
    my $biblionumber = shift;
350
    my $biblioitemnumber;
351
    if (@_) {
352
        $biblioitemnumber = shift;
353
    } else {    
354
        my $sth = $dbh->prepare(
355
            "SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
356
        $sth->execute($biblionumber);
357
        ($biblioitemnumber) = $sth->fetchrow_array;
358
        $sth->finish;
359
        unless ($biblioitemnumber) {
360
            warn "failed to get biblioitemnumber for biblio $biblionumber";
361
            return 0;
362
        }
363
    }
364
365
    # FIXME - this is cheating on two levels
366
    # 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
367
    # 2. Making sure that the biblionumber and biblioitemnumber are correct and
368
    #    present in the MARC::Record object ought to be part of GetMarcBiblio.
369
    #
370
    # On the other hand, this better for now than what rebuild_zebra.pl used to
371
    # do, which was duplicate the code for inserting the biblionumber 
372
    # and biblioitemnumber
373
    C4::Biblio::_koha_marc_update_bib_ids($marc, '', $biblionumber, $biblioitemnumber);
374
375
    return 1;
376
}
377
378
sub fix_authority_id {
379
    # FIXME - as with fix_biblio_ids, the authid must be present
380
    #         for Zebra's sake.  However, this really belongs
381
    #         in C4::AuthoritiesMarc.
382
    my ($marc, $authid) = @_;
383
    unless ($marc->field('001') and $marc->field('001')->data() eq $authid){
384
        $marc->delete_field($marc->field('001'));
385
        $marc->insert_fields_ordered(MARC::Field->new('001',$authid));
386
    }
387
}
388
389
sub fix_unimarc_100 {
390
    # FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
391
    my $marc = shift;
392
393
    my $string;
394
    if ( length($marc->subfield( 100, "a" )) == 35 ) {
395
        $string = $marc->subfield( 100, "a" );
396
        my $f100 = $marc->field(100);
397
        $marc->delete_field($f100);
398
    }
399
    else {
400
        $string = POSIX::strftime( "%Y%m%d", localtime );
401
        $string =~ s/\-//g;
402
        $string = sprintf( "%-*s", 35, $string );
403
    }
404
    substr( $string, 22, 6, "frey50" );
405
    unless ( length($marc->subfield( 100, "a" )) == 35 ) {
406
        $marc->delete_field($marc->field(100));
407
        $marc->insert_grouped_field(MARC::Field->new( 100, "", "", "a" => $string ));
408
    }
409
}
410
411
sub do_indexing {
412
    my ($record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt) = @_;
413
414
    my $zebra_server  = ($record_type eq 'biblio') ? 'biblioserver' : 'authorityserver';
415
    my $zebra_db_name = ($record_type eq 'biblio') ? 'biblios' : 'authorities';
416
    my $zebra_config  = C4::Context->zebraconfig($zebra_server)->{'config'};
417
    my $zebra_db_dir  = C4::Context->zebraconfig($zebra_server)->{'directory'};
418
419
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index;
420
    system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir");
421
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow;
422
423
}
424
425
426
1;

Return to bug 5166