View | Details | Raw Unified | Return to bug 7419
Collapse All | Expand All

(-)a/C4/ZebraIndex.pm (+426 lines)
Line 0 Link Here
1
package C4::ZebraIndex;
2
3
# This file is part of Koha.
4
#
5
# Koha is free software; you can redistribute it and/or modify it under the
6
# terms of the GNU General Public License as published by the Free Software
7
# Foundation; either version 2 of the License, or (at your option) any later
8
# version.
9
#
10
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
11
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
12
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
13
#
14
# You should have received a copy of the GNU General Public License along with
15
# Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
16
# Suite 330, Boston, MA  02111-1307 USA
17
18
# This module contains utility functions to index records in Zebra
19
# It is a temporary place to store rebuild_zebra.pl subs in order for them to be
20
# used by other scripts (misc/migration_tools/dedup_authorities.pl for example)
21
# As soon as the new indexation layer is in place, and all scripts using this
22
# module are updated, this module have to be removed.
23
24
use Modern::Perl;
25
use POSIX;
26
use File::Temp qw/tempdir/;
27
use MARC::Record;
28
29
use C4::Context;
30
use C4::Biblio;
31
use C4::AuthoritiesMarc;
32
use C4::Items;
33
use XML::LibXML;
34
use Koha::RecordProcessor;
35
36
sub IndexRecord {
37
    my ($recordtype, $recordids, $options) = @_;
38
39
    my $as_xml = ($options and $options->{as_xml}) ? 1 : 0;
40
    my $noxml = ($options and $options->{noxml}) ? 1 : 0;
41
    my $nosanitize = ($options and $options->{nosanitize}) ? $as_xml : 0;
42
    my $reset_index = ($options and $options->{reset_index}) ? 1 : 0;
43
    my $noshadow = ($options and $options->{noshadow}) ? " -n " : "";
44
    my $record_format = ($options and $options->{record_format})
45
                    ? $options->{record_format}
46
                    : ($as_xml)
47
                        ? 'marcxml'
48
                        : 'iso2709';
49
    my $zebraidx_log_opt = ($options and defined $options->{zebraidx_log_opt})
50
                        ? $options->{zebraidx_log_opt}
51
                        : "";
52
    my $verbose = ($options and $options->{verbose}) ? 1 : 0;
53
    my $skip_export = ($options and $options->{skip_export}) ? 1 : 0;
54
    my $skip_index = ($options and $options->{skip_index}) ? 1 : 0;
55
    my $keep_export = ($options and $options->{keep_export}) ? 1 : 0;
56
    my $directory = ($options and $options->{directory})
57
                    ? $options->{directory}
58
                    : tempdir(CLEANUP => ($keep_export ? 0 : 1));
59
60
    my @recordids = (ref $recordids eq "ARRAY") ? @$recordids : split (/ /, $recordids);
61
    @recordids = map { { biblio_auth_number => $_ } } @recordids;
62
63
    my $num_exported = 0;
64
    unless ($skip_export) {
65
        $num_exported = _export_marc_records_from_list($recordtype, \@recordids,
66
            $directory, $as_xml, $noxml, $nosanitize, $verbose);
67
    }
68
    unless ($skip_index) {
69
        _do_indexing($recordtype, "update", $directory, $reset_index, $noshadow,
70
            $record_format, $zebraidx_log_opt);
71
    }
72
73
    return $num_exported;
74
}
75
76
sub DeleteRecordIndex {
77
    my ($recordtype, $recordids, $options) = @_;
78
79
    my $as_xml = ($options and $options->{as_xml}) ? 1 : 0;
80
    my $noshadow = ($options and $options->{noshadow}) ? " -n " : "";
81
    my $record_format = ($options and $options->{record_format})
82
                    ? $options->{record_format}
83
                    : ($as_xml)
84
                        ? 'marcxml'
85
                        : 'iso2709';
86
    my $zebraidx_log_opt = ($options and defined $options->{zebraidx_log_opt})
87
                        ? $options->{zebraidx_log_opt}
88
                        : "";
89
    my $verbose = ($options and $options->{verbose}) ? 1 : 0;
90
    my $skip_export = ($options and $options->{skip_export}) ? 1 : 0;
91
    my $skip_index = ($options and $options->{skip_index}) ? 1 : 0;
92
    my $keep_export = ($options and $options->{keep_export}) ? 1 : 0;
93
    my $directory = ($options and $options->{directory})
94
                    ? $options->{directory}
95
                    : tempdir(CLEANUP => ($keep_export ? 0 : 1));
96
97
    my @recordids = (ref $recordids eq "ARRAY") ? @$recordids : split (/ /, $recordids);
98
    @recordids = map { { biblio_auth_number => $_ } } @recordids;
99
100
    my $num_exported = 0;
101
    unless ($skip_export) {
102
        $num_exported = _generate_deleted_marc_records($recordtype, \@recordids,
103
            $directory, $as_xml, $verbose);
104
    }
105
    unless ($skip_index) {
106
        _do_indexing($recordtype, "adelete", $directory, 0, $noshadow,
107
            $record_format, $zebraidx_log_opt);
108
    }
109
110
    return $num_exported;
111
}
112
113
114
sub _export_marc_records_from_list {
115
    my ( $record_type, $entries, $directory, $as_xml, $noxml, $nosanitize, $verbose ) = @_;
116
117
    my $num_exported = 0;
118
    open my $fh, ">:encoding(UTF-8) ", "$directory/exported_records" or die $!;
119
    if (_include_xml_wrapper($as_xml, $record_type)) {
120
        # include XML declaration and root element
121
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
122
    }
123
    my $i     = 0;
124
    my %found = ();
125
    my ($itemtag) = GetMarcFromKohaField("items.itemnumber",'');
126
    my $tester = XML::LibXML->new();
127
    foreach my $record_number (
128
        map  { $_->{biblio_auth_number} }
129
        grep { !$found{ $_->{biblio_auth_number} }++ } @$entries
130
    ) {
131
        if($nosanitize) {
132
            my $marcxml = $record_type eq 'biblio'
133
                        ? GetXmlBiblio($record_number)
134
                        : GetAuthorityXML($record_number);
135
            if ($record_type eq 'biblio'){
136
                my @items = GetItemsInfo($record_number);
137
                if (@items){
138
                    my $record = MARC::Record->new;
139
                    $record->encoding('UTF-8');
140
                    my @itemsrecord;
141
                    foreach my $item (@items){
142
                        my $record = Item2Marc($item, $record_number);
143
                        push @itemsrecord, $record->field($itemtag);
144
                    }
145
                    $record->insert_fields_ordered(@itemsrecord);
146
                    my $itemsxml = $record->as_xml_record();
147
                    $marcxml =
148
                        substr($marcxml, 0, length($marcxml)-10) .
149
                        substr($itemsxml, index($itemsxml, "</leader>\n", 0) + 10);
150
                }
151
            }
152
            # extra test to ensure that result is valid XML; otherwise
153
            # Zebra won't parse it in DOM mode
154
            eval {
155
                my $doc = $tester->parse_string($marcxml);
156
            };
157
            if ($@) {
158
                warn "Error exporting record $record_number ($record_type): $@\n";
159
                next;
160
            }
161
            if ($marcxml) {
162
                $marcxml =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
163
                print {$fh} $marcxml;
164
                $num_exported++;
165
            }
166
        } else {
167
            my ($marc) = _get_corrected_marc_record( $record_type, $record_number, $noxml );
168
            if ( defined $marc ) {
169
                eval {
170
                    my $rec;
171
                    if ($as_xml) {
172
                        $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
173
                        eval {
174
                            my $doc = $tester->parse_string($rec);
175
                        };
176
                        if ($@) {
177
                            die "invalid XML: $@";
178
                        }
179
                        $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
180
                    } else {
181
                        $rec = $marc->as_usmarc();
182
                    }
183
                    print {$fh} $rec;
184
                    $num_exported++;
185
                };
186
                if ($@) {
187
                    warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
188
                    warn "... specific error is $@" if $verbose;
189
                }
190
            }
191
        }
192
        $i++;
193
        if($verbose) {
194
            print ".";
195
            print "$i\n" unless($i % 100);
196
        }
197
    }
198
    print {$fh} '</collection>' if (_include_xml_wrapper($as_xml, $record_type));
199
    close $fh;
200
    print "\n$num_exported records exported to $directory/exported_records\n" if $verbose;
201
    return $num_exported;
202
}
203
204
sub _generate_deleted_marc_records {
205
    my ( $record_type, $entries, $directory, $as_xml, $verbose ) = @_;
206
207
    my $num_exported = 0;
208
    open my $fh, ">:encoding(UTF-8)", "$directory/exported_records" or die $!;
209
    if (_include_xml_wrapper($as_xml, $record_type)) {
210
        # include XML declaration and root element
211
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
212
    }
213
    my $i = 0;
214
    foreach my $record_number ( map { $_->{biblio_auth_number} } @$entries ) {
215
        my $marc = MARC::Record->new();
216
        if ( $record_type eq 'biblio' ) {
217
            _fix_biblio_ids( $marc, $record_number, $record_number );
218
        } else {
219
            _fix_authority_id( $marc, $record_number );
220
        }
221
        if ( C4::Context->preference("marcflavour") eq "UNIMARC" ) {
222
            _fix_unimarc_100($marc);
223
        }
224
225
        my $rec;
226
        if ($as_xml) {
227
            $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
228
            $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
229
        } else {
230
            $rec = $marc->as_usmarc();
231
        }
232
        print {$fh} $rec;
233
234
        $num_exported++;
235
        $i++;
236
        if($verbose) {
237
            print ".";
238
            print "$i\n" unless($i % 100);
239
        }
240
    }
241
    print {$fh} '</collection>' if (_include_xml_wrapper($as_xml, $record_type));
242
    close $fh;
243
244
    return $num_exported;
245
}
246
247
sub _get_corrected_marc_record {
248
    my ( $record_type, $record_number, $noxml ) = @_;
249
250
    my $marc = _get_raw_marc_record( $record_type, $record_number, $noxml );
251
252
    if ( defined $marc ) {
253
        _fix_leader($marc);
254
        if ( $record_type eq 'authority' ) {
255
            _fix_authority_id( $marc, $record_number );
256
        } elsif ($record_type eq 'biblio' && C4::Context->preference('IncludeSeeFromInSearches')) {
257
            my $normalizer = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } );
258
            $marc = $normalizer->process($marc);
259
        }
260
        if ( C4::Context->preference("marcflavour") eq "UNIMARC" ) {
261
            _fix_unimarc_100($marc);
262
        }
263
    }
264
265
    return $marc;
266
}
267
268
sub _get_raw_marc_record {
269
    my ( $record_type, $record_number, $noxml ) = @_;
270
271
    my $dbh = C4::Context->dbh;
272
    my $marc;
273
    if ( $record_type eq 'biblio' ) {
274
        if ($noxml) {
275
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems
276
                WHERE biblionumber = ?");
277
            $fetch_sth->execute($record_number);
278
            if ( my ($blob) = $fetch_sth->fetchrow_array ) {
279
                $marc = MARC::Record->new_from_usmarc($blob);
280
                unless ($marc) {
281
                    warn "error creating MARC::Record from $blob";
282
                }
283
            }
284
            # failure to find a bib is not a problem -
285
            # a delete could have been done before
286
            # trying to process a record update
287
288
            $fetch_sth->finish();
289
            return unless $marc;
290
        } else {
291
            eval { $marc = GetMarcBiblio($record_number, 1); };
292
            if ($@ || !$marc) {
293
                # here we do warn since catching an exception
294
                # means that the bib was found but failed
295
                # to be parsed
296
                warn "error retrieving biblio $record_number: $@";
297
                return;
298
            }
299
        }
300
    } else {
301
        eval { $marc = C4::AuthoritiesMarc::GetAuthority($record_number); };
302
        if ($@) {
303
            warn "error retrieving authority $record_number: $@";
304
            return;
305
        }
306
    }
307
    return $marc;
308
}
309
310
sub _fix_leader {
311
312
    # FIXME - this routine is suspect
313
    # It blanks the Leader/00-05 and Leader/12-16 to
314
    # force them to be recalculated correct when
315
    # the $marc->as_usmarc() or $marc->as_xml() is called.
316
    # But why is this necessary?  It would be a serious bug
317
    # in MARC::Record (definitely) and MARC::File::XML (arguably)
318
    # if they are emitting incorrect leader values.
319
    my $marc = shift;
320
321
    my $leader = $marc->leader;
322
    substr( $leader, 0,  5 ) = '     ';
323
    substr( $leader, 10, 7 ) = '22     ';
324
    $marc->leader( substr( $leader, 0, 24 ) );
325
}
326
327
328
sub _fix_biblio_ids {
329
330
    # FIXME - it is essential to ensure that the biblionumber is present,
331
    #         otherwise, Zebra will choke on the record.  However, this
332
    #         logic belongs in the relevant C4::Biblio APIs.
333
    my $marc         = shift;
334
    my $biblionumber = shift;
335
    my $dbh = C4::Context->dbh;
336
    my $biblioitemnumber;
337
    if (@_) {
338
        $biblioitemnumber = shift;
339
    } else {
340
        my $sth = $dbh->prepare(
341
            "SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
342
        $sth->execute($biblionumber);
343
        ($biblioitemnumber) = $sth->fetchrow_array;
344
        $sth->finish;
345
        unless ($biblioitemnumber) {
346
            warn "failed to get biblioitemnumber for biblio $biblionumber";
347
            return 0;
348
        }
349
    }
350
351
    # FIXME - this is cheating on two levels
352
    # 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
353
    # 2. Making sure that the biblionumber and biblioitemnumber are correct and
354
    #    present in the MARC::Record object ought to be part of GetMarcBiblio.
355
    #
356
    # On the other hand, this better for now than what rebuild_zebra.pl used to
357
    # do, which was duplicate the code for inserting the biblionumber
358
    # and biblioitemnumber
359
    C4::Biblio::_koha_marc_update_bib_ids( $marc, '', $biblionumber, $biblioitemnumber );
360
361
    return 1;
362
}
363
364
sub _fix_authority_id {
365
366
    # FIXME - as with fix_biblio_ids, the authid must be present
367
    #         for Zebra's sake.  However, this really belongs
368
    #         in C4::AuthoritiesMarc.
369
    my ( $marc, $authid ) = @_;
370
    unless ( $marc->field('001') and $marc->field('001')->data() eq $authid ) {
371
        $marc->delete_field( $marc->field('001') );
372
        $marc->insert_fields_ordered( MARC::Field->new( '001', $authid ) );
373
    }
374
}
375
376
sub _fix_unimarc_100 {
377
378
    # FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
379
    my $marc = shift;
380
381
    my $string;
382
    if ( length( $marc->subfield( 100, "a" ) ) == 36 ) {
383
        $string = $marc->subfield( 100, "a" );
384
        my $f100 = $marc->field(100);
385
        $marc->delete_field($f100);
386
    } else {
387
        $string = POSIX::strftime( "%Y%m%d", localtime );
388
        $string =~ s/\-//g;
389
        $string = sprintf( "%-*s", 35, $string );
390
    }
391
    substr( $string, 22, 6, "frey50" );
392
    unless ( length( $marc->subfield( 100, "a" ) ) == 36 ) {
393
        $marc->delete_field( $marc->field(100) );
394
        $marc->insert_grouped_field( MARC::Field->new( 100, "", "", "a" => $string ) );
395
    }
396
}
397
398
sub _do_indexing {
399
    my ( $record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt ) = @_;
400
401
    my $zebra_server  = ( $record_type eq 'biblio' ) ? 'biblioserver' : 'authorityserver';
402
    my $zebra_db_name = ( $record_type eq 'biblio' ) ? 'biblios'      : 'authorities';
403
    my $zebra_config  = C4::Context->zebraconfig($zebra_server)->{'config'};
404
    my $zebra_db_dir  = C4::Context->zebraconfig($zebra_server)->{'directory'};
405
406
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index;
407
    system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir");
408
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow;
409
410
}
411
412
sub _include_xml_wrapper {
413
    my $as_xml = shift;
414
    my $record_type = shift;
415
416
    my $bib_index_mode = C4::Context->config('zebra_bib_index_mode') || 'grs1';
417
    my $auth_index_mode = C4::Context->config('zebra_auth_index_mode') || 'dom';
418
419
    return 0 unless $as_xml;
420
    return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom';
421
    return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom';
422
    return 0;
423
424
}
425
426
1;
(-)a/misc/migration_tools/rebuild_zebra.pl (-396 / +52 lines)
Lines 10-17 use File::Path; Link Here
10
use C4::Biblio;
10
use C4::Biblio;
11
use C4::AuthoritiesMarc;
11
use C4::AuthoritiesMarc;
12
use C4::Items;
12
use C4::Items;
13
use Koha::RecordProcessor;
13
use C4::ZebraIndex;
14
use XML::LibXML;
15
14
16
# script that checks zebradir structure & create directories & mandatory files if needed
15
# script that checks zebradir structure & create directories & mandatory files if needed
17
#
16
#
Lines 181-192 if ( $verbose_logging ) { Link Here
181
}
180
}
182
if ($keep_export) {
181
if ($keep_export) {
183
    print "NOTHING cleaned : the export $directory has been kept.\n";
182
    print "NOTHING cleaned : the export $directory has been kept.\n";
184
    print "You can re-run this script with the -s ";
183
    print "You can re-run this script with the -s and -d $directory parameters";
185
    if ($use_tempdir) {
186
        print " and -d $directory parameters";
187
    } else {
188
        print "parameter";
189
    }
190
    print "\n";
184
    print "\n";
191
    print "if you just want to rebuild zebra after changing the record.abs\n";
185
    print "if you just want to rebuild zebra after changing the record.abs\n";
192
    print "or another zebra config file\n";
186
    print "or another zebra config file\n";
Lines 257-324 sub check_zebra_dirs { Link Here
257
sub index_records {
251
sub index_records {
258
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
252
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
259
253
260
    my $num_records_exported = 0;
261
    my $records_deleted;
262
    my $need_reset = check_zebra_dirs($server_dir);
254
    my $need_reset = check_zebra_dirs($server_dir);
263
    if ($need_reset) {
255
    if ($need_reset) {
264
        print "$0: found broken zebra server directories: forcing a rebuild\n";
256
        print "$0: found broken zebra server directories: forcing a rebuild\n";
265
        $reset = 1;
257
        $reset = 1;
266
    }
258
    }
267
    if ($skip_export && $verbose_logging) {
259
260
    if ( $verbose_logging ) {
268
        print "====================\n";
261
        print "====================\n";
269
        print "SKIPPING $record_type export\n";
262
        print "REINDEXING zebra\n";
270
        print "====================\n";
263
        print "====================\n";
271
    } else {
272
        if ( $verbose_logging ) {
273
            print "====================\n";
274
            print "exporting $record_type\n";
275
            print "====================\n";
276
        }
277
        mkdir "$directory" unless (-d $directory);
278
        mkdir "$directory/$record_type" unless (-d "$directory/$record_type");
279
        if ($process_zebraqueue) {
280
            my $entries = select_zebraqueue_records($record_type, 'deleted');
281
            mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type");
282
            $records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_xml);
283
            mark_zebraqueue_batch_done($entries);
284
            $entries = select_zebraqueue_records($record_type, 'updated');
285
            mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type");
286
            $num_records_exported = export_marc_records_from_list($record_type,
287
                                                                  $entries, "$directory/upd_$record_type", $as_xml, $noxml, $records_deleted);
288
            mark_zebraqueue_batch_done($entries);
289
        } else {
290
            my $sth = select_all_records($record_type);
291
            $num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_xml, $noxml, $nosanitize);
292
            unless ($do_not_clear_zebraqueue) {
293
                mark_all_zebraqueue_done($record_type);
294
            }
295
        }
296
    }
264
    }
297
265
298
    #
266
    my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
299
    # and reindexing everything
267
    if ($process_zebraqueue) {
300
    #
268
        # Process 'deleted' records
301
    if ($skip_index) {
269
        my $entries = select_zebraqueue_records( $record_type, 'deleted' );
302
        if ($verbose_logging) {
270
        my @entries_to_delete = map { $_->{biblio_auth_id} } @$entries;
303
            print "====================\n";
271
304
            print "SKIPPING $record_type indexing\n";
272
        C4::ZebraIndex::DeleteRecordIndex($record_type, \@entries_to_delete, {
305
            print "====================\n";
273
            as_xml => $as_xml, noshadow => $noshadow, record_format => $record_fmt,
306
        }
274
            zebraidx_log_opt => $zebraidx_log_opt, verbose => $verbose_logging,
275
            skip_export => $skip_export, skip_index => $skip_index,
276
            keep_export => $keep_export, directory => $directory
277
        });
278
279
        mark_zebraqueue_batch_done($entries);
280
281
        # Process 'updated' records'
282
        $entries = select_zebraqueue_records( $record_type, 'updated' );
283
        my @entries_to_update = map {
284
            my $id = $_->{biblio_auth_id};
285
            # do not try to update deleted records
286
            (0 == grep { $id == $_ } @entries_to_delete) ? $id : ()
287
        } @$entries;
288
289
        C4::ZebraIndex::IndexRecord($record_type, \@entries_to_update, {
290
            as_xml => $as_xml, noxml => $noxml, reset_index => $reset,
291
            noshadow => $noshadow, record_format => $record_fmt,
292
            zebraidx_log_opt => $zebraidx_log_opt, verbose => $verbose_logging,
293
            skip_export => $skip_export, skip_index => $skip_index,
294
            keep_export => $keep_export, directory => $directory
295
        });
296
297
        mark_zebraqueue_batch_done($entries);
307
    } else {
298
    } else {
308
        if ( $verbose_logging ) {
299
        my $sth = select_all_records($record_type);
309
            print "====================\n";
300
        my $entries = $sth->fetchall_arrayref([]);
310
            print "REINDEXING zebra\n";
301
        my @entries_to_update = map { $_->[0] } @$entries;
311
            print "====================\n";
302
312
        }
303
        C4::ZebraIndex::IndexRecord($record_type, \@entries_to_update, {
313
        my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
304
            as_xml => $as_xml, noxml => $noxml, reset_index => $reset,
314
        if ($process_zebraqueue) {
305
            noshadow => $noshadow, record_format => $record_fmt,
315
            do_indexing($record_type, 'adelete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
306
            zebraidx_log_opt => $zebraidx_log_opt, nosanitize => $nosanitize,
316
                if %$records_deleted;
307
            verbose => $verbose_logging, skip_export => $skip_export,
317
            do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
308
            skip_index => $skip_index, keep_export => $keep_export,
318
                if $num_records_exported;
309
            directory => $directory
319
        } else {
310
        });
320
            do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
311
321
                if ($num_records_exported or $skip_export);
312
        unless ($do_not_clear_zebraqueue) {
313
            mark_all_zebraqueue_done($record_type);
322
        }
314
        }
323
    }
315
    }
324
}
316
}
Lines 388-728 sub select_all_biblios { Link Here
388
    return $sth;
380
    return $sth;
389
}
381
}
390
382
391
sub include_xml_wrapper {
392
    my $as_xml = shift;
393
    my $record_type = shift;
394
395
    return 0 unless $as_xml;
396
    return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom';
397
    return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom';
398
    return 0;
399
400
}
401
402
sub export_marc_records_from_sth {
403
    my ($record_type, $sth, $directory, $as_xml, $noxml, $nosanitize) = @_;
404
405
    my $num_exported = 0;
406
    open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!;
407
    if (include_xml_wrapper($as_xml, $record_type)) {
408
        # include XML declaration and root element
409
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
410
    }
411
    my $i = 0;
412
    my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",'');
413
    while (my ($record_number) = $sth->fetchrow_array) {
414
        print "." if ( $verbose_logging );
415
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
416
        if ( $nosanitize ) {
417
            my $marcxml = $record_type eq 'biblio'
418
                          ? GetXmlBiblio( $record_number )
419
                          : GetAuthorityXML( $record_number );
420
            if ($record_type eq 'biblio'){
421
                my @items = GetItemsInfo($record_number);
422
                if (@items){
423
                    my $record = MARC::Record->new;
424
                    $record->encoding('UTF-8');
425
                    my @itemsrecord;
426
                    foreach my $item (@items){
427
                        my $record = Item2Marc($item, $record_number);
428
                        push @itemsrecord, $record->field($itemtag);
429
                    }
430
                    $record->insert_fields_ordered(@itemsrecord);
431
                    my $itemsxml = $record->as_xml_record();
432
                    $marcxml =
433
                        substr($marcxml, 0, length($marcxml)-10) .
434
                        substr($itemsxml, index($itemsxml, "</leader>\n", 0) + 10);
435
                }
436
            }
437
            # extra test to ensure that result is valid XML; otherwise
438
            # Zebra won't parse it in DOM mode
439
            eval {
440
                my $doc = $tester->parse_string($marcxml);
441
            };
442
            if ($@) {
443
                warn "Error exporting record $record_number ($record_type): $@\n";
444
                next;
445
            }
446
            if ( $marcxml ) {
447
                $marcxml =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
448
                print {$fh} $marcxml;
449
                $num_exported++;
450
            }
451
            next;
452
        }
453
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
454
        if (defined $marc) {
455
            eval {
456
                my $rec;
457
                if ($as_xml) {
458
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
459
                    eval {
460
                        my $doc = $tester->parse_string($rec);
461
                    };
462
                    if ($@) {
463
                        die "invalid XML: $@";
464
                    }
465
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
466
                } else {
467
                    $rec = $marc->as_usmarc();
468
                }
469
                print {$fh} $rec;
470
                $num_exported++;
471
            };
472
            if ($@) {
473
                warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
474
                warn "... specific error is $@" if $verbose_logging;
475
            }
476
        }
477
    }
478
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
479
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
480
    close $fh;
481
    return $num_exported;
482
}
483
484
sub export_marc_records_from_list {
485
    my ($record_type, $entries, $directory, $as_xml, $noxml, $records_deleted) = @_;
486
487
    my $num_exported = 0;
488
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
489
    if (include_xml_wrapper($as_xml, $record_type)) {
490
        # include XML declaration and root element
491
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
492
    }
493
    my $i = 0;
494
495
    # Skip any deleted records. We check for this anyway, but this reduces error spam
496
    my %found = %$records_deleted;
497
    foreach my $record_number ( map { $_->{biblio_auth_number} }
498
                                grep { !$found{ $_->{biblio_auth_number} }++ }
499
                                @$entries ) {
500
        print "." if ( $verbose_logging );
501
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
502
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
503
        if (defined $marc) {
504
            eval {
505
                my $rec;
506
                if ($as_xml) {
507
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
508
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
509
                } else {
510
                    $rec = $marc->as_usmarc();
511
                }
512
                print {$fh} $rec;
513
                $num_exported++;
514
            };
515
            if ($@) {
516
              warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
517
            }
518
        }
519
    }
520
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
521
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
522
    close $fh;
523
    return $num_exported;
524
}
525
526
sub generate_deleted_marc_records {
527
    my ($record_type, $entries, $directory, $as_xml) = @_;
528
529
    my $records_deleted = {};
530
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
531
    if (include_xml_wrapper($as_xml, $record_type)) {
532
        # include XML declaration and root element
533
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
534
    }
535
    my $i = 0;
536
    foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) {
537
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
538
        print "." if ( $verbose_logging );
539
540
        my $marc = MARC::Record->new();
541
        if ($record_type eq 'biblio') {
542
            fix_biblio_ids($marc, $record_number, $record_number);
543
        } else {
544
            fix_authority_id($marc, $record_number);
545
        }
546
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
547
            fix_unimarc_100($marc);
548
        }
549
550
        my $rec;
551
        if ($as_xml) {
552
            $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
553
            $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
554
        } else {
555
            $rec = $marc->as_usmarc();
556
        }
557
        print {$fh} $rec;
558
559
        $records_deleted->{$record_number} = 1;
560
    }
561
    print "\nRecords exported: $i\n" if ( $verbose_logging );
562
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
563
    close $fh;
564
    return $records_deleted;
565
566
567
}
568
569
sub get_corrected_marc_record {
570
    my ($record_type, $record_number, $noxml) = @_;
571
572
    my $marc = get_raw_marc_record($record_type, $record_number, $noxml);
573
574
    if (defined $marc) {
575
        fix_leader($marc);
576
        if ($record_type eq 'authority') {
577
            fix_authority_id($marc, $record_number);
578
        } elsif ($record_type eq 'biblio' && C4::Context->preference('IncludeSeeFromInSearches')) {
579
            my $normalizer = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } );
580
            $marc = $normalizer->process($marc);
581
        }
582
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
583
            fix_unimarc_100($marc);
584
        }
585
    }
586
587
    return $marc;
588
}
589
590
sub get_raw_marc_record {
591
    my ($record_type, $record_number, $noxml) = @_;
592
593
    my $marc;
594
    if ($record_type eq 'biblio') {
595
        if ($noxml) {
596
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems WHERE biblionumber = ?");
597
            $fetch_sth->execute($record_number);
598
            if (my ($blob) = $fetch_sth->fetchrow_array) {
599
                $marc = MARC::Record->new_from_usmarc($blob);
600
                unless ($marc) {
601
                    warn "error creating MARC::Record from $blob";
602
                }
603
            }
604
            # failure to find a bib is not a problem -
605
            # a delete could have been done before
606
            # trying to process a record update
607
608
            $fetch_sth->finish();
609
            return unless $marc;
610
        } else {
611
            eval { $marc = GetMarcBiblio($record_number, 1); };
612
            if ($@ || !$marc) {
613
                # here we do warn since catching an exception
614
                # means that the bib was found but failed
615
                # to be parsed
616
                warn "error retrieving biblio $record_number";
617
                return;
618
            }
619
        }
620
    } else {
621
        eval { $marc = GetAuthority($record_number); };
622
        if ($@) {
623
            warn "error retrieving authority $record_number";
624
            return;
625
        }
626
    }
627
    return $marc;
628
}
629
630
sub fix_leader {
631
    # FIXME - this routine is suspect
632
    # It blanks the Leader/00-05 and Leader/12-16 to
633
    # force them to be recalculated correct when
634
    # the $marc->as_usmarc() or $marc->as_xml() is called.
635
    # But why is this necessary?  It would be a serious bug
636
    # in MARC::Record (definitely) and MARC::File::XML (arguably)
637
    # if they are emitting incorrect leader values.
638
    my $marc = shift;
639
640
    my $leader = $marc->leader;
641
    substr($leader,  0, 5) = '     ';
642
    substr($leader, 10, 7) = '22     ';
643
    $marc->leader(substr($leader, 0, 24));
644
}
645
646
sub fix_biblio_ids {
647
    # FIXME - it is essential to ensure that the biblionumber is present,
648
    #         otherwise, Zebra will choke on the record.  However, this
649
    #         logic belongs in the relevant C4::Biblio APIs.
650
    my $marc = shift;
651
    my $biblionumber = shift;
652
    my $biblioitemnumber;
653
    if (@_) {
654
        $biblioitemnumber = shift;
655
    } else {
656
        my $sth = $dbh->prepare(
657
            "SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
658
        $sth->execute($biblionumber);
659
        ($biblioitemnumber) = $sth->fetchrow_array;
660
        $sth->finish;
661
        unless ($biblioitemnumber) {
662
            warn "failed to get biblioitemnumber for biblio $biblionumber";
663
            return 0;
664
        }
665
    }
666
667
    # FIXME - this is cheating on two levels
668
    # 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
669
    # 2. Making sure that the biblionumber and biblioitemnumber are correct and
670
    #    present in the MARC::Record object ought to be part of GetMarcBiblio.
671
    #
672
    # On the other hand, this better for now than what rebuild_zebra.pl used to
673
    # do, which was duplicate the code for inserting the biblionumber
674
    # and biblioitemnumber
675
    C4::Biblio::_koha_marc_update_bib_ids($marc, '', $biblionumber, $biblioitemnumber);
676
677
    return 1;
678
}
679
680
sub fix_authority_id {
681
    # FIXME - as with fix_biblio_ids, the authid must be present
682
    #         for Zebra's sake.  However, this really belongs
683
    #         in C4::AuthoritiesMarc.
684
    my ($marc, $authid) = @_;
685
    unless ($marc->field('001') and $marc->field('001')->data() eq $authid){
686
        $marc->delete_field($marc->field('001'));
687
        $marc->insert_fields_ordered(MARC::Field->new('001',$authid));
688
    }
689
}
690
691
sub fix_unimarc_100 {
692
    # FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
693
    my $marc = shift;
694
695
    my $string;
696
    if ( length($marc->subfield( 100, "a" )) == 36 ) {
697
        $string = $marc->subfield( 100, "a" );
698
        my $f100 = $marc->field(100);
699
        $marc->delete_field($f100);
700
    }
701
    else {
702
        $string = POSIX::strftime( "%Y%m%d", localtime );
703
        $string =~ s/\-//g;
704
        $string = sprintf( "%-*s", 35, $string );
705
    }
706
    substr( $string, 22, 6, "frey50" );
707
    unless ( length($marc->subfield( 100, "a" )) == 36 ) {
708
        $marc->delete_field($marc->field(100));
709
        $marc->insert_grouped_field(MARC::Field->new( 100, "", "", "a" => $string ));
710
    }
711
}
712
713
sub do_indexing {
714
    my ($record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt) = @_;
715
716
    my $zebra_server  = ($record_type eq 'biblio') ? 'biblioserver' : 'authorityserver';
717
    my $zebra_db_name = ($record_type eq 'biblio') ? 'biblios' : 'authorities';
718
    my $zebra_config  = C4::Context->zebraconfig($zebra_server)->{'config'};
719
    my $zebra_db_dir  = C4::Context->zebraconfig($zebra_server)->{'directory'};
720
721
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index;
722
    system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir");
723
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow;
724
725
}
726
383
727
sub print_usage {
384
sub print_usage {
728
    print <<_USAGE_;
385
    print <<_USAGE_;
729
- 

Return to bug 7419