View | Details | Raw Unified | Return to bug 7419
Collapse All | Expand All

(-)a/C4/ZebraIndex.pm (+426 lines)
Line 0 Link Here
1
package C4::ZebraIndex;
2
3
# This file is part of Koha.
4
#
5
# Koha is free software; you can redistribute it and/or modify it under the
6
# terms of the GNU General Public License as published by the Free Software
7
# Foundation; either version 2 of the License, or (at your option) any later
8
# version.
9
#
10
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
11
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
12
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
13
#
14
# You should have received a copy of the GNU General Public License along with
15
# Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
16
# Suite 330, Boston, MA  02111-1307 USA
17
18
# This module contains utility functions to index records in Zebra
19
# It is a temporary place to store rebuild_zebra.pl subs in order for them to be
20
# used by other scripts (misc/migration_tools/dedup_authorities.pl for example)
21
# As soon as the new indexation layer is in place, and all scripts using this
22
# module are updated, this module have to be removed.
23
24
use Modern::Perl;
25
use POSIX;
26
use File::Temp qw/tempdir/;
27
use MARC::Record;
28
29
use C4::Context;
30
use C4::Biblio;
31
use C4::AuthoritiesMarc;
32
use C4::Items;
33
use XML::LibXML;
34
use Koha::RecordProcessor;
35
36
sub IndexRecord {
37
    my ($recordtype, $recordids, $options) = @_;
38
39
    my $as_xml = ($options and $options->{as_xml}) ? 1 : 0;
40
    my $noxml = ($options and $options->{noxml}) ? 1 : 0;
41
    my $nosanitize = ($options and $options->{nosanitize}) ? $as_xml : 0;
42
    my $reset_index = ($options and $options->{reset_index}) ? 1 : 0;
43
    my $noshadow = ($options and $options->{noshadow}) ? " -n " : "";
44
    my $record_format = ($options and $options->{record_format})
45
                    ? $options->{record_format}
46
                    : ($as_xml)
47
                        ? 'marcxml'
48
                        : 'iso2709';
49
    my $zebraidx_log_opt = ($options and defined $options->{zebraidx_log_opt})
50
                        ? $options->{zebraidx_log_opt}
51
                        : "";
52
    my $verbose = ($options and $options->{verbose}) ? 1 : 0;
53
    my $skip_export = ($options and $options->{skip_export}) ? 1 : 0;
54
    my $skip_index = ($options and $options->{skip_index}) ? 1 : 0;
55
    my $keep_export = ($options and $options->{keep_export}) ? 1 : 0;
56
    my $directory = ($options and $options->{directory})
57
                    ? $options->{directory}
58
                    : tempdir(CLEANUP => ($keep_export ? 0 : 1));
59
60
    my @recordids = (ref $recordids eq "ARRAY") ? @$recordids : split (/ /, $recordids);
61
    @recordids = map { { biblio_auth_number => $_ } } @recordids;
62
63
    my $num_exported = 0;
64
    unless ($skip_export) {
65
        $num_exported = _export_marc_records_from_list($recordtype, \@recordids,
66
            $directory, $as_xml, $noxml, $nosanitize, $verbose);
67
    }
68
    unless ($skip_index) {
69
        _do_indexing($recordtype, "update", $directory, $reset_index, $noshadow,
70
            $record_format, $zebraidx_log_opt);
71
    }
72
73
    return $num_exported;
74
}
75
76
sub DeleteRecordIndex {
77
    my ($recordtype, $recordids, $options) = @_;
78
79
    my $as_xml = ($options and $options->{as_xml}) ? 1 : 0;
80
    my $noshadow = ($options and $options->{noshadow}) ? " -n " : "";
81
    my $record_format = ($options and $options->{record_format})
82
                    ? $options->{record_format}
83
                    : ($as_xml)
84
                        ? 'marcxml'
85
                        : 'iso2709';
86
    my $zebraidx_log_opt = ($options and defined $options->{zebraidx_log_opt})
87
                        ? $options->{zebraidx_log_opt}
88
                        : "";
89
    my $verbose = ($options and $options->{verbose}) ? 1 : 0;
90
    my $skip_export = ($options and $options->{skip_export}) ? 1 : 0;
91
    my $skip_index = ($options and $options->{skip_index}) ? 1 : 0;
92
    my $keep_export = ($options and $options->{keep_export}) ? 1 : 0;
93
    my $directory = ($options and $options->{directory})
94
                    ? $options->{directory}
95
                    : tempdir(CLEANUP => ($keep_export ? 0 : 1));
96
97
    my @recordids = (ref $recordids eq "ARRAY") ? @$recordids : split (/ /, $recordids);
98
    @recordids = map { { biblio_auth_number => $_ } } @recordids;
99
100
    my $num_exported = 0;
101
    unless ($skip_export) {
102
        $num_exported = _generate_deleted_marc_records($recordtype, \@recordids,
103
            $directory, $as_xml, $verbose);
104
    }
105
    unless ($skip_index) {
106
        _do_indexing($recordtype, "adelete", $directory, 0, $noshadow,
107
            $record_format, $zebraidx_log_opt);
108
    }
109
110
    return $num_exported;
111
}
112
113
114
sub _export_marc_records_from_list {
115
    my ( $record_type, $entries, $directory, $as_xml, $noxml, $nosanitize, $verbose ) = @_;
116
117
    my $num_exported = 0;
118
    open my $fh, ">:encoding(UTF-8) ", "$directory/exported_records" or die $!;
119
    if (_include_xml_wrapper($as_xml, $record_type)) {
120
        # include XML declaration and root element
121
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
122
    }
123
    my $i     = 0;
124
    my %found = ();
125
    my ($itemtag) = GetMarcFromKohaField("items.itemnumber",'');
126
    my $tester = XML::LibXML->new();
127
    foreach my $record_number (
128
        map  { $_->{biblio_auth_number} }
129
        grep { !$found{ $_->{biblio_auth_number} }++ } @$entries
130
    ) {
131
        if($nosanitize) {
132
            my $marcxml = $record_type eq 'biblio'
133
                        ? GetXmlBiblio($record_number)
134
                        : GetAuthorityXML($record_number);
135
            if ($record_type eq 'biblio'){
136
                my @items = GetItemsInfo($record_number);
137
                if (@items){
138
                    my $record = MARC::Record->new;
139
                    $record->encoding('UTF-8');
140
                    my @itemsrecord;
141
                    foreach my $item (@items){
142
                        my $record = Item2Marc($item, $record_number);
143
                        push @itemsrecord, $record->field($itemtag);
144
                    }
145
                    $record->insert_fields_ordered(@itemsrecord);
146
                    my $itemsxml = $record->as_xml_record();
147
                    $marcxml =
148
                        substr($marcxml, 0, length($marcxml)-10) .
149
                        substr($itemsxml, index($itemsxml, "</leader>\n", 0) + 10);
150
                }
151
            }
152
            # extra test to ensure that result is valid XML; otherwise
153
            # Zebra won't parse it in DOM mode
154
            eval {
155
                my $doc = $tester->parse_string($marcxml);
156
            };
157
            if ($@) {
158
                warn "Error exporting record $record_number ($record_type): $@\n";
159
                next;
160
            }
161
            if ($marcxml) {
162
                $marcxml =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
163
                print {$fh} $marcxml;
164
                $num_exported++;
165
            }
166
        } else {
167
            my ($marc) = _get_corrected_marc_record( $record_type, $record_number, $noxml );
168
            if ( defined $marc ) {
169
                eval {
170
                    my $rec;
171
                    if ($as_xml) {
172
                        $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
173
                        eval {
174
                            my $doc = $tester->parse_string($rec);
175
                        };
176
                        if ($@) {
177
                            die "invalid XML: $@";
178
                        }
179
                        $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
180
                    } else {
181
                        $rec = $marc->as_usmarc();
182
                    }
183
                    print {$fh} $rec;
184
                    $num_exported++;
185
                };
186
                if ($@) {
187
                    warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
188
                    warn "... specific error is $@" if $verbose;
189
                }
190
            }
191
        }
192
        $i++;
193
        if($verbose) {
194
            print ".";
195
            print "$i\n" unless($i % 100);
196
        }
197
    }
198
    print {$fh} '</collection>' if (_include_xml_wrapper($as_xml, $record_type));
199
    close $fh;
200
    print "\n$num_exported records exported to $directory/exported_records\n" if $verbose;
201
    return $num_exported;
202
}
203
204
sub _generate_deleted_marc_records {
205
    my ( $record_type, $entries, $directory, $as_xml, $verbose ) = @_;
206
207
    my $num_exported = 0;
208
    open my $fh, ">:encoding(UTF-8)", "$directory/exported_records" or die $!;
209
    if (_include_xml_wrapper($as_xml, $record_type)) {
210
        # include XML declaration and root element
211
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
212
    }
213
    my $i = 0;
214
    foreach my $record_number ( map { $_->{biblio_auth_number} } @$entries ) {
215
        my $marc = MARC::Record->new();
216
        if ( $record_type eq 'biblio' ) {
217
            _fix_biblio_ids( $marc, $record_number, $record_number );
218
        } else {
219
            _fix_authority_id( $marc, $record_number );
220
        }
221
        if ( C4::Context->preference("marcflavour") eq "UNIMARC" ) {
222
            _fix_unimarc_100($marc);
223
        }
224
225
        my $rec;
226
        if ($as_xml) {
227
            $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
228
            $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
229
        } else {
230
            $rec = $marc->as_usmarc();
231
        }
232
        print {$fh} $rec;
233
234
        $num_exported++;
235
        $i++;
236
        if($verbose) {
237
            print ".";
238
            print "$i\n" unless($i % 100);
239
        }
240
    }
241
    print {$fh} '</collection>' if (_include_xml_wrapper($as_xml, $record_type));
242
    close $fh;
243
244
    return $num_exported;
245
}
246
247
sub _get_corrected_marc_record {
248
    my ( $record_type, $record_number, $noxml ) = @_;
249
250
    my $marc = _get_raw_marc_record( $record_type, $record_number, $noxml );
251
252
    if ( defined $marc ) {
253
        _fix_leader($marc);
254
        if ( $record_type eq 'authority' ) {
255
            _fix_authority_id( $marc, $record_number );
256
        } elsif ($record_type eq 'biblio' && C4::Context->preference('IncludeSeeFromInSearches')) {
257
            my $normalizer = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } );
258
            $marc = $normalizer->process($marc);
259
        }
260
        if ( C4::Context->preference("marcflavour") eq "UNIMARC" ) {
261
            _fix_unimarc_100($marc);
262
        }
263
    }
264
265
    return $marc;
266
}
267
268
sub _get_raw_marc_record {
269
    my ( $record_type, $record_number, $noxml ) = @_;
270
271
    my $dbh = C4::Context->dbh;
272
    my $marc;
273
    if ( $record_type eq 'biblio' ) {
274
        if ($noxml) {
275
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems
276
                WHERE biblionumber = ?");
277
            $fetch_sth->execute($record_number);
278
            if ( my ($blob) = $fetch_sth->fetchrow_array ) {
279
                $marc = MARC::Record->new_from_usmarc($blob);
280
                unless ($marc) {
281
                    warn "error creating MARC::Record from $blob";
282
                }
283
            }
284
            # failure to find a bib is not a problem -
285
            # a delete could have been done before
286
            # trying to process a record update
287
288
            $fetch_sth->finish();
289
            return unless $marc;
290
        } else {
291
            eval { $marc = GetMarcBiblio($record_number, 1); };
292
            if ($@ || !$marc) {
293
                # here we do warn since catching an exception
294
                # means that the bib was found but failed
295
                # to be parsed
296
                warn "error retrieving biblio $record_number: $@";
297
                return;
298
            }
299
        }
300
    } else {
301
        eval { $marc = C4::AuthoritiesMarc::GetAuthority($record_number); };
302
        if ($@) {
303
            warn "error retrieving authority $record_number: $@";
304
            return;
305
        }
306
    }
307
    return $marc;
308
}
309
310
sub _fix_leader {
311
312
    # FIXME - this routine is suspect
313
    # It blanks the Leader/00-05 and Leader/12-16 to
314
    # force them to be recalculated correct when
315
    # the $marc->as_usmarc() or $marc->as_xml() is called.
316
    # But why is this necessary?  It would be a serious bug
317
    # in MARC::Record (definitely) and MARC::File::XML (arguably)
318
    # if they are emitting incorrect leader values.
319
    my $marc = shift;
320
321
    my $leader = $marc->leader;
322
    substr( $leader, 0,  5 ) = '     ';
323
    substr( $leader, 10, 7 ) = '22     ';
324
    $marc->leader( substr( $leader, 0, 24 ) );
325
}
326
327
328
sub _fix_biblio_ids {
329
330
    # FIXME - it is essential to ensure that the biblionumber is present,
331
    #         otherwise, Zebra will choke on the record.  However, this
332
    #         logic belongs in the relevant C4::Biblio APIs.
333
    my $marc         = shift;
334
    my $biblionumber = shift;
335
    my $dbh = C4::Context->dbh;
336
    my $biblioitemnumber;
337
    if (@_) {
338
        $biblioitemnumber = shift;
339
    } else {
340
        my $sth = $dbh->prepare(
341
            "SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
342
        $sth->execute($biblionumber);
343
        ($biblioitemnumber) = $sth->fetchrow_array;
344
        $sth->finish;
345
        unless ($biblioitemnumber) {
346
            warn "failed to get biblioitemnumber for biblio $biblionumber";
347
            return 0;
348
        }
349
    }
350
351
    # FIXME - this is cheating on two levels
352
    # 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
353
    # 2. Making sure that the biblionumber and biblioitemnumber are correct and
354
    #    present in the MARC::Record object ought to be part of GetMarcBiblio.
355
    #
356
    # On the other hand, this better for now than what rebuild_zebra.pl used to
357
    # do, which was duplicate the code for inserting the biblionumber
358
    # and biblioitemnumber
359
    C4::Biblio::_koha_marc_update_bib_ids( $marc, '', $biblionumber, $biblioitemnumber );
360
361
    return 1;
362
}
363
364
sub _fix_authority_id {
365
366
    # FIXME - as with fix_biblio_ids, the authid must be present
367
    #         for Zebra's sake.  However, this really belongs
368
    #         in C4::AuthoritiesMarc.
369
    my ( $marc, $authid ) = @_;
370
    unless ( $marc->field('001') and $marc->field('001')->data() eq $authid ) {
371
        $marc->delete_field( $marc->field('001') );
372
        $marc->insert_fields_ordered( MARC::Field->new( '001', $authid ) );
373
    }
374
}
375
376
sub _fix_unimarc_100 {
377
378
    # FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
379
    my $marc = shift;
380
381
    my $string;
382
    if ( length( $marc->subfield( 100, "a" ) ) == 36 ) {
383
        $string = $marc->subfield( 100, "a" );
384
        my $f100 = $marc->field(100);
385
        $marc->delete_field($f100);
386
    } else {
387
        $string = POSIX::strftime( "%Y%m%d", localtime );
388
        $string =~ s/\-//g;
389
        $string = sprintf( "%-*s", 35, $string );
390
    }
391
    substr( $string, 22, 6, "frey50" );
392
    unless ( length( $marc->subfield( 100, "a" ) ) == 36 ) {
393
        $marc->delete_field( $marc->field(100) );
394
        $marc->insert_grouped_field( MARC::Field->new( 100, "", "", "a" => $string ) );
395
    }
396
}
397
398
sub _do_indexing {
399
    my ( $record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt ) = @_;
400
401
    my $zebra_server  = ( $record_type eq 'biblio' ) ? 'biblioserver' : 'authorityserver';
402
    my $zebra_db_name = ( $record_type eq 'biblio' ) ? 'biblios'      : 'authorities';
403
    my $zebra_config  = C4::Context->zebraconfig($zebra_server)->{'config'};
404
    my $zebra_db_dir  = C4::Context->zebraconfig($zebra_server)->{'directory'};
405
406
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index;
407
    system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir");
408
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow;
409
410
}
411
412
sub _include_xml_wrapper {
413
    my $as_xml = shift;
414
    my $record_type = shift;
415
416
    my $bib_index_mode = C4::Context->config('zebra_bib_index_mode') || 'grs1';
417
    my $auth_index_mode = C4::Context->config('zebra_auth_index_mode') || 'dom';
418
419
    return 0 unless $as_xml;
420
    return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom';
421
    return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom';
422
    return 0;
423
424
}
425
426
1;
(-)a/misc/migration_tools/rebuild_zebra.pl (-396 / +52 lines)
Lines 10-17 use File::Path; Link Here
10
use C4::Biblio;
10
use C4::Biblio;
11
use C4::AuthoritiesMarc;
11
use C4::AuthoritiesMarc;
12
use C4::Items;
12
use C4::Items;
13
use Koha::RecordProcessor;
13
use C4::ZebraIndex;
14
use XML::LibXML;
15
14
16
# script that checks zebradir structure & create directories & mandatory files if needed
15
# script that checks zebradir structure & create directories & mandatory files if needed
17
#
16
#
Lines 172-183 if ( $verbose_logging ) { Link Here
172
}
171
}
173
if ($keep_export) {
172
if ($keep_export) {
174
    print "NOTHING cleaned : the export $directory has been kept.\n";
173
    print "NOTHING cleaned : the export $directory has been kept.\n";
175
    print "You can re-run this script with the -s ";
174
    print "You can re-run this script with the -s and -d $directory parameters";
176
    if ($use_tempdir) {
177
        print " and -d $directory parameters";
178
    } else {
179
        print "parameter";
180
    }
181
    print "\n";
175
    print "\n";
182
    print "if you just want to rebuild zebra after changing the record.abs\n";
176
    print "if you just want to rebuild zebra after changing the record.abs\n";
183
    print "or another zebra config file\n";
177
    print "or another zebra config file\n";
Lines 212-279 sub check_zebra_dirs { Link Here
212
sub index_records {
206
sub index_records {
213
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
207
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
214
208
215
    my $num_records_exported = 0;
216
    my $records_deleted;
217
    my $need_reset = check_zebra_dirs($server_dir);
209
    my $need_reset = check_zebra_dirs($server_dir);
218
    if ($need_reset) {
210
    if ($need_reset) {
219
        print "$0: found broken zebra server directories: forcing a rebuild\n";
211
        print "$0: found broken zebra server directories: forcing a rebuild\n";
220
        $reset = 1;
212
        $reset = 1;
221
    }
213
    }
222
    if ($skip_export && $verbose_logging) {
214
215
    if ( $verbose_logging ) {
223
        print "====================\n";
216
        print "====================\n";
224
        print "SKIPPING $record_type export\n";
217
        print "REINDEXING zebra\n";
225
        print "====================\n";
218
        print "====================\n";
226
    } else {
227
        if ( $verbose_logging ) {
228
            print "====================\n";
229
            print "exporting $record_type\n";
230
            print "====================\n";
231
        }
232
        mkdir "$directory" unless (-d $directory);
233
        mkdir "$directory/$record_type" unless (-d "$directory/$record_type");
234
        if ($process_zebraqueue) {
235
            my $entries = select_zebraqueue_records($record_type, 'deleted');
236
            mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type");
237
            $records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_xml);
238
            mark_zebraqueue_batch_done($entries);
239
            $entries = select_zebraqueue_records($record_type, 'updated');
240
            mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type");
241
            $num_records_exported = export_marc_records_from_list($record_type,
242
                                                                  $entries, "$directory/upd_$record_type", $as_xml, $noxml, $records_deleted);
243
            mark_zebraqueue_batch_done($entries);
244
        } else {
245
            my $sth = select_all_records($record_type);
246
            $num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_xml, $noxml, $nosanitize);
247
            unless ($do_not_clear_zebraqueue) {
248
                mark_all_zebraqueue_done($record_type);
249
            }
250
        }
251
    }
219
    }
252
220
253
    #
221
    my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
254
    # and reindexing everything
222
    if ($process_zebraqueue) {
255
    #
223
        # Process 'deleted' records
256
    if ($skip_index) {
224
        my $entries = select_zebraqueue_records( $record_type, 'deleted' );
257
        if ($verbose_logging) {
225
        my @entries_to_delete = map { $_->{biblio_auth_id} } @$entries;
258
            print "====================\n";
226
259
            print "SKIPPING $record_type indexing\n";
227
        C4::ZebraIndex::DeleteRecordIndex($record_type, \@entries_to_delete, {
260
            print "====================\n";
228
            as_xml => $as_xml, noshadow => $noshadow, record_format => $record_fmt,
261
        }
229
            zebraidx_log_opt => $zebraidx_log_opt, verbose => $verbose_logging,
230
            skip_export => $skip_export, skip_index => $skip_index,
231
            keep_export => $keep_export, directory => $directory
232
        });
233
234
        mark_zebraqueue_batch_done($entries);
235
236
        # Process 'updated' records'
237
        $entries = select_zebraqueue_records( $record_type, 'updated' );
238
        my @entries_to_update = map {
239
            my $id = $_->{biblio_auth_id};
240
            # do not try to update deleted records
241
            (0 == grep { $id == $_ } @entries_to_delete) ? $id : ()
242
        } @$entries;
243
244
        C4::ZebraIndex::IndexRecord($record_type, \@entries_to_update, {
245
            as_xml => $as_xml, noxml => $noxml, reset_index => $reset,
246
            noshadow => $noshadow, record_format => $record_fmt,
247
            zebraidx_log_opt => $zebraidx_log_opt, verbose => $verbose_logging,
248
            skip_export => $skip_export, skip_index => $skip_index,
249
            keep_export => $keep_export, directory => $directory
250
        });
251
252
        mark_zebraqueue_batch_done($entries);
262
    } else {
253
    } else {
263
        if ( $verbose_logging ) {
254
        my $sth = select_all_records($record_type);
264
            print "====================\n";
255
        my $entries = $sth->fetchall_arrayref([]);
265
            print "REINDEXING zebra\n";
256
        my @entries_to_update = map { $_->[0] } @$entries;
266
            print "====================\n";
257
267
        }
258
        C4::ZebraIndex::IndexRecord($record_type, \@entries_to_update, {
268
        my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
259
            as_xml => $as_xml, noxml => $noxml, reset_index => $reset,
269
        if ($process_zebraqueue) {
260
            noshadow => $noshadow, record_format => $record_fmt,
270
            do_indexing($record_type, 'adelete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
261
            zebraidx_log_opt => $zebraidx_log_opt, nosanitize => $nosanitize,
271
                if %$records_deleted;
262
            verbose => $verbose_logging, skip_export => $skip_export,
272
            do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
263
            skip_index => $skip_index, keep_export => $keep_export,
273
                if $num_records_exported;
264
            directory => $directory
274
        } else {
265
        });
275
            do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
266
276
                if ($num_records_exported or $skip_export);
267
        unless ($do_not_clear_zebraqueue) {
268
            mark_all_zebraqueue_done($record_type);
277
        }
269
        }
278
    }
270
    }
279
}
271
}
Lines 343-683 sub select_all_biblios { Link Here
343
    return $sth;
335
    return $sth;
344
}
336
}
345
337
346
sub include_xml_wrapper {
347
    my $as_xml = shift;
348
    my $record_type = shift;
349
350
    return 0 unless $as_xml;
351
    return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom';
352
    return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom';
353
    return 0;
354
355
}
356
357
sub export_marc_records_from_sth {
358
    my ($record_type, $sth, $directory, $as_xml, $noxml, $nosanitize) = @_;
359
360
    my $num_exported = 0;
361
    open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!;
362
    if (include_xml_wrapper($as_xml, $record_type)) {
363
        # include XML declaration and root element
364
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
365
    }
366
    my $i = 0;
367
    my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",'');
368
    while (my ($record_number) = $sth->fetchrow_array) {
369
        print "." if ( $verbose_logging );
370
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
371
        if ( $nosanitize ) {
372
            my $marcxml = $record_type eq 'biblio'
373
                          ? GetXmlBiblio( $record_number )
374
                          : GetAuthorityXML( $record_number );
375
            if ($record_type eq 'biblio'){
376
                my @items = GetItemsInfo($record_number);
377
                if (@items){
378
                    my $record = MARC::Record->new;
379
                    $record->encoding('UTF-8');
380
                    my @itemsrecord;
381
                    foreach my $item (@items){
382
                        my $record = Item2Marc($item, $record_number);
383
                        push @itemsrecord, $record->field($itemtag);
384
                    }
385
                    $record->insert_fields_ordered(@itemsrecord);
386
                    my $itemsxml = $record->as_xml_record();
387
                    $marcxml =
388
                        substr($marcxml, 0, length($marcxml)-10) .
389
                        substr($itemsxml, index($itemsxml, "</leader>\n", 0) + 10);
390
                }
391
            }
392
            # extra test to ensure that result is valid XML; otherwise
393
            # Zebra won't parse it in DOM mode
394
            eval {
395
                my $doc = $tester->parse_string($marcxml);
396
            };
397
            if ($@) {
398
                warn "Error exporting record $record_number ($record_type): $@\n";
399
                next;
400
            }
401
            if ( $marcxml ) {
402
                $marcxml =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
403
                print {$fh} $marcxml;
404
                $num_exported++;
405
            }
406
            next;
407
        }
408
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
409
        if (defined $marc) {
410
            eval {
411
                my $rec;
412
                if ($as_xml) {
413
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
414
                    eval {
415
                        my $doc = $tester->parse_string($rec);
416
                    };
417
                    if ($@) {
418
                        die "invalid XML: $@";
419
                    }
420
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
421
                } else {
422
                    $rec = $marc->as_usmarc();
423
                }
424
                print {$fh} $rec;
425
                $num_exported++;
426
            };
427
            if ($@) {
428
                warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
429
                warn "... specific error is $@" if $verbose_logging;
430
            }
431
        }
432
    }
433
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
434
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
435
    close $fh;
436
    return $num_exported;
437
}
438
439
sub export_marc_records_from_list {
440
    my ($record_type, $entries, $directory, $as_xml, $noxml, $records_deleted) = @_;
441
442
    my $num_exported = 0;
443
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
444
    if (include_xml_wrapper($as_xml, $record_type)) {
445
        # include XML declaration and root element
446
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
447
    }
448
    my $i = 0;
449
450
    # Skip any deleted records. We check for this anyway, but this reduces error spam
451
    my %found = %$records_deleted;
452
    foreach my $record_number ( map { $_->{biblio_auth_number} }
453
                                grep { !$found{ $_->{biblio_auth_number} }++ }
454
                                @$entries ) {
455
        print "." if ( $verbose_logging );
456
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
457
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
458
        if (defined $marc) {
459
            eval {
460
                my $rec;
461
                if ($as_xml) {
462
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
463
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
464
                } else {
465
                    $rec = $marc->as_usmarc();
466
                }
467
                print {$fh} $rec;
468
                $num_exported++;
469
            };
470
            if ($@) {
471
              warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
472
            }
473
        }
474
    }
475
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
476
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
477
    close $fh;
478
    return $num_exported;
479
}
480
481
sub generate_deleted_marc_records {
482
    my ($record_type, $entries, $directory, $as_xml) = @_;
483
484
    my $records_deleted = {};
485
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
486
    if (include_xml_wrapper($as_xml, $record_type)) {
487
        # include XML declaration and root element
488
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
489
    }
490
    my $i = 0;
491
    foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) {
492
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
493
        print "." if ( $verbose_logging );
494
495
        my $marc = MARC::Record->new();
496
        if ($record_type eq 'biblio') {
497
            fix_biblio_ids($marc, $record_number, $record_number);
498
        } else {
499
            fix_authority_id($marc, $record_number);
500
        }
501
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
502
            fix_unimarc_100($marc);
503
        }
504
505
        my $rec;
506
        if ($as_xml) {
507
            $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
508
            $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
509
        } else {
510
            $rec = $marc->as_usmarc();
511
        }
512
        print {$fh} $rec;
513
514
        $records_deleted->{$record_number} = 1;
515
    }
516
    print "\nRecords exported: $i\n" if ( $verbose_logging );
517
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
518
    close $fh;
519
    return $records_deleted;
520
521
522
}
523
524
sub get_corrected_marc_record {
525
    my ($record_type, $record_number, $noxml) = @_;
526
527
    my $marc = get_raw_marc_record($record_type, $record_number, $noxml);
528
529
    if (defined $marc) {
530
        fix_leader($marc);
531
        if ($record_type eq 'authority') {
532
            fix_authority_id($marc, $record_number);
533
        } elsif ($record_type eq 'biblio' && C4::Context->preference('IncludeSeeFromInSearches')) {
534
            my $normalizer = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } );
535
            $marc = $normalizer->process($marc);
536
        }
537
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
538
            fix_unimarc_100($marc);
539
        }
540
    }
541
542
    return $marc;
543
}
544
545
sub get_raw_marc_record {
546
    my ($record_type, $record_number, $noxml) = @_;
547
548
    my $marc;
549
    if ($record_type eq 'biblio') {
550
        if ($noxml) {
551
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems WHERE biblionumber = ?");
552
            $fetch_sth->execute($record_number);
553
            if (my ($blob) = $fetch_sth->fetchrow_array) {
554
                $marc = MARC::Record->new_from_usmarc($blob);
555
                unless ($marc) {
556
                    warn "error creating MARC::Record from $blob";
557
                }
558
            }
559
            # failure to find a bib is not a problem -
560
            # a delete could have been done before
561
            # trying to process a record update
562
563
            $fetch_sth->finish();
564
            return unless $marc;
565
        } else {
566
            eval { $marc = GetMarcBiblio($record_number, 1); };
567
            if ($@ || !$marc) {
568
                # here we do warn since catching an exception
569
                # means that the bib was found but failed
570
                # to be parsed
571
                warn "error retrieving biblio $record_number";
572
                return;
573
            }
574
        }
575
    } else {
576
        eval { $marc = GetAuthority($record_number); };
577
        if ($@) {
578
            warn "error retrieving authority $record_number";
579
            return;
580
        }
581
    }
582
    return $marc;
583
}
584
585
sub fix_leader {
586
    # FIXME - this routine is suspect
587
    # It blanks the Leader/00-05 and Leader/12-16 to
588
    # force them to be recalculated correct when
589
    # the $marc->as_usmarc() or $marc->as_xml() is called.
590
    # But why is this necessary?  It would be a serious bug
591
    # in MARC::Record (definitely) and MARC::File::XML (arguably)
592
    # if they are emitting incorrect leader values.
593
    my $marc = shift;
594
595
    my $leader = $marc->leader;
596
    substr($leader,  0, 5) = '     ';
597
    substr($leader, 10, 7) = '22     ';
598
    $marc->leader(substr($leader, 0, 24));
599
}
600
601
sub fix_biblio_ids {
602
    # FIXME - it is essential to ensure that the biblionumber is present,
603
    #         otherwise, Zebra will choke on the record.  However, this
604
    #         logic belongs in the relevant C4::Biblio APIs.
605
    my $marc = shift;
606
    my $biblionumber = shift;
607
    my $biblioitemnumber;
608
    if (@_) {
609
        $biblioitemnumber = shift;
610
    } else {
611
        my $sth = $dbh->prepare(
612
            "SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
613
        $sth->execute($biblionumber);
614
        ($biblioitemnumber) = $sth->fetchrow_array;
615
        $sth->finish;
616
        unless ($biblioitemnumber) {
617
            warn "failed to get biblioitemnumber for biblio $biblionumber";
618
            return 0;
619
        }
620
    }
621
622
    # FIXME - this is cheating on two levels
623
    # 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
624
    # 2. Making sure that the biblionumber and biblioitemnumber are correct and
625
    #    present in the MARC::Record object ought to be part of GetMarcBiblio.
626
    #
627
    # On the other hand, this better for now than what rebuild_zebra.pl used to
628
    # do, which was duplicate the code for inserting the biblionumber
629
    # and biblioitemnumber
630
    C4::Biblio::_koha_marc_update_bib_ids($marc, '', $biblionumber, $biblioitemnumber);
631
632
    return 1;
633
}
634
635
sub fix_authority_id {
636
    # FIXME - as with fix_biblio_ids, the authid must be present
637
    #         for Zebra's sake.  However, this really belongs
638
    #         in C4::AuthoritiesMarc.
639
    my ($marc, $authid) = @_;
640
    unless ($marc->field('001') and $marc->field('001')->data() eq $authid){
641
        $marc->delete_field($marc->field('001'));
642
        $marc->insert_fields_ordered(MARC::Field->new('001',$authid));
643
    }
644
}
645
646
sub fix_unimarc_100 {
647
    # FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
648
    my $marc = shift;
649
650
    my $string;
651
    if ( length($marc->subfield( 100, "a" )) == 36 ) {
652
        $string = $marc->subfield( 100, "a" );
653
        my $f100 = $marc->field(100);
654
        $marc->delete_field($f100);
655
    }
656
    else {
657
        $string = POSIX::strftime( "%Y%m%d", localtime );
658
        $string =~ s/\-//g;
659
        $string = sprintf( "%-*s", 35, $string );
660
    }
661
    substr( $string, 22, 6, "frey50" );
662
    unless ( length($marc->subfield( 100, "a" )) == 36 ) {
663
        $marc->delete_field($marc->field(100));
664
        $marc->insert_grouped_field(MARC::Field->new( 100, "", "", "a" => $string ));
665
    }
666
}
667
668
sub do_indexing {
669
    my ($record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt) = @_;
670
671
    my $zebra_server  = ($record_type eq 'biblio') ? 'biblioserver' : 'authorityserver';
672
    my $zebra_db_name = ($record_type eq 'biblio') ? 'biblios' : 'authorities';
673
    my $zebra_config  = C4::Context->zebraconfig($zebra_server)->{'config'};
674
    my $zebra_db_dir  = C4::Context->zebraconfig($zebra_server)->{'directory'};
675
676
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index;
677
    system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir");
678
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow;
679
680
}
681
338
682
sub print_usage {
339
sub print_usage {
683
    print <<_USAGE_;
340
    print <<_USAGE_;
684
- 

Return to bug 7419