View | Details | Raw Unified | Return to bug 13705
Collapse All | Expand All

(-)a/C4/ZebraIndex.pm (+426 lines)
Line 0 Link Here
1
package C4::ZebraIndex;
2
3
# This file is part of Koha.
4
#
5
# Koha is free software; you can redistribute it and/or modify it under the
6
# terms of the GNU General Public License as published by the Free Software
7
# Foundation; either version 2 of the License, or (at your option) any later
8
# version.
9
#
10
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
11
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
12
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
13
#
14
# You should have received a copy of the GNU General Public License along with
15
# Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
16
# Suite 330, Boston, MA  02111-1307 USA
17
18
# This module contains utility functions to index records in Zebra
19
# It is a temporary place to store rebuild_zebra.pl subs in order for them to be
20
# used by other scripts (misc/migration_tools/dedup_authorities.pl for example)
21
# As soon as the new indexation layer is in place, and all scripts using this
22
# module are updated, this module have to be removed.
23
24
use Modern::Perl;
25
use POSIX;
26
use File::Temp qw/tempdir/;
27
use MARC::Record;
28
29
use C4::Context;
30
use C4::Biblio;
31
use C4::AuthoritiesMarc;
32
use C4::Items;
33
use XML::LibXML;
34
use Koha::RecordProcessor;
35
36
sub IndexRecord {
37
    my ($recordtype, $recordids, $options) = @_;
38
39
    my $as_xml = ($options and $options->{as_xml}) ? 1 : 0;
40
    my $noxml = ($options and $options->{noxml}) ? 1 : 0;
41
    my $nosanitize = ($options and $options->{nosanitize}) ? $as_xml : 0;
42
    my $reset_index = ($options and $options->{reset_index}) ? 1 : 0;
43
    my $noshadow = ($options and $options->{noshadow}) ? " -n " : "";
44
    my $record_format = ($options and $options->{record_format})
45
                    ? $options->{record_format}
46
                    : ($as_xml)
47
                        ? 'marcxml'
48
                        : 'iso2709';
49
    my $zebraidx_log_opt = ($options and defined $options->{zebraidx_log_opt})
50
                        ? $options->{zebraidx_log_opt}
51
                        : "";
52
    my $verbose = ($options and $options->{verbose}) ? 1 : 0;
53
    my $skip_export = ($options and $options->{skip_export}) ? 1 : 0;
54
    my $skip_index = ($options and $options->{skip_index}) ? 1 : 0;
55
    my $keep_export = ($options and $options->{keep_export}) ? 1 : 0;
56
    my $directory = ($options and $options->{directory})
57
                    ? $options->{directory}
58
                    : tempdir(CLEANUP => ($keep_export ? 0 : 1));
59
60
    my @recordids = (ref $recordids eq "ARRAY") ? @$recordids : split (/ /, $recordids);
61
    @recordids = map { { biblio_auth_number => $_ } } @recordids;
62
63
    my $num_exported = 0;
64
    unless ($skip_export) {
65
        $num_exported = _export_marc_records_from_list($recordtype, \@recordids,
66
            $directory, $as_xml, $noxml, $nosanitize, $verbose);
67
    }
68
    unless ($skip_index) {
69
        _do_indexing($recordtype, "update", $directory, $reset_index, $noshadow,
70
            $record_format, $zebraidx_log_opt);
71
    }
72
73
    return $num_exported;
74
}
75
76
sub DeleteRecordIndex {
77
    my ($recordtype, $recordids, $options) = @_;
78
79
    my $as_xml = ($options and $options->{as_xml}) ? 1 : 0;
80
    my $noshadow = ($options and $options->{noshadow}) ? " -n " : "";
81
    my $record_format = ($options and $options->{record_format})
82
                    ? $options->{record_format}
83
                    : ($as_xml)
84
                        ? 'marcxml'
85
                        : 'iso2709';
86
    my $zebraidx_log_opt = ($options and defined $options->{zebraidx_log_opt})
87
                        ? $options->{zebraidx_log_opt}
88
                        : "";
89
    my $verbose = ($options and $options->{verbose}) ? 1 : 0;
90
    my $skip_export = ($options and $options->{skip_export}) ? 1 : 0;
91
    my $skip_index = ($options and $options->{skip_index}) ? 1 : 0;
92
    my $keep_export = ($options and $options->{keep_export}) ? 1 : 0;
93
    my $directory = ($options and $options->{directory})
94
                    ? $options->{directory}
95
                    : tempdir(CLEANUP => ($keep_export ? 0 : 1));
96
97
    my @recordids = (ref $recordids eq "ARRAY") ? @$recordids : split (/ /, $recordids);
98
    @recordids = map { { biblio_auth_number => $_ } } @recordids;
99
100
    my $num_exported = 0;
101
    unless ($skip_export) {
102
        $num_exported = _generate_deleted_marc_records($recordtype, \@recordids,
103
            $directory, $as_xml, $verbose);
104
    }
105
    unless ($skip_index) {
106
        _do_indexing($recordtype, "adelete", $directory, 0, $noshadow,
107
            $record_format, $zebraidx_log_opt);
108
    }
109
110
    return $num_exported;
111
}
112
113
114
sub _export_marc_records_from_list {
115
    my ( $record_type, $entries, $directory, $as_xml, $noxml, $nosanitize, $verbose ) = @_;
116
117
    my $num_exported = 0;
118
    open my $fh, ">:encoding(UTF-8) ", "$directory/exported_records" or die $!;
119
    if (_include_xml_wrapper($as_xml, $record_type)) {
120
        # include XML declaration and root element
121
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
122
    }
123
    my $i     = 0;
124
    my %found = ();
125
    my ($itemtag) = GetMarcFromKohaField("items.itemnumber",'');
126
    my $tester = XML::LibXML->new();
127
    foreach my $record_number (
128
        map  { $_->{biblio_auth_number} }
129
        grep { !$found{ $_->{biblio_auth_number} }++ } @$entries
130
    ) {
131
        if($nosanitize) {
132
            my $marcxml = $record_type eq 'biblio'
133
                        ? GetXmlBiblio($record_number)
134
                        : GetAuthorityXML($record_number);
135
            if ($record_type eq 'biblio'){
136
                my @items = GetItemsInfo($record_number);
137
                if (@items){
138
                    my $record = MARC::Record->new;
139
                    $record->encoding('UTF-8');
140
                    my @itemsrecord;
141
                    foreach my $item (@items){
142
                        my $record = Item2Marc($item, $record_number);
143
                        push @itemsrecord, $record->field($itemtag);
144
                    }
145
                    $record->insert_fields_ordered(@itemsrecord);
146
                    my $itemsxml = $record->as_xml_record();
147
                    $marcxml =
148
                        substr($marcxml, 0, length($marcxml)-10) .
149
                        substr($itemsxml, index($itemsxml, "</leader>\n", 0) + 10);
150
                }
151
            }
152
            # extra test to ensure that result is valid XML; otherwise
153
            # Zebra won't parse it in DOM mode
154
            eval {
155
                my $doc = $tester->parse_string($marcxml);
156
            };
157
            if ($@) {
158
                warn "Error exporting record $record_number ($record_type): $@\n";
159
                next;
160
            }
161
            if ($marcxml) {
162
                $marcxml =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
163
                print {$fh} $marcxml;
164
                $num_exported++;
165
            }
166
        } else {
167
            my ($marc) = _get_corrected_marc_record( $record_type, $record_number, $noxml );
168
            if ( defined $marc ) {
169
                eval {
170
                    my $rec;
171
                    if ($as_xml) {
172
                        $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
173
                        eval {
174
                            my $doc = $tester->parse_string($rec);
175
                        };
176
                        if ($@) {
177
                            die "invalid XML: $@";
178
                        }
179
                        $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
180
                    } else {
181
                        $rec = $marc->as_usmarc();
182
                    }
183
                    print {$fh} $rec;
184
                    $num_exported++;
185
                };
186
                if ($@) {
187
                    warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
188
                    warn "... specific error is $@" if $verbose;
189
                }
190
            }
191
        }
192
        $i++;
193
        if($verbose) {
194
            print ".";
195
            print "$i\n" unless($i % 100);
196
        }
197
    }
198
    print {$fh} '</collection>' if (_include_xml_wrapper($as_xml, $record_type));
199
    close $fh;
200
    print "\n$num_exported records exported to $directory/exported_records\n" if $verbose;
201
    return $num_exported;
202
}
203
204
sub _generate_deleted_marc_records {
205
    my ( $record_type, $entries, $directory, $as_xml, $verbose ) = @_;
206
207
    my $num_exported = 0;
208
    open my $fh, ">:encoding(UTF-8)", "$directory/exported_records" or die $!;
209
    if (_include_xml_wrapper($as_xml, $record_type)) {
210
        # include XML declaration and root element
211
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
212
    }
213
    my $i = 0;
214
    foreach my $record_number ( map { $_->{biblio_auth_number} } @$entries ) {
215
        my $marc = MARC::Record->new();
216
        if ( $record_type eq 'biblio' ) {
217
            _fix_biblio_ids( $marc, $record_number, $record_number );
218
        } else {
219
            _fix_authority_id( $marc, $record_number );
220
        }
221
        if ( C4::Context->preference("marcflavour") eq "UNIMARC" ) {
222
            _fix_unimarc_100($marc);
223
        }
224
225
        my $rec;
226
        if ($as_xml) {
227
            $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
228
            $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
229
        } else {
230
            $rec = $marc->as_usmarc();
231
        }
232
        print {$fh} $rec;
233
234
        $num_exported++;
235
        $i++;
236
        if($verbose) {
237
            print ".";
238
            print "$i\n" unless($i % 100);
239
        }
240
    }
241
    print {$fh} '</collection>' if (_include_xml_wrapper($as_xml, $record_type));
242
    close $fh;
243
244
    return $num_exported;
245
}
246
247
sub _get_corrected_marc_record {
248
    my ( $record_type, $record_number, $noxml ) = @_;
249
250
    my $marc = _get_raw_marc_record( $record_type, $record_number, $noxml );
251
252
    if ( defined $marc ) {
253
        _fix_leader($marc);
254
        if ( $record_type eq 'authority' ) {
255
            _fix_authority_id( $marc, $record_number );
256
        } elsif ($record_type eq 'biblio' && C4::Context->preference('IncludeSeeFromInSearches')) {
257
            my $normalizer = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } );
258
            $marc = $normalizer->process($marc);
259
        }
260
        if ( C4::Context->preference("marcflavour") eq "UNIMARC" ) {
261
            _fix_unimarc_100($marc);
262
        }
263
    }
264
265
    return $marc;
266
}
267
268
sub _get_raw_marc_record {
269
    my ( $record_type, $record_number, $noxml ) = @_;
270
271
    my $dbh = C4::Context->dbh;
272
    my $marc;
273
    if ( $record_type eq 'biblio' ) {
274
        if ($noxml) {
275
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems
276
                WHERE biblionumber = ?");
277
            $fetch_sth->execute($record_number);
278
            if ( my ($blob) = $fetch_sth->fetchrow_array ) {
279
                $marc = MARC::Record->new_from_usmarc($blob);
280
                unless ($marc) {
281
                    warn "error creating MARC::Record from $blob";
282
                }
283
            }
284
            # failure to find a bib is not a problem -
285
            # a delete could have been done before
286
            # trying to process a record update
287
288
            $fetch_sth->finish();
289
            return unless $marc;
290
        } else {
291
            eval { $marc = GetMarcBiblio($record_number, 1); };
292
            if ($@ || !$marc) {
293
                # here we do warn since catching an exception
294
                # means that the bib was found but failed
295
                # to be parsed
296
                warn "error retrieving biblio $record_number: $@";
297
                return;
298
            }
299
        }
300
    } else {
301
        eval { $marc = C4::AuthoritiesMarc::GetAuthority($record_number); };
302
        if ($@) {
303
            warn "error retrieving authority $record_number: $@";
304
            return;
305
        }
306
    }
307
    return $marc;
308
}
309
310
sub _fix_leader {
311
312
    # FIXME - this routine is suspect
313
    # It blanks the Leader/00-05 and Leader/12-16 to
314
    # force them to be recalculated correct when
315
    # the $marc->as_usmarc() or $marc->as_xml() is called.
316
    # But why is this necessary?  It would be a serious bug
317
    # in MARC::Record (definitely) and MARC::File::XML (arguably)
318
    # if they are emitting incorrect leader values.
319
    my $marc = shift;
320
321
    my $leader = $marc->leader;
322
    substr( $leader, 0,  5 ) = '     ';
323
    substr( $leader, 10, 7 ) = '22     ';
324
    $marc->leader( substr( $leader, 0, 24 ) );
325
}
326
327
328
sub _fix_biblio_ids {
329
330
    # FIXME - it is essential to ensure that the biblionumber is present,
331
    #         otherwise, Zebra will choke on the record.  However, this
332
    #         logic belongs in the relevant C4::Biblio APIs.
333
    my $marc         = shift;
334
    my $biblionumber = shift;
335
    my $dbh = C4::Context->dbh;
336
    my $biblioitemnumber;
337
    if (@_) {
338
        $biblioitemnumber = shift;
339
    } else {
340
        my $sth = $dbh->prepare(
341
            "SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
342
        $sth->execute($biblionumber);
343
        ($biblioitemnumber) = $sth->fetchrow_array;
344
        $sth->finish;
345
        unless ($biblioitemnumber) {
346
            warn "failed to get biblioitemnumber for biblio $biblionumber";
347
            return 0;
348
        }
349
    }
350
351
    # FIXME - this is cheating on two levels
352
    # 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
353
    # 2. Making sure that the biblionumber and biblioitemnumber are correct and
354
    #    present in the MARC::Record object ought to be part of GetMarcBiblio.
355
    #
356
    # On the other hand, this better for now than what rebuild_zebra.pl used to
357
    # do, which was duplicate the code for inserting the biblionumber
358
    # and biblioitemnumber
359
    C4::Biblio::_koha_marc_update_bib_ids( $marc, '', $biblionumber, $biblioitemnumber );
360
361
    return 1;
362
}
363
364
sub _fix_authority_id {
365
366
    # FIXME - as with fix_biblio_ids, the authid must be present
367
    #         for Zebra's sake.  However, this really belongs
368
    #         in C4::AuthoritiesMarc.
369
    my ( $marc, $authid ) = @_;
370
    unless ( $marc->field('001') and $marc->field('001')->data() eq $authid ) {
371
        $marc->delete_field( $marc->field('001') );
372
        $marc->insert_fields_ordered( MARC::Field->new( '001', $authid ) );
373
    }
374
}
375
376
sub _fix_unimarc_100 {
377
378
    # FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
379
    my $marc = shift;
380
381
    my $string;
382
    if ( length( $marc->subfield( 100, "a" ) ) == 36 ) {
383
        $string = $marc->subfield( 100, "a" );
384
        my $f100 = $marc->field(100);
385
        $marc->delete_field($f100);
386
    } else {
387
        $string = POSIX::strftime( "%Y%m%d", localtime );
388
        $string =~ s/\-//g;
389
        $string = sprintf( "%-*s", 35, $string );
390
    }
391
    substr( $string, 22, 6, "frey50" );
392
    unless ( length( $marc->subfield( 100, "a" ) ) == 36 ) {
393
        $marc->delete_field( $marc->field(100) );
394
        $marc->insert_grouped_field( MARC::Field->new( 100, "", "", "a" => $string ) );
395
    }
396
}
397
398
sub _do_indexing {
399
    my ( $record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt ) = @_;
400
401
    my $zebra_server  = ( $record_type eq 'biblio' ) ? 'biblioserver' : 'authorityserver';
402
    my $zebra_db_name = ( $record_type eq 'biblio' ) ? 'biblios'      : 'authorities';
403
    my $zebra_config  = C4::Context->zebraconfig($zebra_server)->{'config'};
404
    my $zebra_db_dir  = C4::Context->zebraconfig($zebra_server)->{'directory'};
405
406
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index;
407
    system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir");
408
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow;
409
410
}
411
412
sub _include_xml_wrapper {
413
    my $as_xml = shift;
414
    my $record_type = shift;
415
416
    my $bib_index_mode = C4::Context->config('zebra_bib_index_mode') // 'dom';
417
    my $auth_index_mode = C4::Context->config('zebra_auth_index_mode') // 'dom';
418
419
    return 0 unless $as_xml;
420
    return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom';
421
    return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom';
422
    return 0;
423
424
}
425
426
1;
(-)a/misc/migration_tools/rebuild_zebra.pl (-400 / +54 lines)
Lines 11-18 use File::Path; Link Here
11
use C4::Biblio;
11
use C4::Biblio;
12
use C4::AuthoritiesMarc;
12
use C4::AuthoritiesMarc;
13
use C4::Items;
13
use C4::Items;
14
use Koha::RecordProcessor;
14
use C4::ZebraIndex;
15
use XML::LibXML;
16
15
17
use constant LOCK_FILENAME => 'rebuild..LCK';
16
use constant LOCK_FILENAME => 'rebuild..LCK';
18
17
Lines 239-250 if ( $verbose_logging ) { Link Here
239
}
238
}
240
if ($keep_export) {
239
if ($keep_export) {
241
    print "NOTHING cleaned : the export $directory has been kept.\n";
240
    print "NOTHING cleaned : the export $directory has been kept.\n";
242
    print "You can re-run this script with the -s ";
241
    print "You can re-run this script with the -s and -d $directory parameters";
243
    if ($use_tempdir) {
244
        print " and -d $directory parameters";
245
    } else {
246
        print "parameter";
247
    }
248
    print "\n";
242
    print "\n";
249
    print "if you just want to rebuild zebra after changing the record.abs\n";
243
    print "if you just want to rebuild zebra after changing the record.abs\n";
250
    print "or another zebra config file\n";
244
    print "or another zebra config file\n";
Lines 315-387 sub check_zebra_dirs { Link Here
315
sub index_records {
309
sub index_records {
316
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
310
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
317
311
318
    my $num_records_exported = 0;
319
    my $records_deleted = {};
320
    my $need_reset = check_zebra_dirs($server_dir);
312
    my $need_reset = check_zebra_dirs($server_dir);
321
    if ($need_reset) {
313
    if ($need_reset) {
322
        print "$0: found broken zebra server directories: forcing a rebuild\n";
314
        print "$0: found broken zebra server directories: forcing a rebuild\n";
323
        $reset = 1;
315
        $reset = 1;
324
    }
316
    }
325
    if ($skip_export && $verbose_logging) {
317
318
    if ( $verbose_logging ) {
326
        print "====================\n";
319
        print "====================\n";
327
        print "SKIPPING $record_type export\n";
320
        print "REINDEXING zebra\n";
328
        print "====================\n";
321
        print "====================\n";
329
    } else {
330
        if ( $verbose_logging ) {
331
            print "====================\n";
332
            print "exporting $record_type\n";
333
            print "====================\n";
334
        }
335
        mkdir "$directory" unless (-d $directory);
336
        mkdir "$directory/$record_type" unless (-d "$directory/$record_type");
337
        if ($process_zebraqueue) {
338
            my $entries;
339
340
            unless ( $process_zebraqueue_skip_deletes ) {
341
                $entries = select_zebraqueue_records($record_type, 'deleted');
342
                mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type");
343
                $records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_xml);
344
                mark_zebraqueue_batch_done($entries);
345
            }
346
347
            $entries = select_zebraqueue_records($record_type, 'updated');
348
            mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type");
349
            $num_records_exported = export_marc_records_from_list($record_type,$entries, "$directory/upd_$record_type", $as_xml, $noxml, $records_deleted);
350
            mark_zebraqueue_batch_done($entries);
351
352
        } else {
353
            my $sth = select_all_records($record_type);
354
            $num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_xml, $noxml, $nosanitize);
355
            unless ($do_not_clear_zebraqueue) {
356
                mark_all_zebraqueue_done($record_type);
357
            }
358
        }
359
    }
322
    }
360
323
361
    #
324
    my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
362
    # and reindexing everything
325
    if ($process_zebraqueue) {
363
    #
326
        my ($entries, @entries_to_delete);
364
    if ($skip_index) {
327
        unless ( $process_zebraqueue_skip_deletes ) {
365
        if ($verbose_logging) {
328
            # Process 'deleted' records
366
            print "====================\n";
329
            $entries = select_zebraqueue_records( $record_type, 'deleted' );
367
            print "SKIPPING $record_type indexing\n";
330
            @entries_to_delete = map { $_->{biblio_auth_id} } @$entries;
368
            print "====================\n";
331
332
            C4::ZebraIndex::DeleteRecordIndex($record_type, \@entries_to_delete, {
333
                as_xml => $as_xml, noshadow => $noshadow, record_format => $record_fmt,
334
                zebraidx_log_opt => $zebraidx_log_opt, verbose => $verbose_logging,
335
                skip_export => $skip_export, skip_index => $skip_index,
336
                keep_export => $keep_export, directory => $directory
337
            });
338
339
            mark_zebraqueue_batch_done($entries);
369
        }
340
        }
341
342
        # Process 'updated' records'
343
        $entries = select_zebraqueue_records( $record_type, 'updated' );
344
        my @entries_to_update = map {
345
            my $id = $_->{biblio_auth_id};
346
            # do not try to update deleted records
347
            (0 == grep { $id == $_ } @entries_to_delete) ? $id : ()
348
        } @$entries;
349
350
        C4::ZebraIndex::IndexRecord($record_type, \@entries_to_update, {
351
            as_xml => $as_xml, noxml => $noxml, reset_index => $reset,
352
            noshadow => $noshadow, record_format => $record_fmt,
353
            zebraidx_log_opt => $zebraidx_log_opt, verbose => $verbose_logging,
354
            skip_export => $skip_export, skip_index => $skip_index,
355
            keep_export => $keep_export, directory => $directory
356
        });
357
358
        mark_zebraqueue_batch_done($entries);
370
    } else {
359
    } else {
371
        if ( $verbose_logging ) {
360
        my $sth = select_all_records($record_type);
372
            print "====================\n";
361
        my $entries = $sth->fetchall_arrayref([]);
373
            print "REINDEXING zebra\n";
362
        my @entries_to_update = map { $_->[0] } @$entries;
374
            print "====================\n";
363
375
        }
364
        C4::ZebraIndex::IndexRecord($record_type, \@entries_to_update, {
376
        my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
365
            as_xml => $as_xml, noxml => $noxml, reset_index => $reset,
377
        if ($process_zebraqueue) {
366
            noshadow => $noshadow, record_format => $record_fmt,
378
            do_indexing($record_type, 'adelete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
367
            zebraidx_log_opt => $zebraidx_log_opt, nosanitize => $nosanitize,
379
                if %$records_deleted;
368
            verbose => $verbose_logging, skip_export => $skip_export,
380
            do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
369
            skip_index => $skip_index, keep_export => $keep_export,
381
                if $num_records_exported;
370
            directory => $directory
382
        } else {
371
        });
383
            do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
372
384
                if ($num_records_exported or $skip_export);
373
        unless ($do_not_clear_zebraqueue) {
374
            mark_all_zebraqueue_done($record_type);
385
        }
375
        }
386
    }
376
    }
387
}
377
}
Lines 451-791 sub select_all_biblios { Link Here
451
    return $sth;
441
    return $sth;
452
}
442
}
453
443
454
sub include_xml_wrapper {
455
    my $as_xml = shift;
456
    my $record_type = shift;
457
458
    return 0 unless $as_xml;
459
    return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom';
460
    return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom';
461
    return 0;
462
463
}
464
465
sub export_marc_records_from_sth {
466
    my ($record_type, $sth, $directory, $as_xml, $noxml, $nosanitize) = @_;
467
468
    my $num_exported = 0;
469
    open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!;
470
    if (include_xml_wrapper($as_xml, $record_type)) {
471
        # include XML declaration and root element
472
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
473
    }
474
    my $i = 0;
475
    my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",'');
476
    while (my ($record_number) = $sth->fetchrow_array) {
477
        print "." if ( $verbose_logging );
478
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
479
        if ( $nosanitize ) {
480
            my $marcxml = $record_type eq 'biblio'
481
                          ? GetXmlBiblio( $record_number )
482
                          : GetAuthorityXML( $record_number );
483
            if ($record_type eq 'biblio'){
484
                my @items = GetItemsInfo($record_number);
485
                if (@items){
486
                    my $record = MARC::Record->new;
487
                    $record->encoding('UTF-8');
488
                    my @itemsrecord;
489
                    foreach my $item (@items){
490
                        my $record = Item2Marc($item, $record_number);
491
                        push @itemsrecord, $record->field($itemtag);
492
                    }
493
                    $record->insert_fields_ordered(@itemsrecord);
494
                    my $itemsxml = $record->as_xml_record();
495
                    $marcxml =
496
                        substr($marcxml, 0, length($marcxml)-10) .
497
                        substr($itemsxml, index($itemsxml, "</leader>\n", 0) + 10);
498
                }
499
            }
500
            # extra test to ensure that result is valid XML; otherwise
501
            # Zebra won't parse it in DOM mode
502
            eval {
503
                my $doc = $tester->parse_string($marcxml);
504
            };
505
            if ($@) {
506
                warn "Error exporting record $record_number ($record_type): $@\n";
507
                next;
508
            }
509
            if ( $marcxml ) {
510
                $marcxml =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
511
                print {$fh} $marcxml;
512
                $num_exported++;
513
            }
514
            next;
515
        }
516
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
517
        if (defined $marc) {
518
            eval {
519
                my $rec;
520
                if ($as_xml) {
521
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
522
                    eval {
523
                        my $doc = $tester->parse_string($rec);
524
                    };
525
                    if ($@) {
526
                        die "invalid XML: $@";
527
                    }
528
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
529
                } else {
530
                    $rec = $marc->as_usmarc();
531
                }
532
                print {$fh} $rec;
533
                $num_exported++;
534
            };
535
            if ($@) {
536
                warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
537
                warn "... specific error is $@" if $verbose_logging;
538
            }
539
        }
540
    }
541
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
542
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
543
    close $fh;
544
    return $num_exported;
545
}
546
547
sub export_marc_records_from_list {
548
    my ($record_type, $entries, $directory, $as_xml, $noxml, $records_deleted) = @_;
549
550
    my $num_exported = 0;
551
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
552
    if (include_xml_wrapper($as_xml, $record_type)) {
553
        # include XML declaration and root element
554
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
555
    }
556
    my $i = 0;
557
558
    # Skip any deleted records. We check for this anyway, but this reduces error spam
559
    my %found = %$records_deleted;
560
    foreach my $record_number ( map { $_->{biblio_auth_number} }
561
                                grep { !$found{ $_->{biblio_auth_number} }++ }
562
                                @$entries ) {
563
        print "." if ( $verbose_logging );
564
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
565
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
566
        if (defined $marc) {
567
            eval {
568
                my $rec;
569
                if ($as_xml) {
570
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
571
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
572
                } else {
573
                    $rec = $marc->as_usmarc();
574
                }
575
                print {$fh} $rec;
576
                $num_exported++;
577
            };
578
            if ($@) {
579
              warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
580
            }
581
        }
582
    }
583
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
584
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
585
    close $fh;
586
    return $num_exported;
587
}
588
589
sub generate_deleted_marc_records {
590
    my ($record_type, $entries, $directory, $as_xml) = @_;
591
592
    my $records_deleted = {};
593
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
594
    if (include_xml_wrapper($as_xml, $record_type)) {
595
        # include XML declaration and root element
596
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
597
    }
598
    my $i = 0;
599
    foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) {
600
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
601
        print "." if ( $verbose_logging );
602
603
        my $marc = MARC::Record->new();
604
        if ($record_type eq 'biblio') {
605
            fix_biblio_ids($marc, $record_number, $record_number);
606
        } else {
607
            fix_authority_id($marc, $record_number);
608
        }
609
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
610
            fix_unimarc_100($marc);
611
        }
612
613
        my $rec;
614
        if ($as_xml) {
615
            $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
616
            $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
617
        } else {
618
            $rec = $marc->as_usmarc();
619
        }
620
        print {$fh} $rec;
621
622
        $records_deleted->{$record_number} = 1;
623
    }
624
    print "\nRecords exported: $i\n" if ( $verbose_logging );
625
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
626
    close $fh;
627
    return $records_deleted;
628
629
630
}
631
632
sub get_corrected_marc_record {
633
    my ($record_type, $record_number, $noxml) = @_;
634
635
    my $marc = get_raw_marc_record($record_type, $record_number, $noxml);
636
637
    if (defined $marc) {
638
        fix_leader($marc);
639
        if ($record_type eq 'authority') {
640
            fix_authority_id($marc, $record_number);
641
        } elsif ($record_type eq 'biblio' && C4::Context->preference('IncludeSeeFromInSearches')) {
642
            my $normalizer = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } );
643
            $marc = $normalizer->process($marc);
644
        }
645
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
646
            fix_unimarc_100($marc);
647
        }
648
    }
649
650
    return $marc;
651
}
652
653
sub get_raw_marc_record {
654
    my ($record_type, $record_number, $noxml) = @_;
655
656
    my $marc;
657
    if ($record_type eq 'biblio') {
658
        if ($noxml) {
659
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems WHERE biblionumber = ?");
660
            $fetch_sth->execute($record_number);
661
            if (my ($blob) = $fetch_sth->fetchrow_array) {
662
                $marc = MARC::Record->new_from_usmarc($blob);
663
                unless ($marc) {
664
                    warn "error creating MARC::Record from $blob";
665
                }
666
            }
667
            # failure to find a bib is not a problem -
668
            # a delete could have been done before
669
            # trying to process a record update
670
671
            $fetch_sth->finish();
672
            return unless $marc;
673
        } else {
674
            eval { $marc = GetMarcBiblio($record_number, 1); };
675
            if ($@ || !$marc) {
676
                # here we do warn since catching an exception
677
                # means that the bib was found but failed
678
                # to be parsed
679
                warn "error retrieving biblio $record_number";
680
                return;
681
            }
682
        }
683
    } else {
684
        eval { $marc = GetAuthority($record_number); };
685
        if ($@) {
686
            warn "error retrieving authority $record_number";
687
            return;
688
        }
689
    }
690
    return $marc;
691
}
692
693
sub fix_leader {
694
    # FIXME - this routine is suspect
695
    # It blanks the Leader/00-05 and Leader/12-16 to
696
    # force them to be recalculated correct when
697
    # the $marc->as_usmarc() or $marc->as_xml() is called.
698
    # But why is this necessary?  It would be a serious bug
699
    # in MARC::Record (definitely) and MARC::File::XML (arguably)
700
    # if they are emitting incorrect leader values.
701
    my $marc = shift;
702
703
    my $leader = $marc->leader;
704
    substr($leader,  0, 5) = '     ';
705
    substr($leader, 10, 7) = '22     ';
706
    $marc->leader(substr($leader, 0, 24));
707
}
708
709
sub fix_biblio_ids {
710
    # FIXME - it is essential to ensure that the biblionumber is present,
711
    #         otherwise, Zebra will choke on the record.  However, this
712
    #         logic belongs in the relevant C4::Biblio APIs.
713
    my $marc = shift;
714
    my $biblionumber = shift;
715
    my $biblioitemnumber;
716
    if (@_) {
717
        $biblioitemnumber = shift;
718
    } else {
719
        my $sth = $dbh->prepare(
720
            "SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
721
        $sth->execute($biblionumber);
722
        ($biblioitemnumber) = $sth->fetchrow_array;
723
        $sth->finish;
724
        unless ($biblioitemnumber) {
725
            warn "failed to get biblioitemnumber for biblio $biblionumber";
726
            return 0;
727
        }
728
    }
729
730
    # FIXME - this is cheating on two levels
731
    # 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
732
    # 2. Making sure that the biblionumber and biblioitemnumber are correct and
733
    #    present in the MARC::Record object ought to be part of GetMarcBiblio.
734
    #
735
    # On the other hand, this better for now than what rebuild_zebra.pl used to
736
    # do, which was duplicate the code for inserting the biblionumber
737
    # and biblioitemnumber
738
    C4::Biblio::_koha_marc_update_bib_ids($marc, '', $biblionumber, $biblioitemnumber);
739
740
    return 1;
741
}
742
743
sub fix_authority_id {
744
    # FIXME - as with fix_biblio_ids, the authid must be present
745
    #         for Zebra's sake.  However, this really belongs
746
    #         in C4::AuthoritiesMarc.
747
    my ($marc, $authid) = @_;
748
    unless ($marc->field('001') and $marc->field('001')->data() eq $authid){
749
        $marc->delete_field($marc->field('001'));
750
        $marc->insert_fields_ordered(MARC::Field->new('001',$authid));
751
    }
752
}
753
754
sub fix_unimarc_100 {
755
    # FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
756
    my $marc = shift;
757
758
    my $string;
759
    if ( length($marc->subfield( 100, "a" )) == 36 ) {
760
        $string = $marc->subfield( 100, "a" );
761
        my $f100 = $marc->field(100);
762
        $marc->delete_field($f100);
763
    }
764
    else {
765
        $string = POSIX::strftime( "%Y%m%d", localtime );
766
        $string =~ s/\-//g;
767
        $string = sprintf( "%-*s", 35, $string );
768
    }
769
    substr( $string, 22, 6, "frey50" );
770
    unless ( length($marc->subfield( 100, "a" )) == 36 ) {
771
        $marc->delete_field($marc->field(100));
772
        $marc->insert_grouped_field(MARC::Field->new( 100, "", "", "a" => $string ));
773
    }
774
}
775
776
sub do_indexing {
777
    my ($record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt) = @_;
778
779
    my $zebra_server  = ($record_type eq 'biblio') ? 'biblioserver' : 'authorityserver';
780
    my $zebra_db_name = ($record_type eq 'biblio') ? 'biblios' : 'authorities';
781
    my $zebra_config  = C4::Context->zebraconfig($zebra_server)->{'config'};
782
    my $zebra_db_dir  = C4::Context->zebraconfig($zebra_server)->{'directory'};
783
784
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index;
785
    system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir");
786
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow;
787
788
}
789
444
790
sub _flock {
445
sub _flock {
791
    # test if flock is present; if so, use it; if not, return true
446
    # test if flock is present; if so, use it; if not, return true
792
- 

Return to bug 13705