View | Details | Raw Unified | Return to bug 13705
Collapse All | Expand All

(-)a/C4/ZebraIndex.pm (+426 lines)
Line 0 Link Here
1
package C4::ZebraIndex;
2
3
# This file is part of Koha.
4
#
5
# Koha is free software; you can redistribute it and/or modify it under the
6
# terms of the GNU General Public License as published by the Free Software
7
# Foundation; either version 2 of the License, or (at your option) any later
8
# version.
9
#
10
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
11
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
12
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
13
#
14
# You should have received a copy of the GNU General Public License along with
15
# Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
16
# Suite 330, Boston, MA  02111-1307 USA
17
18
# This module contains utility functions to index records in Zebra
19
# It is a temporary place to store rebuild_zebra.pl subs in order for them to be
20
# used by other scripts (misc/migration_tools/dedup_authorities.pl for example)
21
# As soon as the new indexation layer is in place, and all scripts using this
22
# module are updated, this module have to be removed.
23
24
use Modern::Perl;
25
use POSIX;
26
use File::Temp qw/tempdir/;
27
use MARC::Record;
28
29
use C4::Context;
30
use C4::Biblio;
31
use C4::AuthoritiesMarc;
32
use C4::Items;
33
use XML::LibXML;
34
use Koha::RecordProcessor;
35
36
sub IndexRecord {
37
    my ($recordtype, $recordids, $options) = @_;
38
39
    my $as_xml = ($options and $options->{as_xml}) ? 1 : 0;
40
    my $noxml = ($options and $options->{noxml}) ? 1 : 0;
41
    my $nosanitize = ($options and $options->{nosanitize}) ? $as_xml : 0;
42
    my $reset_index = ($options and $options->{reset_index}) ? 1 : 0;
43
    my $noshadow = ($options and $options->{noshadow}) ? " -n " : "";
44
    my $record_format = ($options and $options->{record_format})
45
                    ? $options->{record_format}
46
                    : ($as_xml)
47
                        ? 'marcxml'
48
                        : 'iso2709';
49
    my $zebraidx_log_opt = ($options and defined $options->{zebraidx_log_opt})
50
                        ? $options->{zebraidx_log_opt}
51
                        : "";
52
    my $verbose = ($options and $options->{verbose}) ? 1 : 0;
53
    my $skip_export = ($options and $options->{skip_export}) ? 1 : 0;
54
    my $skip_index = ($options and $options->{skip_index}) ? 1 : 0;
55
    my $keep_export = ($options and $options->{keep_export}) ? 1 : 0;
56
    my $directory = ($options and $options->{directory})
57
                    ? $options->{directory}
58
                    : tempdir(CLEANUP => ($keep_export ? 0 : 1));
59
60
    my @recordids = (ref $recordids eq "ARRAY") ? @$recordids : split (/ /, $recordids);
61
    @recordids = map { { biblio_auth_number => $_ } } @recordids;
62
63
    my $num_exported = 0;
64
    unless ($skip_export) {
65
        $num_exported = _export_marc_records_from_list($recordtype, \@recordids,
66
            $directory, $as_xml, $noxml, $nosanitize, $verbose);
67
    }
68
    unless ($skip_index) {
69
        _do_indexing($recordtype, "update", $directory, $reset_index, $noshadow,
70
            $record_format, $zebraidx_log_opt);
71
    }
72
73
    return $num_exported;
74
}
75
76
sub DeleteRecordIndex {
77
    my ($recordtype, $recordids, $options) = @_;
78
79
    my $as_xml = ($options and $options->{as_xml}) ? 1 : 0;
80
    my $noshadow = ($options and $options->{noshadow}) ? " -n " : "";
81
    my $record_format = ($options and $options->{record_format})
82
                    ? $options->{record_format}
83
                    : ($as_xml)
84
                        ? 'marcxml'
85
                        : 'iso2709';
86
    my $zebraidx_log_opt = ($options and defined $options->{zebraidx_log_opt})
87
                        ? $options->{zebraidx_log_opt}
88
                        : "";
89
    my $verbose = ($options and $options->{verbose}) ? 1 : 0;
90
    my $skip_export = ($options and $options->{skip_export}) ? 1 : 0;
91
    my $skip_index = ($options and $options->{skip_index}) ? 1 : 0;
92
    my $keep_export = ($options and $options->{keep_export}) ? 1 : 0;
93
    my $directory = ($options and $options->{directory})
94
                    ? $options->{directory}
95
                    : tempdir(CLEANUP => ($keep_export ? 0 : 1));
96
97
    my @recordids = (ref $recordids eq "ARRAY") ? @$recordids : split (/ /, $recordids);
98
    @recordids = map { { biblio_auth_number => $_ } } @recordids;
99
100
    my $num_exported = 0;
101
    unless ($skip_export) {
102
        $num_exported = _generate_deleted_marc_records($recordtype, \@recordids,
103
            $directory, $as_xml, $verbose);
104
    }
105
    unless ($skip_index) {
106
        _do_indexing($recordtype, "adelete", $directory, 0, $noshadow,
107
            $record_format, $zebraidx_log_opt);
108
    }
109
110
    return $num_exported;
111
}
112
113
114
sub _export_marc_records_from_list {
115
    my ( $record_type, $entries, $directory, $as_xml, $noxml, $nosanitize, $verbose ) = @_;
116
117
    my $num_exported = 0;
118
    open my $fh, ">:encoding(UTF-8) ", "$directory/exported_records" or die $!;
119
    if (_include_xml_wrapper($as_xml, $record_type)) {
120
        # include XML declaration and root element
121
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
122
    }
123
    my $i     = 0;
124
    my %found = ();
125
    my ($itemtag) = GetMarcFromKohaField("items.itemnumber",'');
126
    my $tester = XML::LibXML->new();
127
    foreach my $record_number (
128
        map  { $_->{biblio_auth_number} }
129
        grep { !$found{ $_->{biblio_auth_number} }++ } @$entries
130
    ) {
131
        if($nosanitize) {
132
            my $marcxml = $record_type eq 'biblio'
133
                        ? GetXmlBiblio($record_number)
134
                        : GetAuthorityXML($record_number);
135
            if ($record_type eq 'biblio'){
136
                my @items = GetItemsInfo($record_number);
137
                if (@items){
138
                    my $record = MARC::Record->new;
139
                    $record->encoding('UTF-8');
140
                    my @itemsrecord;
141
                    foreach my $item (@items){
142
                        my $record = Item2Marc($item, $record_number);
143
                        push @itemsrecord, $record->field($itemtag);
144
                    }
145
                    $record->insert_fields_ordered(@itemsrecord);
146
                    my $itemsxml = $record->as_xml_record();
147
                    $marcxml =
148
                        substr($marcxml, 0, length($marcxml)-10) .
149
                        substr($itemsxml, index($itemsxml, "</leader>\n", 0) + 10);
150
                }
151
            }
152
            # extra test to ensure that result is valid XML; otherwise
153
            # Zebra won't parse it in DOM mode
154
            eval {
155
                my $doc = $tester->parse_string($marcxml);
156
            };
157
            if ($@) {
158
                warn "Error exporting record $record_number ($record_type): $@\n";
159
                next;
160
            }
161
            if ($marcxml) {
162
                $marcxml =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
163
                print {$fh} $marcxml;
164
                $num_exported++;
165
            }
166
        } else {
167
            my ($marc) = _get_corrected_marc_record( $record_type, $record_number, $noxml );
168
            if ( defined $marc ) {
169
                eval {
170
                    my $rec;
171
                    if ($as_xml) {
172
                        $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
173
                        eval {
174
                            my $doc = $tester->parse_string($rec);
175
                        };
176
                        if ($@) {
177
                            die "invalid XML: $@";
178
                        }
179
                        $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
180
                    } else {
181
                        $rec = $marc->as_usmarc();
182
                    }
183
                    print {$fh} $rec;
184
                    $num_exported++;
185
                };
186
                if ($@) {
187
                    warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
188
                    warn "... specific error is $@" if $verbose;
189
                }
190
            }
191
        }
192
        $i++;
193
        if($verbose) {
194
            print ".";
195
            print "$i\n" unless($i % 100);
196
        }
197
    }
198
    print {$fh} '</collection>' if (_include_xml_wrapper($as_xml, $record_type));
199
    close $fh;
200
    print "\n$num_exported records exported to $directory/exported_records\n" if $verbose;
201
    return $num_exported;
202
}
203
204
sub _generate_deleted_marc_records {
205
    my ( $record_type, $entries, $directory, $as_xml, $verbose ) = @_;
206
207
    my $num_exported = 0;
208
    open my $fh, ">:encoding(UTF-8)", "$directory/exported_records" or die $!;
209
    if (_include_xml_wrapper($as_xml, $record_type)) {
210
        # include XML declaration and root element
211
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
212
    }
213
    my $i = 0;
214
    foreach my $record_number ( map { $_->{biblio_auth_number} } @$entries ) {
215
        my $marc = MARC::Record->new();
216
        if ( $record_type eq 'biblio' ) {
217
            _fix_biblio_ids( $marc, $record_number, $record_number );
218
        } else {
219
            _fix_authority_id( $marc, $record_number );
220
        }
221
        if ( C4::Context->preference("marcflavour") eq "UNIMARC" ) {
222
            _fix_unimarc_100($marc);
223
        }
224
225
        my $rec;
226
        if ($as_xml) {
227
            $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
228
            $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
229
        } else {
230
            $rec = $marc->as_usmarc();
231
        }
232
        print {$fh} $rec;
233
234
        $num_exported++;
235
        $i++;
236
        if($verbose) {
237
            print ".";
238
            print "$i\n" unless($i % 100);
239
        }
240
    }
241
    print {$fh} '</collection>' if (_include_xml_wrapper($as_xml, $record_type));
242
    close $fh;
243
244
    return $num_exported;
245
}
246
247
sub _get_corrected_marc_record {
248
    my ( $record_type, $record_number, $noxml ) = @_;
249
250
    my $marc = _get_raw_marc_record( $record_type, $record_number, $noxml );
251
252
    if ( defined $marc ) {
253
        _fix_leader($marc);
254
        if ( $record_type eq 'authority' ) {
255
            _fix_authority_id( $marc, $record_number );
256
        } elsif ($record_type eq 'biblio' && C4::Context->preference('IncludeSeeFromInSearches')) {
257
            my $normalizer = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } );
258
            $marc = $normalizer->process($marc);
259
        }
260
        if ( C4::Context->preference("marcflavour") eq "UNIMARC" ) {
261
            _fix_unimarc_100($marc);
262
        }
263
    }
264
265
    return $marc;
266
}
267
268
sub _get_raw_marc_record {
269
    my ( $record_type, $record_number, $noxml ) = @_;
270
271
    my $dbh = C4::Context->dbh;
272
    my $marc;
273
    if ( $record_type eq 'biblio' ) {
274
        if ($noxml) {
275
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems
276
                WHERE biblionumber = ?");
277
            $fetch_sth->execute($record_number);
278
            if ( my ($blob) = $fetch_sth->fetchrow_array ) {
279
                $marc = MARC::Record->new_from_usmarc($blob);
280
                unless ($marc) {
281
                    warn "error creating MARC::Record from $blob";
282
                }
283
            }
284
            # failure to find a bib is not a problem -
285
            # a delete could have been done before
286
            # trying to process a record update
287
288
            $fetch_sth->finish();
289
            return unless $marc;
290
        } else {
291
            eval { $marc = GetMarcBiblio($record_number, 1); };
292
            if ($@ || !$marc) {
293
                # here we do warn since catching an exception
294
                # means that the bib was found but failed
295
                # to be parsed
296
                warn "error retrieving biblio $record_number: $@";
297
                return;
298
            }
299
        }
300
    } else {
301
        eval { $marc = C4::AuthoritiesMarc::GetAuthority($record_number); };
302
        if ($@) {
303
            warn "error retrieving authority $record_number: $@";
304
            return;
305
        }
306
    }
307
    return $marc;
308
}
309
310
sub _fix_leader {
311
312
    # FIXME - this routine is suspect
313
    # It blanks the Leader/00-05 and Leader/12-16 to
314
    # force them to be recalculated correct when
315
    # the $marc->as_usmarc() or $marc->as_xml() is called.
316
    # But why is this necessary?  It would be a serious bug
317
    # in MARC::Record (definitely) and MARC::File::XML (arguably)
318
    # if they are emitting incorrect leader values.
319
    my $marc = shift;
320
321
    my $leader = $marc->leader;
322
    substr( $leader, 0,  5 ) = '     ';
323
    substr( $leader, 10, 7 ) = '22     ';
324
    $marc->leader( substr( $leader, 0, 24 ) );
325
}
326
327
328
sub _fix_biblio_ids {
329
330
    # FIXME - it is essential to ensure that the biblionumber is present,
331
    #         otherwise, Zebra will choke on the record.  However, this
332
    #         logic belongs in the relevant C4::Biblio APIs.
333
    my $marc         = shift;
334
    my $biblionumber = shift;
335
    my $dbh = C4::Context->dbh;
336
    my $biblioitemnumber;
337
    if (@_) {
338
        $biblioitemnumber = shift;
339
    } else {
340
        my $sth = $dbh->prepare(
341
            "SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
342
        $sth->execute($biblionumber);
343
        ($biblioitemnumber) = $sth->fetchrow_array;
344
        $sth->finish;
345
        unless ($biblioitemnumber) {
346
            warn "failed to get biblioitemnumber for biblio $biblionumber";
347
            return 0;
348
        }
349
    }
350
351
    # FIXME - this is cheating on two levels
352
    # 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
353
    # 2. Making sure that the biblionumber and biblioitemnumber are correct and
354
    #    present in the MARC::Record object ought to be part of GetMarcBiblio.
355
    #
356
    # On the other hand, this better for now than what rebuild_zebra.pl used to
357
    # do, which was duplicate the code for inserting the biblionumber
358
    # and biblioitemnumber
359
    C4::Biblio::_koha_marc_update_bib_ids( $marc, '', $biblionumber, $biblioitemnumber );
360
361
    return 1;
362
}
363
364
sub _fix_authority_id {
365
366
    # FIXME - as with fix_biblio_ids, the authid must be present
367
    #         for Zebra's sake.  However, this really belongs
368
    #         in C4::AuthoritiesMarc.
369
    my ( $marc, $authid ) = @_;
370
    unless ( $marc->field('001') and $marc->field('001')->data() eq $authid ) {
371
        $marc->delete_field( $marc->field('001') );
372
        $marc->insert_fields_ordered( MARC::Field->new( '001', $authid ) );
373
    }
374
}
375
376
sub _fix_unimarc_100 {
377
378
    # FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
379
    my $marc = shift;
380
381
    my $string;
382
    if ( length( $marc->subfield( 100, "a" ) ) == 36 ) {
383
        $string = $marc->subfield( 100, "a" );
384
        my $f100 = $marc->field(100);
385
        $marc->delete_field($f100);
386
    } else {
387
        $string = POSIX::strftime( "%Y%m%d", localtime );
388
        $string =~ s/\-//g;
389
        $string = sprintf( "%-*s", 35, $string );
390
    }
391
    substr( $string, 22, 6, "frey50" );
392
    unless ( length( $marc->subfield( 100, "a" ) ) == 36 ) {
393
        $marc->delete_field( $marc->field(100) );
394
        $marc->insert_grouped_field( MARC::Field->new( 100, "", "", "a" => $string ) );
395
    }
396
}
397
398
sub _do_indexing {
399
    my ( $record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt ) = @_;
400
401
    my $zebra_server  = ( $record_type eq 'biblio' ) ? 'biblioserver' : 'authorityserver';
402
    my $zebra_db_name = ( $record_type eq 'biblio' ) ? 'biblios'      : 'authorities';
403
    my $zebra_config  = C4::Context->zebraconfig($zebra_server)->{'config'};
404
    my $zebra_db_dir  = C4::Context->zebraconfig($zebra_server)->{'directory'};
405
406
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index;
407
    system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir");
408
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow;
409
410
}
411
412
sub _include_xml_wrapper {
413
    my $as_xml = shift;
414
    my $record_type = shift;
415
416
    my $bib_index_mode = C4::Context->config('zebra_bib_index_mode') // 'dom';
417
    my $auth_index_mode = C4::Context->config('zebra_auth_index_mode') // 'dom';
418
419
    return 0 unless $as_xml;
420
    return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom';
421
    return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom';
422
    return 0;
423
424
}
425
426
1;
(-)a/misc/migration_tools/rebuild_zebra.pl (-400 / +54 lines)
Lines 11-18 use File::Path; Link Here
11
use C4::Biblio;
11
use C4::Biblio;
12
use C4::AuthoritiesMarc;
12
use C4::AuthoritiesMarc;
13
use C4::Items;
13
use C4::Items;
14
use Koha::RecordProcessor;
14
use C4::ZebraIndex;
15
use XML::LibXML;
16
15
17
use constant LOCK_FILENAME => 'rebuild..LCK';
16
use constant LOCK_FILENAME => 'rebuild..LCK';
18
17
Lines 248-259 if ( $verbose_logging ) { Link Here
248
}
247
}
249
if ($keep_export) {
248
if ($keep_export) {
250
    print "NOTHING cleaned : the export $directory has been kept.\n";
249
    print "NOTHING cleaned : the export $directory has been kept.\n";
251
    print "You can re-run this script with the -s ";
250
    print "You can re-run this script with the -s and -d $directory parameters";
252
    if ($use_tempdir) {
253
        print " and -d $directory parameters";
254
    } else {
255
        print "parameter";
256
    }
257
    print "\n";
251
    print "\n";
258
    print "if you just want to rebuild zebra after changing the record.abs\n";
252
    print "if you just want to rebuild zebra after changing the record.abs\n";
259
    print "or another zebra config file\n";
253
    print "or another zebra config file\n";
Lines 324-396 sub check_zebra_dirs { Link Here
324
sub index_records {
318
sub index_records {
325
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
319
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
326
320
327
    my $num_records_exported = 0;
328
    my $records_deleted = {};
329
    my $need_reset = check_zebra_dirs($server_dir);
321
    my $need_reset = check_zebra_dirs($server_dir);
330
    if ($need_reset) {
322
    if ($need_reset) {
331
        print "$0: found broken zebra server directories: forcing a rebuild\n";
323
        print "$0: found broken zebra server directories: forcing a rebuild\n";
332
        $reset = 1;
324
        $reset = 1;
333
    }
325
    }
334
    if ($skip_export && $verbose_logging) {
326
327
    if ( $verbose_logging ) {
335
        print "====================\n";
328
        print "====================\n";
336
        print "SKIPPING $record_type export\n";
329
        print "REINDEXING zebra\n";
337
        print "====================\n";
330
        print "====================\n";
338
    } else {
339
        if ( $verbose_logging ) {
340
            print "====================\n";
341
            print "exporting $record_type\n";
342
            print "====================\n";
343
        }
344
        mkdir "$directory" unless (-d $directory);
345
        mkdir "$directory/$record_type" unless (-d "$directory/$record_type");
346
        if ($process_zebraqueue) {
347
            my $entries;
348
349
            unless ( $process_zebraqueue_skip_deletes ) {
350
                $entries = select_zebraqueue_records($record_type, 'deleted');
351
                mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type");
352
                $records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_xml);
353
                mark_zebraqueue_batch_done($entries);
354
            }
355
356
            $entries = select_zebraqueue_records($record_type, 'updated');
357
            mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type");
358
            $num_records_exported = export_marc_records_from_list($record_type,$entries, "$directory/upd_$record_type", $as_xml, $noxml, $records_deleted);
359
            mark_zebraqueue_batch_done($entries);
360
361
        } else {
362
            my $sth = select_all_records($record_type);
363
            $num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_xml, $noxml, $nosanitize);
364
            unless ($do_not_clear_zebraqueue) {
365
                mark_all_zebraqueue_done($record_type);
366
            }
367
        }
368
    }
331
    }
369
332
370
    #
333
    my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
371
    # and reindexing everything
334
    if ($process_zebraqueue) {
372
    #
335
        my ($entries, @entries_to_delete);
373
    if ($skip_index) {
336
        unless ( $process_zebraqueue_skip_deletes ) {
374
        if ($verbose_logging) {
337
            # Process 'deleted' records
375
            print "====================\n";
338
            $entries = select_zebraqueue_records( $record_type, 'deleted' );
376
            print "SKIPPING $record_type indexing\n";
339
            @entries_to_delete = map { $_->{biblio_auth_id} } @$entries;
377
            print "====================\n";
340
341
            C4::ZebraIndex::DeleteRecordIndex($record_type, \@entries_to_delete, {
342
                as_xml => $as_xml, noshadow => $noshadow, record_format => $record_fmt,
343
                zebraidx_log_opt => $zebraidx_log_opt, verbose => $verbose_logging,
344
                skip_export => $skip_export, skip_index => $skip_index,
345
                keep_export => $keep_export, directory => $directory
346
            });
347
348
            mark_zebraqueue_batch_done($entries);
378
        }
349
        }
350
351
        # Process 'updated' records'
352
        $entries = select_zebraqueue_records( $record_type, 'updated' );
353
        my @entries_to_update = map {
354
            my $id = $_->{biblio_auth_id};
355
            # do not try to update deleted records
356
            (0 == grep { $id == $_ } @entries_to_delete) ? $id : ()
357
        } @$entries;
358
359
        C4::ZebraIndex::IndexRecord($record_type, \@entries_to_update, {
360
            as_xml => $as_xml, noxml => $noxml, reset_index => $reset,
361
            noshadow => $noshadow, record_format => $record_fmt,
362
            zebraidx_log_opt => $zebraidx_log_opt, verbose => $verbose_logging,
363
            skip_export => $skip_export, skip_index => $skip_index,
364
            keep_export => $keep_export, directory => $directory
365
        });
366
367
        mark_zebraqueue_batch_done($entries);
379
    } else {
368
    } else {
380
        if ( $verbose_logging ) {
369
        my $sth = select_all_records($record_type);
381
            print "====================\n";
370
        my $entries = $sth->fetchall_arrayref([]);
382
            print "REINDEXING zebra\n";
371
        my @entries_to_update = map { $_->[0] } @$entries;
383
            print "====================\n";
372
384
        }
373
        C4::ZebraIndex::IndexRecord($record_type, \@entries_to_update, {
385
        my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
374
            as_xml => $as_xml, noxml => $noxml, reset_index => $reset,
386
        if ($process_zebraqueue) {
375
            noshadow => $noshadow, record_format => $record_fmt,
387
            do_indexing($record_type, 'adelete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
376
            zebraidx_log_opt => $zebraidx_log_opt, nosanitize => $nosanitize,
388
                if %$records_deleted;
377
            verbose => $verbose_logging, skip_export => $skip_export,
389
            do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
378
            skip_index => $skip_index, keep_export => $keep_export,
390
                if $num_records_exported;
379
            directory => $directory
391
        } else {
380
        });
392
            do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
381
393
                if ($num_records_exported or $skip_export);
382
        unless ($do_not_clear_zebraqueue) {
383
            mark_all_zebraqueue_done($record_type);
394
        }
384
        }
395
    }
385
    }
396
}
386
}
Lines 462-802 sub select_all_biblios { Link Here
462
    return $sth;
452
    return $sth;
463
}
453
}
464
454
465
sub include_xml_wrapper {
466
    my $as_xml = shift;
467
    my $record_type = shift;
468
469
    return 0 unless $as_xml;
470
    return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom';
471
    return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom';
472
    return 0;
473
474
}
475
476
sub export_marc_records_from_sth {
477
    my ($record_type, $sth, $directory, $as_xml, $noxml, $nosanitize) = @_;
478
479
    my $num_exported = 0;
480
    open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!;
481
    if (include_xml_wrapper($as_xml, $record_type)) {
482
        # include XML declaration and root element
483
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
484
    }
485
    my $i = 0;
486
    my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",'');
487
    while (my ($record_number) = $sth->fetchrow_array) {
488
        print "." if ( $verbose_logging );
489
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
490
        if ( $nosanitize ) {
491
            my $marcxml = $record_type eq 'biblio'
492
                          ? GetXmlBiblio( $record_number )
493
                          : GetAuthorityXML( $record_number );
494
            if ($record_type eq 'biblio'){
495
                my @items = GetItemsInfo($record_number);
496
                if (@items){
497
                    my $record = MARC::Record->new;
498
                    $record->encoding('UTF-8');
499
                    my @itemsrecord;
500
                    foreach my $item (@items){
501
                        my $record = Item2Marc($item, $record_number);
502
                        push @itemsrecord, $record->field($itemtag);
503
                    }
504
                    $record->insert_fields_ordered(@itemsrecord);
505
                    my $itemsxml = $record->as_xml_record();
506
                    $marcxml =
507
                        substr($marcxml, 0, length($marcxml)-10) .
508
                        substr($itemsxml, index($itemsxml, "</leader>\n", 0) + 10);
509
                }
510
            }
511
            # extra test to ensure that result is valid XML; otherwise
512
            # Zebra won't parse it in DOM mode
513
            eval {
514
                my $doc = $tester->parse_string($marcxml);
515
            };
516
            if ($@) {
517
                warn "Error exporting record $record_number ($record_type): $@\n";
518
                next;
519
            }
520
            if ( $marcxml ) {
521
                $marcxml =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
522
                print {$fh} $marcxml;
523
                $num_exported++;
524
            }
525
            next;
526
        }
527
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
528
        if (defined $marc) {
529
            eval {
530
                my $rec;
531
                if ($as_xml) {
532
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
533
                    eval {
534
                        my $doc = $tester->parse_string($rec);
535
                    };
536
                    if ($@) {
537
                        die "invalid XML: $@";
538
                    }
539
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
540
                } else {
541
                    $rec = $marc->as_usmarc();
542
                }
543
                print {$fh} $rec;
544
                $num_exported++;
545
            };
546
            if ($@) {
547
                warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
548
                warn "... specific error is $@" if $verbose_logging;
549
            }
550
        }
551
    }
552
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
553
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
554
    close $fh;
555
    return $num_exported;
556
}
557
558
sub export_marc_records_from_list {
559
    my ($record_type, $entries, $directory, $as_xml, $noxml, $records_deleted) = @_;
560
561
    my $num_exported = 0;
562
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
563
    if (include_xml_wrapper($as_xml, $record_type)) {
564
        # include XML declaration and root element
565
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
566
    }
567
    my $i = 0;
568
569
    # Skip any deleted records. We check for this anyway, but this reduces error spam
570
    my %found = %$records_deleted;
571
    foreach my $record_number ( map { $_->{biblio_auth_number} }
572
                                grep { !$found{ $_->{biblio_auth_number} }++ }
573
                                @$entries ) {
574
        print "." if ( $verbose_logging );
575
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
576
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
577
        if (defined $marc) {
578
            eval {
579
                my $rec;
580
                if ($as_xml) {
581
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
582
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
583
                } else {
584
                    $rec = $marc->as_usmarc();
585
                }
586
                print {$fh} $rec;
587
                $num_exported++;
588
            };
589
            if ($@) {
590
              warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
591
            }
592
        }
593
    }
594
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
595
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
596
    close $fh;
597
    return $num_exported;
598
}
599
600
sub generate_deleted_marc_records {
601
    my ($record_type, $entries, $directory, $as_xml) = @_;
602
603
    my $records_deleted = {};
604
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
605
    if (include_xml_wrapper($as_xml, $record_type)) {
606
        # include XML declaration and root element
607
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
608
    }
609
    my $i = 0;
610
    foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) {
611
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
612
        print "." if ( $verbose_logging );
613
614
        my $marc = MARC::Record->new();
615
        if ($record_type eq 'biblio') {
616
            fix_biblio_ids($marc, $record_number, $record_number);
617
        } else {
618
            fix_authority_id($marc, $record_number);
619
        }
620
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
621
            fix_unimarc_100($marc);
622
        }
623
624
        my $rec;
625
        if ($as_xml) {
626
            $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
627
            $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
628
        } else {
629
            $rec = $marc->as_usmarc();
630
        }
631
        print {$fh} $rec;
632
633
        $records_deleted->{$record_number} = 1;
634
    }
635
    print "\nRecords exported: $i\n" if ( $verbose_logging );
636
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
637
    close $fh;
638
    return $records_deleted;
639
640
641
}
642
643
sub get_corrected_marc_record {
644
    my ($record_type, $record_number, $noxml) = @_;
645
646
    my $marc = get_raw_marc_record($record_type, $record_number, $noxml);
647
648
    if (defined $marc) {
649
        fix_leader($marc);
650
        if ($record_type eq 'authority') {
651
            fix_authority_id($marc, $record_number);
652
        } elsif ($record_type eq 'biblio' && C4::Context->preference('IncludeSeeFromInSearches')) {
653
            my $normalizer = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } );
654
            $marc = $normalizer->process($marc);
655
        }
656
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
657
            fix_unimarc_100($marc);
658
        }
659
    }
660
661
    return $marc;
662
}
663
664
sub get_raw_marc_record {
665
    my ($record_type, $record_number, $noxml) = @_;
666
667
    my $marc;
668
    if ($record_type eq 'biblio') {
669
        if ($noxml) {
670
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems WHERE biblionumber = ?");
671
            $fetch_sth->execute($record_number);
672
            if (my ($blob) = $fetch_sth->fetchrow_array) {
673
                $marc = MARC::Record->new_from_usmarc($blob);
674
                unless ($marc) {
675
                    warn "error creating MARC::Record from $blob";
676
                }
677
            }
678
            # failure to find a bib is not a problem -
679
            # a delete could have been done before
680
            # trying to process a record update
681
682
            $fetch_sth->finish();
683
            return unless $marc;
684
        } else {
685
            eval { $marc = GetMarcBiblio($record_number, 1); };
686
            if ($@ || !$marc) {
687
                # here we do warn since catching an exception
688
                # means that the bib was found but failed
689
                # to be parsed
690
                warn "error retrieving biblio $record_number";
691
                return;
692
            }
693
        }
694
    } else {
695
        eval { $marc = GetAuthority($record_number); };
696
        if ($@) {
697
            warn "error retrieving authority $record_number";
698
            return;
699
        }
700
    }
701
    return $marc;
702
}
703
704
sub fix_leader {
705
    # FIXME - this routine is suspect
706
    # It blanks the Leader/00-05 and Leader/12-16 to
707
    # force them to be recalculated correct when
708
    # the $marc->as_usmarc() or $marc->as_xml() is called.
709
    # But why is this necessary?  It would be a serious bug
710
    # in MARC::Record (definitely) and MARC::File::XML (arguably)
711
    # if they are emitting incorrect leader values.
712
    my $marc = shift;
713
714
    my $leader = $marc->leader;
715
    substr($leader,  0, 5) = '     ';
716
    substr($leader, 10, 7) = '22     ';
717
    $marc->leader(substr($leader, 0, 24));
718
}
719
720
sub fix_biblio_ids {
721
    # FIXME - it is essential to ensure that the biblionumber is present,
722
    #         otherwise, Zebra will choke on the record.  However, this
723
    #         logic belongs in the relevant C4::Biblio APIs.
724
    my $marc = shift;
725
    my $biblionumber = shift;
726
    my $biblioitemnumber;
727
    if (@_) {
728
        $biblioitemnumber = shift;
729
    } else {
730
        my $sth = $dbh->prepare(
731
            "SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
732
        $sth->execute($biblionumber);
733
        ($biblioitemnumber) = $sth->fetchrow_array;
734
        $sth->finish;
735
        unless ($biblioitemnumber) {
736
            warn "failed to get biblioitemnumber for biblio $biblionumber";
737
            return 0;
738
        }
739
    }
740
741
    # FIXME - this is cheating on two levels
742
    # 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
743
    # 2. Making sure that the biblionumber and biblioitemnumber are correct and
744
    #    present in the MARC::Record object ought to be part of GetMarcBiblio.
745
    #
746
    # On the other hand, this better for now than what rebuild_zebra.pl used to
747
    # do, which was duplicate the code for inserting the biblionumber
748
    # and biblioitemnumber
749
    C4::Biblio::_koha_marc_update_bib_ids($marc, '', $biblionumber, $biblioitemnumber);
750
751
    return 1;
752
}
753
754
sub fix_authority_id {
755
    # FIXME - as with fix_biblio_ids, the authid must be present
756
    #         for Zebra's sake.  However, this really belongs
757
    #         in C4::AuthoritiesMarc.
758
    my ($marc, $authid) = @_;
759
    unless ($marc->field('001') and $marc->field('001')->data() eq $authid){
760
        $marc->delete_field($marc->field('001'));
761
        $marc->insert_fields_ordered(MARC::Field->new('001',$authid));
762
    }
763
}
764
765
sub fix_unimarc_100 {
766
    # FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
767
    my $marc = shift;
768
769
    my $string;
770
    if ( length($marc->subfield( 100, "a" )) == 36 ) {
771
        $string = $marc->subfield( 100, "a" );
772
        my $f100 = $marc->field(100);
773
        $marc->delete_field($f100);
774
    }
775
    else {
776
        $string = POSIX::strftime( "%Y%m%d", localtime );
777
        $string =~ s/\-//g;
778
        $string = sprintf( "%-*s", 35, $string );
779
    }
780
    substr( $string, 22, 6, "frey50" );
781
    unless ( length($marc->subfield( 100, "a" )) == 36 ) {
782
        $marc->delete_field($marc->field(100));
783
        $marc->insert_grouped_field(MARC::Field->new( 100, "", "", "a" => $string ));
784
    }
785
}
786
787
sub do_indexing {
788
    my ($record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt) = @_;
789
790
    my $zebra_server  = ($record_type eq 'biblio') ? 'biblioserver' : 'authorityserver';
791
    my $zebra_db_name = ($record_type eq 'biblio') ? 'biblios' : 'authorities';
792
    my $zebra_config  = C4::Context->zebraconfig($zebra_server)->{'config'};
793
    my $zebra_db_dir  = C4::Context->zebraconfig($zebra_server)->{'directory'};
794
795
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index;
796
    system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir");
797
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow;
798
799
}
800
455
801
sub _flock {
456
sub _flock {
802
    # test if flock is present; if so, use it; if not, return true
457
    # test if flock is present; if so, use it; if not, return true
803
- 

Return to bug 13705