View | Details | Raw Unified | Return to bug 7419
Collapse All | Expand All

(-)a/C4/ZebraIndex.pm (+406 lines)
Line 0 Link Here
1
package C4::ZebraIndex;
2
3
# This file is part of Koha.
4
#
5
# Koha is free software; you can redistribute it and/or modify it under the
6
# terms of the GNU General Public License as published by the Free Software
7
# Foundation; either version 2 of the License, or (at your option) any later
8
# version.
9
#
10
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
11
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
12
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
13
#
14
# You should have received a copy of the GNU General Public License along with
15
# Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
16
# Suite 330, Boston, MA  02111-1307 USA
17
18
# This module contains utility functions to index records in Zebra
19
# It is a temporary place to store rebuild_zebra.pl subs in order for them to be
20
# used by other scripts (misc/migration_tools/dedup_authorities.pl for example)
21
# As soon as the new indexation layer is in place, and all scripts using this
22
# module are updated, this module have to be removed.
23
24
use Modern::Perl;
25
use POSIX;
26
use File::Temp qw/tempdir/;
27
use MARC::Record;
28
29
use C4::Context;
30
use C4::Biblio;
31
use C4::AuthoritiesMarc;
32
use C4::Items;
33
34
sub IndexRecord {
35
    my ($recordtype, $recordids, $options) = @_;
36
37
    my $as_xml = ($options and $options->{as_xml}) ? 1 : 0;
38
    my $noxml = ($options and $options->{noxml}) ? 1 : 0;
39
    my $nosanitize = ($options and $options->{nosanitize}) ? $as_xml : 0;
40
    my $reset_index = ($options and $options->{reset_index}) ? 1 : 0;
41
    my $noshadow = ($options and $options->{noshadow}) ? " -n " : "";
42
    my $record_format = ($options and $options->{record_format})
43
                    ? $options->{record_format}
44
                    : ($as_xml)
45
                        ? 'marcxml'
46
                        : 'iso2709';
47
    my $zebraidx_log_opt = ($options and defined $options->{zebraidx_log_opt})
48
                        ? $options->{zebraidx_log_opt}
49
                        : "";
50
    my $verbose = ($options and $options->{verbose}) ? 1 : 0;
51
    my $skip_export = ($options and $options->{skip_export}) ? 1 : 0;
52
    my $skip_index = ($options and $options->{skip_index}) ? 1 : 0;
53
    my $keep_export = ($options and $options->{keep_export}) ? 1 : 0;
54
    my $directory = ($options and $options->{directory})
55
                    ? $options->{directory}
56
                    : tempdir(CLEANUP => ($keep_export ? 0 : 1));
57
58
    my @recordids = (ref $recordids eq "ARRAY") ? @$recordids : split (/ /, $recordids);
59
    @recordids = map { { biblio_auth_number => $_ } } @recordids;
60
61
    my $num_exported = 0;
62
    unless ($skip_export) {
63
        $num_exported = _export_marc_records_from_list($recordtype, \@recordids,
64
            $directory, $as_xml, $noxml, $nosanitize, $verbose);
65
    }
66
    unless ($skip_index) {
67
        _do_indexing($recordtype, "update", $directory, $reset_index, $noshadow,
68
            $record_format, $zebraidx_log_opt);
69
    }
70
71
    return $num_exported;
72
}
73
74
sub DeleteRecordIndex {
75
    my ($recordtype, $recordids, $options) = @_;
76
77
    my $as_xml = ($options and $options->{as_xml}) ? 1 : 0;
78
    my $noshadow = ($options and $options->{noshadow}) ? " -n " : "";
79
    my $record_format = ($options and $options->{record_format})
80
                    ? $options->{record_format}
81
                    : ($as_xml)
82
                        ? 'marcxml'
83
                        : 'iso2709';
84
    my $zebraidx_log_opt = ($options and defined $options->{zebraidx_log_opt})
85
                        ? $options->{zebraidx_log_opt}
86
                        : "";
87
    my $verbose = ($options and $options->{verbose}) ? 1 : 0;
88
    my $skip_export = ($options and $options->{skip_export}) ? 1 : 0;
89
    my $skip_index = ($options and $options->{skip_index}) ? 1 : 0;
90
    my $keep_export = ($options and $options->{keep_export}) ? 1 : 0;
91
    my $directory = ($options and $options->{directory})
92
                    ? $options->{directory}
93
                    : tempdir(CLEANUP => ($keep_export ? 0 : 1));
94
95
    my @recordids = (ref $recordids eq "ARRAY") ? @$recordids : split (/ /, $recordids);
96
    @recordids = map { { biblio_auth_number => $_ } } @recordids;
97
98
    my $num_exported = 0;
99
    unless ($skip_export) {
100
        $num_exported = _generate_deleted_marc_records($recordtype, \@recordids,
101
            $directory, $as_xml, $verbose);
102
    }
103
    unless ($skip_index) {
104
        _do_indexing($recordtype, "delete", $directory, 0, $noshadow,
105
            $record_format, $zebraidx_log_opt);
106
    }
107
108
    return $num_exported;
109
}
110
111
112
sub _export_marc_records_from_list {
113
    my ( $record_type, $entries, $directory, $as_xml, $noxml, $nosanitize, $verbose ) = @_;
114
115
    my $num_exported = 0;
116
    open my $fh, ">:encoding(UTF-8) ", "$directory/exported_records" or die $!;
117
    if (_include_xml_wrapper($as_xml, $record_type)) {
118
        # include XML declaration and root element
119
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
120
    }
121
    my $i     = 0;
122
    my %found = ();
123
    my ($itemtag) = GetMarcFromKohaField("items.itemnumber",'');
124
    foreach my $record_number (
125
        map  { $_->{biblio_auth_number} }
126
        grep { !$found{ $_->{biblio_auth_number} }++ } @$entries
127
    ) {
128
        if($nosanitize) {
129
            my $marcxml = $record_type eq 'biblio'
130
                        ? GetXmlBiblio($record_number)
131
                        : GetAuthorityXML($record_number);
132
            if ($record_type eq 'biblio'){
133
                my @items = GetItemsInfo($record_number);
134
                if (@items){
135
                    my $record = MARC::Record->new;
136
                    $record->encoding('UTF-8');
137
                    my @itemsrecord;
138
                    foreach my $item (@items){
139
                        my $record = Item2Marc($item, $record_number);
140
                        push @itemsrecord, $record->field($itemtag);
141
                    }
142
                    $record->insert_fields_ordered(@itemsrecord);
143
                    my $itemsxml = $record->as_xml_record();
144
                    $marcxml =
145
                        substr($marcxml, 0, length($marcxml)-10) .
146
                        substr($itemsxml, index($itemsxml, "</leader>\n", 0) + 10);
147
                }
148
            }
149
            if ($marcxml) {
150
                print {$fh} $marcxml;
151
                $num_exported++;
152
            }
153
        } else {
154
            my ($marc) = _get_corrected_marc_record( $record_type, $record_number, $noxml );
155
            if ( defined $marc ) {
156
                eval {
157
                    my $rec;
158
                    if ($as_xml) {
159
                        $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
160
                        $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
161
                    } else {
162
                        $rec = $marc->as_usmarc();
163
                    }
164
                    print {$fh} $rec;
165
                    $num_exported++;
166
                };
167
                if ($@) {
168
                    warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
169
                }
170
            }
171
        }
172
        $i++;
173
        if($verbose) {
174
            print ".";
175
            print "$i\n" unless($i % 100);
176
        }
177
    }
178
    print {$fh} '</collection>' if (_include_xml_wrapper($as_xml, $record_type));
179
    close $fh;
180
    print "\n$num_exported records exported to $directory/exported_records\n" if $verbose;
181
    return $num_exported;
182
}
183
184
sub _generate_deleted_marc_records {
185
    my ( $record_type, $entries, $directory, $as_xml, $verbose ) = @_;
186
187
    my $num_exported = 0;
188
    open my $fh, ">:encoding(UTF-8)", "$directory/exported_records" or die $!;
189
    if (_include_xml_wrapper($as_xml, $record_type)) {
190
        # include XML declaration and root element
191
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
192
    }
193
    my $i = 0;
194
    foreach my $record_number ( map { $_->{biblio_auth_number} } @$entries ) {
195
        my $marc = MARC::Record->new();
196
        if ( $record_type eq 'biblio' ) {
197
            _fix_biblio_ids( $marc, $record_number, $record_number );
198
        } else {
199
            _fix_authority_id( $marc, $record_number );
200
        }
201
        if ( C4::Context->preference("marcflavour") eq "UNIMARC" ) {
202
            _fix_unimarc_100($marc);
203
        }
204
205
        my $rec;
206
        if ($as_xml) {
207
            $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
208
            $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
209
        } else {
210
            $rec = $marc->as_usmarc();
211
        }
212
        print {$fh} $rec;
213
214
        $num_exported++;
215
        $i++;
216
        if($verbose) {
217
            print ".";
218
            print "$i\n" unless($i % 100);
219
        }
220
    }
221
    print {$fh} '</collection>' if (_include_xml_wrapper($as_xml, $record_type));
222
    close $fh;
223
224
    return $num_exported;
225
}
226
227
sub _get_corrected_marc_record {
228
    my ( $record_type, $record_number, $noxml ) = @_;
229
230
    my $marc = _get_raw_marc_record( $record_type, $record_number, $noxml );
231
232
    if ( defined $marc ) {
233
        _fix_leader($marc);
234
        if ( $record_type eq 'biblio' ) {
235
            my $succeeded = _fix_biblio_ids( $marc, $record_number );
236
            return unless $succeeded;
237
        } else {
238
            _fix_authority_id( $marc, $record_number );
239
        }
240
        if ( C4::Context->preference("marcflavour") eq "UNIMARC" ) {
241
            _fix_unimarc_100($marc);
242
        }
243
    }
244
245
    return $marc;
246
}
247
248
sub _get_raw_marc_record {
249
    my ( $record_type, $record_number, $noxml ) = @_;
250
251
    my $dbh = C4::Context->dbh;
252
    my $marc;
253
    if ( $record_type eq 'biblio' ) {
254
        if ($noxml) {
255
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems
256
                WHERE biblionumber = ?");
257
            $fetch_sth->execute($record_number);
258
            if ( my ($blob) = $fetch_sth->fetchrow_array ) {
259
                $marc = MARC::Record->new_from_usmarc($blob);
260
                unless ($marc) {
261
                    warn "error creating MARC::Record from $blob";
262
                }
263
            }
264
            # failure to find a bib is not a problem -
265
            # a delete could have been done before
266
            # trying to process a record update
267
268
            $fetch_sth->finish();
269
            return unless $marc;
270
        } else {
271
            eval { $marc = GetMarcBiblio($record_number, 1); };
272
            if ($@ || !$marc) {
273
                # here we do warn since catching an exception
274
                # means that the bib was found but failed
275
                # to be parsed
276
                warn "error retrieving biblio $record_number: $@";
277
                return;
278
            }
279
        }
280
    } else {
281
        eval { $marc = C4::AuthoritiesMarc::GetAuthority($record_number); };
282
        if ($@) {
283
            warn "error retrieving authority $record_number: $@";
284
            return;
285
        }
286
    }
287
    return $marc;
288
}
289
290
sub _fix_leader {
291
292
    # FIXME - this routine is suspect
293
    # It blanks the Leader/00-05 and Leader/12-16 to
294
    # force them to be recalculated correct when
295
    # the $marc->as_usmarc() or $marc->as_xml() is called.
296
    # But why is this necessary?  It would be a serious bug
297
    # in MARC::Record (definitely) and MARC::File::XML (arguably)
298
    # if they are emitting incorrect leader values.
299
    my $marc = shift;
300
301
    my $leader = $marc->leader;
302
    substr( $leader, 0,  5 ) = '     ';
303
    substr( $leader, 10, 7 ) = '22     ';
304
    $marc->leader( substr( $leader, 0, 24 ) );
305
}
306
307
308
sub _fix_biblio_ids {
309
310
    # FIXME - it is essential to ensure that the biblionumber is present,
311
    #         otherwise, Zebra will choke on the record.  However, this
312
    #         logic belongs in the relevant C4::Biblio APIs.
313
    my $marc         = shift;
314
    my $biblionumber = shift;
315
    my $dbh = C4::Context->dbh;
316
    my $biblioitemnumber;
317
    if (@_) {
318
        $biblioitemnumber = shift;
319
    } else {
320
        my $sth = $dbh->prepare(
321
            "SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
322
        $sth->execute($biblionumber);
323
        ($biblioitemnumber) = $sth->fetchrow_array;
324
        $sth->finish;
325
        unless ($biblioitemnumber) {
326
            warn "failed to get biblioitemnumber for biblio $biblionumber";
327
            return 0;
328
        }
329
    }
330
331
    # FIXME - this is cheating on two levels
332
    # 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
333
    # 2. Making sure that the biblionumber and biblioitemnumber are correct and
334
    #    present in the MARC::Record object ought to be part of GetMarcBiblio.
335
    #
336
    # On the other hand, this better for now than what rebuild_zebra.pl used to
337
    # do, which was duplicate the code for inserting the biblionumber
338
    # and biblioitemnumber
339
    C4::Biblio::_koha_marc_update_bib_ids( $marc, '', $biblionumber, $biblioitemnumber );
340
341
    return 1;
342
}
343
344
sub _fix_authority_id {
345
346
    # FIXME - as with fix_biblio_ids, the authid must be present
347
    #         for Zebra's sake.  However, this really belongs
348
    #         in C4::AuthoritiesMarc.
349
    my ( $marc, $authid ) = @_;
350
    unless ( $marc->field('001') and $marc->field('001')->data() eq $authid ) {
351
        $marc->delete_field( $marc->field('001') );
352
        $marc->insert_fields_ordered( MARC::Field->new( '001', $authid ) );
353
    }
354
}
355
356
sub _fix_unimarc_100 {
357
358
    # FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
359
    my $marc = shift;
360
361
    my $string;
362
    if ( length( $marc->subfield( 100, "a" ) ) == 36 ) {
363
        $string = $marc->subfield( 100, "a" );
364
        my $f100 = $marc->field(100);
365
        $marc->delete_field($f100);
366
    } else {
367
        $string = POSIX::strftime( "%Y%m%d", localtime );
368
        $string =~ s/\-//g;
369
        $string = sprintf( "%-*s", 35, $string );
370
    }
371
    substr( $string, 22, 6, "frey50" );
372
    unless ( length( $marc->subfield( 100, "a" ) ) == 36 ) {
373
        $marc->delete_field( $marc->field(100) );
374
        $marc->insert_grouped_field( MARC::Field->new( 100, "", "", "a" => $string ) );
375
    }
376
}
377
378
sub _do_indexing {
379
    my ( $record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt ) = @_;
380
381
    my $zebra_server  = ( $record_type eq 'biblio' ) ? 'biblioserver' : 'authorityserver';
382
    my $zebra_db_name = ( $record_type eq 'biblio' ) ? 'biblios'      : 'authorities';
383
    my $zebra_config  = C4::Context->zebraconfig($zebra_server)->{'config'};
384
    my $zebra_db_dir  = C4::Context->zebraconfig($zebra_server)->{'directory'};
385
386
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index;
387
    system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir");
388
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow;
389
390
}
391
392
sub _include_xml_wrapper {
393
    my $as_xml = shift;
394
    my $record_type = shift;
395
396
    my $bib_index_mode = C4::Context->config('zebra_bib_index_mode') || 'grs1';
397
    my $auth_index_mode = C4::Context->config('zebra_auth_index_mode') || 'dom';
398
399
    return 0 unless $as_xml;
400
    return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom';
401
    return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom';
402
    return 0;
403
404
}
405
406
1;
(-)a/misc/migration_tools/rebuild_zebra.pl (-377 / +54 lines)
Lines 10-15 use File::Path; Link Here
10
use C4::Biblio;
10
use C4::Biblio;
11
use C4::AuthoritiesMarc;
11
use C4::AuthoritiesMarc;
12
use C4::Items;
12
use C4::Items;
13
use C4::ZebraIndex;
13
14
14
# 
15
# 
15
# script that checks zebradir structure & create directories & mandatory files if needed
16
# script that checks zebradir structure & create directories & mandatory files if needed
Lines 155-166 if ( $verbose_logging ) { Link Here
155
}
156
}
156
if ($keep_export) {
157
if ($keep_export) {
157
    print "NOTHING cleaned : the export $directory has been kept.\n";
158
    print "NOTHING cleaned : the export $directory has been kept.\n";
158
    print "You can re-run this script with the -s ";
159
    print "You can re-run this script with the -s and -d $directory parameters";
159
    if ($use_tempdir) {
160
        print " and -d $directory parameters";
161
    } else {
162
        print "parameter";
163
    }
164
    print "\n";
160
    print "\n";
165
    print "if you just want to rebuild zebra after changing the record.abs\n";
161
    print "if you just want to rebuild zebra after changing the record.abs\n";
166
    print "or another zebra config file\n";
162
    print "or another zebra config file\n";
Lines 195-262 sub check_zebra_dirs { Link Here
195
sub index_records {
191
sub index_records {
196
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
192
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
197
193
198
    my $num_records_exported = 0;
199
    my $records_deleted;
200
    my $need_reset = check_zebra_dirs($server_dir);
194
    my $need_reset = check_zebra_dirs($server_dir);
201
    if ($need_reset) {
195
    if ($need_reset) {
202
    	print "$0: found broken zebra server directories: forcing a rebuild\n";
196
        print "$0: found broken zebra server directories: forcing a rebuild\n";
203
    	$reset = 1;
197
        $reset = 1;
204
    }
198
    }
205
    if ($skip_export && $verbose_logging) {
199
200
    if ( $verbose_logging ) {
206
        print "====================\n";
201
        print "====================\n";
207
        print "SKIPPING $record_type export\n";
202
        print "REINDEXING zebra\n";
208
        print "====================\n";
203
        print "====================\n";
209
    } else {
210
        if ( $verbose_logging ) {
211
            print "====================\n";
212
            print "exporting $record_type\n";
213
            print "====================\n";
214
        }
215
        mkdir "$directory" unless (-d $directory);
216
        mkdir "$directory/$record_type" unless (-d "$directory/$record_type");
217
        if ($process_zebraqueue) {
218
            my $entries = select_zebraqueue_records($record_type, 'deleted');
219
            mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type");
220
            $records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_xml);
221
            mark_zebraqueue_batch_done($entries);
222
            $entries = select_zebraqueue_records($record_type, 'updated');
223
            mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type");
224
            $num_records_exported = export_marc_records_from_list($record_type, 
225
                                                                  $entries, "$directory/upd_$record_type", $as_xml, $noxml, $records_deleted);
226
            mark_zebraqueue_batch_done($entries);
227
        } else {
228
            my $sth = select_all_records($record_type);
229
            $num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_xml, $noxml, $nosanitize);
230
            unless ($do_not_clear_zebraqueue) {
231
                mark_all_zebraqueue_done($record_type);
232
            }
233
        }
234
    }
204
    }
235
205
236
    #
206
    my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
237
    # and reindexing everything
207
    if ($process_zebraqueue) {
238
    #
208
        # Process 'deleted' records
239
    if ($skip_index) {
209
        my $entries = select_zebraqueue_records( $record_type, 'deleted' );
240
        if ($verbose_logging) {
210
        my @entries_to_delete = map { $_->{biblio_auth_id} } @$entries;
241
            print "====================\n";
211
242
            print "SKIPPING $record_type indexing\n";
212
        C4::ZebraIndex::DeleteRecordIndex($record_type, \@entries_to_delete, {
243
            print "====================\n";
213
            as_xml => $as_xml, noshadow => $noshadow, record_format => $record_fmt,
244
        }
214
            zebraidx_log_opt => $zebraidx_log_opt, verbose => $verbose_logging,
215
            skip_export => $skip_export, skip_index => $skip_index,
216
            keep_export => $keep_export, directory => $directory
217
        });
218
219
        mark_zebraqueue_batch_done($entries);
220
221
        # Process 'updated' records'
222
        $entries = select_zebraqueue_records( $record_type, 'updated' );
223
        my @entries_to_update = map {
224
            my $id = $_->{biblio_auth_id};
225
            # do not try to update deleted records
226
            (0 == grep { $id == $_ } @entries_to_delete) ? $id : ()
227
        } @$entries;
228
229
        C4::ZebraIndex::IndexRecord($record_type, \@entries_to_update, {
230
            as_xml => $as_xml, noxml => $noxml, reset_index => $reset,
231
            noshadow => $noshadow, record_format => $record_fmt,
232
            zebraidx_log_opt => $zebraidx_log_opt, verbose => $verbose_logging,
233
            skip_export => $skip_export, skip_index => $skip_index,
234
            keep_export => $keep_export, directory => $directory
235
        });
236
237
        mark_zebraqueue_batch_done($entries);
245
    } else {
238
    } else {
246
        if ( $verbose_logging ) {
239
        my $sth = select_all_records($record_type);
247
            print "====================\n";
240
        my $entries = $sth->fetchall_arrayref([]);
248
            print "REINDEXING zebra\n";
241
        my @entries_to_update = map { $_->[0] } @$entries;
249
            print "====================\n";
242
250
        }
243
        C4::ZebraIndex::IndexRecord($record_type, \@entries_to_update, {
251
        my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
244
            as_xml => $as_xml, noxml => $noxml, reset_index => $reset,
252
        if ($process_zebraqueue) {
245
            noshadow => $noshadow, record_format => $record_fmt,
253
            do_indexing($record_type, 'delete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
246
            zebraidx_log_opt => $zebraidx_log_opt, nosanitize => $nosanitize,
254
                if %$records_deleted;
247
            verbose => $verbose_logging, skip_export => $skip_export,
255
            do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
248
            skip_index => $skip_index, keep_export => $keep_export,
256
                if $num_records_exported;
249
            directory => $directory
257
        } else {
250
        });
258
            do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
251
259
                if ($num_records_exported or $skip_export);
252
        unless ($do_not_clear_zebraqueue) {
253
            mark_all_zebraqueue_done($record_type);
260
        }
254
        }
261
    }
255
    }
262
}
256
}
Lines 326-647 sub select_all_biblios { Link Here
326
    return $sth;
320
    return $sth;
327
}
321
}
328
322
329
sub include_xml_wrapper {
330
    my $as_xml = shift;
331
    my $record_type = shift;
332
333
    return 0 unless $as_xml;
334
    return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom';
335
    return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom';
336
    return 0;
337
338
}
339
340
sub export_marc_records_from_sth {
341
    my ($record_type, $sth, $directory, $as_xml, $noxml, $nosanitize) = @_;
342
343
    my $num_exported = 0;
344
    open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!;
345
    if (include_xml_wrapper($as_xml, $record_type)) {
346
        # include XML declaration and root element
347
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
348
    }
349
    my $i = 0;
350
    my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",'');
351
    while (my ($record_number) = $sth->fetchrow_array) {
352
        print "." if ( $verbose_logging );
353
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
354
        if ( $nosanitize ) {
355
            my $marcxml = $record_type eq 'biblio'
356
                          ? GetXmlBiblio( $record_number )
357
                          : GetAuthorityXML( $record_number );
358
            if ($record_type eq 'biblio'){
359
                my @items = GetItemsInfo($record_number);
360
                if (@items){
361
                    my $record = MARC::Record->new;
362
                    $record->encoding('UTF-8');
363
                    my @itemsrecord;
364
                    foreach my $item (@items){
365
                        my $record = Item2Marc($item, $record_number);                        
366
                        push @itemsrecord, $record->field($itemtag);
367
                    }
368
                    $record->insert_fields_ordered(@itemsrecord);
369
                    my $itemsxml = $record->as_xml_record();
370
                    $marcxml =
371
                        substr($marcxml, 0, length($marcxml)-10) .
372
                        substr($itemsxml, index($itemsxml, "</leader>\n", 0) + 10);
373
                }
374
            }
375
            if ( $marcxml ) {
376
                print {$fh} $marcxml if $marcxml;
377
                $num_exported++;
378
            }
379
            next;
380
        }
381
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
382
        if (defined $marc) {
383
            eval {
384
                my $rec;
385
                if ($as_xml) {
386
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
387
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
388
                } else {
389
                    $rec = $marc->as_usmarc();
390
                }
391
                print {$fh} $rec;
392
                $num_exported++;
393
            };
394
            if ($@) {
395
              warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
396
            }
397
        }
398
    }
399
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
400
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
401
    close $fh;
402
    return $num_exported;
403
}
404
405
sub export_marc_records_from_list {
406
    my ($record_type, $entries, $directory, $as_xml, $noxml, $records_deleted) = @_;
407
408
    my $num_exported = 0;
409
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
410
    if (include_xml_wrapper($as_xml, $record_type)) {
411
        # include XML declaration and root element
412
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
413
    }
414
    my $i = 0;
415
416
    # Skip any deleted records. We check for this anyway, but this reduces error spam
417
    my %found = %$records_deleted;
418
    foreach my $record_number ( map { $_->{biblio_auth_number} }
419
                                grep { !$found{ $_->{biblio_auth_number} }++ }
420
                                @$entries ) {
421
        print "." if ( $verbose_logging );
422
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
423
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
424
        if (defined $marc) {
425
            eval {
426
                my $rec;
427
                if ($as_xml) {
428
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
429
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
430
                } else {
431
                    $rec = $marc->as_usmarc();
432
                }
433
                print {$fh} $rec;
434
                $num_exported++;
435
            };
436
            if ($@) {
437
              warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
438
            }
439
            $num_exported++;
440
        }
441
    }
442
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
443
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
444
    close $fh;
445
    return $num_exported;
446
}
447
448
sub generate_deleted_marc_records {
449
    my ($record_type, $entries, $directory, $as_xml) = @_;
450
451
    my $records_deleted = {};
452
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
453
    if (include_xml_wrapper($as_xml, $record_type)) {
454
        # include XML declaration and root element
455
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
456
    }
457
    my $i = 0;
458
    foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) {
459
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
460
        print "." if ( $verbose_logging );
461
462
        my $marc = MARC::Record->new();
463
        if ($record_type eq 'biblio') {
464
            fix_biblio_ids($marc, $record_number, $record_number);
465
        } else {
466
            fix_authority_id($marc, $record_number);
467
        }
468
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
469
            fix_unimarc_100($marc);
470
        }
471
472
        my $rec;
473
        if ($as_xml) {
474
            $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
475
            $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
476
        } else {
477
            $rec = $marc->as_usmarc();
478
        }
479
        print {$fh} $rec;
480
481
        $records_deleted->{$record_number} = 1;
482
    }
483
    print "\nRecords exported: $i\n" if ( $verbose_logging );
484
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
485
    close $fh;
486
    return $records_deleted;
487
    
488
489
}
490
491
sub get_corrected_marc_record {
492
    my ($record_type, $record_number, $noxml) = @_;
493
494
    my $marc = get_raw_marc_record($record_type, $record_number, $noxml); 
495
496
    if (defined $marc) {
497
        fix_leader($marc);
498
        if ($record_type eq 'authority') {
499
            fix_authority_id($marc, $record_number);
500
        }
501
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
502
            fix_unimarc_100($marc);
503
        }
504
    }
505
506
    return $marc;
507
}
508
509
sub get_raw_marc_record {
510
    my ($record_type, $record_number, $noxml) = @_;
511
  
512
    my $marc; 
513
    if ($record_type eq 'biblio') {
514
        if ($noxml) {
515
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems WHERE biblionumber = ?");
516
            $fetch_sth->execute($record_number);
517
            if (my ($blob) = $fetch_sth->fetchrow_array) {
518
                $marc = MARC::Record->new_from_usmarc($blob);
519
                unless ($marc) {
520
                    warn "error creating MARC::Record from $blob";
521
                }
522
            }
523
            # failure to find a bib is not a problem -
524
            # a delete could have been done before
525
            # trying to process a record update
526
527
            $fetch_sth->finish();
528
            return unless $marc;
529
        } else {
530
            eval { $marc = GetMarcBiblio($record_number, 1); };
531
            if ($@ || !$marc) {
532
                # here we do warn since catching an exception
533
                # means that the bib was found but failed
534
                # to be parsed
535
                warn "error retrieving biblio $record_number";
536
                return;
537
            }
538
        }
539
    } else {
540
        eval { $marc = GetAuthority($record_number); };
541
        if ($@) {
542
            warn "error retrieving authority $record_number";
543
            return;
544
        }
545
    }
546
    return $marc;
547
}
548
549
sub fix_leader {
550
    # FIXME - this routine is suspect
551
    # It blanks the Leader/00-05 and Leader/12-16 to
552
    # force them to be recalculated correct when
553
    # the $marc->as_usmarc() or $marc->as_xml() is called.
554
    # But why is this necessary?  It would be a serious bug
555
    # in MARC::Record (definitely) and MARC::File::XML (arguably) 
556
    # if they are emitting incorrect leader values.
557
    my $marc = shift;
558
559
    my $leader = $marc->leader;
560
    substr($leader,  0, 5) = '     ';
561
    substr($leader, 10, 7) = '22     ';
562
    $marc->leader(substr($leader, 0, 24));
563
}
564
565
sub fix_biblio_ids {
566
    # FIXME - it is essential to ensure that the biblionumber is present,
567
    #         otherwise, Zebra will choke on the record.  However, this
568
    #         logic belongs in the relevant C4::Biblio APIs.
569
    my $marc = shift;
570
    my $biblionumber = shift;
571
    my $biblioitemnumber;
572
    if (@_) {
573
        $biblioitemnumber = shift;
574
    } else {    
575
        my $sth = $dbh->prepare(
576
            "SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
577
        $sth->execute($biblionumber);
578
        ($biblioitemnumber) = $sth->fetchrow_array;
579
        $sth->finish;
580
        unless ($biblioitemnumber) {
581
            warn "failed to get biblioitemnumber for biblio $biblionumber";
582
            return 0;
583
        }
584
    }
585
586
    # FIXME - this is cheating on two levels
587
    # 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
588
    # 2. Making sure that the biblionumber and biblioitemnumber are correct and
589
    #    present in the MARC::Record object ought to be part of GetMarcBiblio.
590
    #
591
    # On the other hand, this better for now than what rebuild_zebra.pl used to
592
    # do, which was duplicate the code for inserting the biblionumber 
593
    # and biblioitemnumber
594
    C4::Biblio::_koha_marc_update_bib_ids($marc, '', $biblionumber, $biblioitemnumber);
595
596
    return 1;
597
}
598
599
sub fix_authority_id {
600
    # FIXME - as with fix_biblio_ids, the authid must be present
601
    #         for Zebra's sake.  However, this really belongs
602
    #         in C4::AuthoritiesMarc.
603
    my ($marc, $authid) = @_;
604
    unless ($marc->field('001') and $marc->field('001')->data() eq $authid){
605
        $marc->delete_field($marc->field('001'));
606
        $marc->insert_fields_ordered(MARC::Field->new('001',$authid));
607
    }
608
}
609
610
sub fix_unimarc_100 {
611
    # FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
612
    my $marc = shift;
613
614
    my $string;
615
    if ( length($marc->subfield( 100, "a" )) == 36 ) {
616
        $string = $marc->subfield( 100, "a" );
617
        my $f100 = $marc->field(100);
618
        $marc->delete_field($f100);
619
    }
620
    else {
621
        $string = POSIX::strftime( "%Y%m%d", localtime );
622
        $string =~ s/\-//g;
623
        $string = sprintf( "%-*s", 35, $string );
624
    }
625
    substr( $string, 22, 6, "frey50" );
626
    unless ( length($marc->subfield( 100, "a" )) == 36 ) {
627
        $marc->delete_field($marc->field(100));
628
        $marc->insert_grouped_field(MARC::Field->new( 100, "", "", "a" => $string ));
629
    }
630
}
631
632
sub do_indexing {
633
    my ($record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt) = @_;
634
635
    my $zebra_server  = ($record_type eq 'biblio') ? 'biblioserver' : 'authorityserver';
636
    my $zebra_db_name = ($record_type eq 'biblio') ? 'biblios' : 'authorities';
637
    my $zebra_config  = C4::Context->zebraconfig($zebra_server)->{'config'};
638
    my $zebra_db_dir  = C4::Context->zebraconfig($zebra_server)->{'directory'};
639
640
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index;
641
    system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir");
642
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow;
643
644
}
645
323
646
sub print_usage {
324
sub print_usage {
647
    print <<_USAGE_;
325
    print <<_USAGE_;
648
- 

Return to bug 7419