View | Details | Raw Unified | Return to bug 7419
Collapse All | Expand All

(-)a/C4/ZebraIndex.pm (+407 lines)
Line 0 Link Here
1
package C4::ZebraIndex;
2
3
# This file is part of Koha.
4
#
5
# Koha is free software; you can redistribute it and/or modify it under the
6
# terms of the GNU General Public License as published by the Free Software
7
# Foundation; either version 2 of the License, or (at your option) any later
8
# version.
9
#
10
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
11
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
12
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
13
#
14
# You should have received a copy of the GNU General Public License along with
15
# Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
16
# Suite 330, Boston, MA  02111-1307 USA
17
18
# This module contains utility functions to index records in Zebra
19
# It is a temporary place to store rebuild_zebra.pl subs in order for them to be
20
# used by other scripts (misc/migration_tools/dedup_authorities.pl for example)
21
# As soon as the new indexation layer is in place, and all scripts using this
22
# module are updated, this module have to be removed.
23
24
use Modern::Perl;
25
use POSIX;
26
use File::Temp qw/tempdir/;
27
use MARC::Record;
28
29
use C4::Context;
30
use C4::Biblio;
31
use C4::AuthoritiesMarc;
32
use C4::Items;
33
use Koha::RecordProcessor;
34
35
sub IndexRecord {
36
    my ($recordtype, $recordids, $options) = @_;
37
38
    my $as_xml = ($options and $options->{as_xml}) ? 1 : 0;
39
    my $noxml = ($options and $options->{noxml}) ? 1 : 0;
40
    my $nosanitize = ($options and $options->{nosanitize}) ? $as_xml : 0;
41
    my $reset_index = ($options and $options->{reset_index}) ? 1 : 0;
42
    my $noshadow = ($options and $options->{noshadow}) ? " -n " : "";
43
    my $record_format = ($options and $options->{record_format})
44
                    ? $options->{record_format}
45
                    : ($as_xml)
46
                        ? 'marcxml'
47
                        : 'iso2709';
48
    my $zebraidx_log_opt = ($options and defined $options->{zebraidx_log_opt})
49
                        ? $options->{zebraidx_log_opt}
50
                        : "";
51
    my $verbose = ($options and $options->{verbose}) ? 1 : 0;
52
    my $skip_export = ($options and $options->{skip_export}) ? 1 : 0;
53
    my $skip_index = ($options and $options->{skip_index}) ? 1 : 0;
54
    my $keep_export = ($options and $options->{keep_export}) ? 1 : 0;
55
    my $directory = ($options and $options->{directory})
56
                    ? $options->{directory}
57
                    : tempdir(CLEANUP => ($keep_export ? 0 : 1));
58
59
    my @recordids = (ref $recordids eq "ARRAY") ? @$recordids : split (/ /, $recordids);
60
    @recordids = map { { biblio_auth_number => $_ } } @recordids;
61
62
    my $num_exported = 0;
63
    unless ($skip_export) {
64
        $num_exported = _export_marc_records_from_list($recordtype, \@recordids,
65
            $directory, $as_xml, $noxml, $nosanitize, $verbose);
66
    }
67
    unless ($skip_index) {
68
        _do_indexing($recordtype, "update", $directory, $reset_index, $noshadow,
69
            $record_format, $zebraidx_log_opt);
70
    }
71
72
    return $num_exported;
73
}
74
75
sub DeleteRecordIndex {
76
    my ($recordtype, $recordids, $options) = @_;
77
78
    my $as_xml = ($options and $options->{as_xml}) ? 1 : 0;
79
    my $noshadow = ($options and $options->{noshadow}) ? " -n " : "";
80
    my $record_format = ($options and $options->{record_format})
81
                    ? $options->{record_format}
82
                    : ($as_xml)
83
                        ? 'marcxml'
84
                        : 'iso2709';
85
    my $zebraidx_log_opt = ($options and defined $options->{zebraidx_log_opt})
86
                        ? $options->{zebraidx_log_opt}
87
                        : "";
88
    my $verbose = ($options and $options->{verbose}) ? 1 : 0;
89
    my $skip_export = ($options and $options->{skip_export}) ? 1 : 0;
90
    my $skip_index = ($options and $options->{skip_index}) ? 1 : 0;
91
    my $keep_export = ($options and $options->{keep_export}) ? 1 : 0;
92
    my $directory = ($options and $options->{directory})
93
                    ? $options->{directory}
94
                    : tempdir(CLEANUP => ($keep_export ? 0 : 1));
95
96
    my @recordids = (ref $recordids eq "ARRAY") ? @$recordids : split (/ /, $recordids);
97
    @recordids = map { { biblio_auth_number => $_ } } @recordids;
98
99
    my $num_exported = 0;
100
    unless ($skip_export) {
101
        $num_exported = _generate_deleted_marc_records($recordtype, \@recordids,
102
            $directory, $as_xml, $verbose);
103
    }
104
    unless ($skip_index) {
105
        _do_indexing($recordtype, "adelete", $directory, 0, $noshadow,
106
            $record_format, $zebraidx_log_opt);
107
    }
108
109
    return $num_exported;
110
}
111
112
113
sub _export_marc_records_from_list {
114
    my ( $record_type, $entries, $directory, $as_xml, $noxml, $nosanitize, $verbose ) = @_;
115
116
    my $num_exported = 0;
117
    open my $fh, ">:encoding(UTF-8) ", "$directory/exported_records" or die $!;
118
    if (_include_xml_wrapper($as_xml, $record_type)) {
119
        # include XML declaration and root element
120
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
121
    }
122
    my $i     = 0;
123
    my %found = ();
124
    my ($itemtag) = GetMarcFromKohaField("items.itemnumber",'');
125
    foreach my $record_number (
126
        map  { $_->{biblio_auth_number} }
127
        grep { !$found{ $_->{biblio_auth_number} }++ } @$entries
128
    ) {
129
        if($nosanitize) {
130
            my $marcxml = $record_type eq 'biblio'
131
                        ? GetXmlBiblio($record_number)
132
                        : GetAuthorityXML($record_number);
133
            if ($record_type eq 'biblio'){
134
                my @items = GetItemsInfo($record_number);
135
                if (@items){
136
                    my $record = MARC::Record->new;
137
                    $record->encoding('UTF-8');
138
                    my @itemsrecord;
139
                    foreach my $item (@items){
140
                        my $record = Item2Marc($item, $record_number);
141
                        push @itemsrecord, $record->field($itemtag);
142
                    }
143
                    $record->insert_fields_ordered(@itemsrecord);
144
                    my $itemsxml = $record->as_xml_record();
145
                    $marcxml =
146
                        substr($marcxml, 0, length($marcxml)-10) .
147
                        substr($itemsxml, index($itemsxml, "</leader>\n", 0) + 10);
148
                }
149
            }
150
            if ($marcxml) {
151
                print {$fh} $marcxml;
152
                $num_exported++;
153
            }
154
        } else {
155
            my ($marc) = _get_corrected_marc_record( $record_type, $record_number, $noxml );
156
            if ( defined $marc ) {
157
                eval {
158
                    my $rec;
159
                    if ($as_xml) {
160
                        $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
161
                        $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
162
                    } else {
163
                        $rec = $marc->as_usmarc();
164
                    }
165
                    print {$fh} $rec;
166
                    $num_exported++;
167
                };
168
                if ($@) {
169
                    warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
170
                }
171
            }
172
        }
173
        $i++;
174
        if($verbose) {
175
            print ".";
176
            print "$i\n" unless($i % 100);
177
        }
178
    }
179
    print {$fh} '</collection>' if (_include_xml_wrapper($as_xml, $record_type));
180
    close $fh;
181
    print "\n$num_exported records exported to $directory/exported_records\n" if $verbose;
182
    return $num_exported;
183
}
184
185
sub _generate_deleted_marc_records {
186
    my ( $record_type, $entries, $directory, $as_xml, $verbose ) = @_;
187
188
    my $num_exported = 0;
189
    open my $fh, ">:encoding(UTF-8)", "$directory/exported_records" or die $!;
190
    if (_include_xml_wrapper($as_xml, $record_type)) {
191
        # include XML declaration and root element
192
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
193
    }
194
    my $i = 0;
195
    foreach my $record_number ( map { $_->{biblio_auth_number} } @$entries ) {
196
        my $marc = MARC::Record->new();
197
        if ( $record_type eq 'biblio' ) {
198
            _fix_biblio_ids( $marc, $record_number, $record_number );
199
        } else {
200
            _fix_authority_id( $marc, $record_number );
201
        }
202
        if ( C4::Context->preference("marcflavour") eq "UNIMARC" ) {
203
            _fix_unimarc_100($marc);
204
        }
205
206
        my $rec;
207
        if ($as_xml) {
208
            $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
209
            $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
210
        } else {
211
            $rec = $marc->as_usmarc();
212
        }
213
        print {$fh} $rec;
214
215
        $num_exported++;
216
        $i++;
217
        if($verbose) {
218
            print ".";
219
            print "$i\n" unless($i % 100);
220
        }
221
    }
222
    print {$fh} '</collection>' if (_include_xml_wrapper($as_xml, $record_type));
223
    close $fh;
224
225
    return $num_exported;
226
}
227
228
sub _get_corrected_marc_record {
229
    my ( $record_type, $record_number, $noxml ) = @_;
230
231
    my $marc = _get_raw_marc_record( $record_type, $record_number, $noxml );
232
233
    if ( defined $marc ) {
234
        _fix_leader($marc);
235
        if ( $record_type eq 'authority' ) {
236
            _fix_authority_id( $marc, $record_number );
237
        } elsif ($record_type eq 'biblio' && C4::Context->preference('IncludeSeeFromInSearches')) {
238
            my $normalizer = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } );
239
            $marc = $normalizer->process($marc);
240
        }
241
        if ( C4::Context->preference("marcflavour") eq "UNIMARC" ) {
242
            _fix_unimarc_100($marc);
243
        }
244
    }
245
246
    return $marc;
247
}
248
249
sub _get_raw_marc_record {
250
    my ( $record_type, $record_number, $noxml ) = @_;
251
252
    my $dbh = C4::Context->dbh;
253
    my $marc;
254
    if ( $record_type eq 'biblio' ) {
255
        if ($noxml) {
256
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems
257
                WHERE biblionumber = ?");
258
            $fetch_sth->execute($record_number);
259
            if ( my ($blob) = $fetch_sth->fetchrow_array ) {
260
                $marc = MARC::Record->new_from_usmarc($blob);
261
                unless ($marc) {
262
                    warn "error creating MARC::Record from $blob";
263
                }
264
            }
265
            # failure to find a bib is not a problem -
266
            # a delete could have been done before
267
            # trying to process a record update
268
269
            $fetch_sth->finish();
270
            return unless $marc;
271
        } else {
272
            eval { $marc = GetMarcBiblio($record_number, 1); };
273
            if ($@ || !$marc) {
274
                # here we do warn since catching an exception
275
                # means that the bib was found but failed
276
                # to be parsed
277
                warn "error retrieving biblio $record_number: $@";
278
                return;
279
            }
280
        }
281
    } else {
282
        eval { $marc = C4::AuthoritiesMarc::GetAuthority($record_number); };
283
        if ($@) {
284
            warn "error retrieving authority $record_number: $@";
285
            return;
286
        }
287
    }
288
    return $marc;
289
}
290
291
sub _fix_leader {
292
293
    # FIXME - this routine is suspect
294
    # It blanks the Leader/00-05 and Leader/12-16 to
295
    # force them to be recalculated correct when
296
    # the $marc->as_usmarc() or $marc->as_xml() is called.
297
    # But why is this necessary?  It would be a serious bug
298
    # in MARC::Record (definitely) and MARC::File::XML (arguably)
299
    # if they are emitting incorrect leader values.
300
    my $marc = shift;
301
302
    my $leader = $marc->leader;
303
    substr( $leader, 0,  5 ) = '     ';
304
    substr( $leader, 10, 7 ) = '22     ';
305
    $marc->leader( substr( $leader, 0, 24 ) );
306
}
307
308
309
sub _fix_biblio_ids {
310
311
    # FIXME - it is essential to ensure that the biblionumber is present,
312
    #         otherwise, Zebra will choke on the record.  However, this
313
    #         logic belongs in the relevant C4::Biblio APIs.
314
    my $marc         = shift;
315
    my $biblionumber = shift;
316
    my $dbh = C4::Context->dbh;
317
    my $biblioitemnumber;
318
    if (@_) {
319
        $biblioitemnumber = shift;
320
    } else {
321
        my $sth = $dbh->prepare(
322
            "SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
323
        $sth->execute($biblionumber);
324
        ($biblioitemnumber) = $sth->fetchrow_array;
325
        $sth->finish;
326
        unless ($biblioitemnumber) {
327
            warn "failed to get biblioitemnumber for biblio $biblionumber";
328
            return 0;
329
        }
330
    }
331
332
    # FIXME - this is cheating on two levels
333
    # 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
334
    # 2. Making sure that the biblionumber and biblioitemnumber are correct and
335
    #    present in the MARC::Record object ought to be part of GetMarcBiblio.
336
    #
337
    # On the other hand, this better for now than what rebuild_zebra.pl used to
338
    # do, which was duplicate the code for inserting the biblionumber
339
    # and biblioitemnumber
340
    C4::Biblio::_koha_marc_update_bib_ids( $marc, '', $biblionumber, $biblioitemnumber );
341
342
    return 1;
343
}
344
345
sub _fix_authority_id {
346
347
    # FIXME - as with fix_biblio_ids, the authid must be present
348
    #         for Zebra's sake.  However, this really belongs
349
    #         in C4::AuthoritiesMarc.
350
    my ( $marc, $authid ) = @_;
351
    unless ( $marc->field('001') and $marc->field('001')->data() eq $authid ) {
352
        $marc->delete_field( $marc->field('001') );
353
        $marc->insert_fields_ordered( MARC::Field->new( '001', $authid ) );
354
    }
355
}
356
357
sub _fix_unimarc_100 {
358
359
    # FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
360
    my $marc = shift;
361
362
    my $string;
363
    if ( length( $marc->subfield( 100, "a" ) ) == 36 ) {
364
        $string = $marc->subfield( 100, "a" );
365
        my $f100 = $marc->field(100);
366
        $marc->delete_field($f100);
367
    } else {
368
        $string = POSIX::strftime( "%Y%m%d", localtime );
369
        $string =~ s/\-//g;
370
        $string = sprintf( "%-*s", 35, $string );
371
    }
372
    substr( $string, 22, 6, "frey50" );
373
    unless ( length( $marc->subfield( 100, "a" ) ) == 36 ) {
374
        $marc->delete_field( $marc->field(100) );
375
        $marc->insert_grouped_field( MARC::Field->new( 100, "", "", "a" => $string ) );
376
    }
377
}
378
379
sub _do_indexing {
380
    my ( $record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt ) = @_;
381
382
    my $zebra_server  = ( $record_type eq 'biblio' ) ? 'biblioserver' : 'authorityserver';
383
    my $zebra_db_name = ( $record_type eq 'biblio' ) ? 'biblios'      : 'authorities';
384
    my $zebra_config  = C4::Context->zebraconfig($zebra_server)->{'config'};
385
    my $zebra_db_dir  = C4::Context->zebraconfig($zebra_server)->{'directory'};
386
387
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index;
388
    system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir");
389
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow;
390
391
}
392
393
sub _include_xml_wrapper {
394
    my $as_xml = shift;
395
    my $record_type = shift;
396
397
    my $bib_index_mode = C4::Context->config('zebra_bib_index_mode') || 'grs1';
398
    my $auth_index_mode = C4::Context->config('zebra_auth_index_mode') || 'dom';
399
400
    return 0 unless $as_xml;
401
    return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom';
402
    return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom';
403
    return 0;
404
405
}
406
407
1;
(-)a/misc/migration_tools/rebuild_zebra.pl (-381 / +54 lines)
Lines 10-16 use File::Path; Link Here
10
use C4::Biblio;
10
use C4::Biblio;
11
use C4::AuthoritiesMarc;
11
use C4::AuthoritiesMarc;
12
use C4::Items;
12
use C4::Items;
13
use Koha::RecordProcessor;
13
use C4::ZebraIndex;
14
14
15
# 
15
# 
16
# script that checks zebradir structure & create directories & mandatory files if needed
16
# script that checks zebradir structure & create directories & mandatory files if needed
Lines 156-167 if ( $verbose_logging ) { Link Here
156
}
156
}
157
if ($keep_export) {
157
if ($keep_export) {
158
    print "NOTHING cleaned : the export $directory has been kept.\n";
158
    print "NOTHING cleaned : the export $directory has been kept.\n";
159
    print "You can re-run this script with the -s ";
159
    print "You can re-run this script with the -s and -d $directory parameters";
160
    if ($use_tempdir) {
161
        print " and -d $directory parameters";
162
    } else {
163
        print "parameter";
164
    }
165
    print "\n";
160
    print "\n";
166
    print "if you just want to rebuild zebra after changing the record.abs\n";
161
    print "if you just want to rebuild zebra after changing the record.abs\n";
167
    print "or another zebra config file\n";
162
    print "or another zebra config file\n";
Lines 196-263 sub check_zebra_dirs { Link Here
196
sub index_records {
191
sub index_records {
197
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
192
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
198
193
199
    my $num_records_exported = 0;
200
    my $records_deleted;
201
    my $need_reset = check_zebra_dirs($server_dir);
194
    my $need_reset = check_zebra_dirs($server_dir);
202
    if ($need_reset) {
195
    if ($need_reset) {
203
    	print "$0: found broken zebra server directories: forcing a rebuild\n";
196
        print "$0: found broken zebra server directories: forcing a rebuild\n";
204
    	$reset = 1;
197
        $reset = 1;
205
    }
198
    }
206
    if ($skip_export && $verbose_logging) {
199
200
    if ( $verbose_logging ) {
207
        print "====================\n";
201
        print "====================\n";
208
        print "SKIPPING $record_type export\n";
202
        print "REINDEXING zebra\n";
209
        print "====================\n";
203
        print "====================\n";
210
    } else {
211
        if ( $verbose_logging ) {
212
            print "====================\n";
213
            print "exporting $record_type\n";
214
            print "====================\n";
215
        }
216
        mkdir "$directory" unless (-d $directory);
217
        mkdir "$directory/$record_type" unless (-d "$directory/$record_type");
218
        if ($process_zebraqueue) {
219
            my $entries = select_zebraqueue_records($record_type, 'deleted');
220
            mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type");
221
            $records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_xml);
222
            mark_zebraqueue_batch_done($entries);
223
            $entries = select_zebraqueue_records($record_type, 'updated');
224
            mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type");
225
            $num_records_exported = export_marc_records_from_list($record_type, 
226
                                                                  $entries, "$directory/upd_$record_type", $as_xml, $noxml, $records_deleted);
227
            mark_zebraqueue_batch_done($entries);
228
        } else {
229
            my $sth = select_all_records($record_type);
230
            $num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_xml, $noxml, $nosanitize);
231
            unless ($do_not_clear_zebraqueue) {
232
                mark_all_zebraqueue_done($record_type);
233
            }
234
        }
235
    }
204
    }
236
205
237
    #
206
    my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
238
    # and reindexing everything
207
    if ($process_zebraqueue) {
239
    #
208
        # Process 'deleted' records
240
    if ($skip_index) {
209
        my $entries = select_zebraqueue_records( $record_type, 'deleted' );
241
        if ($verbose_logging) {
210
        my @entries_to_delete = map { $_->{biblio_auth_id} } @$entries;
242
            print "====================\n";
211
243
            print "SKIPPING $record_type indexing\n";
212
        C4::ZebraIndex::DeleteRecordIndex($record_type, \@entries_to_delete, {
244
            print "====================\n";
213
            as_xml => $as_xml, noshadow => $noshadow, record_format => $record_fmt,
245
        }
214
            zebraidx_log_opt => $zebraidx_log_opt, verbose => $verbose_logging,
215
            skip_export => $skip_export, skip_index => $skip_index,
216
            keep_export => $keep_export, directory => $directory
217
        });
218
219
        mark_zebraqueue_batch_done($entries);
220
221
        # Process 'updated' records'
222
        $entries = select_zebraqueue_records( $record_type, 'updated' );
223
        my @entries_to_update = map {
224
            my $id = $_->{biblio_auth_id};
225
            # do not try to update deleted records
226
            (0 == grep { $id == $_ } @entries_to_delete) ? $id : ()
227
        } @$entries;
228
229
        C4::ZebraIndex::IndexRecord($record_type, \@entries_to_update, {
230
            as_xml => $as_xml, noxml => $noxml, reset_index => $reset,
231
            noshadow => $noshadow, record_format => $record_fmt,
232
            zebraidx_log_opt => $zebraidx_log_opt, verbose => $verbose_logging,
233
            skip_export => $skip_export, skip_index => $skip_index,
234
            keep_export => $keep_export, directory => $directory
235
        });
236
237
        mark_zebraqueue_batch_done($entries);
246
    } else {
238
    } else {
247
        if ( $verbose_logging ) {
239
        my $sth = select_all_records($record_type);
248
            print "====================\n";
240
        my $entries = $sth->fetchall_arrayref([]);
249
            print "REINDEXING zebra\n";
241
        my @entries_to_update = map { $_->[0] } @$entries;
250
            print "====================\n";
242
251
        }
243
        C4::ZebraIndex::IndexRecord($record_type, \@entries_to_update, {
252
        my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
244
            as_xml => $as_xml, noxml => $noxml, reset_index => $reset,
253
        if ($process_zebraqueue) {
245
            noshadow => $noshadow, record_format => $record_fmt,
254
            do_indexing($record_type, 'adelete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
246
            zebraidx_log_opt => $zebraidx_log_opt, nosanitize => $nosanitize,
255
                if %$records_deleted;
247
            verbose => $verbose_logging, skip_export => $skip_export,
256
            do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
248
            skip_index => $skip_index, keep_export => $keep_export,
257
                if $num_records_exported;
249
            directory => $directory
258
        } else {
250
        });
259
            do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
251
260
                if ($num_records_exported or $skip_export);
252
        unless ($do_not_clear_zebraqueue) {
253
            mark_all_zebraqueue_done($record_type);
261
        }
254
        }
262
    }
255
    }
263
}
256
}
Lines 327-651 sub select_all_biblios { Link Here
327
    return $sth;
320
    return $sth;
328
}
321
}
329
322
330
sub include_xml_wrapper {
331
    my $as_xml = shift;
332
    my $record_type = shift;
333
334
    return 0 unless $as_xml;
335
    return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom';
336
    return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom';
337
    return 0;
338
339
}
340
341
sub export_marc_records_from_sth {
342
    my ($record_type, $sth, $directory, $as_xml, $noxml, $nosanitize) = @_;
343
344
    my $num_exported = 0;
345
    open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!;
346
    if (include_xml_wrapper($as_xml, $record_type)) {
347
        # include XML declaration and root element
348
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
349
    }
350
    my $i = 0;
351
    my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",'');
352
    while (my ($record_number) = $sth->fetchrow_array) {
353
        print "." if ( $verbose_logging );
354
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
355
        if ( $nosanitize ) {
356
            my $marcxml = $record_type eq 'biblio'
357
                          ? GetXmlBiblio( $record_number )
358
                          : GetAuthorityXML( $record_number );
359
            if ($record_type eq 'biblio'){
360
                my @items = GetItemsInfo($record_number);
361
                if (@items){
362
                    my $record = MARC::Record->new;
363
                    $record->encoding('UTF-8');
364
                    my @itemsrecord;
365
                    foreach my $item (@items){
366
                        my $record = Item2Marc($item, $record_number);                        
367
                        push @itemsrecord, $record->field($itemtag);
368
                    }
369
                    $record->insert_fields_ordered(@itemsrecord);
370
                    my $itemsxml = $record->as_xml_record();
371
                    $marcxml =
372
                        substr($marcxml, 0, length($marcxml)-10) .
373
                        substr($itemsxml, index($itemsxml, "</leader>\n", 0) + 10);
374
                }
375
            }
376
            if ( $marcxml ) {
377
                print {$fh} $marcxml if $marcxml;
378
                $num_exported++;
379
            }
380
            next;
381
        }
382
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
383
        if (defined $marc) {
384
            eval {
385
                my $rec;
386
                if ($as_xml) {
387
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
388
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
389
                } else {
390
                    $rec = $marc->as_usmarc();
391
                }
392
                print {$fh} $rec;
393
                $num_exported++;
394
            };
395
            if ($@) {
396
              warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
397
            }
398
        }
399
    }
400
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
401
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
402
    close $fh;
403
    return $num_exported;
404
}
405
406
sub export_marc_records_from_list {
407
    my ($record_type, $entries, $directory, $as_xml, $noxml, $records_deleted) = @_;
408
409
    my $num_exported = 0;
410
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
411
    if (include_xml_wrapper($as_xml, $record_type)) {
412
        # include XML declaration and root element
413
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
414
    }
415
    my $i = 0;
416
417
    # Skip any deleted records. We check for this anyway, but this reduces error spam
418
    my %found = %$records_deleted;
419
    foreach my $record_number ( map { $_->{biblio_auth_number} }
420
                                grep { !$found{ $_->{biblio_auth_number} }++ }
421
                                @$entries ) {
422
        print "." if ( $verbose_logging );
423
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
424
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
425
        if (defined $marc) {
426
            eval {
427
                my $rec;
428
                if ($as_xml) {
429
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
430
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
431
                } else {
432
                    $rec = $marc->as_usmarc();
433
                }
434
                print {$fh} $rec;
435
                $num_exported++;
436
            };
437
            if ($@) {
438
              warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
439
            }
440
            $num_exported++;
441
        }
442
    }
443
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
444
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
445
    close $fh;
446
    return $num_exported;
447
}
448
449
sub generate_deleted_marc_records {
450
    my ($record_type, $entries, $directory, $as_xml) = @_;
451
452
    my $records_deleted = {};
453
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
454
    if (include_xml_wrapper($as_xml, $record_type)) {
455
        # include XML declaration and root element
456
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
457
    }
458
    my $i = 0;
459
    foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) {
460
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
461
        print "." if ( $verbose_logging );
462
463
        my $marc = MARC::Record->new();
464
        if ($record_type eq 'biblio') {
465
            fix_biblio_ids($marc, $record_number, $record_number);
466
        } else {
467
            fix_authority_id($marc, $record_number);
468
        }
469
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
470
            fix_unimarc_100($marc);
471
        }
472
473
        my $rec;
474
        if ($as_xml) {
475
            $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
476
            $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
477
        } else {
478
            $rec = $marc->as_usmarc();
479
        }
480
        print {$fh} $rec;
481
482
        $records_deleted->{$record_number} = 1;
483
    }
484
    print "\nRecords exported: $i\n" if ( $verbose_logging );
485
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
486
    close $fh;
487
    return $records_deleted;
488
    
489
490
}
491
492
sub get_corrected_marc_record {
493
    my ($record_type, $record_number, $noxml) = @_;
494
495
    my $marc = get_raw_marc_record($record_type, $record_number, $noxml); 
496
497
    if (defined $marc) {
498
        fix_leader($marc);
499
        if ($record_type eq 'authority') {
500
            fix_authority_id($marc, $record_number);
501
        } elsif ($record_type eq 'biblio' && C4::Context->preference('IncludeSeeFromInSearches')) {
502
            my $normalizer = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } );
503
            $marc = $normalizer->process($marc);
504
        }
505
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
506
            fix_unimarc_100($marc);
507
        }
508
    }
509
510
    return $marc;
511
}
512
513
sub get_raw_marc_record {
514
    my ($record_type, $record_number, $noxml) = @_;
515
  
516
    my $marc; 
517
    if ($record_type eq 'biblio') {
518
        if ($noxml) {
519
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems WHERE biblionumber = ?");
520
            $fetch_sth->execute($record_number);
521
            if (my ($blob) = $fetch_sth->fetchrow_array) {
522
                $marc = MARC::Record->new_from_usmarc($blob);
523
                unless ($marc) {
524
                    warn "error creating MARC::Record from $blob";
525
                }
526
            }
527
            # failure to find a bib is not a problem -
528
            # a delete could have been done before
529
            # trying to process a record update
530
531
            $fetch_sth->finish();
532
            return unless $marc;
533
        } else {
534
            eval { $marc = GetMarcBiblio($record_number, 1); };
535
            if ($@ || !$marc) {
536
                # here we do warn since catching an exception
537
                # means that the bib was found but failed
538
                # to be parsed
539
                warn "error retrieving biblio $record_number";
540
                return;
541
            }
542
        }
543
    } else {
544
        eval { $marc = GetAuthority($record_number); };
545
        if ($@) {
546
            warn "error retrieving authority $record_number";
547
            return;
548
        }
549
    }
550
    return $marc;
551
}
552
553
sub fix_leader {
554
    # FIXME - this routine is suspect
555
    # It blanks the Leader/00-05 and Leader/12-16 to
556
    # force them to be recalculated correct when
557
    # the $marc->as_usmarc() or $marc->as_xml() is called.
558
    # But why is this necessary?  It would be a serious bug
559
    # in MARC::Record (definitely) and MARC::File::XML (arguably) 
560
    # if they are emitting incorrect leader values.
561
    my $marc = shift;
562
563
    my $leader = $marc->leader;
564
    substr($leader,  0, 5) = '     ';
565
    substr($leader, 10, 7) = '22     ';
566
    $marc->leader(substr($leader, 0, 24));
567
}
568
569
sub fix_biblio_ids {
570
    # FIXME - it is essential to ensure that the biblionumber is present,
571
    #         otherwise, Zebra will choke on the record.  However, this
572
    #         logic belongs in the relevant C4::Biblio APIs.
573
    my $marc = shift;
574
    my $biblionumber = shift;
575
    my $biblioitemnumber;
576
    if (@_) {
577
        $biblioitemnumber = shift;
578
    } else {    
579
        my $sth = $dbh->prepare(
580
            "SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
581
        $sth->execute($biblionumber);
582
        ($biblioitemnumber) = $sth->fetchrow_array;
583
        $sth->finish;
584
        unless ($biblioitemnumber) {
585
            warn "failed to get biblioitemnumber for biblio $biblionumber";
586
            return 0;
587
        }
588
    }
589
590
    # FIXME - this is cheating on two levels
591
    # 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
592
    # 2. Making sure that the biblionumber and biblioitemnumber are correct and
593
    #    present in the MARC::Record object ought to be part of GetMarcBiblio.
594
    #
595
    # On the other hand, this better for now than what rebuild_zebra.pl used to
596
    # do, which was duplicate the code for inserting the biblionumber 
597
    # and biblioitemnumber
598
    C4::Biblio::_koha_marc_update_bib_ids($marc, '', $biblionumber, $biblioitemnumber);
599
600
    return 1;
601
}
602
603
sub fix_authority_id {
604
    # FIXME - as with fix_biblio_ids, the authid must be present
605
    #         for Zebra's sake.  However, this really belongs
606
    #         in C4::AuthoritiesMarc.
607
    my ($marc, $authid) = @_;
608
    unless ($marc->field('001') and $marc->field('001')->data() eq $authid){
609
        $marc->delete_field($marc->field('001'));
610
        $marc->insert_fields_ordered(MARC::Field->new('001',$authid));
611
    }
612
}
613
614
sub fix_unimarc_100 {
615
    # FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
616
    my $marc = shift;
617
618
    my $string;
619
    if ( length($marc->subfield( 100, "a" )) == 36 ) {
620
        $string = $marc->subfield( 100, "a" );
621
        my $f100 = $marc->field(100);
622
        $marc->delete_field($f100);
623
    }
624
    else {
625
        $string = POSIX::strftime( "%Y%m%d", localtime );
626
        $string =~ s/\-//g;
627
        $string = sprintf( "%-*s", 35, $string );
628
    }
629
    substr( $string, 22, 6, "frey50" );
630
    unless ( length($marc->subfield( 100, "a" )) == 36 ) {
631
        $marc->delete_field($marc->field(100));
632
        $marc->insert_grouped_field(MARC::Field->new( 100, "", "", "a" => $string ));
633
    }
634
}
635
636
sub do_indexing {
637
    my ($record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt) = @_;
638
639
    my $zebra_server  = ($record_type eq 'biblio') ? 'biblioserver' : 'authorityserver';
640
    my $zebra_db_name = ($record_type eq 'biblio') ? 'biblios' : 'authorities';
641
    my $zebra_config  = C4::Context->zebraconfig($zebra_server)->{'config'};
642
    my $zebra_db_dir  = C4::Context->zebraconfig($zebra_server)->{'directory'};
643
644
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index;
645
    system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir");
646
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow;
647
648
}
649
323
650
sub print_usage {
324
sub print_usage {
651
    print <<_USAGE_;
325
    print <<_USAGE_;
652
- 

Return to bug 7419