View | Details | Raw Unified | Return to bug 37020
Collapse All | Expand All

(-)a/misc/migration_tools/bulkmarcimport.pl (-391 / +525 lines)
Lines 1-11 Link Here
1
#!/usr/bin/perl
1
#!/usr/bin/perl
2
# Import an iso2709 file into Koha 3
2
# Import an iso2709 or MARCXML file into Koha
3
3
4
use Modern::Perl;
4
use Modern::Perl;
5
5
6
#use diagnostics;
7
8
# Koha modules used
6
# Koha modules used
7
use MARC::Record;
9
use MARC::File::USMARC;
8
use MARC::File::USMARC;
10
use MARC::File::XML;
9
use MARC::File::XML;
11
use MARC::Batch;
10
use MARC::Batch;
Lines 29-35 use C4::MarcModificationTemplates qw( Link Here
29
    GetModificationTemplates
28
    GetModificationTemplates
30
    ModifyRecordWithTemplate
29
    ModifyRecordWithTemplate
31
);
30
);
32
use C4::AuthoritiesMarc qw( GuessAuthTypeCode GuessAuthId GetAuthority ModAuthority AddAuthority );
31
use C4::AuthoritiesMarc qw(
32
    GuessAuthTypeCode
33
    GuessAuthId
34
    GetAuthority
35
    ModAuthority
36
    AddAuthority
37
);
33
38
34
use YAML::XS;
39
use YAML::XS;
35
use Time::HiRes  qw( gettimeofday );
40
use Time::HiRes  qw( gettimeofday );
Lines 45-50 use Koha::SearchEngine::Search; Link Here
45
50
46
use open qw( :std :encoding(UTF-8) );
51
use open qw( :std :encoding(UTF-8) );
47
binmode( STDOUT, ":encoding(UTF-8)" );
52
binmode( STDOUT, ":encoding(UTF-8)" );
53
54
### CHANGED: We add XML::Twig for streaming MARCXML
55
use XML::Twig;
56
48
my ( $input_marc_file, $number, $offset, $cleanisbn ) = ( '', 0, 0, 1 );
57
my ( $input_marc_file, $number, $offset, $cleanisbn ) = ( '', 0, 0, 1 );
49
my $version;
58
my $version;
50
my $delete;
59
my $delete;
Lines 66-72 my $filters; Link Here
66
my $update;
75
my $update;
67
my $all;
76
my $all;
68
my $yamlfile;
77
my $yamlfile;
69
my $authtypes;
70
my $append;
78
my $append;
71
my $sourcetag;
79
my $sourcetag;
72
my $sourcesubfield;
80
my $sourcesubfield;
Lines 78-83 my $marc_mod_template = ''; Link Here
78
my $marc_mod_template_id = -1;
86
my $marc_mod_template_id = -1;
79
my $skip_indexing        = 0;
87
my $skip_indexing        = 0;
80
my $skip_bad_records;
88
my $skip_bad_records;
89
81
$| = 1;
90
$| = 1;
82
91
83
GetOptions(
92
GetOptions(
Lines 120-129 GetOptions( Link Here
120
129
121
$biblios ||= !$authorities;
130
$biblios ||= !$authorities;
122
$insert  ||= !$update;
131
$insert  ||= !$update;
123
my $writemode = ($append) ? "a" : "w";
132
my $writemode = ( $append ) ? "a" : "w";
124
133
125
pod2usage( -msg => "\nYou must specify either --biblios or --authorities, not both.\n", -exitval )
134
pod2usage(
126
    if $biblios && $authorities;
135
    -msg => "\nYou must specify either --biblios or --authorities, not both.\n",
136
    -exitval => 1
137
) if $biblios && $authorities;
127
138
128
if ($all) {
139
if ($all) {
129
    $insert = 1;
140
    $insert = 1;
Lines 134-147 my $using_elastic_search = ( C4::Context->preference('SearchEngine') eq 'Elastic Link Here
134
my $mod_biblio_options   = {
145
my $mod_biblio_options   = {
135
    disable_autolink  => $using_elastic_search,
146
    disable_autolink  => $using_elastic_search,
136
    skip_record_index => $using_elastic_search || $skip_indexing,
147
    skip_record_index => $using_elastic_search || $skip_indexing,
137
    overlay_context   => { source => 'bulkmarcimport' }
148
    overlay_context   => { source => 'bulkmarcimport' },
138
};
149
};
139
my $add_biblio_options = {
150
my $add_biblio_options = {
140
    disable_autolink  => $using_elastic_search,
151
    disable_autolink  => $using_elastic_search,
152
    skip_record_index => $using_elastic_search || $skip_indexing,
153
};
154
my $mod_authority_options = {
155
    skip_record_index => $using_elastic_search || $skip_indexing
156
};
157
my $add_authority_options = {
141
    skip_record_index => $using_elastic_search || $skip_indexing
158
    skip_record_index => $using_elastic_search || $skip_indexing
142
};
159
};
143
my $mod_authority_options = { skip_record_index => $using_elastic_search || $skip_indexing };
144
my $add_authority_options = { skip_record_index => $using_elastic_search || $skip_indexing };
145
160
146
my @search_engine_record_ids;
161
my @search_engine_record_ids;
147
my @search_engine_records;
162
my @search_engine_records;
Lines 151-176 if ($using_elastic_search) { Link Here
151
    $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new(
166
    $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new(
152
        {
167
        {
153
            index => $authorities
168
            index => $authorities
154
            ? $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX
169
                ? $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX
155
            : $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX
170
                : $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX
156
        }
171
        }
157
    );
172
    );
158
}
173
}
159
174
160
if ( $version || ( $input_marc_file eq '' ) ) {
175
if ( $version || $input_marc_file eq '' ) {
161
    pod2usage( -verbose => 2 );
176
    pod2usage( -verbose => 2 );
162
    exit;
177
    exit;
163
}
178
}
164
if ( $update && !( $match || $isbn_check ) ) {
179
if ( $update && !( $match || $isbn_check ) ) {
165
    warn "Using --update without --match or --isbn seems to be useless.\n";
180
    warn "Using --update without --match or --isbn is likely not useful.\n";
166
}
181
}
167
182
168
if ( defined $localcust ) {    #local customize module
183
if ( defined $localcust ) {
169
    if ( !-e $localcust ) {
184
    if ( !-e $localcust ) {
170
        $localcust = $localcust || 'LocalChanges';        #default name
185
        $localcust = $localcust || 'LocalChanges';
171
        $localcust =~ s/^.*\/([^\/]+)$/$1/;               #extract file name only
186
        $localcust =~ s/^.*\/([^\/]+)$/$1/;
172
        $localcust =~ s/\.pm$//;                          #remove extension
187
        $localcust =~ s/\.pm$//;
173
        my $fqcust = $FindBin::Bin . "/$localcust.pm";    #try migration_tools dir
188
        my $fqcust = $FindBin::Bin . "/$localcust.pm";
174
        if ( -e $fqcust ) {
189
        if ( -e $fqcust ) {
175
            $localcust = $fqcust;
190
            $localcust = $fqcust;
176
        } else {
191
        } else {
Lines 190-212 if ( $marc_mod_template ne '' ) { Link Here
190
                $marc_mod_template_id = $this_template->{'template_id'};
205
                $marc_mod_template_id = $this_template->{'template_id'};
191
            } else {
206
            } else {
192
                print "WARNING: MARC modification template name "
207
                print "WARNING: MARC modification template name "
193
                    . "'$marc_mod_template' matches multiple templates. "
208
                      . "'$marc_mod_template' matches multiple templates.\n";
194
                    . "Please rename these templates\n";
195
                exit 1;
209
                exit 1;
196
            }
210
            }
197
        }
211
        }
198
    }
212
    }
199
    if ( $marc_mod_template_id < 0 ) {
213
    if ( $marc_mod_template_id < 0 ) {
200
        die "Can't located MARC modification template '$marc_mod_template'\n";
214
        die "Can't locate MARC modification template '$marc_mod_template'\n";
201
    } else {
215
    } else {
202
        print "Records will be modified using MARC modification template: $marc_mod_template\n" if $verbose;
216
        print "MARC modification template: $marc_mod_template\n" if $verbose;
203
    }
217
    }
204
}
218
}
205
219
206
my $dbh            = C4::Context->dbh;
220
my $dbh            = C4::Context->dbh;
207
my $heading_fields = get_heading_fields();
221
my $heading_fields = get_heading_fields($authtypes);
208
my $idmapfh;
222
my $idmapfh;
209
210
if ( defined $idmapfl ) {
223
if ( defined $idmapfl ) {
211
    open( $idmapfh, '>', $idmapfl ) or die "cannot open $idmapfl \n";
224
    open( $idmapfh, '>', $idmapfl ) or die "cannot open $idmapfl \n";
212
}
225
}
Lines 219-224 if ( ( not defined $sourcesubfield ) && ( not defined $sourcetag ) ) { Link Here
219
# Disable logging for the biblios and authorities import operation. It would unnecessarily
232
# Disable logging for the biblios and authorities import operation. It would unnecessarily
220
# slow the import
233
# slow the import
221
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0;
234
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0;
235
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0;
222
$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0;
236
$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0;
223
237
224
if ($fk_off) {
238
if ($fk_off) {
Lines 236-318 if ($delete) { Link Here
236
        $dbh->do("ALTER TABLE items AUTO_INCREMENT = 1");
250
        $dbh->do("ALTER TABLE items AUTO_INCREMENT = 1");
237
    } else {
251
    } else {
238
        print "Deleting authorities\n";
252
        print "Deleting authorities\n";
239
        $dbh->do("truncate auth_header");
253
        $dbh->do("TRUNCATE auth_header");
240
    }
254
    }
241
    $dbh->do("truncate zebraqueue");
255
    $dbh->do("TRUNCATE zebraqueue");
242
}
256
}
243
257
244
if ($test_parameter) {
258
if ($test_parameter) {
245
    print "TESTING MODE ONLY\n    DOING NOTHING\n===============\n";
259
    print "TESTING MODE ONLY\n    DOING NOTHING\n===============\n";
246
}
260
}
247
261
248
my $batch;
249
my $marc_flavour = C4::Context->preference('marcflavour') || 'MARC21';
262
my $marc_flavour = C4::Context->preference('marcflavour') || 'MARC21';
263
print "MARC flavour: $marc_flavour\n" if $verbose;
250
264
251
# The definition of $searcher must be before MARC::Batch->new
265
# The definition of $searcher must be before MARC::Batch->new
252
my $searcher = Koha::SearchEngine::Search->new(
266
my $searcher = Koha::SearchEngine::Search->new(
253
    {
267
    {
254
        index => (
268
        index => $authorities
255
              $authorities
256
            ? $Koha::SearchEngine::AUTHORITIES_INDEX
269
            ? $Koha::SearchEngine::AUTHORITIES_INDEX
257
            : $Koha::SearchEngine::BIBLIOS_INDEX
270
            : $Koha::SearchEngine::BIBLIOS_INDEX
258
        )
259
    }
271
    }
260
);
272
);
261
273
262
print "Characteristic MARC flavour: $marc_flavour\n" if $verbose;
274
print "Characteristic MARC flavour: $marc_flavour\n" if $verbose;
263
my $starttime = gettimeofday;
275
my $starttime  = gettimeofday;
264
276
my $logger     = Koha::Logger->get;
265
# don't let MARC::Batch open the file, as it applies the ':utf8' IO layer
277
my $schema     = Koha::Database->schema;
266
my $fh = IO::File->new($input_marc_file) or die "Could not open $input_marc_file: $!";
278
my $lint       = MARC::Lint->new;
267
268
if ( defined $format && $format =~ /XML/i ) {
269
270
    # ugly hack follows -- MARC::File::XML, when used by MARC::Batch,
271
    # appears to try to convert incoming XML records from MARC-8
272
    # to UTF-8.  Setting the BinaryEncoding key turns that off
273
    # TODO: see what happens to ISO-8859-1 XML files.
274
    # TODO: determine if MARC::Batch can be fixed to handle
275
    #       XML records properly -- it probably should be
276
    #       be using a proper push or pull XML parser to
277
    #       extract the records, not using regexes to look
278
    #       for <record>.*</record>.
279
    $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8';
280
    my $recordformat = ( $marc_flavour eq "MARC21" ? "USMARC" : uc($marc_flavour) );
281
282
    #UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios.
283
    $recordformat = $recordformat . "AUTH" if ( $authorities and $marc_flavour ne "MARC21" );
284
    $MARC::File::XML::_load_args{RecordFormat} = $recordformat;
285
    $batch = MARC::Batch->new( 'XML', $fh );
286
} else {
287
    $batch = MARC::Batch->new( 'USMARC', $fh );
288
}
289
290
$batch->warnings_off();
291
$batch->strict_off();
292
my $commitnum = $commit ? $commit : 50;
293
my $yamlhash;
294
295
# Skip file offset
296
if ($offset) {
297
    print "Skipping file offset: $offset records\n";
298
    $batch->next() while ( $offset-- );
299
}
300
301
my ( $tagid, $subfieldid );
302
if ($authorities) {
303
    $tagid = '001';
304
} else {
305
    ( $tagid, $subfieldid ) = GetMarcFromKohaField("biblio.biblionumber");
306
    $tagid ||= "001";
307
}
308
309
my $sth_isbn;
310
311
# the SQL query to search on isbn
312
if ($isbn_check) {
313
    $sth_isbn = $dbh->prepare("SELECT biblionumber, biblioitemnumber FROM biblioitems WHERE isbn=?");
314
}
315
316
my $loghandle;
279
my $loghandle;
317
if ($logfile) {
280
if ($logfile) {
318
    $loghandle = IO::File->new( $logfile, $writemode );
281
    $loghandle = IO::File->new( $logfile, $writemode );
Lines 320-399 if ($logfile) { Link Here
320
}
283
}
321
284
322
my $record_number = 0;
285
my $record_number = 0;
323
my $logger        = Koha::Logger->get;
286
my $commitnum     = $commit ? $commit : 50;
324
my $schema        = Koha::Database->schema;
287
my $yamlhash;
325
my $marc_records  = [];
326
my $lint          = MARC::Lint->new;
327
RECORD: while () {
328
    my $record;
329
    $record_number++;
330
288
331
    # get record
289
$schema->txn_begin;
332
    eval { $record = $batch->next() };
333
    if ($@) {
334
        print "Bad MARC record $record_number: $@ skipped\n";
335
336
        # FIXME - because MARC::Batch->next() combines grabbing the next
337
        # blob and parsing it into one operation, a correctable condition
338
        # such as a MARC-8 record claiming that it's UTF-8 can't be recovered
339
        # from because we don't have access to the original blob.  Note
340
        # that the staging import can deal with this condition (via
341
        # C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch.
342
        next;
343
    }
344
    if ($record) {
345
290
346
        if ($skip_bad_records) {
291
### CHANGED: A new sub to handle "one MARC::Record" import logic.
347
            my $xml = $record->as_xml_record();
292
### We take the logic from the old while RECORD loop, but wrap in a sub.
348
            eval { MARC::Record::new_from_xml( $xml, 'UTF-8', "MARC21" ); };
293
sub import_single_record {
349
            if ($@) {
294
    my ($record) = @_;
350
                print "Record $record_number generated invalid xml:\n";
295
    return unless $record;
351
                $lint->check_record($record);
296
352
                foreach my $warning ( $lint->warnings ) {
297
    # increment once per record
353
                    print "    " . $warning . "\n";
298
    $record_number++;
354
                }
299
355
                print "    Record skipped!";
300
    # skip Bad records if requested
356
                next;
301
    if ($skip_bad_records) {
302
        my $xml = $record->as_xml_record();
303
        eval { MARC::Record::new_from_xml( $xml, 'UTF-8', "MARC21" ); };
304
        if ($@) {
305
            print "Record $record_number is invalid, skipping.\n";
306
            $lint->check_record($record);
307
            foreach my $warning ( $lint->warnings ) {
308
                print "  Lint: $warning\n";
357
            }
309
            }
310
            return;
358
        }
311
        }
359
        # transcode the record to UTF8 if needed & applicable.
312
    }
360
        if ( $record->encoding() eq 'MARC-8' and not $skip_marc8_conversion ) {
313
361
            my ( $guessed_charset, $charset_errors );
314
    # If record is MARC-8 and not skipping
362
            ( $record, $guessed_charset, $charset_errors ) = MarcToUTF8Record(
315
    if ( $record->encoding() eq 'MARC-8' and not $skip_marc8_conversion ) {
363
                $record,
316
        my ( $newrec, $charset_guess, $charset_err ) = MarcToUTF8Record(
364
                $marc_flavour . ( ( $authorities and $marc_flavour ne "MARC21" ) ? 'AUTH' : '' )
317
            $record,
365
            );
318
            $marc_flavour . ( ($authorities && $marc_flavour ne 'MARC21') ? 'AUTH' : '' )
366
            if ( $guessed_charset eq 'failed' ) {
319
        );
367
                warn "ERROR: failed to perform character conversion for record $record_number\n";
320
        if ( $charset_guess eq 'failed' ) {
368
                next RECORD;
321
            warn "Charset conversion failed for record #$record_number\n";
369
            }
322
            return;
370
        }
323
        }
371
        SetUTF8Flag($record);
324
        $record = $newrec;
372
        &$localcust($record) if $localcust;
373
        push @{$marc_records}, $record;
374
    } else {
375
        last;
376
    }
325
    }
377
}
326
    SetUTF8Flag($record);
378
327
379
$record_number = 0;
328
    # local custom routine
380
my $records_total = @{$marc_records};
329
    &$localcust($record) if $localcust;
381
$schema->txn_begin;
330
382
RECORD: foreach my $record ( @{$marc_records} ) {
331
    if ( ( $verbose // 0 ) == 1 ) {
383
    $record_number++;
384
    if ( ( $verbose // 1 ) == 1 ) {    #no dot for verbose==2
385
        print "." . ( $record_number % 100 == 0 ? "\n$record_number" : '' );
332
        print "." . ( $record_number % 100 == 0 ? "\n$record_number" : '' );
333
    } elsif ( ( $verbose // 0 ) > 1 ) {
334
        print $record->as_formatted(), "\n";
386
    }
335
    }
387
336
337
    # apply MARC mod template
388
    if ( $marc_mod_template_id > 0 ) {
338
    if ( $marc_mod_template_id > 0 ) {
389
        print "Modifying MARC\n" if $verbose;
390
        ModifyRecordWithTemplate( $marc_mod_template_id, $record );
339
        ModifyRecordWithTemplate( $marc_mod_template_id, $record );
391
    }
340
    }
392
341
342
    # Possibly handle ISBN cleaning
393
    my $isbn;
343
    my $isbn;
394
344
    if ($biblios && ( $cleanisbn || $isbn_check )) {
395
    # remove trailing - in isbn (only for biblios, of course)
396
    if ( $biblios && ( $cleanisbn || $isbn_check ) ) {
397
        my $tag   = $marc_flavour eq 'UNIMARC' ? '010' : '020';
345
        my $tag   = $marc_flavour eq 'UNIMARC' ? '010' : '020';
398
        my $field = $record->field($tag);
346
        my $field = $record->field($tag);
399
        $isbn = $field && $field->subfield('a');
347
        $isbn = $field && $field->subfield('a');
Lines 403-430 RECORD: foreach my $record ( @{$marc_records} ) { Link Here
403
        }
351
        }
404
    }
352
    }
405
353
406
    # search for duplicates (based on Local-number)
354
    my ( $tagid, $subfieldid );
355
    if ($authorities) {
356
        $tagid = '001';
357
    } else {
358
        ( $tagid, $subfieldid ) = GetMarcFromKohaField("biblio.biblionumber");
359
        $tagid ||= '001';
360
    }
361
407
    my $originalid        = GetRecordId( $record, $tagid, $subfieldid );
362
    my $originalid        = GetRecordId( $record, $tagid, $subfieldid );
408
    my $matched_record_id = undef;
363
    my $matched_record_id = undef;
364
365
    # handle match logic
409
    if ($match) {
366
    if ($match) {
410
        require C4::Search;
367
        require C4::Search;
411
        my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
368
        my $server = $authorities ? 'authorityserver' : 'biblioserver';
412
        my $query  = build_query( $match, $record );
369
        my $query  = build_query( $match, $record );
413
        $logger->debug("Bulkmarcimport: $query");
370
        $logger->debug("Bulkmarcimport: $query");
414
        my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] );
371
        my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] );
415
416
        # changed to warn so able to continue with one broken record
417
        if ( defined $error ) {
372
        if ( defined $error ) {
418
            warn "unable to search the database for duplicates : $error";
373
            warn "Duplicate search error: $error\n";
419
            printlog( { id => $originalid, op => "match", status => "ERROR" } ) if ($logfile);
374
            printlog({ id => $originalid, op => "match", status => "ERROR" }) if $loghandle;
420
            next RECORD;
375
            return;
421
        }
376
        }
422
        $logger->debug("Bulkmarcimport: $query $server : $totalhits");
377
        $logger->debug("Bulkmarcimport: $query $server : $totalhits");
423
378
424
        # sub SimpleSearch could return undefined, but only on error, so
425
        # should not really need to safeguard here, but do so anyway
426
        $results //= [];
379
        $results //= [];
427
        if ( @{$results} == 1 ) {
380
        if (@$results == 1) {
428
            my $matched_record = C4::Search::new_record_from_zebra( $server, $results->[0] );
381
            my $matched_record = C4::Search::new_record_from_zebra( $server, $results->[0] );
429
            SetUTF8Flag($matched_record);
382
            SetUTF8Flag($matched_record);
430
            $matched_record_id = GetRecordId( $matched_record, $tagid, $subfieldid );
383
            $matched_record_id = GetRecordId( $matched_record, $tagid, $subfieldid );
Lines 443-451 RECORD: foreach my $record ( @{$marc_records} ) { Link Here
443
                    next;
396
                    next;
444
                }
397
                }
445
            }
398
            }
446
        } elsif ( @{$results} > 1 ) {
399
        } elsif (@$results > 1) {
447
            $logger->debug("More than one match for: $query");
400
            $logger->debug("More than one match for: $query");
448
            next;
401
            return;
449
        } else {
402
        } else {
450
            $logger->debug("No match for: $query");
403
            $logger->debug("No match for: $query");
451
        }
404
        }
Lines 455-473 RECORD: foreach my $record ( @{$marc_records} ) { Link Here
455
            if ( length($keepids) == 3 ) {
408
            if ( length($keepids) == 3 ) {
456
                $storeidfield = MARC::Field->new( $keepids, $originalid );
409
                $storeidfield = MARC::Field->new( $keepids, $originalid );
457
            } else {
410
            } else {
458
                $storeidfield =
411
                $storeidfield = MARC::Field->new(
459
                    MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );
412
                    substr($keepids, 0, 3), "",
413
                    "", substr($keepids, 3, 1), $originalid
414
                );
460
            }
415
            }
461
            $record->insert_fields_ordered($storeidfield);
416
            $record->insert_fields_ordered($storeidfield);
462
            $record->delete_field( $record->field($tagid) );
417
            $record->delete_field( $record->field($tagid) );
463
        }
418
        }
464
    }
419
    }
465
420
421
    # possibly remove fields if --filter
466
    foreach my $stringfilter (@$filters) {
422
    foreach my $stringfilter (@$filters) {
467
        if ( length($stringfilter) == 3 ) {
423
        if ( length($stringfilter) == 3 ) {
468
            foreach my $field ( $record->field($stringfilter) ) {
424
            foreach my $field ( $record->field($stringfilter) ) {
469
                $record->delete_field($field);
425
                $record->delete_field($field);
470
                $logger->debug( "Removed: ", $field->as_string );
426
                $logger->debug("Removed field $stringfilter") if $verbose;
471
            }
427
            }
472
        } elsif ( $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/ ) {
428
        } elsif ( $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/ ) {
473
            my $removetag      = $1;
429
            my $removetag      = $1;
Lines 475-756 RECORD: foreach my $record ( @{$marc_records} ) { Link Here
475
            my $removematch    = $3;
431
            my $removematch    = $3;
476
            if ( ( $removetag > "010" ) && $removesubfield ) {
432
            if ( ( $removetag > "010" ) && $removesubfield ) {
477
                foreach my $field ( $record->field($removetag) ) {
433
                foreach my $field ( $record->field($removetag) ) {
478
                    $field->delete_subfield( code => "$removesubfield", match => $removematch );
434
                    $field->delete_subfield( code => $removesubfield, match => $removematch );
479
                    $logger->debug( "Potentially removed: ", $field->subfield($removesubfield) );
480
                }
435
                }
481
            }
436
            }
482
        }
437
        }
483
    }
438
    }
484
    unless ($test_parameter) {
439
485
        if ($authorities) {
440
    # if test only, skip DB changes
486
            my $authtypecode = GuessAuthTypeCode( $record, $heading_fields );
441
    if ($test_parameter) {
487
            my $authid;
442
        return;
488
443
    }
489
            if ($matched_record_id) {
444
490
                if ($update) {
445
    # handle authorities vs biblios
491
                    ## Authority has an id and is in database: update
446
    if ($authorities) {
492
                    eval {
447
        my $authtypecode = GuessAuthTypeCode( $record, $heading_fields );
493
                        ($authid) = ModAuthority(
448
        my $authid;
494
                            $matched_record_id, $record, $authtypecode,
449
495
                            $mod_authority_options,
450
        if ($matched_record_id) {
496
                        );
451
            if ($update) {
497
                    };
452
                ## Authority has an id and is in database: update
498
                    if ($@) {
453
                eval {
499
                        warn "ERROR: Update authority $matched_record_id failed: $@\n";
454
                    ($authid) = ModAuthority(
500
                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
455
                        $matched_record_id,
501
                        next RECORD;
456
                        $record,
502
                    } else {
457
                        $authtypecode,
503
                        printlog( { id => $authid, op => "update", status => "ok" } ) if ($logfile);
458
                        $mod_authority_options
504
                    }
505
                } elsif ($logfile) {
506
                    warn "WARNING: Update authority $originalid skipped";
507
                    printlog(
508
                        {
509
                            id     => $matched_record_id,
510
                            op     => "update",
511
                            status =>
512
                                "warning: authority already in database and option -update not enabled, skipping..."
513
                        }
514
                    );
459
                    );
515
                }
460
                };
516
            } elsif ($insert) {
517
                ## An authid is defined but no authority in database: insert
518
                eval { ($authid) = AddAuthority( $record, undef, $authtypecode, $add_authority_options ); };
519
                if ($@) {
461
                if ($@) {
520
                    warn "ERROR: Insert authority $originalid failed: $@\n";
462
                    warn "Update authority $matched_record_id failed: $@\n";
521
                    printlog( { id => $originalid, op => "insert", status => "ERROR" } ) if ($logfile);
463
                    printlog({ id => $matched_record_id, op => "update", status => "ERROR" }) if $loghandle;
522
                    next RECORD;
464
                    return;
523
                } else {
465
                } else {
524
                    printlog( { id => $authid, op => "insert", status => "ok" } ) if ($logfile);
466
                    printlog({ id => $authid, op => "update", status => "ok" }) if $loghandle;
467
                    push @search_engine_record_ids, $authid;
468
                    push @search_engine_records,    $record;
525
                }
469
                }
526
            } else {
470
            } else {
527
                warn "WARNING: Insert authority $originalid skipped";
471
                warn "Authority $originalid matched but update not enabled.\n";
528
                printlog(
472
                printlog({ id => $matched_record_id, op => "update", status => "skipped" }) if $loghandle;
529
                    {
530
                        id     => $originalid, op => "insert",
531
                        status => "warning : authority not in database and option -insert not enabled, skipping..."
532
                    }
533
                ) if ($logfile);
534
            }
473
            }
535
474
        } elsif ($insert) {
536
            if ($yamlfile) {
475
            ## An authid is defined but no authority in database: insert
537
                $yamlhash->{$originalid} = YAMLFileEntry(
476
            eval {
538
                    $record,
477
                ($authid) = AddAuthority( $record, undef, $authtypecode, $add_authority_options );
539
                    $authid,
478
            };
540
                    1    #@FIXME: Really always updated?
479
            if ($@) {
541
                );
480
                warn "Insert authority $originalid failed: $@\n";
481
                printlog({ id => $originalid, op => "insert", status => "ERROR" }) if $loghandle;
482
                return;
542
            }
483
            }
484
            printlog({ id => $authid, op => "insert", status => "ok" }) if $loghandle;
543
            push @search_engine_record_ids, $authid;
485
            push @search_engine_record_ids, $authid;
544
            push @search_engine_records,    $record;
486
            push @search_engine_records,    $record;
545
        } else {
487
        } else {
546
            my ( $biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id );
488
            warn "Authority $originalid not in DB and --insert not set; skipping.\n";
547
489
            printlog({ id => $originalid, op => "insert", status => "skipped" }) if $loghandle;
548
            # check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter
490
        }
549
            if ( !$matched_record_id && $isbn_check && $isbn ) {
550
                $sth_isbn->execute($isbn);
551
                ( $matched_record_id, $biblioitemnumber ) = $sth_isbn->fetchrow;
552
            }
553
491
554
            if ( defined $idmapfl && $matched_record_id ) {
492
        if ($yamlfile) {
555
                if ( $sourcetag < "010" ) {
493
            $yamlhash->{$originalid} = YAMLFileEntry(
556
                    if ( $record->field($sourcetag) ) {
494
                $record,
557
                        my $source = $record->field($sourcetag)->data();
495
                $authid,
558
                        printf( $idmapfh "%s|%s\n", $source, $matched_record_id );
496
                1    #@FIXME: Really always updated?
559
                    }
497
            );
560
                } else {
498
        }
561
                    my $source = $record->subfield( $sourcetag, $sourcesubfield );
499
    }
562
                    printf( $idmapfh "%s|%s\n", $source, $matched_record_id );
500
    else {
501
        # biblios
502
        my ( $biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id );
503
        if ( !$matched_record_id && $isbn_check && $isbn ) {
504
            my $sth_isbn = $dbh->prepare("SELECT biblionumber, biblioitemnumber FROM biblioitems WHERE isbn=?");
505
            $sth_isbn->execute($isbn);
506
            ( $matched_record_id, $biblioitemnumber ) = $sth_isbn->fetchrow;
507
        }
508
        if ( defined $idmapfh && $matched_record_id ) {
509
            my $sourceval;
510
            if ( $sourcetag < "010" ) {
511
                if ( my $f = $record->field($sourcetag) ) {
512
                    $sourceval = $f->data();
563
                }
513
                }
514
            } else {
515
                $sourceval = $record->subfield( $sourcetag, $sourcesubfield );
516
            }
517
            if ( defined $sourceval ) {
518
                printf( $idmapfh "%s|%s\n", $sourceval, $matched_record_id );
564
            }
519
            }
520
        }
565
521
566
            # Create biblio, unless we already have it (either match or ISBN)
522
        # Create biblio, unless we already have it (either match or ISBN)
567
            if ($matched_record_id) {
523
        if ($matched_record_id) {
568
                eval { $biblioitemnumber = Koha::Biblios->find($matched_record_id)->biblioitem->biblioitemnumber; };
524
            if ($update) {
569
                if ($update) {
525
                eval {
570
                    my $success;
526
                    $biblioitemnumber =
571
                    eval {
527
                      Koha::Biblios->find($matched_record_id)->biblioitem->biblioitemnumber;
572
                        $success = ModBiblio(
528
                };
573
                            $record, $matched_record_id, GetFrameworkCode($matched_record_id),
529
                my $success;
574
                            $mod_biblio_options
530
                eval {
575
                        );
531
                    $success = ModBiblio(
576
                    };
532
                        $record,
577
                    if ($@) {
533
                        $matched_record_id,
578
                        warn "ERROR: Update biblio $matched_record_id failed: $@\n";
534
                        GetFrameworkCode($matched_record_id),
579
                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
535
                        $mod_biblio_options
580
                        next RECORD;
536
                    );
581
                    } elsif ( !$success ) {
537
                };
582
                        warn "ERROR: Update biblio $matched_record_id failed for unknown reason";
583
                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
584
                        next RECORD;
585
                    } else {
586
                        $record_id = $matched_record_id;
587
                        printlog( { id => $record_id, op => "update", status => "ok" } ) if ($logfile);
588
                    }
589
                } else {
590
                    warn "WARNING: Update biblio $originalid skipped";
591
                    printlog(
592
                        {
593
                            id     => $matched_record_id, op => "update",
594
                            status => "warning : already in database and option -update not enabled, skipping..."
595
                        }
596
                    ) if ($logfile);
597
                }
598
            } elsif ($insert) {
599
                my $record_clone = $record->clone();
600
                C4::Biblio::_strip_item_fields($record_clone);
601
                eval { ( $record_id, $biblioitemnumber ) = AddBiblio( $record_clone, $framework, $add_biblio_options ) };
602
                if ($@) {
538
                if ($@) {
603
                    warn "ERROR: Insert biblio $originalid failed: $@\n";
539
                    warn "ERROR updating biblio $matched_record_id: $@\n";
604
                    printlog( { id => $originalid, op => "insert", status => "ERROR" } ) if ($logfile);
540
                    printlog({ id => $matched_record_id, op => "update", status => "ERROR" }) if $loghandle;
605
                    next RECORD;
541
                    return;
542
                } elsif (!$success) {
543
                    warn "Update biblio $matched_record_id returned no success.\n";
544
                    printlog({ id => $matched_record_id, op => "update", status => "ERROR" }) if $loghandle;
545
                    return;
606
                } else {
546
                } else {
607
                    printlog( { id => $originalid, op => "insert", status => "ok" } ) if ($logfile);
547
                    $record_id = $matched_record_id;
548
                    printlog({ id => $record_id, op => "update", status => "ok" }) if $loghandle;
608
                }
549
                }
609
550
            } else {
551
                warn "Biblio $originalid matched but --update not set; skipping.\n";
552
                printlog({ id => $matched_record_id, op => "update", status => "skipped" }) if $loghandle;
553
                return;
554
            }
555
        }
556
        elsif ($insert) {
557
            my $record_clone = $record->clone();
558
            C4::Biblio::_strip_item_fields($record_clone);
559
            eval {
560
                ( $record_id, $biblioitemnumber ) = AddBiblio( $record_clone, $framework, $add_biblio_options );
561
            };
562
            if ($@) {
563
                warn "Insert biblio $originalid failed: $@\n";
564
                printlog({ id => $originalid, op => "insert", status => "ERROR" }) if $loghandle;
565
                return;
566
            } else {
567
                printlog({ id => $originalid, op => "insert", status => "ok" }) if $loghandle;
610
                # If incoming record has bib ids set we need to transfer
568
                # If incoming record has bib ids set we need to transfer
611
                # new ids from record_clone to incoming record to avoid
569
                # new ids from record_clone to incoming record to avoid
612
                # working on wrong record (the original record) later on
570
                # working on wrong record (the original record) later on
613
                # when adding items for example
571
                # when adding items for example
614
                C4::Biblio::_koha_marc_update_bib_ids( $record, $framework, $record_id, $biblioitemnumber );
572
                C4::Biblio::_koha_marc_update_bib_ids( $record, $framework, $record_id, $biblioitemnumber )
615
            } else {
573
                  if $record_id;
616
                warn "WARNING: Insert biblio $originalid skipped";
617
                printlog(
618
                    {
619
                        id     => $originalid, op => "insert",
620
                        status => "warning : biblio not in database and option -insert not enabled, skipping..."
621
                    }
622
                ) if ($logfile);
623
                next RECORD;
624
            }
574
            }
625
            my $record_has_added_items = 0;
575
        } else {
626
            if ($record_id) {
576
            warn "Biblio $originalid not in DB & --insert not set; skipping.\n";
627
                $yamlhash->{$originalid} = $record_id if $yamlfile;
577
            printlog({ id => $originalid, op => "insert", status => "skipped" }) if $loghandle;
628
                eval {
578
            return;
629
                    ( $itemnumbers_ref, $errors_ref ) =
579
        }
630
                        AddItemBatchFromMarc( $record, $record_id, $biblioitemnumber, $framework );
631
                };
632
                my $error_adding = $@;
633
634
                if ($error_adding) {
635
                    warn "ERROR: Adding items to bib $record_id failed: $error_adding";
636
                    printlog( { id => $record_id, op => "insert items", status => "ERROR" } ) if ($logfile);
637
638
                    # if we failed because of an exception, assume that
639
                    # the MARC columns in biblioitems were not set.
640
                    next RECORD;
641
                }
642
643
                $record_has_added_items = @{$itemnumbers_ref};
644
645
                if ( $dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' }
646
                    @$errors_ref )
647
                {
648
                    # Find the record called 'barcode'
649
                    my ( $tag, $sub ) = C4::Biblio::GetMarcFromKohaField('items.barcode');
650
651
                    # Now remove any items that didn't have a duplicate_barcode error,
652
                    # erase the barcodes on items that did, and re-add those items.
653
                    my %dupes;
654
                    foreach my $i ( 0 .. $#{$errors_ref} ) {
655
                        my $ref = $errors_ref->[$i];
656
                        if ( $ref && ( $ref->{error_code} eq 'duplicate_barcode' ) ) {
657
                            $dupes{ $ref->{item_sequence} } = 1;
658
659
                            # Delete the error message because we're going to
660
                            # retry this one.
661
                            delete $errors_ref->[$i];
662
                        }
663
                    }
664
                    my $seq = 0;
665
                    foreach my $field ( $record->field($tag) ) {
666
                        $seq++;
667
                        if ( $dupes{$seq} ) {
668
669
                            # Here we remove the barcode
670
                            $field->delete_subfield( code => $sub );
671
                        } else {
672
673
                            # otherwise we delete the field because we don't want
674
                            # two of them
675
                            $record->delete_fields($field);
676
                        }
677
                    }
678
580
679
                    # Now re-add the record as before, adding errors to the prev list
581
        if ($record_id) {
680
                    my $more_errors;
582
            my $record_has_added_items = 0;
681
                    eval {
583
            $yamlhash->{$originalid} = $record_id if $yamlfile;
682
                        ( $itemnumbers_ref, $more_errors ) =
584
            eval {
683
                            AddItemBatchFromMarc( $record, $record_id, $biblioitemnumber, '' );
585
                ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc(
684
                    };
586
                    $record,
685
                    if ($@) {
587
                    $record_id,
686
                        warn "ERROR: Adding items to bib $record_id failed: $@\n";
588
                    $biblioitemnumber,
687
                        printlog( { id => $record_id, op => "insert items", status => "ERROR" } ) if ($logfile);
589
                    $framework
688
590
                );
689
                        # if we failed because of an exception, assume that
591
            };
690
                        # the MARC columns in biblioitems were not set.
592
            my $error_adding = $@;
691
                        next RECORD;
593
            if ($error_adding) {
594
                warn "ERROR adding items to biblio $record_id: $error_adding";
595
                printlog({ id => $record_id, op => "insert items", status => "ERROR" }) if $loghandle;
596
                # If we failed because of an exception, assume that
597
                # the MARC columns in biblioitems were not set.
598
                return;
599
            }
600
            $record_has_added_items = @{$itemnumbers_ref};
601
602
            if ( $dedup_barcode
603
                && grep { $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref )
604
            {
605
                # Find the record called 'barcode'.
606
                my ( $tag, $sub ) = C4::Biblio::GetMarcFromKohaField('items.barcode');
607
                # Now remove any items that didn't have a duplicate_barcode error,
608
                # erase the barcodes on items that did, and re-add those items.
609
                my %dupes;
610
                foreach my $i ( 0 .. $#{$errors_ref} ) {
611
                    my $ref = $errors_ref->[$i];
612
                    if ( $ref && $ref->{error_code} && $ref->{error_code} eq 'duplicate_barcode' ) {
613
                        $dupes{ $ref->{item_sequence} } = 1;
614
                        # Delete the error message because we're going to
615
                        # retry this one.
616
                        delete $errors_ref->[$i];
692
                    }
617
                    }
693
                    $record_has_added_items ||= @{$itemnumbers_ref};
618
                }
694
                    if ( @{$more_errors} ) {
619
                my $seq = 0;
695
                        push @$errors_ref, @{$more_errors};
620
                foreach my $field ( $record->field($tag) ) {
621
                    $seq++;
622
                    if ( $dupes{$seq} ) {
623
                        # Here we remove the barcode.
624
                        $field->delete_subfield( code => $sub );
625
                    } else {
626
                        # Otherwise, we delete the field because we don't want
627
                        # two of them.
628
                        $record->delete_fields($field);
696
                    }
629
                    }
697
                }
630
                }
698
631
699
                if ($record_has_added_items) {
632
                # Now re-add the record as before, adding errors to the prev list.
700
                    printlog( { id => $record_id, op => "insert items", status => "ok" } ) if ($logfile);
633
                my $more_errors;
634
                eval {
635
                    ( $itemnumbers_ref, $more_errors ) = AddItemBatchFromMarc(
636
                        $record,
637
                        $record_id,
638
                        $biblioitemnumber,
639
                        ''
640
                    );
641
                };
642
                if ($@) {
643
                    warn "ERROR re-adding items biblio $record_id: $@\n";
644
                    printlog({ id => $record_id, op => "insert items", status => "ERROR" }) if $loghandle;
645
                    # If we failed because of an exception, assume that
646
                    # the MARC columns in biblioitems were not set.
647
                    return;
701
                }
648
                }
702
649
                $record_has_added_items ||= @{$itemnumbers_ref};
703
                if ( @{$errors_ref} ) {
650
                if (@{$more_errors}) {
704
                    report_item_errors( $record_id, $errors_ref );
651
                    push @$errors_ref, @{$more_errors};
705
                }
652
                }
653
            }
706
654
707
                my $biblio = Koha::Biblios->find($record_id);
655
            if ($record_has_added_items) {
708
                $record = $biblio->metadata_record( { embed_items => 1 } );
656
                printlog({ id => $record_id, op => "insert items", status => "ok" }) if $loghandle;
657
            }
658
            if (@$errors_ref) {
659
                report_item_errors( $record_id, $errors_ref );
660
            }
709
661
662
            my $biblio_obj = Koha::Biblios->find($record_id);
663
            if ($biblio_obj) {
664
                my $indexed_record = $biblio_obj->metadata_record({ embed_items => 1 });
710
                push @search_engine_record_ids, $record_id;
665
                push @search_engine_record_ids, $record_id;
711
                push @search_engine_records,    $record;
666
                push @search_engine_records,    $indexed_record;
712
            }
667
            }
713
        }
668
        }
714
        if ( $record_number % $commitnum == 0 || $record_number == $number || $record_number == $records_total ) {
669
    } # end authorities vs biblios
715
            $schema->txn_commit;
670
716
            $schema->txn_begin;
671
    # If we've hit a commit boundary
717
            if ($indexer) {
672
    if ( $record_number % $commitnum == 0 ) {
718
                $indexer->update_index( \@search_engine_record_ids, \@search_engine_records ) unless $skip_indexing;
673
        $schema->txn_commit;
719
                if ( C4::Context->preference('AutoLinkBiblios') ) {
674
        $schema->txn_begin;
720
                    foreach my $record (@search_engine_records) {
675
        if ($indexer && !$skip_indexing) {
721
                        BiblioAutoLink( $record, $framework );
676
            $indexer->update_index( \@search_engine_record_ids, \@search_engine_records );
722
                    }
677
            if (!$authorities && C4::Context->preference('AutoLinkBiblios')) {
678
                foreach my $r (@search_engine_records) {
679
                    BiblioAutoLink($r, $framework);
723
                }
680
                }
724
                @search_engine_record_ids = ();
725
                @search_engine_records    = ();
726
            }
681
            }
727
        }
682
        }
683
        @search_engine_record_ids = ();
684
        @search_engine_records    = ();
728
    }
685
    }
729
    print $record->as_formatted() . "\n" if ( $verbose // 0 ) == 2;
686
} # end sub import_single_record
730
    last                                 if $record_number == $number;
731
}
732
$schema->txn_commit;
733
687
734
if ($fk_off) {
735
    $dbh->do("SET FOREIGN_KEY_CHECKS = 1");
736
}
737
688
738
# Restore CataloguingLog and AuthoritiesLog
689
### CHANGED: A function for the older USMARC (non-XML) approach
739
delete $ENV{OVERRIDE_SYSPREF_CataloguingLog};
690
sub parse_nonxml_with_marcbatch {
740
delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog};
691
    my ($fh) = @_;
741
692
742
my $timeneeded = gettimeofday - $starttime;
693
    my $batch = MARC::Batch->new('USMARC', $fh);
743
print "\n$record_number MARC records done in $timeneeded seconds\n";
694
    $batch->warnings_off();
744
if ($logfile) {
695
    $batch->strict_off();
745
    print $loghandle "file : $input_marc_file\n";
696
746
    print $loghandle "$record_number MARC records done in $timeneeded seconds\n";
697
    # Skip file offset
747
    $loghandle->close;
698
    if ($offset) {
699
        print "Skipping offset: $offset records\n" if $verbose;
700
        $batch->next() while $offset-- > 0;
701
    }
702
703
    RECORD: while (1) {
704
        my $record;
705
        eval { $record = $batch->next() };
706
        if ($@) {
707
            warn "Bad MARC record ? : $@\n";
708
            next RECORD;
709
        }
710
        last unless $record;
711
712
        import_single_record($record);
713
        last if ( $number && $record_number == $number );
714
    }
748
}
715
}
749
if ($yamlfile) {
716
750
    open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";
717
# This version dynamically removes any marc prefix in each XML element, but uses more memory.
751
    print $yamlfileout Encode::decode_utf8( YAML::XS::Dump($yamlhash) );
718
# sub parse_xml_with_twig {
719
#     my ($file) = @_;
720
721
#     # We'll read offset records if needed, but doing so with Twig means
722
#     # we have to skip N <record> elements manually. We'll do a counter:
723
#     my $skipcount = $offset;
724
725
#     # We'll parse each <record>, convert to MARC::Record, call import_single_record
726
#     my $twig = XML::Twig->new(
727
#         twig_handlers => {
728
#             'marc:record' => sub {
729
#                 print "In handler!\n";
730
#                 my ($t, $elt) = @_;
731
732
#                 # If offset skipping is not done yet, skip this record
733
#                 if ($skipcount > 0) {
734
#                     $skipcount--;
735
#                     $elt->purge();
736
#                     return;
737
#                 }
738
739
#                 # Convert <record> to string
740
#                 my $xml_chunk = $elt->sprint();
741
742
#                 my $marcrec;
743
#                 eval {
744
#                     # If your Koha is MARC21, use 'MARC21' 
745
#                     # or 'UNIMARC' if you have that
746
#                     $marcrec = MARC::Record->new_from_xml($xml_chunk, 'UTF-8', 'MARC21');
747
#                 };
748
#                 if ($@ || !$marcrec) {
749
#                     warn "Skipping bad MARC XML record: $@\n";
750
#                     $elt->purge();
751
#                     return;
752
#                 }
753
#                 print "Looking to call import record sub\n";
754
#                 import_single_record($marcrec);
755
756
#                 # purge from memory
757
#                 $elt->purge();
758
759
#                 # if we have a record limit
760
#                 if ($number && $record_number >= $number) {
761
#                     $t->finish_now();
762
#                 }
763
#             }
764
#         },
765
#         keep_encoding => 1,
766
#     );
767
768
#     $twig->parsefile($file);
769
# }
770
771
### CHANGED: A function for streaming MARCXML with XML::Twig, with preprocessing the file.
772
sub parse_xml_with_twig {
773
    my ($file) = @_;
774
775
    # We'll read offset records if needed, but doing so with Twig means
776
    # we have to skip N <record> elements manually. We'll do a counter:
777
    my $skipcount = $offset;
778
779
    # Preprocess the XML file to remove namespaces
780
    my $preprocessed_file = 'preprocessed_marc.xml';
781
    open my $in, '<', $file or die "Can't open input file $file: $!";
782
    open my $out, '>', $preprocessed_file or die "Can't open output file $preprocessed_file: $!";
783
784
    while (<$in>) {
785
        s/marc://g;  # Remove 'marc:' prefix
786
        print $out $_;
787
    }
788
789
    close $in;
790
    close $out;
791
792
    print "Namespaces removed. Processing preprocessed file: $preprocessed_file\n";
793
794
    # Now process the preprocessed file
795
    my $twig = XML::Twig->new(
796
        twig_handlers => {
797
            'record' => sub {  # No namespace needed anymore
798
                my ($t, $elt) = @_;
799
800
                # If offset skipping is not done yet, skip this record
801
                if ($skipcount > 0) {
802
                    $skipcount--;
803
                    $elt->purge();
804
                    return;
805
                }
806
807
                # Convert <record> to string
808
                my $xml_chunk = $elt->sprint();
809
810
                my $marcrec;
811
                eval {
812
                    $marcrec = MARC::Record->new_from_xml($xml_chunk, 'UTF-8', 'MARC21');
813
                };
814
                if ($@ || !$marcrec) {
815
                    warn "Skipping bad MARC XML record: $@\n";
816
                    $elt->purge();
817
                    return;
818
                }
819
820
                import_single_record($marcrec);
821
822
                $elt->purge();
823
824
                # if we have a record limit
825
                if ($number && $record_number >= $number) {
826
                    $t->finish_now();
827
                }
828
            }
829
        },
830
        keep_encoding => 1,
831
    );
832
833
    $twig->parsefile($preprocessed_file);
752
}
834
}
753
exit 0;
835
836
837
#-----------------------------------------------------------------
838
# Helper subroutines from original script
839
#-----------------------------------------------------------------
754
840
755
sub YAMLFileEntry {
841
sub YAMLFileEntry {
756
    my ( $record, $record_id, $updated ) = @_;
842
    my ( $record, $record_id, $updated ) = @_;
Lines 859-864 sub get_heading_fields { Link Here
859
    return $headingfields;
945
    return $headingfields;
860
}
946
}
861
947
948
#-----------------------------------------------------------------
949
# MAIN
950
#-----------------------------------------------------------------
951
952
my $fh = IO::File->new($input_marc_file)
953
  or die "Could not open $input_marc_file: $!";
954
955
### CHANGED: if the user specified -m=MARCXML or something matching /XML/i,
956
### we do the streaming twig approach. Otherwise, do old MARC::Batch approach.
957
if ( defined $format && $format =~ /XML/i ) {
958
    print "XML with Twig call...\n";
959
    parse_xml_with_twig($input_marc_file);
960
} else {
961
    parse_nonxml_with_marcbatch($fh);
962
}
963
964
# Final commit
965
$schema->txn_commit;
966
967
# Index after importations.
968
if ($indexer && !$skip_indexing) {
969
    $indexer->update_index( \@search_engine_record_ids, \@search_engine_records );
970
    if (!$authorities && C4::Context->preference('AutoLinkBiblios')) {
971
        foreach my $r (@search_engine_records) {
972
            BiblioAutoLink($r, $framework);
973
        }
974
    }
975
}
976
977
if ($fk_off) {
978
    $dbh->do("SET FOREIGN_KEY_CHECKS = 1");
979
}
980
981
# Restore CataloguingLog and AuthoritiesLog
982
delete $ENV{OVERRIDE_SYSPREF_CataloguingLog};
983
delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog};
984
985
my $timeneeded = gettimeofday - $starttime;
986
print "\n$record_number MARC records done in $timeneeded seconds\n";
987
if ($logfile) {
988
    print $loghandle "file : $input_marc_file\n";
989
    print $loghandle "$record_number MARC records done in $timeneeded seconds\n";
990
    $loghandle->close;
991
}
992
if ($yamlfile) {
993
    open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";
994
    print $yamlfileout Encode::decode_utf8( YAML::XS::Dump($yamlhash) );
995
}
996
exit 0;
997
862
=head1 NAME
998
=head1 NAME
863
999
864
bulkmarcimport.pl - Import bibliographic/authority records into Koha
1000
bulkmarcimport.pl - Import bibliographic/authority records into Koha
Lines 1054-1058 this option bad records may kill the job. Link Here
1054
1190
1055
=back
1191
=back
1056
1192
1057
=cut
1193
=cut
1058
1059
- 

Return to bug 37020