View | Details | Raw Unified | Return to bug 10407
Collapse All | Expand All

(-)a/C4/ImportBatch.pm (-18 / +64 lines)
Lines 326-340 sub ModAuthInBatch { Link Here
326
=cut
326
=cut
327
327
328
sub BatchStageMarcRecords {
328
sub BatchStageMarcRecords {
329
    my $record_type = shift;
329
	my ($record_type, $encoding, $marc_records, $file_name,
330
    my $encoding = shift;
330
		$marc_modification_template, $comments, $branch_code, $parse_items,
331
    my $marc_records = shift;
331
		$leave_as_staging) = @_;
332
    my $file_name = shift;
333
    my $marc_modification_template = shift;
334
    my $comments = shift;
335
    my $branch_code = shift;
336
    my $parse_items = shift;
337
    my $leave_as_staging = shift;
338
332
339
    # optional callback to monitor status 
333
    # optional callback to monitor status 
340
    # of job
334
    # of job
Lines 368-391 sub BatchStageMarcRecords { Link Here
368
    my $num_items = 0;
362
    my $num_items = 0;
369
    # FIXME - for now, we're dealing only with bibs
363
    # FIXME - for now, we're dealing only with bibs
370
    my $rec_num = 0;
364
    my $rec_num = 0;
371
    foreach my $marc_blob (split(/\x1D/, $marc_records)) {
365
372
        $marc_blob =~ s/^\s+//g;
366
    foreach my $marc_record (@$marc_records) {
373
        $marc_blob =~ s/\s+$//g;
374
        next unless $marc_blob;
375
        $rec_num++;
367
        $rec_num++;
376
        if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
368
        if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
377
            &$progress_callback($rec_num);
369
            &$progress_callback($rec_num);
378
        }
370
        }
379
        my ($marc_record, $charset_guessed, $char_errors) =
380
            MarcToUTF8Record($marc_blob, $marc_type, $encoding);
381
382
        $encoding = $charset_guessed unless $encoding;
383
371
384
        ModifyRecordWithTemplate( $marc_modification_template, $marc_record ) if ( $marc_modification_template );
372
        ModifyRecordWithTemplate( $marc_modification_template, $marc_record ) if ( $marc_modification_template );
385
373
386
        my $import_record_id;
374
        my $import_record_id;
387
        if (scalar($marc_record->fields()) == 0) {
375
        if (scalar($marc_record->fields()) == 0) {
388
            push @invalid_records, $marc_blob;
376
            push @invalid_records, $marc_record;
389
        } else {
377
        } else {
390
378
391
            # Normalize the record so it doesn't have separated diacritics
379
            # Normalize the record so it doesn't have separated diacritics
Lines 1425-1430 sub SetImportRecordBiblioMatch { Link Here
1425
    SetImportRecordMatches( $import_record_id, $matchedBiblioStub );
1413
    SetImportRecordMatches( $import_record_id, $matchedBiblioStub );
1426
}
1414
}
1427
1415
1416
=head RecordsFromISO2709File
1417
1418
    my $records = C4::ImportBatch::RecordsFromISO2709File($input_file, $record_type, $encoding);
1419
1420
Reads ISO2709 binary porridge from the given file and creates MARC::Record-objects out of it.
1421
1422
@PARAM1, String, absolute path to the ISO2709 file.
1423
@PARAM2, String, see stage_file.pl
1424
@PARAM3, String, should be utf8
1425
1426
=cut
1427
1428
sub RecordsFromISO2709File {
1429
    my ($input_file, $record_type, $encoding) = @_;
1430
    my $errors;
1431
1432
    my $marc_type = C4::Context->preference('marcflavour');
1433
    $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth');
1434
1435
    open IN, "<$input_file" or die "$0: cannot open input file $input_file: $!\n";
1436
    my @marc_records;
1437
    $/ = "\035";
1438
    while (<IN>) {
1439
        s/^\s+//;
1440
        s/\s+$//;
1441
        next unless $_; # skip if record has only whitespace, as might occur
1442
                        # if file includes newlines between each MARC record
1443
        my ($marc_record, $charset_guessed, $char_errors) = MarcToUTF8Record($_, $marc_type, $encoding);
1444
        push @marc_records, $marc_record;
1445
        if ($charset_guessed ne $encoding) {
1446
            $errors = '' unless $errors;
1447
            $errors .= "Unexpected charset $charset_guessed, expecting $encoding\n";
1448
        }
1449
    }
1450
    close IN;
1451
    return ($errors, \@marc_records);
1452
}
1453
1454
=head RecordsFromMARCXMLFile
1455
1456
    my $records = C4::ImportBatch::RecordsFromMARCXMLFile($input_file, $encoding);
1457
1458
Creates MARC::Record-objects out of the given MARCXML-file.
1459
1460
@PARAM1, String, absolute path to the ISO2709 file.
1461
@PARAM2, String, should be utf8
1462
1463
=cut
1464
1465
sub RecordsFromMARCXMLFile {
1466
    my ($filename, $encoding) = @_;
1467
    my $batch = MARC::File::XML->in( $filename );
1468
    my @marcRecords;
1469
    while (my $record = $batch->next($encoding)) { #How incredibly hard can be these MARC-libraries be to use?
1470
        push @marcRecords, $record;
1471
    }
1472
    return \@marcRecords;
1473
}
1428
1474
1429
# internal functions
1475
# internal functions
1430
1476
(-)a/misc/stage_file.pl (-17 / +12 lines)
Lines 45-54 my $input_file = ""; Link Here
45
my $batch_comment = "";
45
my $batch_comment = "";
46
my $want_help = 0;
46
my $want_help = 0;
47
my $no_replace ;
47
my $no_replace ;
48
my $format = 'ISO2709';
48
my $item_action = 'always_add';
49
my $item_action = 'always_add';
49
50
50
my $result = GetOptions(
51
my $result = GetOptions(
51
    'encoding:s'    => \$encoding,
52
    'encoding:s'    => \$encoding,
53
    'format:s'      => \$format,
52
    'file:s'        => \$input_file,
54
    'file:s'        => \$input_file,
53
    'match|match-bibs:s'  => \$match,
55
    'match|match-bibs:s'  => \$match,
54
    'add-items'     => \$add_items,
56
    'add-items'     => \$add_items,
Lines 76-103 unless (-r $input_file) { Link Here
76
78
77
my $dbh = C4::Context->dbh;
79
my $dbh = C4::Context->dbh;
78
$dbh->{AutoCommit} = 0;
80
$dbh->{AutoCommit} = 0;
79
process_batch($input_file, $record_type, $match, $add_items, $batch_comment);
81
process_batch($format, $input_file, $record_type, $match, $add_items, $batch_comment);
80
$dbh->commit();
82
$dbh->commit();
81
83
82
exit 0;
84
exit 0;
83
85
84
sub process_batch {
86
sub process_batch {
85
    my ($input_file, $record_type, $match, $add_items, $batch_comment) = @_;
87
    my ($format, $input_file, $record_type, $match, $add_items, $batch_comment) = @_;
86
88
87
    open IN, "<$input_file" or die "$0: cannot open input file $input_file: $!\n";
89
    my ($errors, $marc_records) = C4::ImportBatch::RecordsFromISO2709File($input_file) if $format eq 'ISO2709';
88
    my $marc_records = "";
90
    warn $errors if $errors;
89
    $/ = "\035";
91
    $marc_records = C4::ImportBatch::RecordsFromMARCXMLFile($input_file, $encoding) if $format eq 'MARCXML';
90
    my $num_input_records = 0;
92
    my $num_input_records = ($marc_records) ? scalar(@$marc_records) : 0;
91
    while (<IN>) {
92
        s/^\s+//;
93
        s/\s+$//;
94
        next unless $_; # skip if record has only whitespace, as might occur
95
                        # if file includes newlines between each MARC record
96
        $marc_records .= $_; # FIXME - this sort of string concatenation
97
                             # is probably rather inefficient
98
        $num_input_records++;
99
    }
100
    close IN;
101
93
102
    print "... staging MARC records -- please wait\n";
94
    print "... staging MARC records -- please wait\n";
103
    my ($batch_id, $num_valid_records, $num_items, @import_errors) =
95
    my ($batch_id, $num_valid_records, $num_items, @import_errors) =
Lines 179-184 Parameters: Link Here
179
    --encoding <encoding>   encoding of MARC records, default is utf8.
171
    --encoding <encoding>   encoding of MARC records, default is utf8.
180
                            Other possible options are: MARC-8,
172
                            Other possible options are: MARC-8,
181
                            ISO_5426, ISO_6937, ISO_8859-1, EUC-KR
173
                            ISO_5426, ISO_6937, ISO_8859-1, EUC-KR
174
    --format                The MARC transport format to use?
175
                            Defaults to ISO2709.
176
                            Available values, MARCXML, ISO2709.
182
    --match <match_id>      use this option to match records
177
    --match <match_id>      use this option to match records
183
                            in the file with records already in
178
                            in the file with records already in
184
                            the database for future overlay.
179
                            the database for future overlay.
(-)a/tools/stage-marc-import.pl (-11 / +4 lines)
Lines 58-63 my $item_action = $input->param('item_action'); Link Here
58
my $comments = $input->param('comments');
58
my $comments = $input->param('comments');
59
my $record_type = $input->param('record_type');
59
my $record_type = $input->param('record_type');
60
my $encoding = $input->param('encoding');
60
my $encoding = $input->param('encoding');
61
my $format = $input->param('format') || 'ISO2709';
61
my $marc_modification_template = $input->param('marc_modification_template_id');
62
my $marc_modification_template = $input->param('marc_modification_template_id');
62
63
63
my ($template, $loggedinuser, $cookie)
64
my ($template, $loggedinuser, $cookie)
Lines 80-100 if ($completedJobID) { Link Here
80
    $template->param(map { $_ => $results->{$_} } keys %{ $results });
81
    $template->param(map { $_ => $results->{$_} } keys %{ $results });
81
} elsif ($fileID) {
82
} elsif ($fileID) {
82
    my $uploaded_file = C4::UploadedFile->fetch($sessionID, $fileID);
83
    my $uploaded_file = C4::UploadedFile->fetch($sessionID, $fileID);
83
    my $fh = $uploaded_file->fh();
84
    my ($errors, $marcrecords) = C4::ImportBatch::RecordsFromISO2709File($uploaded_file->filename(), $record_type, $encoding);
84
	my $marcrecord='';
85
    $/ = "\035";
86
	while (<$fh>) {
87
        s/^\s+//;
88
        s/\s+$//;
89
		$marcrecord.=$_;
90
	}
91
85
92
    my $filename = $uploaded_file->name();
86
    my $filename = $uploaded_file->name();
93
    my $job = undef;
87
    my $job = undef;
94
    my $staging_callback = sub { };
88
    my $staging_callback = sub { };
95
    my $matching_callback = sub { };
89
    my $matching_callback = sub { };
96
    if ($runinbackground) {
90
    if ($runinbackground) {
97
        my $job_size = () = $marcrecord =~ /\035/g;
91
        my $job_size = scalar(@$marcrecords);
98
        # if we're matching, job size is doubled
92
        # if we're matching, job size is doubled
99
        $job_size *= 2 if ($matcher_id ne "");
93
        $job_size *= 2 if ($matcher_id ne "");
100
        $job = C4::BackgroundJob->new($sessionID, $filename, $ENV{'SCRIPT_NAME'}, $job_size);
94
        $job = C4::BackgroundJob->new($sessionID, $filename, $ENV{'SCRIPT_NAME'}, $job_size);
Lines 134-140 if ($completedJobID) { Link Here
134
    }
128
    }
135
129
136
    # FIXME branch code
130
    # FIXME branch code
137
    my ($batch_id, $num_valid, $num_items, @import_errors) = BatchStageMarcRecords($record_type, $encoding, $marcrecord, $filename, $marc_modification_template, $comments, '', $parse_items, 0, 50, staging_progress_callback($job, $dbh));
131
    my ($batch_id, $num_valid, $num_items, @import_errors) = BatchStageMarcRecords($record_type, $encoding, $marcrecords, $filename, $marc_modification_template, $comments, '', $parse_items, 0, 50, staging_progress_callback($job, $dbh));
138
132
139
    $dbh->commit();
133
    $dbh->commit();
140
134
141
- 

Return to bug 10407