View | Details | Raw Unified | Return to bug 10407
Collapse All | Expand All

(-)a/C4/ImportBatch.pm (-10 / +69 lines)
Lines 413-436 sub BatchStageMarcRecords { Link Here
413
    my $num_items = 0;
413
    my $num_items = 0;
414
    # FIXME - for now, we're dealing only with bibs
414
    # FIXME - for now, we're dealing only with bibs
415
    my $rec_num = 0;
415
    my $rec_num = 0;
416
    foreach my $marc_blob (split(/\x1D/, $marc_records)) {
416
    foreach my $marc_record (@$marc_records) {
417
        $marc_blob =~ s/^\s+//g;
418
        $marc_blob =~ s/\s+$//g;
419
        next unless $marc_blob;
420
        $rec_num++;
417
        $rec_num++;
421
        if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
418
        if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
422
            &$progress_callback($rec_num);
419
            &$progress_callback($rec_num);
423
        }
420
        }
424
        my ($marc_record, $charset_guessed, $char_errors) =
425
            MarcToUTF8Record($marc_blob, $marc_type, $encoding);
426
427
        $encoding = $charset_guessed unless $encoding;
428
421
429
        ModifyRecordWithTemplate( $marc_modification_template, $marc_record ) if ( $marc_modification_template );
422
        ModifyRecordWithTemplate( $marc_modification_template, $marc_record ) if ( $marc_modification_template );
430
423
431
        my $import_record_id;
424
        my $import_record_id;
432
        if (scalar($marc_record->fields()) == 0) {
425
        if (scalar($marc_record->fields()) == 0) {
433
            push @invalid_records, $marc_blob;
426
            push @invalid_records, $marc_record;
434
        } else {
427
        } else {
435
428
436
            # Normalize the record so it doesn't have separated diacritics
429
            # Normalize the record so it doesn't have separated diacritics
Lines 1448-1454 sub GetImportRecordMatches { Link Here
1448
    
1441
    
1449
}
1442
}
1450
1443
1451
1452
=head2 SetImportRecordMatches
1444
=head2 SetImportRecordMatches
1453
1445
1454
  SetImportRecordMatches($import_record_id, @matches);
1446
  SetImportRecordMatches($import_record_id, @matches);
Lines 1471-1476 sub SetImportRecordMatches { Link Here
1471
    }
1463
    }
1472
}
1464
}
1473
1465
1466
=head2 RecordsFromISO2709File
1467
1468
    my ($errors, $records) = C4::ImportBatch::RecordsFromISO2709File($input_file, $record_type, $encoding);
1469
1470
Reads ISO2709 binary porridge from the given file and creates MARC::Record-objects out of it.
1471
1472
@PARAM1, String, absolute path to the ISO2709 file.
1473
@PARAM2, String, see stage_file.pl
1474
@PARAM3, String, should be utf8
1475
1476
=cut
1477
1478
sub RecordsFromISO2709File {
1479
    my ($input_file, $record_type, $encoding) = @_;
1480
    my $errors;
1481
1482
    my $marc_type = C4::Context->preference('marcflavour');
1483
    $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth');
1484
1485
    open IN, "<$input_file" or die "$0: cannot open input file $input_file: $!\n";
1486
    my @marc_records;
1487
    $/ = "\035";
1488
    while (<IN>) {
1489
        s/^\s+//;
1490
        s/\s+$//;
1491
        next unless $_; # skip if record has only whitespace, as might occur
1492
                        # if file includes newlines between each MARC record
1493
        my ($marc_record, $charset_guessed, $char_errors) = MarcToUTF8Record($_, $marc_type, $encoding);
1494
        push @marc_records, $marc_record;
1495
        if ($charset_guessed ne $encoding) {
1496
            $errors = '' unless $errors;
1497
            $errors .= "Unexpected charset $charset_guessed, expecting $encoding\n";
1498
        }
1499
    }
1500
    close IN;
1501
    return ($errors, \@marc_records);
1502
}
1503
1504
=head2 RecordsFromMARCXMLFile
1505
1506
    my ($errors, $records) = C4::ImportBatch::RecordsFromMARCXMLFile($input_file, $encoding);
1507
1508
1509
1510
Creates MARC::Record-objects out of the given MARCXML-file.
1511
1512
@PARAM1, String, absolute path to the ISO2709 file.
1513
@PARAM2, String, should be utf8
1514
1515
=cut
1516
1517
sub RecordsFromMARCXMLFile {
1518
    my ( $filename, $encoding ) = @_;
1519
    my $batch = MARC::File::XML->in( $filename );
1520
    my @marcRecords;
1521
    my @errors;
1522
    do {
1523
        eval {
1524
            my $record = $batch->next($encoding);
1525
            push @marcRecords, $record if $record;
1526
        };
1527
        if ($@) {
1528
            push @errors, $@;
1529
        }
1530
    } while( $record );
1531
    return (\@errors, \@marcRecords);
1532
}
1474
1533
1475
# internal functions
1534
# internal functions
1476
1535
(-)a/misc/stage_file.pl (-18 / +18 lines)
Lines 44-56 my $add_items = 0; Link Here
44
my $input_file = "";
44
my $input_file = "";
45
my $batch_comment = "";
45
my $batch_comment = "";
46
my $want_help = 0;
46
my $want_help = 0;
47
my $no_replace ;
47
my $no_replace;
48
my $format = 'ISO2709';
48
my $no_create;
49
my $no_create;
49
my $item_action = 'always_add';
50
my $item_action = 'always_add';
50
51
51
my $result = GetOptions(
52
my $result = GetOptions(
52
    'encoding:s'    => \$encoding,
53
    'encoding:s'    => \$encoding,
53
    'file:s'        => \$input_file,
54
    'file:s'        => \$input_file,
55
    'format:s'      => \$format,
54
    'match|match-bibs:s'  => \$match,
56
    'match|match-bibs:s'  => \$match,
55
    'add-items'     => \$add_items,
57
    'add-items'     => \$add_items,
56
    'item-action:s' => \$item_action,
58
    'item-action:s' => \$item_action,
Lines 71-76 if (not $result or $input_file eq "" or $want_help) { Link Here
71
    print_usage();
73
    print_usage();
72
    exit 0;
74
    exit 0;
73
}
75
}
76
if ( $format !~ /^(MARCXML|ISO2709)$/i ) {
77
    print "\n --format must be MARCXML or ISO2709\n";
78
    print_usage();
79
    exit 0;
80
}
74
81
75
unless (-r $input_file) {
82
unless (-r $input_file) {
76
    die "$0: cannot open input file $input_file: $!\n";
83
    die "$0: cannot open input file $input_file: $!\n";
Lines 78-105 unless (-r $input_file) { Link Here
78
85
79
my $dbh = C4::Context->dbh;
86
my $dbh = C4::Context->dbh;
80
$dbh->{AutoCommit} = 0;
87
$dbh->{AutoCommit} = 0;
81
process_batch($input_file, $record_type, $match, $add_items, $batch_comment);
88
process_batch($format, $input_file, $record_type, $match, $add_items, $batch_comment);
82
$dbh->commit();
89
$dbh->commit();
83
90
84
exit 0;
91
exit 0;
85
92
86
sub process_batch {
93
sub process_batch {
87
    my ($input_file, $record_type, $match, $add_items, $batch_comment) = @_;
94
    my ($format, $input_file, $record_type, $match, $add_items, $batch_comment) = @_;
88
95
89
    open IN, "<$input_file" or die "$0: cannot open input file $input_file: $!\n";
96
    my ($errors, $marc_records) = C4::ImportBatch::RecordsFromISO2709File($input_file, $record_type, $encoding) if $format eq 'ISO2709';
90
    my $marc_records = "";
97
    warn $errors if $errors;
91
    $/ = "\035";
98
    $marc_records = C4::ImportBatch::RecordsFromMARCXMLFile($input_file, $encoding) if $format eq 'MARCXML';
92
    my $num_input_records = 0;
99
    my $num_input_records = ($marc_records) ? scalar(@$marc_records) : 0;
93
    while (<IN>) {
94
        s/^\s+//;
95
        s/\s+$//;
96
        next unless $_; # skip if record has only whitespace, as might occur
97
                        # if file includes newlines between each MARC record
98
        $marc_records .= $_; # FIXME - this sort of string concatenation
99
                             # is probably rather inefficient
100
        $num_input_records++;
101
    }
102
    close IN;
103
100
104
    print "... staging MARC records -- please wait\n";
101
    print "... staging MARC records -- please wait\n";
105
    #FIXME: We should really allow the use of marc modification frameworks and to_marc plugins here if possible
102
    #FIXME: We should really allow the use of marc modification frameworks and to_marc plugins here if possible
Lines 182-187 Parameters: Link Here
182
    --encoding <encoding>   encoding of MARC records, default is utf8.
179
    --encoding <encoding>   encoding of MARC records, default is utf8.
183
                            Other possible options are: MARC-8,
180
                            Other possible options are: MARC-8,
184
                            ISO_5426, ISO_6937, ISO_8859-1, EUC-KR
181
                            ISO_5426, ISO_6937, ISO_8859-1, EUC-KR
182
    --format                The MARC transport format to use?
183
                            Defaults to ISO2709.
184
                            Available values, MARCXML, ISO2709.
185
    --match <match_id>      use this option to match records
185
    --match <match_id>      use this option to match records
186
                            in the file with records already in
186
                            in the file with records already in
187
                            the database for future overlay.
187
                            the database for future overlay.
(-)a/tools/stage-marc-import.pl (-14 / +7 lines)
Lines 56-62 my $parse_items = $input->param('parse_items'); Link Here
56
my $item_action                = $input->param('item_action');
56
my $item_action                = $input->param('item_action');
57
my $comments                   = $input->param('comments');
57
my $comments                   = $input->param('comments');
58
my $record_type                = $input->param('record_type');
58
my $record_type                = $input->param('record_type');
59
my $encoding                   = $input->param('encoding');
59
my $encoding                   = $input->param('encoding') || 'utf8';
60
my $format                     = $input->param('format') || 'ISO2709';
60
my $to_marc_plugin             = $input->param('to_marc_plugin');
61
my $to_marc_plugin             = $input->param('to_marc_plugin');
61
my $marc_modification_template = $input->param('marc_modification_template_id');
62
my $marc_modification_template = $input->param('marc_modification_template_id');
62
63
Lines 84-105 if ($completedJobID) { Link Here
84
    my $results = $job->results();
85
    my $results = $job->results();
85
    $template->param(map { $_ => $results->{$_} } keys %{ $results });
86
    $template->param(map { $_ => $results->{$_} } keys %{ $results });
86
} elsif ($fileID) {
87
} elsif ($fileID) {
87
    my $upload = Koha::Upload->new->get({ id => $fileID, filehandle => 1 });
88
    my $upload = Koha::Upload->new->get({ id => $fileID });
88
    my $fh = $upload->{fh};
89
    my $filename = $upload->{path};
89
    my $filename = $upload->{name}; # filename only, no path
90
	my $marcrecord='';
90
	my $marcrecord='';
91
    $/ = "\035";
91
    my ($errors, $marcrecords) = C4::ImportBatch::RecordsFromISO2709File($uploaded_file->filename(), $record_type, $encoding);
92
	while (<$fh>) {
93
        s/^\s+//;
94
        s/\s+$//;
95
		$marcrecord.=$_;
96
	}
97
    $fh->close;
98
92
99
    my $job = undef;
93
    my $job = undef;
100
    my $dbh;
94
    my $dbh;
101
    if ($runinbackground) {
95
    if ($runinbackground) {
102
        my $job_size = () = $marcrecord =~ /\035/g;
96
        my $job_size = scalar(@$marcrecords);
103
        # if we're matching, job size is doubled
97
        # if we're matching, job size is doubled
104
        $job_size *= 2 if ($matcher_id ne "");
98
        $job_size *= 2 if ($matcher_id ne "");
105
        $job = C4::BackgroundJob->new($sessionID, $filename, '/cgi-bin/koha/tools/stage-marc-import.pl', $job_size);
99
        $job = C4::BackgroundJob->new($sessionID, $filename, '/cgi-bin/koha/tools/stage-marc-import.pl', $job_size);
Lines 137-143 if ($completedJobID) { Link Here
137
    my ( $batch_id, $num_valid, $num_items, @import_errors ) =
131
    my ( $batch_id, $num_valid, $num_items, @import_errors ) =
138
      BatchStageMarcRecords(
132
      BatchStageMarcRecords(
139
        $record_type,    $encoding,
133
        $record_type,    $encoding,
140
        $marcrecord,     $filename,
134
        $marcrecords,    $filename,
141
        $to_marc_plugin, $marc_modification_template,
135
        $to_marc_plugin, $marc_modification_template,
142
        $comments,       '',
136
        $comments,       '',
143
        $parse_items,    0,
137
        $parse_items,    0,
144
- 

Return to bug 10407