View | Details | Raw Unified | Return to bug 10407
Collapse All | Expand All

(-)a/C4/ImportBatch.pm (-10 / +69 lines)
Lines 414-437 sub BatchStageMarcRecords { Link Here
414
    my $num_items = 0;
414
    my $num_items = 0;
415
    # FIXME - for now, we're dealing only with bibs
415
    # FIXME - for now, we're dealing only with bibs
416
    my $rec_num = 0;
416
    my $rec_num = 0;
417
    foreach my $marc_blob (split(/\x1D/, $marc_records)) {
417
    foreach my $marc_record (@$marc_records) {
418
        $marc_blob =~ s/^\s+//g;
419
        $marc_blob =~ s/\s+$//g;
420
        next unless $marc_blob;
421
        $rec_num++;
418
        $rec_num++;
422
        if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
419
        if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
423
            &$progress_callback($rec_num);
420
            &$progress_callback($rec_num);
424
        }
421
        }
425
        my ($marc_record, $charset_guessed, $char_errors) =
426
            MarcToUTF8Record($marc_blob, $marc_type, $encoding);
427
428
        $encoding = $charset_guessed unless $encoding;
429
422
430
        ModifyRecordWithTemplate( $marc_modification_template, $marc_record ) if ( $marc_modification_template );
423
        ModifyRecordWithTemplate( $marc_modification_template, $marc_record ) if ( $marc_modification_template );
431
424
432
        my $import_record_id;
425
        my $import_record_id;
433
        if (scalar($marc_record->fields()) == 0) {
426
        if (scalar($marc_record->fields()) == 0) {
434
            push @invalid_records, $marc_blob;
427
            push @invalid_records, $marc_record;
435
        } else {
428
        } else {
436
429
437
            # Normalize the record so it doesn't have separated diacritics
430
            # Normalize the record so it doesn't have separated diacritics
Lines 1467-1473 sub GetImportRecordMatches { Link Here
1467
    
1460
    
1468
}
1461
}
1469
1462
1470
1471
=head2 SetImportRecordMatches
1463
=head2 SetImportRecordMatches
1472
1464
1473
  SetImportRecordMatches($import_record_id, @matches);
1465
  SetImportRecordMatches($import_record_id, @matches);
Lines 1490-1495 sub SetImportRecordMatches { Link Here
1490
    }
1482
    }
1491
}
1483
}
1492
1484
1485
=head2 RecordsFromISO2709File
1486
1487
    my ($errors, $records) = C4::ImportBatch::RecordsFromISO2709File($input_file, $record_type, $encoding);
1488
1489
Reads ISO2709 binary porridge from the given file and creates MARC::Record-objects out of it.
1490
1491
@PARAM1, String, absolute path to the ISO2709 file.
1492
@PARAM2, String, see stage_file.pl
1493
@PARAM3, String, should be utf8
1494
1495
=cut
1496
1497
sub RecordsFromISO2709File {
1498
    my ($input_file, $record_type, $encoding) = @_;
1499
    my $errors;
1500
1501
    my $marc_type = C4::Context->preference('marcflavour');
1502
    $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth');
1503
1504
    open IN, "<$input_file" or die "$0: cannot open input file $input_file: $!\n";
1505
    my @marc_records;
1506
    $/ = "\035";
1507
    while (<IN>) {
1508
        s/^\s+//;
1509
        s/\s+$//;
1510
        next unless $_; # skip if record has only whitespace, as might occur
1511
                        # if file includes newlines between each MARC record
1512
        my ($marc_record, $charset_guessed, $char_errors) = MarcToUTF8Record($_, $marc_type, $encoding);
1513
        push @marc_records, $marc_record;
1514
        if ($charset_guessed ne $encoding) {
1515
            $errors = '' unless $errors;
1516
            $errors .= "Unexpected charset $charset_guessed, expecting $encoding\n";
1517
        }
1518
    }
1519
    close IN;
1520
    return ($errors, \@marc_records);
1521
}
1522
1523
=head2 RecordsFromMARCXMLFile
1524
1525
    my ($errors, $records) = C4::ImportBatch::RecordsFromMARCXMLFile($input_file, $encoding);
1526
1527
1528
1529
Creates MARC::Record-objects out of the given MARCXML-file.
1530
1531
@PARAM1, String, absolute path to the ISO2709 file.
1532
@PARAM2, String, should be utf8
1533
1534
=cut
1535
1536
sub RecordsFromMARCXMLFile {
1537
    my ( $filename, $encoding ) = @_;
1538
    my $batch = MARC::File::XML->in( $filename );
1539
    my @marcRecords;
1540
    my @errors;
1541
    do {
1542
        eval {
1543
            my $record = $batch->next($encoding);
1544
            push @marcRecords, $record if $record;
1545
        };
1546
        if ($@) {
1547
            push @errors, $@;
1548
        }
1549
    } while( $record );
1550
    return (\@errors, \@marcRecords);
1551
}
1493
1552
1494
# internal functions
1553
# internal functions
1495
1554
(-)a/misc/stage_file.pl (-18 / +18 lines)
Lines 44-56 my $add_items = 0; Link Here
44
my $input_file = "";
44
my $input_file = "";
45
my $batch_comment = "";
45
my $batch_comment = "";
46
my $want_help = 0;
46
my $want_help = 0;
47
my $no_replace ;
47
my $no_replace;
48
my $format = 'ISO2709';
48
my $no_create;
49
my $no_create;
49
my $item_action = 'always_add';
50
my $item_action = 'always_add';
50
51
51
my $result = GetOptions(
52
my $result = GetOptions(
52
    'encoding:s'    => \$encoding,
53
    'encoding:s'    => \$encoding,
53
    'file:s'        => \$input_file,
54
    'file:s'        => \$input_file,
55
    'format:s'      => \$format,
54
    'match|match-bibs:s'  => \$match,
56
    'match|match-bibs:s'  => \$match,
55
    'add-items'     => \$add_items,
57
    'add-items'     => \$add_items,
56
    'item-action:s' => \$item_action,
58
    'item-action:s' => \$item_action,
Lines 71-76 if (not $result or $input_file eq "" or $want_help) { Link Here
71
    print_usage();
73
    print_usage();
72
    exit 0;
74
    exit 0;
73
}
75
}
76
if ( $format !~ /^(MARCXML|ISO2709)$/i ) {
77
    print "\n --format must be MARCXML or ISO2709\n";
78
    print_usage();
79
    exit 0;
80
}
74
81
75
unless (-r $input_file) {
82
unless (-r $input_file) {
76
    die "$0: cannot open input file $input_file: $!\n";
83
    die "$0: cannot open input file $input_file: $!\n";
Lines 78-105 unless (-r $input_file) { Link Here
78
85
79
my $dbh = C4::Context->dbh;
86
my $dbh = C4::Context->dbh;
80
$dbh->{AutoCommit} = 0;
87
$dbh->{AutoCommit} = 0;
81
process_batch($input_file, $record_type, $match, $add_items, $batch_comment);
88
process_batch($format, $input_file, $record_type, $match, $add_items, $batch_comment);
82
$dbh->commit();
89
$dbh->commit();
83
90
84
exit 0;
91
exit 0;
85
92
86
sub process_batch {
93
sub process_batch {
87
    my ($input_file, $record_type, $match, $add_items, $batch_comment) = @_;
94
    my ($format, $input_file, $record_type, $match, $add_items, $batch_comment) = @_;
88
95
89
    open IN, "<$input_file" or die "$0: cannot open input file $input_file: $!\n";
96
    my ($errors, $marc_records) = C4::ImportBatch::RecordsFromISO2709File($input_file, $record_type, $encoding) if $format eq 'ISO2709';
90
    my $marc_records = "";
97
    warn $errors if $errors;
91
    $/ = "\035";
98
    $marc_records = C4::ImportBatch::RecordsFromMARCXMLFile($input_file, $encoding) if $format eq 'MARCXML';
92
    my $num_input_records = 0;
99
    my $num_input_records = ($marc_records) ? scalar(@$marc_records) : 0;
93
    while (<IN>) {
94
        s/^\s+//;
95
        s/\s+$//;
96
        next unless $_; # skip if record has only whitespace, as might occur
97
                        # if file includes newlines between each MARC record
98
        $marc_records .= $_; # FIXME - this sort of string concatenation
99
                             # is probably rather inefficient
100
        $num_input_records++;
101
    }
102
    close IN;
103
100
104
    print "... staging MARC records -- please wait\n";
101
    print "... staging MARC records -- please wait\n";
105
    #FIXME: We should really allow the use of marc modification frameworks and to_marc plugins here if possible
102
    #FIXME: We should really allow the use of marc modification frameworks and to_marc plugins here if possible
Lines 182-187 Parameters: Link Here
182
    --encoding <encoding>   encoding of MARC records, default is utf8.
179
    --encoding <encoding>   encoding of MARC records, default is utf8.
183
                            Other possible options are: MARC-8,
180
                            Other possible options are: MARC-8,
184
                            ISO_5426, ISO_6937, ISO_8859-1, EUC-KR
181
                            ISO_5426, ISO_6937, ISO_8859-1, EUC-KR
182
    --format                The MARC transport format to use?
183
                            Defaults to ISO2709.
184
                            Available values, MARCXML, ISO2709.
185
    --match <match_id>      use this option to match records
185
    --match <match_id>      use this option to match records
186
                            in the file with records already in
186
                            in the file with records already in
187
                            the database for future overlay.
187
                            the database for future overlay.
(-)a/tools/stage-marc-import.pl (-14 / +7 lines)
Lines 56-62 my $parse_items = $input->param('parse_items'); Link Here
56
my $item_action                = $input->param('item_action');
56
my $item_action                = $input->param('item_action');
57
my $comments                   = $input->param('comments');
57
my $comments                   = $input->param('comments');
58
my $record_type                = $input->param('record_type');
58
my $record_type                = $input->param('record_type');
59
my $encoding                   = $input->param('encoding');
59
my $encoding                   = $input->param('encoding') || 'utf8';
60
my $format                     = $input->param('format') || 'ISO2709';
60
my $to_marc_plugin             = $input->param('to_marc_plugin');
61
my $to_marc_plugin             = $input->param('to_marc_plugin');
61
my $marc_modification_template = $input->param('marc_modification_template_id');
62
my $marc_modification_template = $input->param('marc_modification_template_id');
62
63
Lines 84-105 if ($completedJobID) { Link Here
84
    my $results = $job->results();
85
    my $results = $job->results();
85
    $template->param(map { $_ => $results->{$_} } keys %{ $results });
86
    $template->param(map { $_ => $results->{$_} } keys %{ $results });
86
} elsif ($fileID) {
87
} elsif ($fileID) {
87
    my $upload = Koha::Upload->new->get({ id => $fileID, filehandle => 1 });
88
    my $upload = Koha::Upload->new->get({ id => $fileID });
88
    my $fh = $upload->{fh};
89
    my $filename = $upload->{path};
89
    my $filename = $upload->{name}; # filename only, no path
90
	my $marcrecord='';
90
	my $marcrecord='';
91
    $/ = "\035";
91
    my ($errors, $marcrecords) = C4::ImportBatch::RecordsFromISO2709File($uploaded_file->filename(), $record_type, $encoding);
92
	while (<$fh>) {
93
        s/^\s+//;
94
        s/\s+$//;
95
		$marcrecord.=$_;
96
	}
97
    $fh->close;
98
92
99
    my $job = undef;
93
    my $job = undef;
100
    my $dbh;
94
    my $dbh;
101
    if ($runinbackground) {
95
    if ($runinbackground) {
102
        my $job_size = () = $marcrecord =~ /\035/g;
96
        my $job_size = scalar(@$marcrecords);
103
        # if we're matching, job size is doubled
97
        # if we're matching, job size is doubled
104
        $job_size *= 2 if ($matcher_id ne "");
98
        $job_size *= 2 if ($matcher_id ne "");
105
        $job = C4::BackgroundJob->new($sessionID, $filename, '/cgi-bin/koha/tools/stage-marc-import.pl', $job_size);
99
        $job = C4::BackgroundJob->new($sessionID, $filename, '/cgi-bin/koha/tools/stage-marc-import.pl', $job_size);
Lines 137-143 if ($completedJobID) { Link Here
137
    my ( $batch_id, $num_valid, $num_items, @import_errors ) =
131
    my ( $batch_id, $num_valid, $num_items, @import_errors ) =
138
      BatchStageMarcRecords(
132
      BatchStageMarcRecords(
139
        $record_type,    $encoding,
133
        $record_type,    $encoding,
140
        $marcrecord,     $filename,
134
        $marcrecords,    $filename,
141
        $to_marc_plugin, $marc_modification_template,
135
        $to_marc_plugin, $marc_modification_template,
142
        $comments,       '',
136
        $comments,       '',
143
        $parse_items,    0,
137
        $parse_items,    0,
144
- 

Return to bug 10407