From 7c39b583c42a7849b00f67738bac4a45b22ad796 Mon Sep 17 00:00:00 2001 From: Olli-Antti Kivilahti Date: Fri, 17 Apr 2015 20:12:49 +0300 Subject: [PATCH] Bug 10407 - allow MARCXML records to be imported via Koha's GUI Initial commit --- C4/ImportBatch.pm | 77 +++++++++++++++++++++++++++++++++----------- misc/stage_file.pl | 30 +++++++---------- tools/stage-marc-import.pl | 14 +++----- 3 files changed, 75 insertions(+), 46 deletions(-) diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm index 5cd0306..1d5e089 100644 --- a/C4/ImportBatch.pm +++ b/C4/ImportBatch.pm @@ -326,15 +326,9 @@ sub ModAuthInBatch { =cut sub BatchStageMarcRecords { - my $record_type = shift; - my $encoding = shift; - my $marc_records = shift; - my $file_name = shift; - my $marc_modification_template = shift; - my $comments = shift; - my $branch_code = shift; - my $parse_items = shift; - my $leave_as_staging = shift; + my ($format, $record_type, $encoding, $marc_records, $file_name, + $marc_modification_template, $comments, $branch_code, $parse_items, + $leave_as_staging) = @_; # optional callback to monitor status # of job @@ -368,24 +362,18 @@ sub BatchStageMarcRecords { my $num_items = 0; # FIXME - for now, we're dealing only with bibs my $rec_num = 0; - foreach my $marc_blob (split(/\x1D/, $marc_records)) { - $marc_blob =~ s/^\s+//g; - $marc_blob =~ s/\s+$//g; - next unless $marc_blob; + + foreach my $marc_record (@$marc_records) { $rec_num++; if ($progress_interval and (0 == ($rec_num % $progress_interval))) { &$progress_callback($rec_num); } - my ($marc_record, $charset_guessed, $char_errors) = - MarcToUTF8Record($marc_blob, $marc_type, $encoding); - - $encoding = $charset_guessed unless $encoding; ModifyRecordWithTemplate( $marc_modification_template, $marc_record ) if ( $marc_modification_template ); my $import_record_id; if (scalar($marc_record->fields()) == 0) { - push @invalid_records, $marc_blob; + push @invalid_records, $marc_record; } else { # Normalize the record so it doesn't have separated diacritics @@ -1425,6 +1413,59 @@ sub SetImportRecordBiblioMatch { SetImportRecordMatches( $import_record_id, $matchedBiblioStub ); } +=head RecordsFromISO2709File + + my $records = C4::ImportBatch::RecordsFromISO2709File($input_file, $record_type, $encoding); + +Reads ISO2709 binary porridge from the given file and creates MARC::Record-objects out of it. + +@PARAM1, String, absolute path to the ISO2709 file. +@PARAM2, String, see stage_file.pl +@PARAM3, String, should be utf8 + +=cut + +sub RecordsFromISO2709File { + my ($input_file, $record_type, $encoding) = @_; + + my $marc_type = C4::Context->preference('marcflavour'); + $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth'); + + open IN, "<$input_file" or die "$0: cannot open input file $input_file: $!\n"; + my @marc_records; + $/ = "\035"; + while () { + s/^\s+//; + s/\s+$//; + next unless $_; # skip if record has only whitespace, as might occur + # if file includes newlines between each MARC record + my ($marc_record, $charset_guessed, $char_errors) = MarcToUTF8Record($_, $marc_type, $encoding); + push @marc_records, $marc_record; + } + close IN; + return \@marc_records; +} + +=head RecordsFromMARCXMLFile + + my $records = C4::ImportBatch::RecordsFromMARCXMLFile($input_file, $encoding); + +Creates MARC::Record-objects out of the given MARCXML-file. + +@PARAM1, String, absolute path to the ISO2709 file. +@PARAM2, String, should be utf8 + +=cut + +sub RecordsFromMARCXMLFile { + my ($filename, $encoding) = @_; + my $batch = MARC::File::XML->in( $filename, $encoding ); + my @marcRecords; + while (my $record = $batch->next()) { + push @marcRecords, $record; + } + return \@marcRecords; +} # internal functions diff --git a/misc/stage_file.pl b/misc/stage_file.pl index 2e15466..89dbe2a 100755 --- a/misc/stage_file.pl +++ b/misc/stage_file.pl @@ -45,10 +45,12 @@ my $input_file = ""; my $batch_comment = ""; my $want_help = 0; my $no_replace ; +my $format = 'ISO2709'; my $item_action = 'always_add'; my $result = GetOptions( 'encoding:s' => \$encoding, + 'format:s' => \$format, 'file:s' => \$input_file, 'match|match-bibs:s' => \$match, 'add-items' => \$add_items, @@ -76,32 +78,21 @@ unless (-r $input_file) { my $dbh = C4::Context->dbh; $dbh->{AutoCommit} = 0; -process_batch($input_file, $record_type, $match, $add_items, $batch_comment); +process_batch($format, $input_file, $record_type, $match, $add_items, $batch_comment); $dbh->commit(); exit 0; sub process_batch { - my ($input_file, $record_type, $match, $add_items, $batch_comment) = @_; - - open IN, "<$input_file" or die "$0: cannot open input file $input_file: $!\n"; - my $marc_records = ""; - $/ = "\035"; - my $num_input_records = 0; - while () { - s/^\s+//; - s/\s+$//; - next unless $_; # skip if record has only whitespace, as might occur - # if file includes newlines between each MARC record - $marc_records .= $_; # FIXME - this sort of string concatenation - # is probably rather inefficient - $num_input_records++; - } - close IN; + my ($format, $input_file, $record_type, $match, $add_items, $batch_comment) = @_; + + my $marc_records = C4::ImportBatch::RecordsFromISO2709File($input_file) if $format eq 'ISO2709'; + $marc_records = C4::ImportBatch::RecordsFromMARCXMLFile($input_file, $encoding) if $format eq 'MARCXML'; + my $num_input_records = ($marc_records) ? scalar(@$marc_records) : 0; print "... staging MARC records -- please wait\n"; my ($batch_id, $num_valid_records, $num_items, @import_errors) = - BatchStageMarcRecords($record_type, $encoding, $marc_records, $input_file, undef, $batch_comment, '', $add_items, 0, + BatchStageMarcRecords($format, $record_type, $encoding, $marc_records, $input_file, undef, $batch_comment, '', $add_items, 0, 100, \&print_progress_and_commit); print "... finished staging MARC records\n"; @@ -179,6 +170,9 @@ Parameters: --encoding encoding of MARC records, default is utf8. Other possible options are: MARC-8, ISO_5426, ISO_6937, ISO_8859-1, EUC-KR + --format The MARC transport format to use? + Defaults to ISO2709. + Available values, MARCXML, ISO2709. --match use this option to match records in the file with records already in the database for future overlay. diff --git a/tools/stage-marc-import.pl b/tools/stage-marc-import.pl index 8d6ab1a..b249838 100755 --- a/tools/stage-marc-import.pl +++ b/tools/stage-marc-import.pl @@ -58,6 +58,7 @@ my $item_action = $input->param('item_action'); my $comments = $input->param('comments'); my $record_type = $input->param('record_type'); my $encoding = $input->param('encoding'); +my $format = $input->param('format') || 'ISO2709'; my $marc_modification_template = $input->param('marc_modification_template_id'); my ($template, $loggedinuser, $cookie) @@ -80,21 +81,14 @@ if ($completedJobID) { $template->param(map { $_ => $results->{$_} } keys %{ $results }); } elsif ($fileID) { my $uploaded_file = C4::UploadedFile->fetch($sessionID, $fileID); - my $fh = $uploaded_file->fh(); - my $marcrecord=''; - $/ = "\035"; - while (<$fh>) { - s/^\s+//; - s/\s+$//; - $marcrecord.=$_; - } + my $marcrecords = C4::ImportBatch::RecordsFromISO2709File($uploaded_file->filename(), $record_type, $encoding); my $filename = $uploaded_file->name(); my $job = undef; my $staging_callback = sub { }; my $matching_callback = sub { }; if ($runinbackground) { - my $job_size = () = $marcrecord =~ /\035/g; + my $job_size = scalar(@$marcrecords); # if we're matching, job size is doubled $job_size *= 2 if ($matcher_id ne ""); $job = C4::BackgroundJob->new($sessionID, $filename, $ENV{'SCRIPT_NAME'}, $job_size); @@ -134,7 +128,7 @@ if ($completedJobID) { } # FIXME branch code - my ($batch_id, $num_valid, $num_items, @import_errors) = BatchStageMarcRecords($record_type, $encoding, $marcrecord, $filename, $marc_modification_template, $comments, '', $parse_items, 0, 50, staging_progress_callback($job, $dbh)); + my ($batch_id, $num_valid, $num_items, @import_errors) = BatchStageMarcRecords($format, $record_type, $encoding, $marcrecords, $filename, $marc_modification_template, $comments, '', $parse_items, 0, 50, staging_progress_callback($job, $dbh)); $dbh->commit(); -- 1.7.9.5