Bugzilla – Attachment 38127 Details for
Bug 10407
Allow MARCXML records to be imported with Tools/Stage MARC records for import
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 10407 - allow MARCXML records to be imported via Koha's GUI
Bug-10407---allow-MARCXML-records-to-be-imported-v.patch (text/plain), 9.29 KB, created by
Olli-Antti Kivilahti
on 2015-04-19 08:20:50 UTC
(
hide
)
Description:
Bug 10407 - allow MARCXML records to be imported via Koha's GUI
Filename:
MIME Type:
Creator:
Olli-Antti Kivilahti
Created:
2015-04-19 08:20:50 UTC
Size:
9.29 KB
patch
obsolete
>From a7963af368aa5f85ba4cc0a4a8befcb63244aa86 Mon Sep 17 00:00:00 2001 >From: Olli-Antti Kivilahti <olli-antti.kivilahti@jns.fi> >Date: Fri, 17 Apr 2015 20:12:49 +0300 >Subject: [PATCH] Bug 10407 - allow MARCXML records to be imported via Koha's > GUI > >Initial commit > >Added check to warn if ISO2709-Records' encoding is different than the expected one. >--- > C4/ImportBatch.pm | 82 ++++++++++++++++++++++++++++++++++---------- > misc/stage_file.pl | 31 +++++++---------- > tools/stage-marc-import.pl | 14 +++----- > 3 files changed, 81 insertions(+), 46 deletions(-) > >diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm >index 5cd0306..812d3a6 100644 >--- a/C4/ImportBatch.pm >+++ b/C4/ImportBatch.pm >@@ -326,15 +326,9 @@ sub ModAuthInBatch { > =cut > > sub BatchStageMarcRecords { >- my $record_type = shift; >- my $encoding = shift; >- my $marc_records = shift; >- my $file_name = shift; >- my $marc_modification_template = shift; >- my $comments = shift; >- my $branch_code = shift; >- my $parse_items = shift; >- my $leave_as_staging = shift; >+ my ($format, $record_type, $encoding, $marc_records, $file_name, >+ $marc_modification_template, $comments, $branch_code, $parse_items, >+ $leave_as_staging) = @_; > > # optional callback to monitor status > # of job >@@ -368,24 +362,18 @@ sub BatchStageMarcRecords { > my $num_items = 0; > # FIXME - for now, we're dealing only with bibs > my $rec_num = 0; >- foreach my $marc_blob (split(/\x1D/, $marc_records)) { >- $marc_blob =~ s/^\s+//g; >- $marc_blob =~ s/\s+$//g; >- next unless $marc_blob; >+ >+ foreach my $marc_record (@$marc_records) { > $rec_num++; > if ($progress_interval and (0 == ($rec_num % $progress_interval))) { > &$progress_callback($rec_num); > } >- my ($marc_record, $charset_guessed, $char_errors) = >- MarcToUTF8Record($marc_blob, $marc_type, $encoding); >- >- $encoding = $charset_guessed unless $encoding; > > ModifyRecordWithTemplate( $marc_modification_template, $marc_record ) if ( $marc_modification_template ); > > my $import_record_id; > if (scalar($marc_record->fields()) == 0) { >- push @invalid_records, $marc_blob; >+ push @invalid_records, $marc_record; > } else { > > # Normalize the record so it doesn't have separated diacritics >@@ -1425,6 +1413,64 @@ sub SetImportRecordBiblioMatch { > SetImportRecordMatches( $import_record_id, $matchedBiblioStub ); > } > >+=head RecordsFromISO2709File >+ >+ my $records = C4::ImportBatch::RecordsFromISO2709File($input_file, $record_type, $encoding); >+ >+Reads ISO2709 binary porridge from the given file and creates MARC::Record-objects out of it. >+ >+@PARAM1, String, absolute path to the ISO2709 file. >+@PARAM2, String, see stage_file.pl >+@PARAM3, String, should be utf8 >+ >+=cut >+ >+sub RecordsFromISO2709File { >+ my ($input_file, $record_type, $encoding) = @_; >+ my $errors; >+ >+ my $marc_type = C4::Context->preference('marcflavour'); >+ $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth'); >+ >+ open IN, "<$input_file" or die "$0: cannot open input file $input_file: $!\n"; >+ my @marc_records; >+ $/ = "\035"; >+ while (<IN>) { >+ s/^\s+//; >+ s/\s+$//; >+ next unless $_; # skip if record has only whitespace, as might occur >+ # if file includes newlines between each MARC record >+ my ($marc_record, $charset_guessed, $char_errors) = MarcToUTF8Record($_, $marc_type, $encoding); >+ push @marc_records, $marc_record; >+ if (EXPR) { >+ $errors = '' unless $errors; >+ $errors .= "Unexpected charset $charset_guessed, expecting $encoding\n"; >+ } >+ } >+ close IN; >+ return ($errors, \@marc_records); >+} >+ >+=head RecordsFromMARCXMLFile >+ >+ my $records = C4::ImportBatch::RecordsFromMARCXMLFile($input_file, $encoding); >+ >+Creates MARC::Record-objects out of the given MARCXML-file. >+ >+@PARAM1, String, absolute path to the ISO2709 file. >+@PARAM2, String, should be utf8 >+ >+=cut >+ >+sub RecordsFromMARCXMLFile { >+ my ($filename, $encoding) = @_; >+ my $batch = MARC::File::XML->in( $filename, $encoding ); >+ my @marcRecords; >+ while (my $record = $batch->next()) { >+ push @marcRecords, $record; >+ } >+ return \@marcRecords; >+} > > # internal functions > >diff --git a/misc/stage_file.pl b/misc/stage_file.pl >index 2e15466..ded38e6 100755 >--- a/misc/stage_file.pl >+++ b/misc/stage_file.pl >@@ -45,10 +45,12 @@ my $input_file = ""; > my $batch_comment = ""; > my $want_help = 0; > my $no_replace ; >+my $format = 'ISO2709'; > my $item_action = 'always_add'; > > my $result = GetOptions( > 'encoding:s' => \$encoding, >+ 'format:s' => \$format, > 'file:s' => \$input_file, > 'match|match-bibs:s' => \$match, > 'add-items' => \$add_items, >@@ -76,32 +78,22 @@ unless (-r $input_file) { > > my $dbh = C4::Context->dbh; > $dbh->{AutoCommit} = 0; >-process_batch($input_file, $record_type, $match, $add_items, $batch_comment); >+process_batch($format, $input_file, $record_type, $match, $add_items, $batch_comment); > $dbh->commit(); > > exit 0; > > sub process_batch { >- my ($input_file, $record_type, $match, $add_items, $batch_comment) = @_; >- >- open IN, "<$input_file" or die "$0: cannot open input file $input_file: $!\n"; >- my $marc_records = ""; >- $/ = "\035"; >- my $num_input_records = 0; >- while (<IN>) { >- s/^\s+//; >- s/\s+$//; >- next unless $_; # skip if record has only whitespace, as might occur >- # if file includes newlines between each MARC record >- $marc_records .= $_; # FIXME - this sort of string concatenation >- # is probably rather inefficient >- $num_input_records++; >- } >- close IN; >+ my ($format, $input_file, $record_type, $match, $add_items, $batch_comment) = @_; >+ >+ my ($errors, $marc_records) = C4::ImportBatch::RecordsFromISO2709File($input_file) if $format eq 'ISO2709'; >+ warn $errors if $errors; >+ $marc_records = C4::ImportBatch::RecordsFromMARCXMLFile($input_file, $encoding) if $format eq 'MARCXML'; >+ my $num_input_records = ($marc_records) ? scalar(@$marc_records) : 0; > > print "... staging MARC records -- please wait\n"; > my ($batch_id, $num_valid_records, $num_items, @import_errors) = >- BatchStageMarcRecords($record_type, $encoding, $marc_records, $input_file, undef, $batch_comment, '', $add_items, 0, >+ BatchStageMarcRecords($format, $record_type, $encoding, $marc_records, $input_file, undef, $batch_comment, '', $add_items, 0, > 100, \&print_progress_and_commit); > print "... finished staging MARC records\n"; > >@@ -179,6 +171,9 @@ Parameters: > --encoding <encoding> encoding of MARC records, default is utf8. > Other possible options are: MARC-8, > ISO_5426, ISO_6937, ISO_8859-1, EUC-KR >+ --format The MARC transport format to use? >+ Defaults to ISO2709. >+ Available values, MARCXML, ISO2709. > --match <match_id> use this option to match records > in the file with records already in > the database for future overlay. >diff --git a/tools/stage-marc-import.pl b/tools/stage-marc-import.pl >index 8d6ab1a..cf5d64f 100755 >--- a/tools/stage-marc-import.pl >+++ b/tools/stage-marc-import.pl >@@ -58,6 +58,7 @@ my $item_action = $input->param('item_action'); > my $comments = $input->param('comments'); > my $record_type = $input->param('record_type'); > my $encoding = $input->param('encoding'); >+my $format = $input->param('format') || 'ISO2709'; > my $marc_modification_template = $input->param('marc_modification_template_id'); > > my ($template, $loggedinuser, $cookie) >@@ -80,21 +81,14 @@ if ($completedJobID) { > $template->param(map { $_ => $results->{$_} } keys %{ $results }); > } elsif ($fileID) { > my $uploaded_file = C4::UploadedFile->fetch($sessionID, $fileID); >- my $fh = $uploaded_file->fh(); >- my $marcrecord=''; >- $/ = "\035"; >- while (<$fh>) { >- s/^\s+//; >- s/\s+$//; >- $marcrecord.=$_; >- } >+ my ($errors, $marcrecords) = C4::ImportBatch::RecordsFromISO2709File($uploaded_file->filename(), $record_type, $encoding); > > my $filename = $uploaded_file->name(); > my $job = undef; > my $staging_callback = sub { }; > my $matching_callback = sub { }; > if ($runinbackground) { >- my $job_size = () = $marcrecord =~ /\035/g; >+ my $job_size = scalar(@$marcrecords); > # if we're matching, job size is doubled > $job_size *= 2 if ($matcher_id ne ""); > $job = C4::BackgroundJob->new($sessionID, $filename, $ENV{'SCRIPT_NAME'}, $job_size); >@@ -134,7 +128,7 @@ if ($completedJobID) { > } > > # FIXME branch code >- my ($batch_id, $num_valid, $num_items, @import_errors) = BatchStageMarcRecords($record_type, $encoding, $marcrecord, $filename, $marc_modification_template, $comments, '', $parse_items, 0, 50, staging_progress_callback($job, $dbh)); >+ my ($batch_id, $num_valid, $num_items, @import_errors) = BatchStageMarcRecords($format, $record_type, $encoding, $marcrecords, $filename, $marc_modification_template, $comments, '', $parse_items, 0, 50, staging_progress_callback($job, $dbh)); > > $dbh->commit(); > >-- >1.7.9.5
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 10407
:
38101
|
38127
|
38129
|
38133
|
42831
|
53164
|
53165
|
53218
|
53219
|
53220
|
53221
|
53223
|
53306
|
53765
|
53766
|
53767
|
53768
|
53769
|
53937
|
53938
|
53939
|
53940
|
53941