Bugzilla – Attachment 122003 Details for
Bug 19266
Add ability to update 005 during import of records and check 001 uniqueness on save to batch
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 19266: Allow 001 checking on save to batch and 005 update on import of records
Bug-19266-Allow-001-checking-on-save-to-batch-and-.patch (text/plain), 21.28 KB, created by
Nick Clemens (kidclamp)
on 2021-06-15 13:16:32 UTC
(
hide
)
Description:
Bug 19266: Allow 001 checking on save to batch and 005 update on import of records
Filename:
MIME Type:
Creator:
Nick Clemens (kidclamp)
Created:
2021-06-15 13:16:32 UTC
Size:
21.28 KB
patch
obsolete
>From a6349bc87f6fe46ca1fb5a9cb35501c8ce87edd6 Mon Sep 17 00:00:00 2001 >From: Jesse Weaver <pianohacker@gmail.com> >Date: Fri, 8 Mar 2019 16:11:33 +0100 >Subject: [PATCH] Bug 19266: Allow 001 checking on save to batch and 005 update > on import of records > >To test: > 1 - Export a set of records with control numbers from your system > 2 - Stage those records > 3 - Note during staging a new option to update the 005 > 4 - Check records after staging > 5 - Note 005 is updated or not as the option is selected > 6 - Open the advanced cataloging editor > 7 - Under settings choose Import Batches > 8 - Set the batch you just imported as a save target > 9 - Select that batch only as save target >10 - Create a new record, give it the control number of a record in the >batch >11 - Attempt to save to the batch >12 - You get an error >13 - Save overriding warnings >14 - The record is then added to the batch >--- > C4/Biblio.pm | 34 ++++-- > C4/ImportBatch.pm | 135 ++++++++++++++++----- > .../prog/en/modules/tools/stage-marc-import.tt | 30 +++++ > misc/stage_file.pl | 40 +++--- > tools/stage-marc-import.pl | 60 +++++---- > 5 files changed, 213 insertions(+), 86 deletions(-) > >diff --git a/C4/Biblio.pm b/C4/Biblio.pm >index d341e8f6c0..744797574c 100644 >--- a/C4/Biblio.pm >+++ b/C4/Biblio.pm >@@ -2907,6 +2907,31 @@ sub _koha_delete_biblio_metadata { > > =head1 UNEXPORTED FUNCTIONS > >+=head2 UpdateMarcTimestamp >+ >+ &UpdateMarcTimestamp({ record => $record, encoding => $encoding }); >+ >+Updates the 005 timestamp of the given record to the current time. >+ >+=cut >+ >+sub UpdateMarcTimestamp { >+ my $params = shift; >+ my $encoding = $params->{encoding}; >+ my $record = $params->{record}; >+ >+ if($encoding =~ /MARC21|UNIMARC/) { >+ my @a= (localtime) [5,4,3,2,1,0]; $a[0]+=1900; $a[1]++; >+ # YY MM DD HH MM SS (update year and month) >+ my $f005= $record->field('005'); >+ unless ($f005) { >+ $f005 = MARC::Field->new('005'); >+ $record->insert_fields_ordered( $f005 ); >+ } >+ $f005->update(sprintf("%4d%02d%02d%02d%02d%04.1f",@a)); >+ } >+} >+ > =head2 ModBiblioMarc > > &ModBiblioMarc($newrec,$biblionumber); >@@ -2952,13 +2977,8 @@ sub ModBiblioMarc { > } > } > >- #enhancement 5374: update transaction date (005) for marc21/unimarc >- if($encoding =~ /MARC21|UNIMARC/) { >- my @a= (localtime) [5,4,3,2,1,0]; $a[0]+=1900; $a[1]++; >- # YY MM DD HH MM SS (update year and month) >- my $f005= $record->field('005'); >- $f005->update(sprintf("%4d%02d%02d%02d%02d%04.1f",@a)) if $f005; >- } >+ UpdateMarcTimestamp({ record => $record, encoding => $encoding }); >+ > > my $metadata = { > biblionumber => $biblionumber, >diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm >index 2a5696d513..ca34966cdd 100644 >--- a/C4/ImportBatch.pm >+++ b/C4/ImportBatch.pm >@@ -48,6 +48,7 @@ BEGIN { > AddItemsToImportBiblio > ModAuthorityInBatch > ModBiblioInBatch >+ CheckRecordControlNumberInBatch > > BatchStageMarcRecords > BatchFindDuplicates >@@ -309,6 +310,41 @@ sub ModBiblioInBatch { > > } > >+=head2 CheckRecordControlNumberInBatch >+ >+ CheckRecordControlNumberInBatch( $marc_record, $batch_id[, $import_record_id] ); >+ >+Check to see if the given record's control number is already in use in the given batch. If provided, >+will check only for conflicts in records besides the given C<$import_batch_id>. >+ >+If a record with the given control number in the given batch exists, will return its C<import_record_id>. >+ >+=cut >+ >+sub CheckRecordControlNumberInBatch { >+ my ( $marc_record, $batch_id, $import_record_id ) = @_; >+ >+ return unless ( $marc_record->field('001') ); >+ >+ my %extra_conditions; >+ >+ $extra_conditions{'import_record.import_record_id'} = { '!=', $import_record_id } if ( $import_record_id ); >+ >+ my $control_number = $marc_record->field('001')->data; >+ >+ my $schema = Koha::Database->new()->schema(); >+ return $schema->resultset('ImportBiblio')->search( >+ { >+ control_number => $control_number, >+ 'import_record.import_batch_id' => $batch_id, >+ %extra_conditions >+ }, >+ { >+ join => 'import_record', >+ } >+ )->get_column('import_record.import_batch_id')->first(); >+} >+ > =head2 AddAuthToBatch > > my $import_record_id = AddAuthToBatch($batch_id, $record_sequence, >@@ -348,7 +384,28 @@ sub ModAuthInBatch { > > =head2 BatchStageMarcRecords > >-( $batch_id, $num_records, $num_items, @invalid_records ) = >+{ >+ batch_id => $batch_id, >+ num_records => $num_records, >+ num_items => $num_items, >+ invalid_record => \@invalid_records >+} = BatchStageMarcRecords({ >+ record_type => $record_type, >+ encoding => $encoding, >+ marc_records => $marc_records, >+ file_name => $file_name, >+ comments => $comments, >+ [ control_number_handling => 'preserve' | 'overwrite' ], >+ [ leave_as_staging => 1 ] >+ [ marc_modification_template => $marc_modification_template, ] >+ [ parse_items => 1 ] >+ [ progress_interval => $progress_interval, >+ progress_callback => \&progress_callback, ] >+ [ timestamp_update => 'now' ], >+ }); >+ >+Any optional parameters are assumed to be no / off if omitted. If C<progress_interval> is specified, C<progress_callback> must be as well. >+ > BatchStageMarcRecords( > $record_type, $encoding, > $marc_records, $file_name, >@@ -361,23 +418,15 @@ sub ModAuthInBatch { > =cut > > sub BatchStageMarcRecords { >- my $record_type = shift; >- my $encoding = shift; >- my $marc_records = shift; >- my $file_name = shift; >- my $marc_modification_template = shift; >- my $comments = shift; >- my $branch_code = shift; >- my $parse_items = shift; >- my $leave_as_staging = shift; >+ my $params = shift; > > # optional callback to monitor status > # of job > my $progress_interval = 0; > my $progress_callback = undef; >- if ($#_ == 1) { >- $progress_interval = shift; >- $progress_callback = shift; >+ if ( $params->{progress_interval} ) { >+ $progress_interval = $params->{progress_interval}; >+ $progress_callback = $params->{progress_callback}; > $progress_interval = 0 unless $progress_interval =~ /^\d+$/ and $progress_interval > 0; > $progress_interval = 0 unless 'CODE' eq ref $progress_callback; > } >@@ -386,19 +435,18 @@ sub BatchStageMarcRecords { > overlay_action => 'create_new', > import_status => 'staging', > batch_type => 'batch', >- file_name => $file_name, >- comments => $comments, >- record_type => $record_type, >+ file_name => $params->{file_name}, >+ comments => $params->{comments}, >+ record_type => $params->{record_type}, > } ); >- if ($parse_items) { >+ if ($params->{parse_items}) { > SetImportBatchItemAction($batch_id, 'always_add'); > } else { > SetImportBatchItemAction($batch_id, 'ignore'); > } > >- > my $marc_type = C4::Context->preference('marcflavour'); >- $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth'); >+ $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $params->{record_type} eq 'auth'); > my @invalid_records = (); > my $num_valid = 0; > my $num_items = 0; >@@ -410,7 +458,7 @@ sub BatchStageMarcRecords { > &$progress_callback($rec_num); > } > >- ModifyRecordWithTemplate( $marc_modification_template, $marc_record ) if ( $marc_modification_template ); >+ ModifyRecordWithTemplate( $params->{marc_modification_template}, $marc_record ) if ( $params->{marc_modification_template} ); > > my $import_record_id; > if (scalar($marc_record->fields()) == 0) { >@@ -419,25 +467,55 @@ sub BatchStageMarcRecords { > > # Normalize the record so it doesn't have separated diacritics > SetUTF8Flag($marc_record); >- > $num_valid++; >- if ($record_type eq 'biblio') { >- $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0); >- if ($parse_items) { >+ >+ C4::Biblio::UpdateMarcTimestamp( $marc_record ) if ( $params->{timestamp_update} eq 'now' ); >+ my $existing_import_record_id; >+ >+ if ( $params->{control_number_handling} ) { >+ $existing_import_record_id = CheckRecordControlNumberInBatch( $marc_record, $batch_id ); >+ >+ if ( $existing_import_record_id ) { >+ $num_matched_control_number++; >+ >+ next if ( $params->{control_number_handling} eq 'preserve' ); >+ } >+ } >+ >+ if ($params->{record_type} eq 'biblio') { >+ if ( $existing_import_record_id ) { >+ $import_record_id = $existing_import_record_id; >+ ModBiblioInBatch( $import_record_id, $marc_record ); >+ } else { >+ $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $params->{encoding}, int(rand(99999)), 0); >+ } >+ >+ if ($params->{parse_items}) { > my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 0); > $num_items += scalar(@import_items_ids); > } >- } elsif ($record_type eq 'auth') { >- $import_record_id = AddAuthToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0, $marc_type); >+ } elsif ($params->{record_type} eq 'auth') { >+ if ( $existing_import_record_id ) { >+ $import_record_id = $existing_import_record_id; >+ ModAuthInBatch( $import_record_id, $marc_record ); >+ } else { >+ $import_record_id = AddAuthToBatch($batch_id, $rec_num, $marc_record, $params->{encoding}, int(rand(99999)), 0, $marc_type); >+ } > } > } > } >- unless ($leave_as_staging) { >+ unless ($params->{leave_as_staging}) { > SetImportBatchStatus($batch_id, 'staged'); > } > # FIXME branch_code, number of bibs, number of items > _update_batch_record_counts($batch_id); >- return ($batch_id, $num_valid, $num_items, @invalid_records); >+ return { >+ batch_id => $batch_id, >+ num_items => $num_items, >+ num_valid => $num_valid, >+ num_matched_control_number => $num_matched_control_number, >+ invalid_records => \@invalid_records >+ }; > } > > =head2 AddItemsToImportBiblio >@@ -456,6 +534,7 @@ sub AddItemsToImportBiblio { > my @import_items_ids = (); > > my $dbh = C4::Context->dbh; >+ $dbh->do( "DELETE FROM import_items WHERE import_record_id = ?", undef, $import_record_id ); > my ($item_tag,$item_subfield) = &GetMarcFromKohaField( "items.itemnumber" ); > foreach my $item_field ($marc_record->field($item_tag)) { > my $item_marc = MARC::Record->new(); >diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tt >index 4cd9e4b456..7fb8541e44 100644 >--- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tt >+++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tt >@@ -68,6 +68,16 @@ > <li>[% total | html %] records in file</li> > <li>[% import_errors | html %] records not staged because of MARC error</li> > <li>[% staged | html %] records staged</li> >+ [% IF control_number_handling %] >+ <li> >+ [% num_matched_control_number | html %] records with matching control number >+ [% IF control_number_handling == 'preserve' %] >+ (preserved) >+ [% ELSIF control_number_handling == 'overwrite' %] >+ (overwritten) >+ [% END %] >+ </li> >+ [% END %] > [% IF ( checked_matches ) %] > <li>[% matched | html %] records with at least one match in catalog per matching rule > "[% matcher_code | html %]"</li> >@@ -216,6 +226,26 @@ > </li> > </ol> > </fieldset> >+ <fieldset class="rows" id="record-update"> >+ <legend>Automatically update records?</legend> >+ <ol> >+ [% IF existing_batch_id %] >+ <li><label for="control_number_handling">How to process 001:</label> >+ <select name="control_number_handling" id="control_number_handling"> >+ <option value="">Ignore</option> >+ <option value="overwrite">Overwrite existing records with same 001</option> >+ <option value="preserve">Preserve existing records with same 001</option> >+ </select> >+ </li> >+ [% END %] >+ <li><label for="timestamp_update">How to process 005:</label> >+ <select name="timestamp_update" id="timestamp_update"> >+ <option value="">Do not update 005</option> >+ <option value="now">Set 005 to current date and time</option> >+ </select> >+ </li> >+ </ol> >+ </fieldset> > <fieldset class="rows" id="items"> > <legend>Check for embedded item record data?</legend> > <ol> >diff --git a/misc/stage_file.pl b/misc/stage_file.pl >index 16a74d0576..aa53d26b94 100755 >--- a/misc/stage_file.pl >+++ b/misc/stage_file.pl >@@ -142,22 +142,24 @@ sub process_batch { > > print "... staging MARC records -- please wait\n"; > #FIXME: We should really allow the use of marc modification frameworks and to_marc plugins here if possible >- my ( $batch_id, $num_valid_records, $num_items, @import_errors ) = >- BatchStageMarcRecords( >- $record_type, $params->{encoding}, >- $marc_records, $params->{input_file}, >- $params->{'marc_mod_template_id'}, $params->{batch_comment}, >- '', $params->{add_items}, >- 0, 100, >- \&print_progress_and_commit >- ); >+ my (@import_errors) = >+ my $stage_results = BatchStageMarcRecords( { >+ record_type => $record_type, >+ encoding => $encoding, >+ marc_records => $marc_records, >+ file_name => $input_file, >+ comments => $batch_comment, >+ parse_items => $add_items, >+ progress_interval => 100, >+ progress_callback => \&print_progress_and_commit >+ } ); > print "... finished staging MARC records\n"; > > my $num_with_matches = 0; > if ( $params->{match} ) { > my $matcher = C4::Matcher->fetch( $params->{match} ); > if (defined $matcher) { >- SetImportBatchMatcher( $batch_id, $params->{match} ); >+ SetImportBatchMatcher($stage_results->{batch_id}, $params->{match}); > } elsif ($record_type eq 'biblio') { > $matcher = C4::Matcher->new($record_type); > $matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, ''); >@@ -165,15 +167,15 @@ sub process_batch { > '245', 'a', -1, 0, ''); > } > # set default record overlay behavior >- SetImportBatchOverlayAction( $batch_id, $params->{no_replace} ? 'ignore' : 'replace' ); >- SetImportBatchNoMatchAction( $batch_id, $params->{no_create} ? 'ignore' : 'create_new' ); >- SetImportBatchItemAction( $batch_id, $params->{item_action} ); >+ SetImportBatchOverlayAction($stage_results->{batch_id}, $params->{no_replace} ? 'ignore' : 'replace'); >+ SetImportBatchNoMatchAction($stage_results->{batch_id}, $params->{no_create} ? 'ignore' : 'create_new'); >+ SetImportBatchItemAction($stage_results->{batch_id}, $params->{item_action} ); > print "... looking for matches with records already in database\n"; >- $num_with_matches = BatchFindDuplicates($batch_id, $matcher, 10, 100, \&print_progress_and_commit); >+ $num_with_matches = BatchFindDuplicates($stage_results->{batch_id}, $matcher, 10, 100, \&print_progress_and_commit); > print "... finished looking for matches\n"; > } > >- my $num_invalid_records = scalar(@import_errors); >+ my $num_invalid_records = scalar( @{ $stage_results->{invalid_records} } ); > print <<_SUMMARY_; > > MARC record staging report >@@ -181,7 +183,7 @@ MARC record staging report > Input file: $params->{input_file} > Record type: $record_type > Number of input records: $num_input_records >-Number of valid records: $num_valid_records >+Number of valid records: $stage_results->{num_valid} > Number of invalid records: $num_invalid_records > _SUMMARY_ > if( $params->{match} ) { >@@ -190,15 +192,15 @@ _SUMMARY_ > print "Incoming records not matched against existing records (--match option not supplied)\n"; > } > if ($record_type eq 'biblio') { >- if ( $params->{add_items} ) { >- print "Number of items parsed: $num_items\n"; >+ if ($params->{add_items} ) { >+ print "Number of items parsed: $stage_results->{num_items}\n"; > } else { > print "No items parsed (--add-items option not supplied)\n"; > } > } > > print "\n"; >- print "Batch number assigned: $batch_id\n"; >+ print "Batch number assigned: $stage_results->{batch_id}\n"; > print "\n"; > } > >diff --git a/tools/stage-marc-import.pl b/tools/stage-marc-import.pl >index 9f285efe87..f9f53a3dcb 100755 >--- a/tools/stage-marc-import.pl >+++ b/tools/stage-marc-import.pl >@@ -142,15 +142,20 @@ if ($completedJobID) { > $schema->storage->txn_begin; > > # FIXME branch code >- my ( $batch_id, $num_valid, $num_items, @import_errors ) = >- BatchStageMarcRecords( >- $record_type, $encoding, >- $marcrecords, $filename, >- $marc_modification_template, >- $comments, '', >- $parse_items, 0, >- 50, staging_progress_callback( $job ) >- ); >+ my $stage_results = BatchStageMarcRecords( { >+ record_type => $record_type, >+ encoding => $encoding, >+ marc_records => $marcrecords, >+ file_name => $filename, >+ marc_modification_template => $marc_modification_template, >+ comments => $comments, >+ parse_items => $parse_items, >+ progress_interval => 50, >+ progress_callback => staging_progress_callback( $job ), >+ existing_batch_id => $existing_batch_id, >+ control_number_handling => $control_number_handling, >+ timestamp_update => $timestamp_update, >+ } ); > > if($profile_id) { > my $ibatch = Koha::ImportBatches->find($batch_id); >@@ -167,12 +172,12 @@ if ($completedJobID) { > $checked_matches = 1; > $matcher_code = $matcher->code(); > $num_with_matches = >- BatchFindDuplicates( $batch_id, $matcher, 10, 50, >+ BatchFindDuplicates( $stage_results->{batch_id}, $matcher, 10, 50, > matching_progress_callback($job) ); >- SetImportBatchMatcher($batch_id, $matcher_id); >- SetImportBatchOverlayAction($batch_id, $overlay_action); >- SetImportBatchNoMatchAction($batch_id, $nomatch_action); >- SetImportBatchItemAction($batch_id, $item_action); >+ SetImportBatchMatcher($stage_results->{batch_id}, $matcher_id); >+ SetImportBatchOverlayAction($stage_results->{batch_id}, $overlay_action); >+ SetImportBatchNoMatchAction($stage_results->{batch_id}, $nomatch_action); >+ SetImportBatchItemAction($stage_results->{batch_id}, $item_action); > $schema->storage->txn_commit; > } else { > $matcher_failed = 1; >@@ -183,34 +188,25 @@ if ($completedJobID) { > } > > my $results = { >- staged => $num_valid, >+ staged => $stage_results->{num_valid}, > matched => $num_with_matches, >- num_items => $num_items, >- import_errors => scalar(@import_errors), >- total => $num_valid + scalar(@import_errors), >+ num_items => $stage_results->{num_items}, >+ import_errors => scalar( @{ $stage_results->{invalid_records} } ), >+ total => $stage_results->{num_valid} + scalar( @{ $stage_results->{invalid_records} } ), > checked_matches => $checked_matches, > matcher_failed => $matcher_failed, > matcher_code => $matcher_code, >- import_batch_id => $batch_id, >+ import_batch_id => $stage_results->{batch_id}, > booksellerid => $booksellerid, >- basketno => $basketno >+ basketno => $basketno, >+ control_number_handling => $control_number_handling, >+ num_matched_control_number => $stage_results->{num_matched_control_number}, > }; > if ($runinbackground) { > $job->finish($results); > exit 0; > } else { >- $template->param(staged => $num_valid, >- matched => $num_with_matches, >- num_items => $num_items, >- import_errors => scalar(@import_errors), >- total => $num_valid + scalar(@import_errors), >- checked_matches => $checked_matches, >- matcher_failed => $matcher_failed, >- matcher_code => $matcher_code, >- import_batch_id => $batch_id, >- booksellerid => $booksellerid, >- basketno => $basketno >- ); >+ $template->param( %$results ); > } > > } else { >-- >2.11.0
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 19266
:
66901
|
66902
|
77905
|
86387
| 122003