Bugzilla – Attachment 135251 Details for
Bug 30822
BatchCommit does not deal with indexation correctly
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 30822: Make BatchCommitRecords update the index in one request
Bug-30822-Make-BatchCommitRecords-update-the-index.patch (text/plain), 8.52 KB, created by
Martin Renvoize (ashimema)
on 2022-05-20 16:06:32 UTC
(
hide
)
Description:
Bug 30822: Make BatchCommitRecords update the index in one request
Filename:
MIME Type:
Creator:
Martin Renvoize (ashimema)
Created:
2022-05-20 16:06:32 UTC
Size:
8.52 KB
patch
obsolete
>From e1e4ff8b1533f2c5f162a38db8806ddcc791ea9e Mon Sep 17 00:00:00 2001 >From: Martin Renvoize <martin.renvoize@ptfs-europe.com> >Date: Fri, 20 May 2022 16:59:23 +0100 >Subject: [PATCH] Bug 30822: Make BatchCommitRecords update the index in one > request > >When committing staged marc imports to the catalogue we will often be >importing a batch of records. We don't want to send one index request >per biblio affected, we want to index them all after the records have >been modified otherwise we will end up with multiple tasks per record >(when items are also affected). > >Test plan: >1) Use the stage marc record tool to stage and commit a set of records and >confirm the behaviour remains correct. >2) If using Elastic, check that only one indexing job is queued to take >place resulting from the committed import. >--- > C4/Biblio.pm | 34 ++++++++++++++++------------------ > C4/ImportBatch.pm | 32 +++++++++++++++++++++++++------- > 2 files changed, 41 insertions(+), 25 deletions(-) > >diff --git a/C4/Biblio.pm b/C4/Biblio.pm >index 459a94510a..9b93f60c4f 100644 >--- a/C4/Biblio.pm >+++ b/C4/Biblio.pm >@@ -187,31 +187,29 @@ The first argument is a C<MARC::Record> object containing the > bib to add, while the second argument is the desired MARC > framework code. > >-This function also accepts a third, optional argument: a hashref >-to additional options. The only defined option is C<defer_marc_save>, >-which if present and mapped to a true value, causes C<AddBiblio> >-to omit the call to save the MARC in C<biblio_metadata.metadata> >-This option is provided B<only> >-for the use of scripts such as C<bulkmarcimport.pl> that may need >-to do some manipulation of the MARC record for item parsing before >-saving it and which cannot afford the performance hit of saving >-the MARC record twice. Consequently, do not use that option >-unless you can guarantee that C<ModBiblioMarc> will be called. >+The C<$options> argument is a hashref with additional parameters: >+ >+=over 4 >+ >+=item B<defer_marc_save>: used when ModBiblioMarc is handled by the caller >+ >+=item B<skip_record_index>: used when the indexing schedulling will be handled by the caller >+ >+=back > > =cut > > sub AddBiblio { >- my $record = shift; >- my $frameworkcode = shift; >- my $options = @_ ? shift : undef; >- my $defer_marc_save = 0; >+ my ( $record, $frameworkcode, $options ) = @_; >+ >+ $options //= {}; >+ my $skip_record_index = $options->{skip_record_index} || 0; >+ my $defer_marc_save = $options->{defer_marc_save} || 0; >+ > if (!$record) { > carp('AddBiblio called with undefined record'); > return; > } >- if ( defined $options and exists $options->{'defer_marc_save'} and $options->{'defer_marc_save'} ) { >- $defer_marc_save = 1; >- } > > my $schema = Koha::Database->schema; > my ( $biblionumber, $biblioitemnumber ); >@@ -292,7 +290,7 @@ sub AddBiblio { > } > > # now add the record >- ModBiblioMarc( $record, $biblionumber ) unless $defer_marc_save; >+ ModBiblioMarc( $record, $biblionumber, { skip_record_index => $skip_record_index } ) unless $defer_marc_save; > > # update OAI-PMH sets > if(C4::Context->preference("OAI-PMH:AutoUpdateSets")) { >diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm >index 845caa3b58..aee25ca549 100644 >--- a/C4/ImportBatch.pm >+++ b/C4/ImportBatch.pm >@@ -35,6 +35,8 @@ use C4::Charset qw( MarcToUTF8Record SetUTF8Flag StripNonXmlChars ); > use C4::AuthoritiesMarc qw( AddAuthority GuessAuthTypeCode GetAuthorityXML ModAuthority DelAuthority ); > use C4::MarcModificationTemplates qw( ModifyRecordWithTemplate ); > use Koha::Items; >+use Koha::SearchEngine; >+use Koha::SearchEngine::Indexer; > use Koha::Plugins::Handler; > use Koha::Logger; > >@@ -607,6 +609,7 @@ sub BatchCommitRecords { > my $logged_in_patron = Koha::Patrons->find( $userenv->{number} ); > > my $rec_num = 0; >+ my @biblio_ids; > while (my $rowref = $sth->fetchrow_hashref) { > $record_type = $rowref->{'record_type'}; > $rec_num++; >@@ -646,7 +649,8 @@ sub BatchCommitRecords { > $num_added++; > if ($record_type eq 'biblio') { > my $biblioitemnumber; >- ($recordid, $biblioitemnumber) = AddBiblio($marc_record, $framework); >+ ($recordid, $biblioitemnumber) = AddBiblio($marc_record, $framework, { skip_record_index => 1 }); >+ push @biblio_ids, $recordid; > $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; # FIXME call SetMatchedBiblionumber instead > if ($item_result eq 'create_new' || $item_result eq 'replace') { > my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result); >@@ -685,9 +689,16 @@ sub BatchCommitRecords { > $context->{userid} = $logged_in_patron->userid; > } > >- ModBiblio($marc_record, $recordid, $oldbiblio->frameworkcode, { >- overlay_context => $context >- }); >+ ModBiblio( >+ $marc_record, >+ $recordid, >+ $oldbiblio->frameworkcode, >+ { >+ overlay_context => $context, >+ skip_record_index => 1 >+ } >+ ); >+ push @biblio_ids, $recordid; > $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; # FIXME call SetMatchedBiblionumber instead > > if ($item_result eq 'create_new' || $item_result eq 'replace') { >@@ -715,6 +726,7 @@ sub BatchCommitRecords { > $num_ignored++; > if ($record_type eq 'biblio' and defined $recordid and ( $item_result eq 'create_new' || $item_result eq 'replace' ) ) { > my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result); >+ push @biblio_ids, $recordid if $bib_items_added || $bib_items_replaced; > $num_items_added += $bib_items_added; > $num_items_replaced += $bib_items_replaced; > $num_items_errored += $bib_items_errored; >@@ -728,6 +740,12 @@ sub BatchCommitRecords { > } > } > $sth->finish(); >+ >+ if ( @biblio_ids ) { >+ my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX }); >+ $indexer->index_records( \@biblio_ids, "specialUpdate", "biblioserver" ); >+ } >+ > SetImportBatchStatus($batch_id, 'imported'); > return ($num_added, $num_updated, $num_items_added, $num_items_replaced, $num_items_errored, $num_ignored); > } >@@ -773,7 +791,7 @@ sub BatchCommitItems { > my $updsth = $dbh->prepare("UPDATE import_items SET status = ?, itemnumber = ? WHERE import_items_id = ?"); > if ( $action eq "replace" && $duplicate_itemnumber ) { > # Duplicate itemnumbers have precedence, that way we can update barcodes by overlaying >- ModItemFromMarc( $item_marc, $biblionumber, $item->{itemnumber} ); >+ ModItemFromMarc( $item_marc, $biblionumber, $item->{itemnumber}, { skip_record_index => 1 } ); > $updsth->bind_param( 1, 'imported' ); > $updsth->bind_param( 2, $item->{itemnumber} ); > $updsth->bind_param( 3, $row->{'import_items_id'} ); >@@ -782,7 +800,7 @@ sub BatchCommitItems { > $num_items_replaced++; > } elsif ( $action eq "replace" && $duplicate_barcode ) { > my $itemnumber = $duplicate_barcode->itemnumber; >- ModItemFromMarc( $item_marc, $biblionumber, $itemnumber ); >+ ModItemFromMarc( $item_marc, $biblionumber, $itemnumber, { skip_record_index => 1 } ); > $updsth->bind_param( 1, 'imported' ); > $updsth->bind_param( 2, $item->{itemnumber} ); > $updsth->bind_param( 3, $row->{'import_items_id'} ); >@@ -800,7 +818,7 @@ sub BatchCommitItems { > my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField( "items.itemnumber" ); > $item_marc->field($itemtag)->delete_subfield( code => $itemsubfield ); > >- my ( $item_biblionumber, $biblioitemnumber, $itemnumber ) = AddItemFromMarc( $item_marc, $biblionumber ); >+ my ( $item_biblionumber, $biblioitemnumber, $itemnumber ) = AddItemFromMarc( $item_marc, $biblionumber, { skip_record_index => 1 } ); > if( $itemnumber ) { > $updsth->bind_param( 1, 'imported' ); > $updsth->bind_param( 2, $itemnumber ); >-- >2.20.1
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 30822
:
135251
|
135372
|
135963
|
135964
|
136218
|
136219
|
136247
|
136248