Bugzilla – Attachment 116408 Details for
Bug 20551
Add option for including deleted records in export_records.pl
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 20551: Add option for including deleted records in export_records.pl
Bug-20551-Add-option-for-including-deleted-records.patch (text/plain), 18.87 KB, created by
Frank Hansen
on 2021-02-05 15:04:39 UTC
(
hide
)
Description:
Bug 20551: Add option for including deleted records in export_records.pl
Filename:
MIME Type:
Creator:
Frank Hansen
Created:
2021-02-05 15:04:39 UTC
Size:
18.87 KB
patch
obsolete
>From 88138eae4114d067bf36b8547a429a835773df2a Mon Sep 17 00:00:00 2001 >From: David Gustafsson <david.gustafsson@ub.gu.se> >Date: Mon, 9 Apr 2018 19:28:41 +0200 >Subject: [PATCH] Bug 20551: Add option for including deleted records in > export_records.pl > >Add option "--include_deleted" to include deleted >biblios in export_records.pl as marc records with >record status (in leader) set to "d" and >"--deleted_only" to export only deleted biblios. > >How to test: >1) Run tests in t/db_dependent/Exporter/Record.t >2) All tests should pass >3) Delete a biblio record in Koha >4) Run the export script as: > `export_records.pl --date=<date-just-before-deletion> > --include_deleted` >5) Open the generated koha.mrc, and verify that contains > the recently deleted record, and that the record header > has record status "d". >6) Perform the same check with: > `export_records.pl --date=<date-just-before-deletion> > --include_deleted --format=xml` >7) Run: > `export_records.pl --date=<date-just-before-deletion> > --deleted_only --format=xml` > and verify that only the deleted biblio was included > in the export. > >Sponsored-by: Gothenburg University Library > >Signed-off-by: Frank Hansen <frank.hansen@ub.lu.se> >--- > Koha/Exporter/Record.pm | 132 ++++++++++++++++++++++++++++++--------- > misc/export_records.pl | 87 +++++++++++++++++++------- > t/db_dependent/Exporter/Record.t | 43 +++++++++---- > 3 files changed, 198 insertions(+), 64 deletions(-) > >diff --git a/Koha/Exporter/Record.pm b/Koha/Exporter/Record.pm >index eaebe57639..a49494d497 100644 >--- a/Koha/Exporter/Record.pm >+++ b/Koha/Exporter/Record.pm >@@ -6,11 +6,16 @@ use MARC::File::USMARC; > > use C4::AuthoritiesMarc; > use C4::Biblio; >+use C4::Charset; > use C4::Record; > use Koha::CsvProfiles; >+use Koha::Database; > use Koha::Logger; > use List::Util qw(all any); > >+use MARC::Record; >+use MARC::File::XML; >+ > sub _get_record_for_export { > my ($params) = @_; > my $record_type = $params->{record_type}; >@@ -105,6 +110,40 @@ sub _get_record_for_export { > return $record; > } > >+sub _get_deleted_biblio_for_export { >+ my ($params) = @_; >+ my $biblionumber = $params->{biblionumber}; >+ # Creating schema is expensive, allow caller to >+ # pass it so don't have to recreate for each call >+ my $resultset = $params->{resultset} || Koha::Database >+ ->new() >+ ->schema() >+ ->resultset('DeletedbiblioMetadata'); >+ my $marc_flavour = C4::Context->preference('marcflavour'); >+ my $biblio_metadata = $resultset->find({ >+ 'biblionumber' => $biblionumber, >+ 'format' => 'marcxml', >+ 'marcflavour' => $marc_flavour >+ }); >+ my $marc_xml = $biblio_metadata->metadata; >+ $marc_xml = StripNonXmlChars($marc_xml); >+ >+ my $record = eval { >+ MARC::Record::new_from_xml($marc_xml, 'UTF-8', $marc_flavour) >+ }; >+ if (!$record) { >+ Koha::Logger->get->warn( >+ "Failed to load MARCXML for deleted biblio with biblionumber \"$biblionumber\": $@" >+ ); >+ return; >+ } >+ # Set deleted flag (record status, position 05) >+ my $leader = $record->leader; >+ substr $leader, 5, 1, 'd'; >+ $record->leader($leader); >+ return $record; >+} >+ > sub _get_authority_for_export { > my ($params) = @_; > my $authid = $params->{authid} || return; >@@ -122,7 +161,12 @@ sub _get_biblio_for_export { > > my $record = eval { C4::Biblio::GetMarcBiblio({ biblionumber => $biblionumber }); }; > >- return if $@ or not defined $record; >+ if (!$record) { >+ Koha::Logger->get->warn( >+ "Failed to load MARCXML for biblio with biblionumber \"$biblionumber\": $@" >+ ); >+ return; >+ } > > if ($export_items) { > C4::Biblio::EmbedItemsInMarcBiblio({ >@@ -149,6 +193,7 @@ sub export { > > my $record_type = $params->{record_type}; > my $record_ids = $params->{record_ids} || []; >+ my $deleted_record_ids = $params->{deleted_record_ids} || []; > my $format = $params->{format}; > my $itemnumbers = $params->{itemnumbers} || []; # Does not make sense with record_type eq auths > my $export_items = $params->{export_items}; >@@ -160,7 +205,7 @@ sub export { > Koha::Logger->get->warn( "No record_type given." ); > return; > } >- return unless @$record_ids; >+ return unless (@{$record_ids} || @{$deleted_record_ids} && $format ne 'csv'); > > my $fh; > if ( $output_filepath ) { >@@ -171,40 +216,71 @@ sub export { > binmode STDOUT, ':encoding(UTF-8)' unless $format eq 'csv'; > } > >- if ( $format eq 'iso2709' ) { >- for my $record_id (@$record_ids) { >- my $record = _get_record_for_export( { %$params, record_id => $record_id } ); >- next unless $record; >- my $errorcount_on_decode = eval { scalar( MARC::File::USMARC->decode( $record->as_usmarc )->warnings() ) }; >- if ( $errorcount_on_decode or $@ ) { >- my $msg = "Record $record_id could not be exported. " . >- ( $@ // '' ); >- chomp $msg; >- Koha::Logger->get->info( $msg ); >- next; >- } >- print $record->as_usmarc(); >+ if ($format eq 'xml' || $format eq 'iso2709') { >+ my @records; >+ @records = map { >+ my $record = _get_record_for_export({ %{$params}, record_id => $_ }); >+ $record ? $record : (); >+ } @{$record_ids}; >+ >+ my @deleted_records; >+ if (@{$deleted_record_ids}) { >+ my $resultset = Koha::Database >+ ->new() >+ ->schema() >+ ->resultset('DeletedbiblioMetadata'); >+ @deleted_records = map { >+ my $record = _get_deleted_biblio_for_export({ >+ biblionumber => $_, >+ resultset => $resultset, >+ }); >+ $record ? $record : (); >+ } @{$deleted_record_ids}; > } >- } elsif ( $format eq 'xml' ) { >- my $marcflavour = C4::Context->preference("marcflavour"); >- MARC::File::XML->default_record_format( ( $marcflavour eq 'UNIMARC' && $record_type eq 'auths' ) ? 'UNIMARCAUTH' : $marcflavour ); >- >- print MARC::File::XML::header(); >- print "\n"; >- for my $record_id (@$record_ids) { >- my $record = _get_record_for_export( { %$params, record_id => $record_id } ); >- next unless $record; >- print MARC::File::XML::record($record); >+ if ( $format eq 'iso2709' ) { >+ my $encoding_validator = sub { >+ my ($record_type) = @_; >+ return sub { >+ my ($record) = @_; >+ my $errorcount_on_decode = eval { scalar(MARC::File::USMARC->decode($record->as_usmarc)->warnings()) }; >+ if ($errorcount_on_decode || $@) { >+ my ($id_tag, $id_subfield) = GetMarcFromKohaField('biblio.biblionumber', ''); >+ my $record_id = $record->subfield($id_tag, $id_subfield); >+ my $msg = "$record_type $record_id could not be USMARC decoded/encoded. " . ($@ // ''); >+ chomp $msg; >+ Koha::Logger->get->warn($msg); >+ return 0; >+ } >+ return 1; >+ } >+ }; >+ my $validator = $encoding_validator->('Record'); >+ for my $record (grep { $validator->($_) } @records) { >+ print $record->as_usmarc(); >+ } >+ if (@deleted_records) { >+ $validator = $encoding_validator->('Deleted record'); >+ for my $deleted_record (grep { $validator->($_) } @deleted_records) { >+ print $deleted_record->as_usmarc(); >+ } >+ } >+ } elsif ( $format eq 'xml' ) { >+ my $marcflavour = C4::Context->preference("marcflavour"); >+ MARC::File::XML->default_record_format( ( $marcflavour eq 'UNIMARC' && $record_type eq 'auths' ) ? 'UNIMARCAUTH' : $marcflavour ); >+ print MARC::File::XML::header(); >+ print "\n"; >+ for my $record (@records, @deleted_records) { >+ print MARC::File::XML::record($record); >+ print "\n"; >+ } >+ print MARC::File::XML::footer(); > print "\n"; > } >- print MARC::File::XML::footer(); >- print "\n"; > } elsif ( $format eq 'csv' ) { > die 'There is no valid csv profile defined for this export' > unless Koha::CsvProfiles->find( $csv_profile_id ); > print marc2csv( $record_ids, $csv_profile_id, $itemnumbers ); > } >- > close $fh if $output_filepath; > } > >diff --git a/misc/export_records.pl b/misc/export_records.pl >index 2a05e794df..3f885f1bb9 100755 >--- a/misc/export_records.pl >+++ b/misc/export_records.pl >@@ -36,6 +36,8 @@ use Koha::DateUtils qw( dt_from_string output_pref ); > my ( > $output_format, > $timestamp, >+ $include_deleted, >+ $deleted_only, > $dont_export_items, > $csv_profile_id, > $deleted_barcodes, >@@ -60,6 +62,8 @@ my ( > GetOptions( > 'format=s' => \$output_format, > 'date=s' => \$timestamp, >+ 'include_deleted' => \$include_deleted, >+ 'deleted_only' => \$deleted_only, > 'dont_export_items' => \$dont_export_items, > 'csv_profile_id=s' => \$csv_profile_id, > 'deleted_barcodes' => \$deleted_barcodes, >@@ -92,6 +96,18 @@ $record_type ||= 'bibs'; > # Retrocompatibility for the format parameter > $output_format = 'iso2709' if $output_format eq 'marc'; > >+if ($include_deleted || $deleted_only) { >+ if ($record_type ne 'bibs') { >+ pod2usage(q|Option "--include_deleted" or "--deleted_only" can only be used with "--record-type=bibs"|); >+ } >+ if (!$timestamp) { >+ pod2usage(q|Option "--include_deleted" or "--deleted_only" requires that "--date" is also set|); >+ } >+ if ($output_format eq 'csv') { >+ pod2usage(q|Option "--include_deleted" or "--deleted_only" cannot be used with "--format=csv"|); >+ } >+} >+ > if ( $output_format eq 'csv' and $record_type eq 'auths' ) { > pod2usage(q|CSV output is only available for biblio records|); > } >@@ -138,34 +154,46 @@ open STDOUT, '>', $filename if $filename; > > > my @record_ids; >+my @deleted_record_ids; > > $timestamp = ($timestamp) ? output_pref({ dt => dt_from_string($timestamp), dateformat => 'iso', dateonly => 0, }): ''; > > if ( $record_type eq 'bibs' ) { > if ( $timestamp ) { >- if (!$dont_export_items) { >- push @record_ids, $_->{biblionumber} for @{ >- $dbh->selectall_arrayref(q| ( >- SELECT biblio_metadata.biblionumber >- FROM biblio_metadata >- LEFT JOIN items USING(biblionumber) >- WHERE biblio_metadata.timestamp >= ? >- OR items.timestamp >= ? >- ) UNION ( >- SELECT biblio_metadata.biblionumber >- FROM biblio_metadata >- LEFT JOIN deleteditems USING(biblionumber) >- WHERE biblio_metadata.timestamp >= ? >- OR deleteditems.timestamp >= ? >- ) |, { Slice => {} }, ( $timestamp ) x 4 ); >- }; >- } else { >- push @record_ids, $_->{biblionumber} for @{ >- $dbh->selectall_arrayref(q| ( >- SELECT biblio_metadata.biblionumber >- FROM biblio_metadata >- WHERE biblio_metadata.timestamp >= ? >- ) |, { Slice => {} }, $timestamp ); >+ unless ($deleted_only) { >+ if (!$dont_export_items) { >+ push @record_ids, $_->{biblionumber} for @{ >+ $dbh->selectall_arrayref(q| ( >+ SELECT biblio_metadata.biblionumber >+ FROM biblio_metadata >+ LEFT JOIN items USING(biblionumber) >+ WHERE biblio_metadata.timestamp >= ? >+ OR items.timestamp >= ? >+ ) UNION ( >+ SELECT biblio_metadata.biblionumber >+ FROM biblio_metadata >+ LEFT JOIN deleteditems USING(biblionumber) >+ WHERE biblio_metadata.timestamp >= ? >+ OR deleteditems.timestamp >= ? >+ ) |, { Slice => {} }, ( $timestamp ) x 4 ); >+ }; >+ } else { >+ push @record_ids, $_->{biblionumber} for @{ >+ $dbh->selectall_arrayref(q| ( >+ SELECT biblio_metadata.biblionumber >+ FROM biblio_metadata >+ WHERE biblio_metadata.timestamp >= ? >+ ) |, { Slice => {} }, $timestamp ); >+ }; >+ } >+ } >+ if ($include_deleted || $deleted_only) { >+ push @deleted_record_ids, $_->{biblionumber} for @{ >+ $dbh->selectall_arrayref(q| >+ SELECT `biblionumber` >+ FROM `deletedbiblio` >+ WHERE `timestamp` >= ? >+ |, { Slice => {} }, $timestamp); > }; > } > } else { >@@ -252,6 +280,7 @@ else { > { record_type => $record_type, > record_ids => \@record_ids, > record_conditions => @marc_conditions ? \@marc_conditions : undef, >+ deleted_record_ids => \@deleted_record_ids, > format => $output_format, > csv_profile_id => $csv_profile_id, > export_items => (not $dont_export_items), >@@ -268,7 +297,7 @@ export records - This script exports record (biblios or authorities) > > =head1 SYNOPSIS > >-export_records.pl [-h|--help] [--format=format] [--date=datetime] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] [--id_list_file=PATH] --filename=outputfile >+export_records.pl [-h|--help] [--format=format] [--date=datetime] [--include_deleted] [--deleted_only] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] [--id_list_file=PATH] --filename=outputfile > > =head1 OPTIONS > >@@ -289,6 +318,16 @@ Print a brief help message. > mm/dd/yyyy[ hh:mm:ss] for us) records exported are the ones that > have been modified since DATETIME. > >+=item B<--include_deleted> >+ >+ --include_deleted If enabled, when using --date option, deleted records will be included in export as marc records >+ with leader record status set to "d" (deleted). >+ >+=item B<--include_deleted> >+ >+ --include_deleted If enabled, when using --date option, only deleted records will be included in export as marc >+ records with leader record status set to "d" (deleted). >+ > =item B<--record-type> > > --record-type=TYPE TYPE is 'bibs' or 'auths'. >diff --git a/t/db_dependent/Exporter/Record.t b/t/db_dependent/Exporter/Record.t >index 9993d7874b..39412e6eca 100755 >--- a/t/db_dependent/Exporter/Record.t >+++ b/t/db_dependent/Exporter/Record.t >@@ -58,6 +58,15 @@ $biblio_2->append_fields( > ); > my ($biblionumber_2, $biblioitemnumber_2) = AddBiblio($biblio_2, ''); > >+my $deleted_biblio = MARC::Record->new(); >+$deleted_biblio->leader('00136nam a22000617a 4500'); >+$deleted_biblio->append_fields( >+ MARC::Field->new('100', ' ', ' ', a => 'Chopra, Deepak'), >+ MARC::Field->new('245', ' ', ' ', a => 'The seven spiritual laws of success'), >+); >+my ($deleted_biblionumber) = AddBiblio($deleted_biblio, ''); >+DelBiblio($deleted_biblionumber); >+ > my $bad_biblio = Koha::Biblio->new()->store(); > Koha::Biblio::Metadata->new( { biblionumber => $bad_biblio->id, format => 'marcxml', metadata => 'something wrong', schema => C4::Context->preference('marcflavour') } )->store(); > my $bad_biblionumber = $bad_biblio->id; >@@ -134,14 +143,15 @@ EOF > }; > > subtest 'export xml' => sub { >- plan tests => 3; >+ plan tests => 4; > my $generated_xml_file = '/tmp/test_export.xml'; > warning_like { > Koha::Exporter::Record::export( >- { record_type => 'bibs', >- record_ids => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ], >- format => 'xml', >- output_filepath => $generated_xml_file, >+ { record_type => 'bibs', >+ record_ids => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ], >+ deleted_record_ids => [ $deleted_biblionumber ], >+ format => 'xml', >+ output_filepath => $generated_xml_file, > } > ); > } >@@ -158,23 +168,28 @@ subtest 'export xml' => sub { > while ( my $record = $records->next ) { > push @records, $record; > } >- is( scalar( @records ), 2, 'Export XML: 2 records should have been exported' ); >+ is( scalar( @records ), 3, 'Export XML: 3 records should have been exported' ); > my $second_record = $records[1]; > my $title = $second_record->subfield(245, 'a'); > $title = Encode::encode('UTF-8', $title); > is( $title, $biblio_2_title, 'Export XML: The title is correctly encoded' ); >+ >+ my $deleted_record = $records[2]; >+ # Leader has the expected value (and record status "d") >+ is( $deleted_record->leader, '00136dam a22000617a 4500', 'Deleted record has the correct leader value' ); > }; > > subtest 'export iso2709' => sub { >- plan tests => 3; >+ plan tests => 4; > my $generated_mrc_file = '/tmp/test_export.mrc'; > # Get all item infos > warning_like { > Koha::Exporter::Record::export( >- { record_type => 'bibs', >- record_ids => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ], >- format => 'iso2709', >- output_filepath => $generated_mrc_file, >+ { record_type => 'bibs', >+ record_ids => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ], >+ deleted_record_ids => [ $deleted_biblionumber ], >+ format => 'iso2709', >+ output_filepath => $generated_mrc_file, > } > ); > } >@@ -185,11 +200,15 @@ subtest 'export iso2709' => sub { > while ( my $record = $records->next ) { > push @records, $record; > } >- is( scalar( @records ), 2, 'Export ISO2709: 2 records should have been exported' ); >+ is( scalar( @records ), 3, 'Export ISO2709: 3 records should have been exported' ); > my $second_record = $records[1]; > my $title = $second_record->subfield(245, 'a'); > $title = Encode::encode('UTF-8', $title); > is( $title, $biblio_2_title, 'Export ISO2709: The title is correctly encoded' ); >+ >+ my $deleted_record = $records[2]; >+ # Leader has the expected value (and record status "d") >+ is( $deleted_record->leader, '00136dam a22000617a 4500', 'Deleted record has the correct leader value' ); > }; > > subtest 'export without record_type' => sub { >-- >2.11.0
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 20551
:
73916
|
73923
|
75113
|
82843
|
82914
|
82918
|
90184
|
90190
|
90220
|
93619
|
93620
|
93621
|
105403
|
116408
|
116409
|
116410
|
116411
|
123099
|
123101
|
123102
|
125211
|
125212
|
125213
|
125214
|
125215
|
125216
|
128895
|
128896
|
128897
|
128898
|
128899
|
128900
|
132936
|
132937
|
176623
|
176624
|
176625
|
176626
|
176627
|
176628
|
176629
|
176630
|
176631
|
177536