From c6c5474f9b6e55e0b10c4d10359336ef32bbf5a8 Mon Sep 17 00:00:00 2001 From: Julian Maurice Date: Tue, 18 Feb 2020 12:41:55 +0100 Subject: [PATCH] Bug 24679: Allow to apply a MARC modification template in export tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Test plan: 1. Create a MARC modification template with a simple action (for instance, "Add new field 999$9 with value foobar") 2. Go to Tools » Export data 3. Select some records (for instance, from biblionumber 1 to 10) 4. In options, select the MARC modification template you just created 5. Select File format 'MARC' 6. Click on 'Export bibliographic records' 7. Verify that exported records contain the 999$9 field 8. Repeat steps 3-7 with other export formats, and with authority records Signed-off-by: Devinim --- C4/Record.pm | 20 ++- Koha/Exporter/Record.pm | 18 ++- .../prog/en/modules/tools/export.tt | 25 ++- t/db_dependent/Exporter/Record.t | 148 +++++++++++++++--- tools/export.pl | 5 + 5 files changed, 186 insertions(+), 30 deletions(-) diff --git a/C4/Record.pm b/C4/Record.pm index 1916a1beca..2fcc43b40e 100644 --- a/C4/Record.pm +++ b/C4/Record.pm @@ -29,6 +29,7 @@ use Unicode::Normalize; # _entity_encode use C4::Biblio; #marc2bibtex use C4::Koha; #marc2csv use C4::XSLT (); +use C4::MarcModificationTemplates; use YAML; #marcrecords2csv use Template; use Text::CSV::Encoded; #marc2csv @@ -403,10 +404,12 @@ C<$csvprofileid> - the id of the CSV profile to use for the export (see export_f C<$itemnumbers> - a list of itemnumbers to export +C<$marc_modification_template_id> - MARC modification template to apply + =cut sub marc2csv { - my ($biblios, $id, $itemnumbers) = @_; + my ($biblios, $id, $itemnumbers, $marc_modification_template_id) = @_; $itemnumbers ||= []; my $output; my $csv = Text::CSV::Encoded->new(); @@ -426,12 +429,12 @@ sub marc2csv { for my $itemnumber ( @$itemnumbers) { my $item = Koha::Items->find( $itemnumber ); my $biblionumber = $item->biblio->biblionumber; - $output .= marcrecord2csv( $biblionumber, $id, $firstpass, $csv, $fieldprocessing, [$itemnumber] ) // ''; + $output .= marcrecord2csv( $biblionumber, $id, $firstpass, $csv, $fieldprocessing, [$itemnumber], $marc_modification_template_id ) // ''; $firstpass = 0; } } else { foreach my $biblio (@$biblios) { - $output .= marcrecord2csv( $biblio, $id, $firstpass, $csv, $fieldprocessing ) // ''; + $output .= marcrecord2csv( $biblio, $id, $firstpass, $csv, $fieldprocessing, undef, $marc_modification_template_id ) // ''; $firstpass = 0; } } @@ -444,7 +447,7 @@ sub marc2csv { =head2 marcrecord2csv - Convert a single record from UNIMARC to CSV - my ($csv) = marcrecord2csv($biblio, $csvprofileid, $header); + my ($csv) = marcrecord2csv($biblio, $csvprofileid, $header, $csv, $fieldprocessing, $itemnumbers, $marc_modification_template_id); Returns a CSV scalar @@ -460,10 +463,12 @@ C<$fieldprocessing> C<$itemnumbers> a list of itemnumbers to export +C<$marc_modification_template_id> - MARC modification template to apply + =cut sub marcrecord2csv { - my ($biblio, $id, $header, $csv, $fieldprocessing, $itemnumbers) = @_; + my ($biblio, $id, $header, $csv, $fieldprocessing, $itemnumbers, $marc_modification_template_id) = @_; my $output; # Getting the record @@ -473,6 +478,11 @@ sub marcrecord2csv { marc_record => $record, biblionumber => $biblio, item_numbers => $itemnumbers }); + + if ($marc_modification_template_id) { + C4::MarcModificationTemplates::ModifyRecordWithTemplate($marc_modification_template_id, $record); + } + # Getting the framework my $frameworkcode = GetFrameworkCode($biblio); diff --git a/Koha/Exporter/Record.pm b/Koha/Exporter/Record.pm index eaebe57639..80423b86bb 100644 --- a/Koha/Exporter/Record.pm +++ b/Koha/Exporter/Record.pm @@ -7,6 +7,7 @@ use MARC::File::USMARC; use C4::AuthoritiesMarc; use C4::Biblio; use C4::Record; +use C4::MarcModificationTemplates; use Koha::CsvProfiles; use Koha::Logger; use List::Util qw(all any); @@ -155,6 +156,7 @@ sub export { my $dont_export_fields = $params->{dont_export_fields}; my $csv_profile_id = $params->{csv_profile_id}; my $output_filepath = $params->{output_filepath}; + my $marc_modification_template_id = $params->{marc_modification_template_id}; if( !$record_type ) { Koha::Logger->get->warn( "No record_type given." ); @@ -183,6 +185,11 @@ sub export { Koha::Logger->get->info( $msg ); next; } + + if ($marc_modification_template_id) { + C4::MarcModificationTemplates::ModifyRecordWithTemplate($marc_modification_template_id, $record); + } + print $record->as_usmarc(); } } elsif ( $format eq 'xml' ) { @@ -194,6 +201,11 @@ sub export { for my $record_id (@$record_ids) { my $record = _get_record_for_export( { %$params, record_id => $record_id } ); next unless $record; + + if ($marc_modification_template_id) { + C4::MarcModificationTemplates::ModifyRecordWithTemplate($marc_modification_template_id, $record); + } + print MARC::File::XML::record($record); print "\n"; } @@ -202,7 +214,7 @@ sub export { } elsif ( $format eq 'csv' ) { die 'There is no valid csv profile defined for this export' unless Koha::CsvProfiles->find( $csv_profile_id ); - print marc2csv( $record_ids, $csv_profile_id, $itemnumbers ); + print marc2csv( $record_ids, $csv_profile_id, $itemnumbers, $marc_modification_template_id ); } close $fh if $output_filepath; @@ -263,6 +275,10 @@ It will displays on STDOUT the generated file. If the format is csv, you have to define a csv_profile_id. +=item marc_modification_template_id + + Apply MARC modification template to records before export + =cut =back diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/export.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/export.tt index 4e9a992c04..3cf86a27f8 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/export.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/export.tt @@ -129,7 +129,19 @@ separate by a blank. (e.g., 100a 200 606) - + + [% IF marc_modification_templates.size > 0 %] +
  • + + +
  • + [% END %] +
    @@ -201,6 +213,17 @@
    Options
      + [% IF marc_modification_templates.size > 0 %] +
    1. + + +
    2. + [% END %]
    3. diff --git a/t/db_dependent/Exporter/Record.t b/t/db_dependent/Exporter/Record.t index 9993d7874b..fb3ef27f18 100755 --- a/t/db_dependent/Exporter/Record.t +++ b/t/db_dependent/Exporter/Record.t @@ -17,7 +17,7 @@ use Modern::Perl; -use Test::More tests => 6; +use Test::More tests => 9; use Test::Warn; use t::lib::TestBuilder; @@ -45,17 +45,33 @@ my $biblio_1_title = 'Silence in the library'; my $biblio_2_title = 'The art of computer programming ກ ຂ ຄ ງ ຈ ຊ ຍ é'; my $biblio_1 = MARC::Record->new(); $biblio_1->leader('00266nam a22001097a 4500'); -$biblio_1->append_fields( - MARC::Field->new('100', ' ', ' ', a => 'Moffat, Steven'), - MARC::Field->new('245', ' ', ' ', a => $biblio_1_title), -); -my ($biblionumber_1, $biblioitemnumber_1) = AddBiblio($biblio_1, ''); my $biblio_2 = MARC::Record->new(); $biblio_2->leader('00266nam a22001097a 4500'); -$biblio_2->append_fields( - MARC::Field->new('100', ' ', ' ', a => 'Knuth, Donald Ervin'), - MARC::Field->new('245', ' ', ' ', a => $biblio_2_title), -); + +my ($title_field, $title_subfield, $item_field, $barcode_subfield, $homebranch_subfield); +if (C4::Context->preference('marcflavour') eq 'UNIMARC') { + $biblio_1->append_fields( + MARC::Field->new('200', ' ', ' ', a => $biblio_1_title, f => 'Moffat, Steven'), + ); + $biblio_2->append_fields( + MARC::Field->new('200', ' ', ' ', a => $biblio_2_title, f => 'Knuth, Donald Ervin'), + ); + ($title_field, $title_subfield) = ('200', 'a'); + ($item_field, $barcode_subfield, $homebranch_subfield) = ('995', 'f', 'b'); +} else { + $biblio_1->append_fields( + MARC::Field->new('100', ' ', ' ', a => 'Moffat, Steven'), + MARC::Field->new('245', ' ', ' ', a => $biblio_1_title), + ); + $biblio_2->append_fields( + MARC::Field->new('100', ' ', ' ', a => 'Knuth, Donald Ervin'), + MARC::Field->new('245', ' ', ' ', a => $biblio_2_title), + ); + ($title_field, $title_subfield) = ('245', 'a'); + ($item_field, $barcode_subfield, $homebranch_subfield) = ('952', 'p', 'a'); +} + +my ($biblionumber_1, $biblioitemnumber_1) = AddBiblio($biblio_1, ''); my ($biblionumber_2, $biblioitemnumber_2) = AddBiblio($biblio_2, ''); my $bad_biblio = Koha::Biblio->new()->store(); @@ -87,7 +103,7 @@ my $bad_item = $builder->build({ # Cannot call build_sample_item, we want incons subtest 'export csv' => sub { plan tests => 3; - my $csv_content = q{Title=245$a|Barcode=952$p}; + my $csv_content = qq{Title=$title_field\$$title_subfield|Barcode=$item_field\$$barcode_subfield}; $dbh->do(q|INSERT INTO export_format(profile, description, content, csv_separator, field_separator, subfield_separator, encoding, type) VALUES (?, ?, ?, ?, ?, ?, ?, ?)|, {}, "TEST_PROFILE_Records.t", "my useless desc", $csv_content, '|', ';', ',', 'utf8', 'marc'); my $csv_profile_id = $dbh->last_insert_id( undef, undef, 'export_format', undef ); my $generated_csv_file = '/tmp/test_export_1.csv'; @@ -160,7 +176,7 @@ subtest 'export xml' => sub { } is( scalar( @records ), 2, 'Export XML: 2 records should have been exported' ); my $second_record = $records[1]; - my $title = $second_record->subfield(245, 'a'); + my $title = $second_record->subfield($title_field, $title_subfield); $title = Encode::encode('UTF-8', $title); is( $title, $biblio_2_title, 'Export XML: The title is correctly encoded' ); }; @@ -187,7 +203,7 @@ subtest 'export iso2709' => sub { } is( scalar( @records ), 2, 'Export ISO2709: 2 records should have been exported' ); my $second_record = $records[1]; - my $title = $second_record->subfield(245, 'a'); + my $title = $second_record->subfield($title_field, $title_subfield); $title = Encode::encode('UTF-8', $title); is( $title, $biblio_2_title, 'Export ISO2709: The title is correctly encoded' ); }; @@ -209,10 +225,16 @@ subtest '_get_biblio_for_export' => sub { my $biblio = MARC::Record->new(); $biblio->leader('00266nam a22001097a 4500'); - $biblio->append_fields( - MARC::Field->new( '100', ' ', ' ', a => 'Thurber, James' ), - MARC::Field->new( '245', ' ', ' ', a => "The 13 Clocks" ), - ); + if (C4::Context->preference('marcflavour') eq 'UNIMARC') { + $biblio->append_fields( + MARC::Field->new( '200', ' ', ' ', a => "The 13 Clocks", f => 'Thurber, James' ), + ); + } else { + $biblio->append_fields( + MARC::Field->new( '100', ' ', ' ', a => 'Thurber, James' ), + MARC::Field->new( '245', ' ', ' ', a => "The 13 Clocks" ), + ); + } my ( $biblionumber, $biblioitemnumber ) = AddBiblio( $biblio, '' ); my $branch_a = $builder->build({source => 'Branch',}); my $branch_b = $builder->build({source => 'Branch',}); @@ -236,7 +258,7 @@ subtest '_get_biblio_for_export' => sub { only_export_items_for_branches => undef } ); - my @items = $record->field('952'); + my @items = $record->field($item_field); is( scalar @items, 2, "We should retrieve all items if we don't pass specific branches and request items" ); $record = Koha::Exporter::Record::_get_biblio_for_export( @@ -246,10 +268,10 @@ subtest '_get_biblio_for_export' => sub { only_export_items_for_branches => [ $branch_b->{branchcode} ] } ); - @items = $record->field('952'); + @items = $record->field($item_field); is( scalar @items, 1, "We should retrieve only item for branch_b item if we request items and pass branch" ); is( - $items[0]->subfield('a'), + $items[0]->subfield($homebranch_subfield), $branch_b->{branchcode}, "And the homebranch for that item should be branch_b branchcode" ); @@ -261,7 +283,7 @@ subtest '_get_biblio_for_export' => sub { only_export_items_for_branches => [ $branch_b->{branchcode} ] } ); - @items = $record->field('952'); + @items = $record->field($item_field); is( scalar @items, 0, "We should not have any items if we don't request items and pass a branch"); }; @@ -272,8 +294,8 @@ subtest '_get_record_for_export MARC field conditions' => sub { my $biblio = MARC::Record->new(); $biblio->leader('00266nam a22001097a 4500'); $biblio->append_fields( - MARC::Field->new( '100', ' ', ' ', a => 'Thurber, James' ), - MARC::Field->new( '245', ' ', ' ', a => 'The 13 Clocks' ), + MARC::Field->new( '200', ' ', ' ', f => 'Thurber, James' ), + MARC::Field->new( '200', ' ', ' ', a => 'The 13 Clocks' ), MARC::Field->new( '080', ' ', ' ', a => '12345' ), MARC::Field->new( '035', ' ', ' ', a => '(TEST)123' ), MARC::Field->new( '035', ' ', ' ', a => '(TEST)1234' ), @@ -391,4 +413,84 @@ subtest '_get_record_for_export MARC field conditions' => sub { is( $record, undef, "Record condition \"not_exists(035a)\" should not match" ); }; +subtest 'export csv with marc modification template' => sub { + plan tests => 1; + my $csv_content = qq{Title=$title_field\$$title_subfield|Test=999\$9}; + $dbh->do(q|INSERT INTO export_format(profile, description, content, csv_separator, field_separator, subfield_separator, encoding, type) VALUES (?, ?, ?, ?, ?, ?, ?, ?)|, {}, "TEST_PROFILE_Records.t", "my useless desc", $csv_content, '|', ';', ',', 'utf8', 'marc'); + my $csv_profile_id = $dbh->last_insert_id( undef, undef, 'export_format', undef ); + + my $template_id = C4::MarcModificationTemplates::AddModificationTemplate('test exporter'); + C4::MarcModificationTemplates::AddModificationTemplateAction($template_id, 'add_field', 0, '999', '9', 'Foobar'); + + my $generated_csv_file = '/tmp/test_export_1.csv'; + Koha::Exporter::Record::export({ + record_type => 'bibs', + record_ids => [ $biblionumber_1 ], + format => 'csv', + csv_profile_id => $csv_profile_id, + marc_modification_template_id => $template_id, + output_filepath => $generated_csv_file, + }); + + my $expected_csv = < sub { + plan tests => 2; + + my $template_id = C4::MarcModificationTemplates::AddModificationTemplate('test exporter'); + C4::MarcModificationTemplates::AddModificationTemplateAction($template_id, 'add_field', 0, '999', '9', 'Foobar'); + + my $generated_xml_file = '/tmp/test_export.xml'; + Koha::Exporter::Record::export({ + record_type => 'bibs', + record_ids => [ $biblionumber_1 ], + format => 'xml', + marc_modification_template_id => $template_id, + output_filepath => $generated_xml_file, + }); + + my $generated_xml_content = read_file( $generated_xml_file ); + $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8'; + open my $fh, '<', $generated_xml_file; + my $records = MARC::Batch->new( 'XML', $fh ); + my @records; + while ( my $record = $records->next ) { + push @records, $record; + } + is( scalar( @records ), 1, 'Export XML: 1 record should have been exported' ); + my $record = $records[0]; + is( $record->subfield('999', '9'), 'Foobar', 'Export XML: marc modification template should have added 999$9' ); +}; + +subtest 'export iso2709 with marc modification template' => sub { + plan tests => 2; + + my $template_id = C4::MarcModificationTemplates::AddModificationTemplate('test exporter'); + C4::MarcModificationTemplates::AddModificationTemplateAction($template_id, 'add_field', 0, '999', '9', 'Foobar'); + + my $generated_mrc_file = '/tmp/test_export.mrc'; + Koha::Exporter::Record::export({ + record_type => 'bibs', + record_ids => [ $biblionumber_1 ], + format => 'iso2709', + marc_modification_template_id => $template_id, + output_filepath => $generated_mrc_file, + }); + + my $records = MARC::File::USMARC->in( $generated_mrc_file ); + my @records; + while ( my $record = $records->next ) { + push @records, $record; + } + is( scalar( @records ), 1, 'Export ISO2709: 1 record should have been exported' ); + my $record = $records[0]; + is( $record->subfield('999', '9'), 'Foobar', 'Export ISO2709: marc modification template should have added 999$9' ); +}; + $schema->storage->txn_rollback; diff --git a/tools/export.pl b/tools/export.pl index 5b252be05e..4cef022def 100755 --- a/tools/export.pl +++ b/tools/export.pl @@ -22,6 +22,7 @@ use MARC::File::XML; use List::MoreUtils qw(uniq); use C4::Auth; use C4::Output; +use C4::MarcModificationTemplates; use Koha::Authority::Types; use Koha::Biblioitems; @@ -211,6 +212,7 @@ if ( $op eq "export" ) { csv_profile_id => $csv_profile_id, export_items => (not $dont_export_items), only_export_items_for_branches => $only_export_items_for_branches, + marc_modification_template_id => scalar $query->param('marc_modification_template_id'), } ); } @@ -297,6 +299,8 @@ else { { directory => "$backupdir", extension => 'tar' } ); } + my @marc_modification_templates = C4::MarcModificationTemplates::GetModificationTemplates(); + $template->param( libraries => $libraries, itemtypes => $itemtypes, @@ -304,6 +308,7 @@ else { export_remove_fields => C4::Context->preference("ExportRemoveFields"), csv_profiles => [ Koha::CsvProfiles->search({ type => 'marc', used_for => 'export_records' }) ], messages => \@messages, + marc_modification_templates => \@marc_modification_templates, ); output_html_with_http_headers $query, $cookie, $template->output; -- 2.20.1