Bugzilla – Attachment 99304 Details for
Bug 24679
Allow to apply a MARC modification template in export tool
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 24679: Allow to apply a MARC modification template in export tool
Bug-24679-Allow-to-apply-a-MARC-modification-templ.patch (text/plain), 20.16 KB, created by
Julian Maurice
on 2020-02-20 09:29:52 UTC
(
hide
)
Description:
Bug 24679: Allow to apply a MARC modification template in export tool
Filename:
MIME Type:
Creator:
Julian Maurice
Created:
2020-02-20 09:29:52 UTC
Size:
20.16 KB
patch
obsolete
>From 988c3f6fbf2c19d385c602d0efad245cad849b2e Mon Sep 17 00:00:00 2001 >From: Julian Maurice <julian.maurice@biblibre.com> >Date: Tue, 18 Feb 2020 12:41:55 +0100 >Subject: [PATCH] Bug 24679: Allow to apply a MARC modification template in > export tool >MIME-Version: 1.0 >Content-Type: text/plain; charset=UTF-8 >Content-Transfer-Encoding: 8bit > >Test plan: >1. Create a MARC modification template with a simple action (for > instance, "Add new field 999$9 with value foobar") >2. Go to Tools » Export data >3. Select some records (for instance, from biblionumber 1 to 10) >4. In options, select the MARC modification template you just created >5. Select File format 'MARC' >6. Click on 'Export bibliographic records' >7. Verify that exported records contain the 999$9 field >8. Repeat steps 3-7 with other export formats, and with authority > records >--- > C4/Record.pm | 20 ++- > Koha/Exporter/Record.pm | 18 ++- > .../prog/en/modules/tools/export.tt | 25 ++- > t/db_dependent/Exporter/Record.t | 148 +++++++++++++++--- > tools/export.pl | 5 + > 5 files changed, 186 insertions(+), 30 deletions(-) > >diff --git a/C4/Record.pm b/C4/Record.pm >index c15c2076f7..5e2da8670a 100644 >--- a/C4/Record.pm >+++ b/C4/Record.pm >@@ -29,6 +29,7 @@ use Unicode::Normalize; # _entity_encode > use C4::Biblio; #marc2bibtex > use C4::Koha; #marc2csv > use C4::XSLT (); >+use C4::MarcModificationTemplates; > use YAML; #marcrecords2csv > use Template; > use Text::CSV::Encoded; #marc2csv >@@ -404,10 +405,12 @@ C<$csvprofileid> - the id of the CSV profile to use for the export (see export_f > > C<$itemnumbers> - a list of itemnumbers to export > >+C<$marc_modification_template_id> - MARC modification template to apply >+ > =cut > > sub marc2csv { >- my ($biblios, $id, $itemnumbers) = @_; >+ my ($biblios, $id, $itemnumbers, $marc_modification_template_id) = @_; > $itemnumbers ||= []; > my $output; > my $csv = Text::CSV::Encoded->new(); >@@ -427,12 +430,12 @@ sub marc2csv { > for my $itemnumber ( @$itemnumbers) { > my $item = Koha::Items->find( $itemnumber ); > my $biblionumber = $item->biblio->biblionumber; >- $output .= marcrecord2csv( $biblionumber, $id, $firstpass, $csv, $fieldprocessing, [$itemnumber] ) // ''; >+ $output .= marcrecord2csv( $biblionumber, $id, $firstpass, $csv, $fieldprocessing, [$itemnumber], $marc_modification_template_id ) // ''; > $firstpass = 0; > } > } else { > foreach my $biblio (@$biblios) { >- $output .= marcrecord2csv( $biblio, $id, $firstpass, $csv, $fieldprocessing ) // ''; >+ $output .= marcrecord2csv( $biblio, $id, $firstpass, $csv, $fieldprocessing, undef, $marc_modification_template_id ) // ''; > $firstpass = 0; > } > } >@@ -445,7 +448,7 @@ sub marc2csv { > > =head2 marcrecord2csv - Convert a single record from UNIMARC to CSV > >- my ($csv) = marcrecord2csv($biblio, $csvprofileid, $header); >+ my ($csv) = marcrecord2csv($biblio, $csvprofileid, $header, $csv, $fieldprocessing, $itemnumbers, $marc_modification_template_id); > > Returns a CSV scalar > >@@ -461,10 +464,12 @@ C<$fieldprocessing> > > C<$itemnumbers> a list of itemnumbers to export > >+C<$marc_modification_template_id> - MARC modification template to apply >+ > =cut > > sub marcrecord2csv { >- my ($biblio, $id, $header, $csv, $fieldprocessing, $itemnumbers) = @_; >+ my ($biblio, $id, $header, $csv, $fieldprocessing, $itemnumbers, $marc_modification_template_id) = @_; > my $output; > > # Getting the record >@@ -474,6 +479,11 @@ sub marcrecord2csv { > marc_record => $record, > biblionumber => $biblio, > item_numbers => $itemnumbers }); >+ >+ if ($marc_modification_template_id) { >+ C4::MarcModificationTemplates::ModifyRecordWithTemplate($marc_modification_template_id, $record); >+ } >+ > # Getting the framework > my $frameworkcode = GetFrameworkCode($biblio); > >diff --git a/Koha/Exporter/Record.pm b/Koha/Exporter/Record.pm >index eaebe57639..80423b86bb 100644 >--- a/Koha/Exporter/Record.pm >+++ b/Koha/Exporter/Record.pm >@@ -7,6 +7,7 @@ use MARC::File::USMARC; > use C4::AuthoritiesMarc; > use C4::Biblio; > use C4::Record; >+use C4::MarcModificationTemplates; > use Koha::CsvProfiles; > use Koha::Logger; > use List::Util qw(all any); >@@ -155,6 +156,7 @@ sub export { > my $dont_export_fields = $params->{dont_export_fields}; > my $csv_profile_id = $params->{csv_profile_id}; > my $output_filepath = $params->{output_filepath}; >+ my $marc_modification_template_id = $params->{marc_modification_template_id}; > > if( !$record_type ) { > Koha::Logger->get->warn( "No record_type given." ); >@@ -183,6 +185,11 @@ sub export { > Koha::Logger->get->info( $msg ); > next; > } >+ >+ if ($marc_modification_template_id) { >+ C4::MarcModificationTemplates::ModifyRecordWithTemplate($marc_modification_template_id, $record); >+ } >+ > print $record->as_usmarc(); > } > } elsif ( $format eq 'xml' ) { >@@ -194,6 +201,11 @@ sub export { > for my $record_id (@$record_ids) { > my $record = _get_record_for_export( { %$params, record_id => $record_id } ); > next unless $record; >+ >+ if ($marc_modification_template_id) { >+ C4::MarcModificationTemplates::ModifyRecordWithTemplate($marc_modification_template_id, $record); >+ } >+ > print MARC::File::XML::record($record); > print "\n"; > } >@@ -202,7 +214,7 @@ sub export { > } elsif ( $format eq 'csv' ) { > die 'There is no valid csv profile defined for this export' > unless Koha::CsvProfiles->find( $csv_profile_id ); >- print marc2csv( $record_ids, $csv_profile_id, $itemnumbers ); >+ print marc2csv( $record_ids, $csv_profile_id, $itemnumbers, $marc_modification_template_id ); > } > > close $fh if $output_filepath; >@@ -263,6 +275,10 @@ It will displays on STDOUT the generated file. > > If the format is csv, you have to define a csv_profile_id. > >+=item marc_modification_template_id >+ >+ Apply MARC modification template to records before export >+ > =cut > > =back >diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/export.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/export.tt >index 5df6fb16bd..8a70bf6d19 100644 >--- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/export.tt >+++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/export.tt >@@ -129,7 +129,19 @@ > <label for="export_remove_fields">Don't export fields:</label> > <input id="export_remove_fields" type="text" name="export_remove_fields" value="[% export_remove_fields | html %]" /> > separate by a blank. (e.g., 100a 200 606) >- </li></ol> >+ </li> >+ [% IF marc_modification_templates.size > 0 %] >+ <li> >+ <label for="marc_modification_template_id">Modify records using template:</label> >+ <select id="marc_modification_template_id" name="marc_modification_template_id"> >+ <option value=""></option> >+ [% FOREACH marc_modification_template IN marc_modification_templates %] >+ <option value="[% marc_modification_template.template_id | html %]">[% marc_modification_template.name | html %]</option> >+ [% END %] >+ </select> >+ </li> >+ [% END %] >+ </ol> > </fieldset> > <fieldset class="rows"> > <legend> >@@ -201,6 +213,17 @@ > <fieldset class="rows"> > <legend>Options</legend> > <ol> >+ [% IF marc_modification_templates.size > 0 %] >+ <li> >+ <label for="marc_modification_template_id">Modify records using template:</label> >+ <select id="marc_modification_template_id" name="marc_modification_template_id"> >+ <option value=""></option> >+ [% FOREACH marc_modification_template IN marc_modification_templates %] >+ <option value="[% marc_modification_template.template_id | html %]">[% marc_modification_template.name | html %]</option> >+ [% END %] >+ </select> >+ </li> >+ [% END %] > <li> > <label for="export_remove_fields">Don't export fields:</label> > <input id="export_remove_fields" type="text" name="export_remove_fields" /> >diff --git a/t/db_dependent/Exporter/Record.t b/t/db_dependent/Exporter/Record.t >index 5768a7e814..f2a35d8db0 100644 >--- a/t/db_dependent/Exporter/Record.t >+++ b/t/db_dependent/Exporter/Record.t >@@ -17,7 +17,7 @@ > > use Modern::Perl; > >-use Test::More tests => 6; >+use Test::More tests => 9; > use Test::Warn; > use t::lib::TestBuilder; > >@@ -45,17 +45,33 @@ my $biblio_1_title = 'Silence in the library'; > my $biblio_2_title = 'The art of computer programming ຠຠຠຠຠຠຠé'; > my $biblio_1 = MARC::Record->new(); > $biblio_1->leader('00266nam a22001097a 4500'); >-$biblio_1->append_fields( >- MARC::Field->new('100', ' ', ' ', a => 'Moffat, Steven'), >- MARC::Field->new('245', ' ', ' ', a => $biblio_1_title), >-); >-my ($biblionumber_1, $biblioitemnumber_1) = AddBiblio($biblio_1, ''); > my $biblio_2 = MARC::Record->new(); > $biblio_2->leader('00266nam a22001097a 4500'); >-$biblio_2->append_fields( >- MARC::Field->new('100', ' ', ' ', a => 'Knuth, Donald Ervin'), >- MARC::Field->new('245', ' ', ' ', a => $biblio_2_title), >-); >+ >+my ($title_field, $title_subfield, $item_field, $barcode_subfield, $homebranch_subfield); >+if (C4::Context->preference('marcflavour') eq 'UNIMARC') { >+ $biblio_1->append_fields( >+ MARC::Field->new('200', ' ', ' ', a => $biblio_1_title, f => 'Moffat, Steven'), >+ ); >+ $biblio_2->append_fields( >+ MARC::Field->new('200', ' ', ' ', a => $biblio_2_title, f => 'Knuth, Donald Ervin'), >+ ); >+ ($title_field, $title_subfield) = ('200', 'a'); >+ ($item_field, $barcode_subfield, $homebranch_subfield) = ('995', 'f', 'b'); >+} else { >+ $biblio_1->append_fields( >+ MARC::Field->new('100', ' ', ' ', a => 'Moffat, Steven'), >+ MARC::Field->new('245', ' ', ' ', a => $biblio_1_title), >+ ); >+ $biblio_2->append_fields( >+ MARC::Field->new('100', ' ', ' ', a => 'Knuth, Donald Ervin'), >+ MARC::Field->new('245', ' ', ' ', a => $biblio_2_title), >+ ); >+ ($title_field, $title_subfield) = ('245', 'a'); >+ ($item_field, $barcode_subfield, $homebranch_subfield) = ('952', 'p', 'a'); >+} >+ >+my ($biblionumber_1, $biblioitemnumber_1) = AddBiblio($biblio_1, ''); > my ($biblionumber_2, $biblioitemnumber_2) = AddBiblio($biblio_2, ''); > > my $bad_biblio = Koha::Biblio->new()->store(); >@@ -94,7 +110,7 @@ my $bad_item = $builder->build({ > > subtest 'export csv' => sub { > plan tests => 3; >- my $csv_content = q{Title=245$a|Barcode=952$p}; >+ my $csv_content = qq{Title=$title_field\$$title_subfield|Barcode=$item_field\$$barcode_subfield}; > $dbh->do(q|INSERT INTO export_format(profile, description, content, csv_separator, field_separator, subfield_separator, encoding, type) VALUES (?, ?, ?, ?, ?, ?, ?, ?)|, {}, "TEST_PROFILE_Records.t", "my useless desc", $csv_content, '|', ';', ',', 'utf8', 'marc'); > my $csv_profile_id = $dbh->last_insert_id( undef, undef, 'export_format', undef ); > my $generated_csv_file = '/tmp/test_export_1.csv'; >@@ -167,7 +183,7 @@ subtest 'export xml' => sub { > } > is( scalar( @records ), 2, 'Export XML: 2 records should have been exported' ); > my $second_record = $records[1]; >- my $title = $second_record->subfield(245, 'a'); >+ my $title = $second_record->subfield($title_field, $title_subfield); > $title = Encode::encode('UTF-8', $title); > is( $title, $biblio_2_title, 'Export XML: The title is correctly encoded' ); > }; >@@ -194,7 +210,7 @@ subtest 'export iso2709' => sub { > } > is( scalar( @records ), 2, 'Export ISO2709: 2 records should have been exported' ); > my $second_record = $records[1]; >- my $title = $second_record->subfield(245, 'a'); >+ my $title = $second_record->subfield($title_field, $title_subfield); > $title = Encode::encode('UTF-8', $title); > is( $title, $biblio_2_title, 'Export ISO2709: The title is correctly encoded' ); > }; >@@ -216,10 +232,16 @@ subtest '_get_biblio_for_export' => sub { > > my $biblio = MARC::Record->new(); > $biblio->leader('00266nam a22001097a 4500'); >- $biblio->append_fields( >- MARC::Field->new( '100', ' ', ' ', a => 'Thurber, James' ), >- MARC::Field->new( '245', ' ', ' ', a => "The 13 Clocks" ), >- ); >+ if (C4::Context->preference('marcflavour') eq 'UNIMARC') { >+ $biblio->append_fields( >+ MARC::Field->new( '200', ' ', ' ', a => "The 13 Clocks", f => 'Thurber, James' ), >+ ); >+ } else { >+ $biblio->append_fields( >+ MARC::Field->new( '100', ' ', ' ', a => 'Thurber, James' ), >+ MARC::Field->new( '245', ' ', ' ', a => "The 13 Clocks" ), >+ ); >+ } > my ( $biblionumber, $biblioitemnumber ) = AddBiblio( $biblio, '' ); > my $branch_a = $builder->build({source => 'Branch',}); > my $branch_b = $builder->build({source => 'Branch',}); >@@ -251,7 +273,7 @@ subtest '_get_biblio_for_export' => sub { > only_export_items_for_branches => undef > } > ); >- my @items = $record->field('952'); >+ my @items = $record->field($item_field); > is( scalar @items, 2, "We should retrieve all items if we don't pass specific branches and request items" ); > > $record = Koha::Exporter::Record::_get_biblio_for_export( >@@ -261,10 +283,10 @@ subtest '_get_biblio_for_export' => sub { > only_export_items_for_branches => [ $branch_b->{branchcode} ] > } > ); >- @items = $record->field('952'); >+ @items = $record->field($item_field); > is( scalar @items, 1, "We should retrieve only item for branch_b item if we request items and pass branch" ); > is( >- $items[0]->subfield('a'), >+ $items[0]->subfield($homebranch_subfield), > $branch_b->{branchcode}, > "And the homebranch for that item should be branch_b branchcode" > ); >@@ -276,7 +298,7 @@ subtest '_get_biblio_for_export' => sub { > only_export_items_for_branches => [ $branch_b->{branchcode} ] > } > ); >- @items = $record->field('952'); >+ @items = $record->field($item_field); > is( scalar @items, 0, "We should not have any items if we don't request items and pass a branch"); > > }; >@@ -287,8 +309,8 @@ subtest '_get_record_for_export MARC field conditions' => sub { > my $biblio = MARC::Record->new(); > $biblio->leader('00266nam a22001097a 4500'); > $biblio->append_fields( >- MARC::Field->new( '100', ' ', ' ', a => 'Thurber, James' ), >- MARC::Field->new( '245', ' ', ' ', a => 'The 13 Clocks' ), >+ MARC::Field->new( '200', ' ', ' ', f => 'Thurber, James' ), >+ MARC::Field->new( '200', ' ', ' ', a => 'The 13 Clocks' ), > MARC::Field->new( '080', ' ', ' ', a => '12345' ), > MARC::Field->new( '035', ' ', ' ', a => '(TEST)123' ), > MARC::Field->new( '035', ' ', ' ', a => '(TEST)1234' ), >@@ -406,4 +428,84 @@ subtest '_get_record_for_export MARC field conditions' => sub { > is( $record, undef, "Record condition \"not_exists(035a)\" should not match" ); > }; > >+subtest 'export csv with marc modification template' => sub { >+ plan tests => 1; >+ my $csv_content = qq{Title=$title_field\$$title_subfield|Test=999\$9}; >+ $dbh->do(q|INSERT INTO export_format(profile, description, content, csv_separator, field_separator, subfield_separator, encoding, type) VALUES (?, ?, ?, ?, ?, ?, ?, ?)|, {}, "TEST_PROFILE_Records.t", "my useless desc", $csv_content, '|', ';', ',', 'utf8', 'marc'); >+ my $csv_profile_id = $dbh->last_insert_id( undef, undef, 'export_format', undef ); >+ >+ my $template_id = C4::MarcModificationTemplates::AddModificationTemplate('test exporter'); >+ C4::MarcModificationTemplates::AddModificationTemplateAction($template_id, 'add_field', 0, '999', '9', 'Foobar'); >+ >+ my $generated_csv_file = '/tmp/test_export_1.csv'; >+ Koha::Exporter::Record::export({ >+ record_type => 'bibs', >+ record_ids => [ $biblionumber_1 ], >+ format => 'csv', >+ csv_profile_id => $csv_profile_id, >+ marc_modification_template_id => $template_id, >+ output_filepath => $generated_csv_file, >+ }); >+ >+ my $expected_csv = <<EOF; >+Title|Test >+"$biblio_1_title"|Foobar >+EOF >+ my $generated_csv_content = read_file( $generated_csv_file ); >+ is( $generated_csv_content, $expected_csv, "Export CSV: Modification template should have been applied" ); >+}; >+ >+subtest 'export xml with marc modification template' => sub { >+ plan tests => 2; >+ >+ my $template_id = C4::MarcModificationTemplates::AddModificationTemplate('test exporter'); >+ C4::MarcModificationTemplates::AddModificationTemplateAction($template_id, 'add_field', 0, '999', '9', 'Foobar'); >+ >+ my $generated_xml_file = '/tmp/test_export.xml'; >+ Koha::Exporter::Record::export({ >+ record_type => 'bibs', >+ record_ids => [ $biblionumber_1 ], >+ format => 'xml', >+ marc_modification_template_id => $template_id, >+ output_filepath => $generated_xml_file, >+ }); >+ >+ my $generated_xml_content = read_file( $generated_xml_file ); >+ $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8'; >+ open my $fh, '<', $generated_xml_file; >+ my $records = MARC::Batch->new( 'XML', $fh ); >+ my @records; >+ while ( my $record = $records->next ) { >+ push @records, $record; >+ } >+ is( scalar( @records ), 1, 'Export XML: 1 record should have been exported' ); >+ my $record = $records[0]; >+ is( $record->subfield('999', '9'), 'Foobar', 'Export XML: marc modification template should have added 999$9' ); >+}; >+ >+subtest 'export iso2709 with marc modification template' => sub { >+ plan tests => 2; >+ >+ my $template_id = C4::MarcModificationTemplates::AddModificationTemplate('test exporter'); >+ C4::MarcModificationTemplates::AddModificationTemplateAction($template_id, 'add_field', 0, '999', '9', 'Foobar'); >+ >+ my $generated_mrc_file = '/tmp/test_export.mrc'; >+ Koha::Exporter::Record::export({ >+ record_type => 'bibs', >+ record_ids => [ $biblionumber_1 ], >+ format => 'iso2709', >+ marc_modification_template_id => $template_id, >+ output_filepath => $generated_mrc_file, >+ }); >+ >+ my $records = MARC::File::USMARC->in( $generated_mrc_file ); >+ my @records; >+ while ( my $record = $records->next ) { >+ push @records, $record; >+ } >+ is( scalar( @records ), 1, 'Export ISO2709: 1 record should have been exported' ); >+ my $record = $records[0]; >+ is( $record->subfield('999', '9'), 'Foobar', 'Export ISO2709: marc modification template should have added 999$9' ); >+}; >+ > $schema->storage->txn_rollback; >diff --git a/tools/export.pl b/tools/export.pl >index e2d75d377f..3013a24ecd 100755 >--- a/tools/export.pl >+++ b/tools/export.pl >@@ -22,6 +22,7 @@ use MARC::File::XML; > use List::MoreUtils qw(uniq); > use C4::Auth; > use C4::Output; >+use C4::MarcModificationTemplates; > > use Koha::Authority::Types; > use Koha::Biblioitems; >@@ -209,6 +210,7 @@ if ( $op eq "export" ) { > csv_profile_id => $csv_profile_id, > export_items => (not $dont_export_items), > only_export_items_for_branches => $only_export_items_for_branches, >+ marc_modification_template_id => scalar $query->param('marc_modification_template_id'), > } > ); > } >@@ -295,6 +297,8 @@ else { > { directory => "$backupdir", extension => 'tar' } ); > } > >+ my @marc_modification_templates = C4::MarcModificationTemplates::GetModificationTemplates(); >+ > $template->param( > libraries => $libraries, > itemtypes => $itemtypes, >@@ -302,6 +306,7 @@ else { > export_remove_fields => C4::Context->preference("ExportRemoveFields"), > csv_profiles => [ Koha::CsvProfiles->search({ type => 'marc', used_for => 'export_records' }) ], > messages => \@messages, >+ marc_modification_templates => \@marc_modification_templates, > ); > > output_html_with_http_headers $query, $cookie, $template->output; >-- >2.20.1
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 24679
:
99175
|
99304
|
99700
|
111519
|
111520
|
111521