Bugzilla – Attachment 81762 Details for
Bug 20486
Add --marc_conditions option to export_records.pl
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 20486: Add --marc_conditions option to export_records.pl
Bug-20486-Add---marcconditions-option-to-exportrec.patch (text/plain), 13.43 KB, created by
David Gustafsson
on 2018-10-31 17:05:59 UTC
(
hide
)
Description:
Bug 20486: Add --marc_conditions option to export_records.pl
Filename:
MIME Type:
Creator:
David Gustafsson
Created:
2018-10-31 17:05:59 UTC
Size:
13.43 KB
patch
obsolete
>From d3f68492e3b55bf7b080e4f0f2beb2fb0819ec15 Mon Sep 17 00:00:00 2001 >From: David Gustafsson <david.gustafsson@ub.gu.se> >Date: Fri, 23 Mar 2018 17:32:30 +0100 >Subject: [PATCH] Bug 20486: Add --marc_conditions option to export_records.pl > >Add --marc_conditions option to export_records.pl script, for >excluding records based on conditions applied to MARC-record >data. > >How to test: >1) Run tests in t/db_dependent/Exporter/Record.t >2) All tests should pass > >Sponsored-by: Gothenburg University Library > >Signed-off-by: Maksim Sen <maksim.sen@inlibro.com> >Signed-off-by: Michal Denar <black23@gmail.com> >Signed-off-by: Chris Cormack <chrisc@catalyst.net.nz> >--- > Koha/Exporter/Record.pm | 63 ++++++++++++++++++-- > misc/export_records.pl | 65 +++++++++++++++++++- > t/db_dependent/Exporter/Record.t | 125 ++++++++++++++++++++++++++++++++++++++- > 3 files changed, 245 insertions(+), 8 deletions(-) > >diff --git a/Koha/Exporter/Record.pm b/Koha/Exporter/Record.pm >index 311f88d4e5..9621836c42 100644 >--- a/Koha/Exporter/Record.pm >+++ b/Koha/Exporter/Record.pm >@@ -9,11 +9,13 @@ use C4::Biblio; > use C4::Record; > use Koha::CsvProfiles; > use Koha::Logger; >+use List::Util qw(all any); > > sub _get_record_for_export { > my ($params) = @_; > my $record_type = $params->{record_type}; > my $record_id = $params->{record_id}; >+ my $conditions = $params->{record_conditions}; > my $dont_export_fields = $params->{dont_export_fields}; > my $clean = $params->{clean}; > >@@ -25,7 +27,60 @@ sub _get_record_for_export { > } else { > Koha::Logger->get->warn( "Record_type $record_type not supported." ); > } >- return unless $record; >+ if ( !$record ) { >+ Koha::Logger->get->warn( "Record $record_id could not be exported." ); >+ return; >+ } >+ >+ # If multiple conditions all are required to match (and) >+ # For matching against multiple marc targets all are also required to match >+ my %operators = ( >+ '=' => sub { >+ return $_[0] eq $_[1]; >+ }, >+ '!=' => sub { >+ return $_[0] ne $_[1]; >+ }, >+ '>' => sub { >+ return $_[0] gt $_[1]; >+ }, >+ '<' => sub { >+ return $_[0] lt $_[1]; >+ }, >+ ); >+ if ($conditions) { >+ foreach my $condition (@{$conditions}) { >+ my ($field_tag, $subfield, $operator, $match_value) = @{$condition}; >+ my @fields = $record->field($field_tag); >+ my $no_target = 0; >+ >+ if (!@fields) { >+ $no_target = 1; >+ } >+ else { >+ if ($operator eq '?') { >+ return unless any { $subfield ? $_->subfield($subfield) : $_->data() } @fields; >+ } elsif ($operator eq '!?') { >+ return if any { $subfield ? $_->subfield($subfield) : $_->data() } @fields; >+ } else { >+ my $op; >+ if (exists $operators{$operator}) { >+ $op = $operators{$operator}; >+ } else { >+ die("Invalid operator: $op"); >+ } >+ my @target_values = map { $subfield ? $_->subfield($subfield) : ($_->data()) } @fields; >+ if (!@target_values) { >+ $no_target = 1; >+ } >+ else { >+ return unless all { $op->($_, $match_value) } @target_values; >+ } >+ } >+ } >+ return if $no_target && $operator ne '!='; >+ } >+ } > > if ($dont_export_fields) { > for my $f ( split / /, $dont_export_fields ) { >@@ -116,6 +171,7 @@ sub export { > if ( $format eq 'iso2709' ) { > for my $record_id (@$record_ids) { > my $record = _get_record_for_export( { %$params, record_id => $record_id } ); >+ next unless $record; > my $errorcount_on_decode = eval { scalar( MARC::File::USMARC->decode( $record->as_usmarc )->warnings() ) }; > if ( $errorcount_on_decode or $@ ) { > my $msg = "Record $record_id could not be exported. " . >@@ -134,10 +190,7 @@ sub export { > print "\n"; > for my $record_id (@$record_ids) { > my $record = _get_record_for_export( { %$params, record_id => $record_id } ); >- if( !$record ) { >- Koha::Logger->get->info( "Record $record_id could not be exported." ); >- next; >- } >+ next unless $record; > print MARC::File::XML::record($record); > print "\n"; > } >diff --git a/misc/export_records.pl b/misc/export_records.pl >index 6e363686a3..a1eecc8eb2 100755 >--- a/misc/export_records.pl >+++ b/misc/export_records.pl >@@ -32,7 +32,30 @@ use Koha::CsvProfiles; > use Koha::Exporter::Record; > use Koha::DateUtils qw( dt_from_string output_pref ); > >-my ( $output_format, $timestamp, $dont_export_items, $csv_profile_id, $deleted_barcodes, $clean, $filename, $record_type, $id_list_file, $starting_authid, $ending_authid, $authtype, $starting_biblionumber, $ending_biblionumber, $itemtype, $starting_callnumber, $ending_callnumber, $start_accession, $end_accession, $help ); >+my ( >+ $output_format, >+ $timestamp, >+ $dont_export_items, >+ $csv_profile_id, >+ $deleted_barcodes, >+ $clean, >+ $filename, >+ $record_type, >+ $id_list_file, >+ $starting_authid, >+ $ending_authid, >+ $authtype, >+ $starting_biblionumber, >+ $ending_biblionumber, >+ $itemtype, >+ $starting_callnumber, >+ $ending_callnumber, >+ $start_accession, >+ $end_accession, >+ $marc_conditions, >+ $help >+); >+ > GetOptions( > 'format=s' => \$output_format, > 'date=s' => \$timestamp, >@@ -53,6 +76,7 @@ GetOptions( > 'ending_callnumber=s' => \$ending_callnumber, > 'start_accession=s' => \$start_accession, > 'end_accession=s' => \$end_accession, >+ 'marc_conditions=s' => \$marc_conditions, > 'h|help|?' => \$help > ) || pod2usage(1); > >@@ -90,6 +114,22 @@ if ( $deleted_barcodes and $record_type ne 'bibs' ) { > $start_accession = dt_from_string( $start_accession ) if $start_accession; > $end_accession = dt_from_string( $end_accession ) if $end_accession; > >+# Parse marc conditions >+my @marc_conditions; >+if ($marc_conditions) { >+ foreach my $condition (split(/,\s*/, $marc_conditions)) { >+ if ($condition =~ /^(\d{3})([\w\d]?)(=|(?:!=)|>|<)([^,]+)$/) { >+ push @marc_conditions, [$1, $2, $3, $4]; >+ } >+ elsif ($condition =~ /^(exists|not_exists)\((\d{3})([\w\d]?)\)$/) { >+ push @marc_conditions, [$2, $3, $1 eq 'exists' ? '?' : '!?']; >+ } >+ else { >+ die("Invalid condititon: $condition"); >+ } >+ } >+} >+ > my $dbh = C4::Context->dbh; > > # Redirect stdout >@@ -199,6 +239,7 @@ else { > Koha::Exporter::Record::export( > { record_type => $record_type, > record_ids => \@record_ids, >+ record_conditions => @marc_conditions ? \@marc_conditions : undef, > format => $output_format, > csv_profile_id => $csv_profile_id, > export_items => (not $dont_export_items), >@@ -310,6 +351,28 @@ Print a brief help message. > > --end_accession=DATE Export biblio with an item accessionned after DATE > >+=item B<--marc_conditions> >+ >+ --marc_conditions=CONDITIONS Only include biblios with MARC data matching CONDITIONS. >+ CONDITIONS is on the format: <marc_target><binary_operator><value>, >+ or <unary_operation>(<marc_target>). >+ with multiple conditions separated by commas (,). >+ For example: --marc_conditions="035a!=(EXAMPLE)123,041a=swe". >+ Multiple conditions are all required to match. >+ If <marc_target> has multiple values all values >+ are also required to match. >+ Valid operators are: = (equal to), != (not equal to), >+ > (great than) and < (less than). >+ >+ Two unary operations are also supported: >+ exists(<marc_target>) and not_exists(<marc_target>). >+ For example: --marc_conditions="exists(035a)". >+ >+ "exists(<marc_target)" will include marc records where >+ <marc_target> exists regardless of target value, and >+ "exists(<marc_target>)" will include marc records where >+ no <marc_target> exists. >+ > =back > > =head1 AUTHOR >diff --git a/t/db_dependent/Exporter/Record.t b/t/db_dependent/Exporter/Record.t >index 3dc25d6afb..68eef06775 100644 >--- a/t/db_dependent/Exporter/Record.t >+++ b/t/db_dependent/Exporter/Record.t >@@ -17,7 +17,7 @@ > > use Modern::Perl; > >-use Test::More tests => 5; >+use Test::More tests => 6; > use Test::Warn; > use t::lib::TestBuilder; > >@@ -281,8 +281,129 @@ subtest '_get_biblio_for_export' => sub { > > }; > >+subtest '_get_record_for_export MARC field conditions' => sub { >+ plan tests => 11; > >+ my $biblio = MARC::Record->new(); >+ $biblio->leader('00266nam a22001097a 4500'); >+ $biblio->append_fields( >+ MARC::Field->new( '100', ' ', ' ', a => 'Thurber, James' ), >+ MARC::Field->new( '245', ' ', ' ', a => 'The 13 Clocks' ), >+ MARC::Field->new( '080', ' ', ' ', a => '12345' ), >+ MARC::Field->new( '035', ' ', ' ', a => '(TEST)123' ), >+ MARC::Field->new( '035', ' ', ' ', a => '(TEST)1234' ), >+ ); >+ my ( $biblionumber ) = AddBiblio( $biblio, '' ); >+ my $record; > >+ $record = Koha::Exporter::Record::_get_record_for_export( >+ { >+ record_id => $biblionumber, >+ record_conditions => [['080', 'a', '=', '12345']], >+ record_type => 'bibs', >+ } >+ ); >+ ok( $record, "Record condition \"080a=12345\" should match" ); > >-$schema->storage->txn_rollback; >+ $record = Koha::Exporter::Record::_get_record_for_export( >+ { >+ record_id => $biblionumber, >+ record_conditions => [['080', 'a', '!=', '12345']], >+ record_type => 'bibs', >+ } >+ ); >+ is( $record, undef, "Record condition \"080a!=12345\" should not match" ); >+ >+ $record = Koha::Exporter::Record::_get_record_for_export( >+ { >+ record_id => $biblionumber, >+ record_conditions => [['080', 'a', '>', '1234']], >+ record_type => 'bibs', >+ } >+ ); >+ ok( $record, "Record condition \"080a>1234\" should match" ); >+ >+ $record = Koha::Exporter::Record::_get_record_for_export( >+ { >+ record_id => $biblionumber, >+ record_conditions => [['080', 'a', '<', '123456']], >+ record_type => 'bibs', >+ } >+ ); >+ ok( $record, "Record condition \"080a<123456\" should match" ); >+ >+ $record = Koha::Exporter::Record::_get_record_for_export( >+ { >+ record_id => $biblionumber, >+ record_conditions => [['080', 'a', '>', '123456']], >+ record_type => 'bibs', >+ } >+ ); >+ is( $record, undef, "Record condition \"080a>123456\" should not match" ); >+ >+ >+ ## Multiple subfields >+ >+ $record = Koha::Exporter::Record::_get_record_for_export( >+ { >+ record_id => $biblionumber, >+ record_conditions => [['035', 'a', '!=', 'TEST(12345)']], >+ record_type => 'bibs', >+ } >+ ); >+ ok( $record, "Record condition \"035a!=TEST(12345)\" should match" ); >+ >+ $record = Koha::Exporter::Record::_get_record_for_export( >+ { >+ record_id => $biblionumber, >+ record_conditions => [['035', 'a', '=', 'TEST(1234)']], >+ record_type => 'bibs', >+ } >+ ); >+ is( $record, undef, "Record condition \"035a=TEST(1234)\" should not match" ); # Since matching all subfields required >+ >+ >+ ## Multiple conditions >+ >+ $record = Koha::Exporter::Record::_get_record_for_export( >+ { >+ record_id => $biblionumber, >+ record_conditions => [['035', 'a', '!=', 'TEST(12345)'], ['080', 'a', '>', '1234']], >+ record_type => 'bibs', >+ } >+ ); >+ ok( $record, "Record condition \"035a!=TEST(12345),080a>1234\" should match" ); >+ >+ $record = Koha::Exporter::Record::_get_record_for_export( >+ { >+ record_id => $biblionumber, >+ record_conditions => [['035', 'a', '!=', 'TEST(12345)'], ['080', 'a', '<', '1234']], >+ record_type => 'bibs', >+ } >+ ); >+ is( $record, undef, "Record condition \"035a!=TEST(12345),080a<1234\" should not match" ); >+ >+ >+ ## exists/not_exists >+ >+ $record = Koha::Exporter::Record::_get_record_for_export( >+ { >+ record_id => $biblionumber, >+ record_conditions => [['035', 'a', '?']], >+ record_type => 'bibs', >+ } >+ ); >+ ok( $record, "Record condition \"exists(035a)\" should match" ); > >+ $record = Koha::Exporter::Record::_get_record_for_export( >+ { >+ record_id => $biblionumber, >+ record_conditions => [['035', 'a', '!?']], >+ record_type => 'bibs', >+ record_type => 'bibs', >+ } >+ ); >+ is( $record, undef, "Record condition \"not_exists(035a)\" should not match" ); >+}; >+ >+$schema->storage->txn_rollback; >-- >2.11.0
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 20486
:
73384
|
73497
|
73999
|
74853
|
75110
|
79266
|
80318
| 81762