Bugzilla – Attachment 10671 Details for
Bug 5600
Bulk MARC biblio export script
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 5600: Command line interface for tools/export.pl
0001-Bug-5600-Command-line-interface-for-tools-export.pl.patch (text/plain), 18.85 KB, created by
Julian Maurice
on 2012-07-06 12:26:11 UTC
(
hide
)
Description:
Bug 5600: Command line interface for tools/export.pl
Filename:
MIME Type:
Creator:
Julian Maurice
Created:
2012-07-06 12:26:11 UTC
Size:
18.85 KB
patch
obsolete
>From e5877fbe87b98c71687cc1fd410d7668fbec2237 Mon Sep 17 00:00:00 2001 >From: Julian Maurice <julian.maurice@biblibre.com> >Date: Wed, 6 Jun 2012 15:28:18 +0200 >Subject: [PATCH] Bug 5600: Command line interface for tools/export.pl > >export.pl [--format=format] [--date=date] [--dont_export_items] > [--deleted_barcodes] [--clean] --filename=outputfile > > * format is either 'xml' or 'marc' (default) > * date should be entered as the 'dateformat' syspref is set > (dd/mm/yyyy for metric, yyyy-mm-dd for iso, mm/dd/yyyy for us) > * records exported are the ones that have been modified since 'date' > * if --deleted_barcodes is used, a list of barcodes of items deleted > since 'date' is produced (or from all deleted items if no date is > specified) > * --clean removes NSE/NSB >--- > C4/Biblio.pm | 45 +++++++ > tools/export.pl | 355 +++++++++++++++++++++++++++++++++++++------------------ > 2 files changed, 282 insertions(+), 118 deletions(-) > >diff --git a/C4/Biblio.pm b/C4/Biblio.pm >index f8ba099..c479502 100644 >--- a/C4/Biblio.pm >+++ b/C4/Biblio.pm >@@ -105,6 +105,7 @@ BEGIN { > &ModBiblioframework > &ModZebra > &UpdateTotalIssues >+ &RemoveAllNsb > ); > > # To delete something >@@ -3876,6 +3877,50 @@ sub UpdateTotalIssues { > return; > } > >+=head2 RemoveAllNsb >+ >+ &RemoveAllNsb($record); >+ >+Removes all nsb/nse chars from a record >+ >+=cut >+ >+sub RemoveAllNsb { >+ my $record = shift; >+ >+ SetUTF8Flag($record); >+ >+ foreach my $field ($record->fields()) { >+ if ($field->is_control_field()) { >+ $field->update(nsb_clean($field->data())); >+ } else { >+ my @subfields = $field->subfields(); >+ my @new_subfields; >+ foreach my $subfield (@subfields) { >+ push @new_subfields, $subfield->[0] => nsb_clean($subfield->[1]); >+ } >+ if (scalar(@new_subfields) > 0) { >+ my $new_field; >+ eval { >+ $new_field = MARC::Field->new( >+ $field->tag(), >+ $field->indicator(1), >+ $field->indicator(2), >+ @new_subfields >+ ); >+ }; >+ if ($@) { >+ warn "error in RemoveAllNsb : $@"; >+ } else { >+ $field->replace_with($new_field); >+ } >+ } >+ } >+ } >+ >+ return $record; >+} >+ > 1; > > >diff --git a/tools/export.pl b/tools/export.pl >index 8c08602..b06ed03 100755 >--- a/tools/export.pl >+++ b/tools/export.pl >@@ -26,45 +26,118 @@ use C4::AuthoritiesMarc; # GetAuthority > use CGI; > use C4::Koha; # GetItemTypes > use C4::Branch; # GetBranches >+use C4::Record; >+use Getopt::Long; > > my $query = new CGI; >-my $op=$query->param("op") || ''; >-my $filename=$query->param("filename"); >-my $dbh=C4::Context->dbh; >+ >+my $commandline = 0; >+my $op; >+my $filename; >+my $dbh = C4::Context->dbh; > my $marcflavour = C4::Context->preference("marcflavour"); >+my $clean; >+my $output_format; >+my $dont_export_items; >+my $deleted_barcodes; >+my $timestamp; >+my $record_type; >+my $help; > >-my ($template, $loggedinuser, $cookie) >- = get_template_and_user >- ( >- { >- template_name => "tools/export.tmpl", >- query => $query, >- type => "intranet", >- authnotrequired => 0, >- flagsrequired => {tools => 'export_catalog'}, >- debug => 1, >- } >+# Checks if the script is called from commandline >+if ( scalar @ARGV > 0 ) { >+ >+ # Getting parameters >+ $op = 'export'; >+ GetOptions( >+ 'format=s' => \$output_format, >+ 'date=s' => \$timestamp, >+ 'dont_export_items' => \$dont_export_items, >+ 'deleted_barcodes' => \$deleted_barcodes, >+ 'clean' => \$clean, >+ 'filename=s' => \$filename, >+ 'record-type=s' => \$record_type, >+ 'help|?' => \$help > ); >+ $commandline = 1; >+ >+ >+ if ($help) { >+ print <<_USAGE_; >+export.pl [--format=format] [--date=date] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] --filename=outputfile >+ >+ >+ --format=FORMAT FORMAT is either 'xml' or 'marc' (default) >+ >+ --date=DATE DATE should be entered as the 'dateformat' syspref is >+ set (dd/mm/yyyy for metric, yyyy-mm-dd for iso, >+ mm/dd/yyyy for us) records exported are the ones that >+ have been modified since DATE >+ >+ --record-type=TYPE TYPE is 'bibs' or 'auths' >+ >+ --deleted_barcodes If used, a list of barcodes of items deleted since DATE >+ is produced (or from all deleted items if no date is >+ specified). Used only if TYPE is 'bibs' >+ >+ --clean removes NSE/NSB >+_USAGE_ >+ exit; >+ } >+ >+ # Default parameters values : >+ $output_format ||= 'marc'; >+ $timestamp ||= ''; >+ $dont_export_items ||= 0; >+ $deleted_barcodes ||= 0; >+ $clean ||= 0; >+ $record_type ||= "bibs"; > >- my $limit_ind_branch=(C4::Context->preference('IndependantBranches') && >- C4::Context->userenv && >- !(C4::Context->userenv->{flags} & 1) && >- C4::Context->userenv->{branch}?1:0); >- my $branches = GetBranches($limit_ind_branch); >- my $branch = $query->param("branch") || ''; >- if ( C4::Context->preference("IndependantBranches") && >- !(C4::Context->userenv->{flags} & 1) ) { >- $branch = C4::Context->userenv->{'branch'}; >- } >+ # Redirect stdout >+ open STDOUT, ">$filename" if $filename; >+ >+} else { >+ >+ $op = $query->param("op") || ''; >+ $filename = $query->param("filename") || 'koha.mrc'; >+ >+} >+ >+my ($template, $loggedinuser, $cookie) = get_template_and_user( >+ { >+ template_name => "tools/export.tmpl", >+ query => $query, >+ type => "intranet", >+ authnotrequired => $commandline, >+ flagsrequired => {tools => 'export_catalog'}, >+ debug => 1, >+ } >+); >+ >+my $limit_ind_branch = ( >+ C4::Context->preference('IndependantBranches') && >+ C4::Context->userenv && >+ !(C4::Context->userenv->{flags} & 1) && >+ C4::Context->userenv->{branch} >+) ? 1 : 0; >+ >+my $branch = $query->param("branch") || ''; >+if ( C4::Context->preference("IndependantBranches") && >+ C4::Context->userenv && >+ !(C4::Context->userenv->{flags} & 1) ) { >+ $branch = C4::Context->userenv->{'branch'}; >+} > > if ($op eq "export") { > binmode STDOUT, ':encoding(UTF-8)'; >- print $query->header( -type => 'application/octet-stream', >- -charset => 'utf-8', >- -attachment=>$filename); >+ print $query->header( >+ -type => 'application/octet-stream', >+ -charset => 'utf-8', >+ -attachment => $filename >+ ) unless ($commandline); > >- my $record_type = $query->param("record_type"); >- my $output_format = $query->param("output_format"); >+ $record_type = $query->param("record_type") unless ($commandline); >+ $output_format = $query->param("output_format") || 'marc' unless ($commandline); > my $dont_export_fields = $query->param("dont_export_fields"); > my @sql_params; > my $sql_query; >@@ -74,6 +147,7 @@ if ($op eq "export") { > my $itemtype = $query->param("itemtype"); > my $start_callnumber = $query->param("start_callnumber"); > my $end_callnumber = $query->param("end_callnumber"); >+ $timestamp = ($timestamp) ? C4::Dates->new($timestamp) : '' if ($commandline); > my $start_accession = > ( $query->param("start_accession") ) > ? C4::Dates->new( $query->param("start_accession") ) >@@ -82,62 +156,92 @@ if ($op eq "export") { > ( $query->param("end_accession") ) > ? C4::Dates->new( $query->param("end_accession") ) > : ''; >- my $dont_export_items = $query->param("dont_export_item"); >+ $dont_export_items = $query->param("dont_export_item") unless ($commandline); > my $strip_nonlocal_items = $query->param("strip_nonlocal_items"); > >+ my $biblioitemstable = ($commandline and $deleted_barcodes) >+ ? 'deletedbiblioitems' >+ : 'biblioitems'; >+ my $itemstable = ($commandline and $deleted_barcodes) >+ ? 'deleteditems' >+ : 'items'; >+ > my $starting_authid = $query->param('starting_authid'); > my $ending_authid = $query->param('ending_authid'); > my $authtype = $query->param('authtype'); > > if ( $record_type eq 'bibs' ) { >- my $items_filter = >- $branch || $start_callnumber || $end_callnumber || >- $start_accession || $end_accession || >- ($itemtype && C4::Context->preference('item-level_itypes')); >- $sql_query = $items_filter ? >- "SELECT DISTINCT biblioitems.biblionumber >- FROM biblioitems JOIN items >- USING (biblionumber) WHERE 1" >- : >- "SELECT biblioitems.biblionumber FROM biblioitems WHERE biblionumber >0 "; >- >- if ( $StartingBiblionumber ) { >- $sql_query .= " AND biblioitems.biblionumber >= ? "; >- push @sql_params, $StartingBiblionumber; >- } >+ if ($timestamp) { >+ # Specific query when timestamp is used >+ # Actually it's used only with CLI and so all previous filters >+ # are not used. >+ # If one day timestamp is used via the web interface, this part will >+ # certainly have to be rewrited >+ $sql_query = " ( >+ SELECT biblionumber >+ FROM $biblioitemstable >+ LEFT JOIN items USING(biblionumber) >+ WHERE $biblioitemstable.timestamp >= ? >+ OR items.timestamp >= ? >+ ) UNION ( >+ SELECT biblionumber >+ FROM $biblioitemstable >+ LEFT JOIN deleteditems USING(biblionumber) >+ WHERE $biblioitemstable.timestamp >= ? >+ OR deleteditems.timestamp >= ? >+ ) "; >+ my $ts = $timestamp->output('iso'); >+ @sql_params = ($ts, $ts, $ts, $ts); >+ } else { >+ my $items_filter = >+ $branch || $start_callnumber || $end_callnumber || >+ $start_accession || $timestamp || $end_accession || >+ ($itemtype && C4::Context->preference('item-level_itypes')); >+ $sql_query = $items_filter ? >+ "SELECT DISTINCT $biblioitemstable.biblionumber >+ FROM $biblioitemstable JOIN $itemstable >+ USING (biblionumber) WHERE 1" >+ : >+ "SELECT $biblioitemstable.biblionumber FROM $biblioitemstable WHERE biblionumber >0 "; > >- if ( $EndingBiblionumber ) { >- $sql_query .= " AND biblioitems.biblionumber <= ? "; >- push @sql_params, $EndingBiblionumber; >- } >+ if ( $StartingBiblionumber ) { >+ $sql_query .= " AND $biblioitemstable.biblionumber >= ? "; >+ push @sql_params, $StartingBiblionumber; >+ } > >- if ($branch) { >- $sql_query .= " AND homebranch = ? "; >- push @sql_params, $branch; >- } >+ if ( $EndingBiblionumber ) { >+ $sql_query .= " AND $biblioitemstable.biblionumber <= ? "; >+ push @sql_params, $EndingBiblionumber; >+ } > >- if ($start_callnumber) { >- $sql_query .= " AND itemcallnumber <= ? "; >- push @sql_params, $start_callnumber; >- } >+ if ($branch) { >+ $sql_query .= " AND homebranch = ? "; >+ push @sql_params, $branch; >+ } > >- if ($end_callnumber) { >- $sql_query .= " AND itemcallnumber >= ? "; >- push @sql_params, $end_callnumber; >- } >- if ($start_accession) { >- $sql_query .= " AND dateaccessioned >= ? "; >- push @sql_params, $start_accession->output('iso'); >- } >+ if ($start_callnumber) { >+ $sql_query .= " AND itemcallnumber <= ? "; >+ push @sql_params, $start_callnumber; >+ } > >- if ($end_accession) { >- $sql_query .= " AND dateaccessioned <= ? "; >- push @sql_params, $end_accession->output('iso'); >- } >+ if ($end_callnumber) { >+ $sql_query .= " AND itemcallnumber >= ? "; >+ push @sql_params, $end_callnumber; >+ } >+ if ($start_accession) { >+ $sql_query .= " AND dateaccessioned >= ? "; >+ push @sql_params, $start_accession->output('iso'); >+ } >+ >+ if ($end_accession) { >+ $sql_query .= " AND dateaccessioned <= ? "; >+ push @sql_params, $end_accession->output('iso'); >+ } > >- if ( $itemtype ) { >- $sql_query .= (C4::Context->preference('item-level_itypes')) ? " AND items.itype = ? " : " AND biblioitems.itemtype = ?"; >- push @sql_params, $itemtype; >+ if ( $itemtype ) { >+ $sql_query .= (C4::Context->preference('item-level_itypes')) ? " AND items.itype = ? " : " AND biblioitems.itemtype = ?"; >+ push @sql_params, $itemtype; >+ } > } > } > elsif ( $record_type eq 'auths' ) { >@@ -164,61 +268,74 @@ if ($op eq "export") { > $sth->execute(@sql_params); > > while ( my ($recordid) = $sth->fetchrow ) { >- my $record; >- if ( $record_type eq 'bibs' ) { >- $record = eval { GetMarcBiblio($recordid); }; >- >- # FIXME: decide how to handle records GetMarcBiblio can't parse or retrieve >- if ($@) { >- next; >+ if ( $deleted_barcodes ) { >+ my $q = " >+ SELECT DISTINCT barcode >+ FROM deleteditems >+ WHERE deleteditems.biblionumber = ? >+ "; >+ my $sth = $dbh->prepare($q); >+ $sth->execute($recordid); >+ while (my $row = $sth->fetchrow_array) { >+ print "$row\n"; > } >- next if not defined $record; >- C4::Biblio::EmbedItemsInMarcBiblio( $record, $recordid ) >- unless $dont_export_items; >- if ( $strip_nonlocal_items || $limit_ind_branch ) { >- my ( $homebranchfield, $homebranchsubfield ) = >- GetMarcFromKohaField( 'items.homebranch', '' ); >- for my $itemfield ( $record->field($homebranchfield) ) { >- >-# if stripping nonlocal items, use loggedinuser's branch if they didn't select one >- $branch = C4::Context->userenv->{'branch'} unless $branch; >- $record->delete_field($itemfield) >- if ( >- $itemfield->subfield($homebranchsubfield) ne $branch ); >+ } else { >+ my $record; >+ if ( $record_type eq 'bibs' ) { >+ $record = eval { GetMarcBiblio($recordid); }; >+ >+ if ($@) { >+ next; >+ } >+ next if not defined $record; >+ C4::Biblio::EmbedItemsInMarcBiblio( $record, $recordid ) >+ unless $dont_export_items; >+ if ( $strip_nonlocal_items || $limit_ind_branch ) { >+ my ( $homebranchfield, $homebranchsubfield ) = >+ GetMarcFromKohaField( 'items.homebranch', '' ); >+ for my $itemfield ( $record->field($homebranchfield) ) { >+ >+ # if stripping nonlocal items, use loggedinuser's branch if they didn't select one >+ $branch = C4::Context->userenv->{'branch'} unless $branch; >+ $record->delete_field($itemfield) >+ if ( >+ $itemfield->subfield($homebranchsubfield) ne $branch ); >+ } > } > } >- } >- elsif ( $record_type eq 'auths' ) { >- $record = C4::AuthoritiesMarc::GetAuthority($recordid); >- next if not defined $record; >- } >+ elsif ( $record_type eq 'auths' ) { >+ $record = C4::AuthoritiesMarc::GetAuthority($recordid); >+ next if not defined $record; >+ } > >- if ( $dont_export_fields ) { >- my @fields = split " ", $dont_export_fields; >- foreach ( @fields ) { >- /^(\d*)(\w)?$/; >- my $field = $1; >- my $subfield = $2; >- # skip if this record doesn't have this field >- next if not defined $record->field($field); >- if( $subfield ) { >- $record->field($field)->delete_subfields($subfield); >+ if ( $dont_export_fields ) { >+ my @fields = split " ", $dont_export_fields; >+ foreach ( @fields ) { >+ /^(\d*)(\w)?$/; >+ my $field = $1; >+ my $subfield = $2; >+ # skip if this record doesn't have this field >+ next if not defined $record->field($field); >+ if( $subfield ) { >+ $record->field($field)->delete_subfields($subfield); >+ } >+ else { >+ $record->delete_field($record->field($field)); >+ } > } >- else { >- $record->delete_field($record->field($field)); >+ } >+ RemoveAllNsb($record) if ($clean); >+ if ( $output_format eq "xml" ) { >+ if ($marcflavour eq 'UNIMARC' && $record_type eq 'auths') { >+ print $record->as_xml_record('UNIMARCAUTH'); >+ } else { >+ print $record->as_xml_record($marcflavour); > } > } >- } >- if ( $output_format eq "xml" ) { >- if ($marcflavour eq 'UNIMARC' && $record_type eq 'auths') { >- print $record->as_xml_record('UNIMARCAUTH'); >- } else { >- print $record->as_xml_record($marcflavour); >+ else { >+ print $record->as_usmarc(); > } > } >- else { >- print $record->as_usmarc(); >- } > } > exit; > >@@ -236,6 +353,7 @@ else { > ); > push @itemtypesloop, \%row; > } >+ my $branches = GetBranches($limit_ind_branch); > my @branchloop; > for my $thisbranch ( > sort { $branches->{$a}->{branchname} cmp $branches->{$b}->{branchname} } >@@ -264,6 +382,7 @@ else { > itemtypeloop => \@itemtypesloop, > DHTMLcalendar_dateformat => C4::Dates->DHTMLcalendar(), > authtypeloop => \@authtypesloop, >+ dont_export_fields => C4::Context->preference("DontExportFields"), > ); > > output_html_with_http_headers $query, $cookie, $template->output; >-- >1.7.10 >
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 5600
:
9955
|
10200
|
10671
|
10676
|
10954
|
10997
|
11099
|
11331
|
11334