From cb500d3948ce36119aa89d1cdc43861baac8c0d4 Mon Sep 17 00:00:00 2001
From: Julian Maurice <julian.maurice@biblibre.com>
Date: Wed, 6 Jun 2012 15:28:18 +0200
Subject: [PATCH] Bug 5600: Command line interface for tools/export.pl

export.pl [--format=format] [--date=date] [--dont_export_items]
  [--deleted_barcodes] [--clean] --filename=outputfile

    * format is either 'xml' or 'marc' (default)
    * date should be entered as the 'dateformat' syspref is set
      (dd/mm/yyyy for metric, yyyy-mm-dd for iso, mm/dd/yyyy for us)
    * records exported are the ones that have been modified since 'date'
    * if --deleted_barcodes is used, a list of barcodes of items deleted
      since 'date' is produced (or from all deleted items if no date is
      specified)
    * --clean removes NSE/NSB
---
 C4/Biblio.pm    |   45 +++++++++++
 tools/export.pl |  234 +++++++++++++++++++++++++++++++++++++++----------------
 2 files changed, 214 insertions(+), 65 deletions(-)

diff --git a/C4/Biblio.pm b/C4/Biblio.pm
index 2aa17d5..fe390d0 100644
--- a/C4/Biblio.pm
+++ b/C4/Biblio.pm
@@ -104,6 +104,7 @@ BEGIN {
       &ModBiblio
       &ModBiblioframework
       &ModZebra
+      &RemoveAllNsb
     );
 
     # To delete something
@@ -3825,6 +3826,50 @@ sub prepare_host_field {
     return;
 }
 
+=head2 RemoveAllNsb
+
+    &RemoveAllNsb($record);
+
+Removes all nsb/nse chars from a record
+
+=cut
+
+sub RemoveAllNsb {
+    my $record = shift;
+
+    SetUTF8Flag($record);
+
+    foreach my $field ($record->fields()) {
+        if ($field->is_control_field()) {
+            $field->update(nsb_clean($field->data()));
+        } else {
+            my @subfields = $field->subfields();
+            my @new_subfields;
+            foreach my $subfield (@subfields) {
+                push @new_subfields, $subfield->[0] => nsb_clean($subfield->[1]);
+            }
+            if (scalar(@new_subfields) > 0) {
+                my $new_field;
+                eval {
+                    $new_field = MARC::Field->new(
+                        $field->tag(),
+                        $field->indicator(1),
+                        $field->indicator(2),
+                        @new_subfields
+                    );
+                };
+                if ($@) {
+                    warn "error in RemoveAllNsb : $@";
+                } else {
+                    $field->replace_with($new_field);
+                }
+            }
+        }
+    }
+
+    return $record;
+}
+
 1;
 
 
diff --git a/tools/export.pl b/tools/export.pl
index d3d8646..118b626 100755
--- a/tools/export.pl
+++ b/tools/export.pl
@@ -25,109 +25,199 @@ use C4::Biblio;  # GetMarcBiblio GetXmlBiblio
 use CGI;
 use C4::Koha;    # GetItemTypes
 use C4::Branch;  # GetBranches
+use C4::Record;
+use Getopt::Long;
 
 my $query = new CGI;
-my $op=$query->param("op") || '';
-my $filename=$query->param("filename");
-my $dbh=C4::Context->dbh;
+
+my $commandline = 0;
+my $op;
+my $filename;
+my $dbh         = C4::Context->dbh;
 my $marcflavour = C4::Context->preference("marcflavour");
+my $clean;
+my $output_format;
+my $dont_export_items;
+my $deleted_barcodes;
+my $timestamp;
+my $help;
 
-my ($template, $loggedinuser, $cookie)
-    = get_template_and_user
-    (
-        {
-            template_name => "tools/export.tmpl",
-            query => $query,
-            type => "intranet",
-            authnotrequired => 0,
-            flagsrequired => {tools => 'export_catalog'},
-            debug => 1,
-            }
+# Checks if the script is called from commandline
+if ( scalar @ARGV > 0 ) {
+
+    # Getting parameters
+    $op = 'export';
+    GetOptions(
+        'format=s' => \$output_format,
+        'date=s' => \$timestamp,
+        'dont_export_items' => \$dont_export_items,
+        'deleted_barcodes' => \$deleted_barcodes,
+        'clean' => \$clean,
+        'filename=s' => \$filename,
+        'help|?' => \$help
     );
+    $commandline = 1;
+
+
+    if ($help) {
+        print "\nexport.pl [--format=format] [--date=date] [--dont_export_items] [--deleted_barcodes] [--clean] --filename=outputfile\n\n";
+        print " * format is either 'xml' or 'marc' (default)\n";
+        print " * date should be entered as the 'dateformat' syspref is set (dd/mm/yyyy for metric, yyyy-mm-dd for iso, mm/dd/yyyy for us)\n";
+        print " * records exported are the ones that have been modified since 'date'\n";
+        print " * if --deleted_barcodes is used, a list of barcodes of items deleted since 'date' is produced (or from all deleted items if no date is specified)\n";
+        print " * --clean removes NSE/NSB\n";
+        exit;
+    }
 
-	my $limit_ind_branch=(C4::Context->preference('IndependantBranches') &&
-              C4::Context->userenv &&
-              !(C4::Context->userenv->{flags} & 1) &&
-              C4::Context->userenv->{branch}?1:0);
-	my $branches = GetBranches($limit_ind_branch);    
-    my $branch                = $query->param("branch") || '';
-	if ( C4::Context->preference("IndependantBranches") &&
-         !(C4::Context->userenv->{flags} & 1) ) {
-    	$branch = C4::Context->userenv->{'branch'};
-	}
+    # Default parameters values :
+    $output_format     ||= 'marc';
+    $timestamp         ||= '';
+    $dont_export_items ||= 0;
+    $deleted_barcodes  ||= 0;
+    $clean             ||= 0;
+    $filename          ||= "koha.mrc";
+
+    # Redirect stdout
+    open STDOUT, ">$filename";
+
+} else {
+
+    $op          = $query->param("op") || '';
+    $filename    = $query->param("filename") || 'koha.mrc';
+
+}
+
+
+my $limit_ind_branch = (
+    C4::Context->preference('IndependantBranches') &&
+    C4::Context->userenv &&
+    !(C4::Context->userenv->{flags} & 1) &&
+    C4::Context->userenv->{branch}
+) ? 1 : 0;
+
+my $branch = $query->param("branch") || '';
+if ( C4::Context->preference("IndependantBranches") &&
+     !(C4::Context->userenv->{flags} & 1) ) {
+    $branch = C4::Context->userenv->{'branch'};
+}
 
 if ($op eq "export") {
     binmode STDOUT, ':encoding(UTF-8)';
-	print $query->header(   -type => 'application/octet-stream', 
-                            -charset => 'utf-8',
-                            -attachment=>$filename);
-     
+    print $query->header(
+        -type => 'application/octet-stream',
+        -charset => 'utf-8',
+        -attachment => $filename
+    ) unless ($commandline);
+
     my $StartingBiblionumber  = $query->param("StartingBiblionumber");
     my $EndingBiblionumber    = $query->param("EndingBiblionumber");
-    my $output_format         = $query->param("output_format");
+    $output_format            = $query->param("output_format") || 'marc' unless ($commandline);
     my $itemtype              = $query->param("itemtype");
     my $start_callnumber      = $query->param("start_callnumber");
     my $end_callnumber        = $query->param("end_callnumber");
+    if ($commandline) {
+        $timestamp = ($timestamp) ? C4::Dates->new($timestamp) : '';
+    }
     my $start_accession      = ($query->param("start_accession")) ? C4::Dates->new($query->param("start_accession")) : '' ;
     my $end_accession        = ($query->param("end_accession")) ? C4::Dates->new($query->param("end_accession")) : '' ;
-    my $dont_export_items     = $query->param("dont_export_item");
-    my $strip_nonlocal_items   = $query->param("strip_nonlocal_items");
-    my $dont_export_fields    = $query->param("dont_export_fields");
+    $dont_export_items       = $query->param("dont_export_item") unless ($commandline);
+    my $strip_nonlocal_items = $query->param("strip_nonlocal_items");
+    my $dont_export_fields   = $query->param("dont_export_fields");
+    my $export_only_borrowed = $query->param("export_only_borrowed");
+    my $borrowernumber       = $query->param("borrowernumber");
+    my $biblioitemstable     = ($commandline and $deleted_barcodes) ? 'deletedbiblioitems' : 'biblioitems';
+    my $itemstable           = ($commandline and $deleted_barcodes) ? 'deleteditems' : 'items';
+
     my @sql_params;
-    
+
     my $items_filter =
-        $branch || $start_callnumber || $end_callnumber ||  
-        $start_accession || $end_accession || 
+        $branch || $start_callnumber || $end_callnumber ||
+        $start_accession || $timestamp || $end_accession ||
         ($itemtype && C4::Context->preference('item-level_itypes'));
-    my $query = $items_filter ?
-        "SELECT DISTINCT biblioitems.biblionumber
-         FROM biblioitems JOIN items
-         USING (biblionumber) WHERE 1"
-        :
-        "SELECT biblioitems.biblionumber FROM biblioitems WHERE biblionumber >0 ";
-                  
+    my $q = $items_filter ?
+        "SELECT DISTINCT $biblioitemstable.biblionumber
+        FROM $biblioitemstable JOIN $itemstable
+        USING (biblionumber) WHERE 1"
+    :
+        "SELECT $biblioitemstable.biblionumber FROM $biblioitemstable WHERE biblionumber >0 ";
+
     if ( $StartingBiblionumber ) {
-        $query .= " AND biblioitems.biblionumber >= ? ";
+        $q .= " AND $biblioitemstable.biblionumber >= ? ";
         push @sql_params, $StartingBiblionumber;
     }
-    
+
     if ( $EndingBiblionumber ) {
-        $query .= " AND biblioitems.biblionumber <= ? ";
-        push @sql_params, $EndingBiblionumber;    
+        $q .= " AND $biblioitemstable.biblionumber <= ? ";
+        push @sql_params, $EndingBiblionumber;
     }
 
     if ($branch) {
-        $query .= " AND homebranch = ? ";
+        $q .= " AND homebranch = ? ";
         push @sql_params, $branch;
     }
 
     if ($start_callnumber) {
-        $query .= " AND itemcallnumber <= ? ";
+        $q .= " AND itemcallnumber <= ? ";
         push @sql_params, $start_callnumber;
     }
 
     if ($end_callnumber) {
-        $query .= " AND itemcallnumber >= ? ";
+        $q .= " AND itemcallnumber >= ? ";
         push @sql_params, $end_callnumber;
     }
     if ($start_accession) {
-        $query .= " AND dateaccessioned >= ? ";
+        $q .= " AND dateaccessioned >= ? ";
         push @sql_params, $start_accession->output('iso');
     }
 
     if ($end_accession) {
-        $query .= " AND dateaccessioned <= ? ";
+        $q .= " AND dateaccessioned <= ? ";
         push @sql_params, $end_accession->output('iso');
     }
-    
+
     if ( $itemtype ) {
-        $query .= (C4::Context->preference('item-level_itypes')) ? " AND items.itype = ? " : " AND biblioitems.itemtype = ?";
+        $q .= (C4::Context->preference('item-level_itypes')) ? " AND $itemstable.itype = ? " : " AND $biblioitemstable.itemtype = ?";
         push @sql_params, $itemtype;
     }
-    my $sth = $dbh->prepare($query);
+
+    # Specific query when timestamp is used
+    # Actually it's used only with CLI and so all previous filters
+    # are not used.
+    # If one day timestamp is used via the web interface, this part will
+    # certainly have to be rewrited
+    if ($timestamp) {
+        $q = " (
+            SELECT biblionumber
+            FROM $biblioitemstable
+              JOIN items USING(biblionumber)
+            WHERE $biblioitemstable.timestamp >= ?
+              OR items.timestamp >= ?
+        ) UNION (
+            SELECT biblionumber
+            FROM $biblioitemstable
+              JOIN deleteditems USING(biblionumber)
+            WHERE $biblioitemstable.timestamp >= ?
+              OR deleteditems.timestamp >= ?
+        ) ";
+        my $ts = $timestamp->output('iso');
+        @sql_params = ($ts, $ts, $ts, $ts);
+    }
+
+    my $sth = $dbh->prepare($q);
     $sth->execute(@sql_params);
-    
+
     while (my ($biblionumber) = $sth->fetchrow) {
+
+        if ( $deleted_barcodes ) {
+            my $q = "SELECT DISTINCT barcode from deleteditems where deleteditems.biblionumber = ?";
+            my $sth = $dbh->prepare($q);
+            $sth->execute($biblionumber);
+            while (my $row = $sth->fetchrow_array) {
+                print $row . "\n";
+            }
+            next;
+        }
+
         my $record = eval{ GetMarcBiblio($biblionumber); };
         # FIXME: decide how to handle records GetMarcBiblio can't parse or retrieve
         if ($@) {
@@ -135,16 +225,18 @@ if ($op eq "export") {
         }
         next if not defined $record;
         C4::Biblio::EmbedItemsInMarcBiblio($record, $biblionumber) unless $dont_export_items;
-        if ($strip_nonlocal_items || $limit_ind_branch) {
+        if ( $dont_export_items || $strip_nonlocal_items || $limit_ind_branch) {
             my ( $homebranchfield, $homebranchsubfield ) =
                 GetMarcFromKohaField( 'items.homebranch', '' );
-			for my $itemfield ($record->field($homebranchfield)){
-				# if stripping nonlocal items, use loggedinuser's branch if they didn't select one
-				$branch = C4::Context->userenv->{'branch'} unless $branch;
-                $record->delete_field($itemfield) if($itemfield->subfield($homebranchsubfield) ne $branch) ;
+            for my $itemfield ($record->field($homebranchfield)){
+                # if stripping nonlocal items, use loggedinuser's branch if they didn't select one
+                if ($strip_nonlocal_items) {
+                    $branch = C4::Context->userenv->{'branch'} unless $branch;
+                }
+                $record->delete_field($itemfield) if($dont_export_items || ($branch ne '' && $itemfield->subfield($homebranchsubfield) ne $branch) );
             }
         }
-        
+
         if ( $dont_export_fields ) {
             my @fields = split " ", $dont_export_fields;
             foreach ( @fields ) {
@@ -161,18 +253,28 @@ if ($op eq "export") {
                 }
             }
         }
+        RemoveAllNsb($record) if ($clean);
         if ( $output_format eq "xml" ) {
             print $record->as_xml_record($marcflavour);
         }
         else {
-            print $record->as_usmarc(); 
+            print $record->as_usmarc();
         }
     }
     exit;
-    
 } # if export
 
 else {
+    my ($template, $loggedinuser, $cookie) = get_template_and_user(
+        {
+            template_name => "tools/export.tmpl",
+            query => $query,
+            type => "intranet",
+            authnotrequired => $commandline,
+            flagsrequired => {tools => 'export_catalog'},
+            debug => 1,
+        }
+    );
 
     my $itemtypes = GetItemTypes;
     my @itemtypesloop;
@@ -184,6 +286,7 @@ else {
             );
        push @itemtypesloop, \%row;
     }
+    my $branches = GetBranches($limit_ind_branch);
     my @branchloop;
     for my $thisbranch (
         sort { $branches->{$a}->{branchname} cmp $branches->{$b}->{branchname} }
@@ -199,8 +302,9 @@ else {
     $template->param(
         branchloop   => \@branchloop,
         itemtypeloop => \@itemtypesloop,
-		DHTMLcalendar_dateformat => C4::Dates->DHTMLcalendar(),
+        DHTMLcalendar_dateformat => C4::Dates->DHTMLcalendar(),
+        dont_export_fields        => C4::Context->preference("DontExportFields"),
     );
-    
+
     output_html_with_http_headers $query, $cookie, $template->output;
 }
-- 
1.7.10