From d7ae49c4e08cde4dda5aa486d968878961251b91 Mon Sep 17 00:00:00 2001 From: Jonathan Druart Date: Mon, 12 Jun 2017 17:04:47 -0300 Subject: [PATCH] Bug 18331: Fix CSV export (once and for all!) I am going to try and explain the problem here in order to make it a have-to-read for next changes. There are several things to test when and most of the time we break something when we try to fix something else. You have to test different CSV export, not only one. There are not all processed the same way. For instance acqui/csv/basket.tt does not contain any strings to translate and the translate script will copy the original file as it. But reports/csv/orders_by_budget.tt will not! Indeed it contains "TOTAL" which will be translated. The generated tt file for the translated language will remove all the carriage returns! That's why we cannot rely on TT for newlines (contrary to bug 16914 assumption). There are two possible methods: 1/ Fix the translate script => Hum... nope 2/ Remove all carriage returns and make them explicit by including an file that only contains 1 carriage return This second method is implemented in this patch. How it works: Use the PRE_CHOMP "[%-" and POST_CHOMP "-%]" (http://www.template-toolkit.org/docs/manual/Config.html#section_PRE_CHOMP_POST_CHOMP) to remove all the newlines that could be added by TT in the original (en) files. Then include the new_line.inc to add a new line. That way original and translated files will behave the same way. The BLOCK in the csv_headers avoid to have the newlines added, in any cases. For instance: by default we will have an empty line at the end of the headers, but the translated headers will not have it. Test plan: At least 2 signoffs will be needed, please test carefully! You will need to generate CSV with at least 2 entries! Test with the non-translated interface (en) and with the language you want. To test with a translated language you will *have to*: - cd misc/translator - perl translate update LANG - manually edit the po file, remove the fuzzy tags, correctly translate the string (do not forget the additional %s in the headers) - perl translate install LANG 1/ Export basket and basketgroup as CSV 2/ Export late orders as CSV 3/ Export items (from the item search) as CSV 4/ Export the 2 reports "Orders by fund" and "Cash register" as CSV --- .../prog/en/includes/csv_headers/acqui/basket.tt | 2 +- .../prog/en/includes/csv_headers/acqui/basketgroup.tt | 2 +- .../prog/en/includes/csv_headers/acqui/lateorders.tt | 2 +- .../prog/en/includes/csv_headers/catalogue/itemsearch.tt | 2 +- .../includes/csv_headers/reports/cash_register_stats.tt | 2 +- .../en/includes/csv_headers/reports/orders_by_budget.tt | 2 +- koha-tmpl/intranet-tmpl/prog/en/includes/empty_line.inc | 1 + .../intranet-tmpl/prog/en/modules/acqui/csv/basket.tt | 6 ++++-- .../prog/en/modules/acqui/csv/basketgroup.tt | 10 ++++++---- .../prog/en/modules/acqui/csv/lateorders.tt | 10 ++++++---- .../prog/en/modules/catalogue/itemsearch_csv.tt | 8 +++++--- .../prog/en/modules/reports/csv/cash_register_stats.tt | 16 +++++++++------- .../prog/en/modules/reports/csv/orders_by_budget.tt | 14 ++++++++------ 13 files changed, 45 insertions(+), 32 deletions(-) create mode 100644 koha-tmpl/intranet-tmpl/prog/en/includes/empty_line.inc diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/basket.tt b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/basket.tt index a2086f65a7..60740d2657 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/basket.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/basket.tt @@ -1 +1 @@ -Contract name,Order number,Entry date,ISBN,Author,Title,Publication year,Publisher,Collection title,Note for vendor,Quantity,RRP,Delivery place,Billing place +[%- BLOCK -%]Contract name,Order number,Entry date,ISBN,Author,Title,Publication year,Publisher,Collection title,Note for vendor,Quantity,RRP,Delivery place,Billing place[%- END -%] diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/basketgroup.tt b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/basketgroup.tt index d28d8f81d0..2f583b9705 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/basketgroup.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/basketgroup.tt @@ -1 +1 @@ -Account number,Basket name,Order number,Author,Title,Publisher,Publication year,Collection title,ISBN,Quantity,RRP tax included, RRP tax excluded,Discount,Estimated cost tax included,Estimated cost tax excluded,Note for vendor,Entry date,Bookseller name,Bookseller physical address,Bookseller postal address,Contract number,Contract name,Basket group delivery place,Basket group billing place,Basket delivery place,Basket billing place +[%- BLOCK -%]Account number,Basket name,Order number,Author,Title,Publisher,Publication year,Collection title,ISBN,Quantity,RRP tax included, RRP tax excluded,Discount,Estimated cost tax included,Estimated cost tax excluded,Note for vendor,Entry date,Bookseller name,Bookseller physical address,Bookseller postal address,Contract number,Contract name,Basket group delivery place,Basket group billing place,Basket delivery place,Basket billing place[%- END -%] diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/lateorders.tt b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/lateorders.tt index a235269f1b..a853d5f9ed 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/lateorders.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/lateorders.tt @@ -1 +1 @@ -ORDER DATE,ESTIMATED DELIVERY DATE,VENDOR,INFORMATION,TOTAL COST,BASKET,CLAIMS COUNT,CLAIMED DATE +[%- BLOCK -%]ORDER DATE,ESTIMATED DELIVERY DATE,VENDOR,INFORMATION,TOTAL COST,BASKET,CLAIMS COUNT,CLAIMED DATE[%- END -%] diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/catalogue/itemsearch.tt b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/catalogue/itemsearch.tt index bac4b147f0..8add507eec 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/catalogue/itemsearch.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/catalogue/itemsearch.tt @@ -1 +1 @@ -Title, Publication date, Publisher, Collection, Barcode, Call number, Home library, Current location, Shelving location, Inventory number, Status, Checkouts +[%- BLOCK -%]Title, Publication date, Publisher, Collection, Barcode, Call number, Home library, Current location, Shelving location, Inventory number, Status, Checkouts[%- END -%] diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/reports/cash_register_stats.tt b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/reports/cash_register_stats.tt index 0746a69bb3..11ec56b9cd 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/reports/cash_register_stats.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/reports/cash_register_stats.tt @@ -1 +1 @@ -mfirstname[% sep %]card number[% sep %]firstname[% sep %]branchname[% sep %]date[% sep %]accounttype[% sep %]amount[% sep %]title[% sep %]barcode[% sep %]itype +[%- BLOCK -%]mfirstname[% sep %]card number[% sep %]firstname[% sep %]branchname[% sep %]date[% sep %]accounttype[% sep %]amount[% sep %]title[% sep %]barcode[% sep %]itype[%- END -%] diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/reports/orders_by_budget.tt b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/reports/orders_by_budget.tt index 2acc7959af..2494d4e9d6 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/reports/orders_by_budget.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/reports/orders_by_budget.tt @@ -1 +1 @@ -Fund[% sep %]"Basket num"[% sep %]"Basket name"[% sep %]"Authorised by"[% sep %]"Biblio number"[% sep %]Title[% sep %]Currency[% sep %]"Vendor price"[% sep %]RRP[% sep %]"Budgeted cost"[% sep %]Quantity[% sep %]"Total RRP"[% sep %]"Total cost"[% sep %]"Entry date"[% sep %]"Date received"[% sep %]"Internal note"[% sep %]"Vendor note" +[%- BLOCK -%]Fund[% sep %]"Basket num"[% sep %]"Basket name"[% sep %]"Authorised by"[% sep %]"Biblio number"[% sep %]Title[% sep %]Currency[% sep %]"Vendor price"[% sep %]RRP[% sep %]"Budgeted cost"[% sep %]Quantity[% sep %]"Total RRP"[% sep %]"Total cost"[% sep %]"Entry date"[% sep %]"Date received"[% sep %]"Internal note"[% sep %]"Vendor note"[%- END -%] diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/empty_line.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/empty_line.inc new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/empty_line.inc @@ -0,0 +1 @@ + diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/basket.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/basket.tt index 6ee81aa2c5..a353e59c0f 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/basket.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/basket.tt @@ -1,4 +1,6 @@ -[% INCLUDE csv_headers/acqui/basket.tt %] +[%- INCLUDE csv_headers/acqui/basket.tt -%] +[% INCLUDE empty_line.inc -%] [%- FOREACH r IN rows -%] "[% r.contractname %]",[% r.ordernumber %],[% r.entrydate %],[% r.isbn %],"[% r.author %]","[% r.title %]",[% r.publicationyear %],"[% r.publishercode %]","[% r.collectiontitle %]","[% r.notes %]",[% r.quantity %],[% r.rrp %],"[% r.deliveryplace %]","[% r.billingplace %]" -[% END -%] +[%- INCLUDE empty_line.inc %] +[%- END -%] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/basketgroup.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/basketgroup.tt index e31419404a..b421ed211f 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/basketgroup.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/basketgroup.tt @@ -1,4 +1,6 @@ -[% USE Price %] -[% INCLUDE csv_headers/acqui/basketgroup.tt %] -[% FOREACH r IN rows %][% r.clientnumber %],"[% r.basketname %]",[% r.ordernumber %],"[% r.author %]","[% r.title %]","[% r.publishercode %]",[% r.publicationyear %],"[% r.collectiontitle %]",[% r.isbn %],[% r.quantity %],[% r.rrp_tax_included | $Price %],[% r.rrp_tax_excluded | $Price %],[% r.discount %],[% r.ecost_tax_included | $Price %],[% r.ecost_tax_excluded | $Price %],"[% r.notes %]",[% r.entrydate %],"[% r.booksellername %]","[% r.bookselleraddress %]","[% r.booksellerpostal %]",[% r.contractnumber %],"[% r.contractname %]","[% r.basketgroupdeliveryplace %]","[% r.basketgroupbillingplace %]","[% r.basketdeliveryplace %]","[% r.basketbillingplace %]" -[% END %] +[%- USE Price -%] +[%- INCLUDE csv_headers/acqui/basketgroup.tt -%] +[%- INCLUDE empty_line.inc -%] +[%- FOREACH r IN rows -%][% r.clientnumber %],"[% r.basketname %]",[% r.ordernumber %],"[% r.author %]","[% r.title %]","[% r.publishercode %]",[% r.publicationyear %],"[% r.collectiontitle %]",[% r.isbn %],[% r.quantity %],[% r.rrp_tax_included | $Price %],[% r.rrp_tax_excluded | $Price %],[% r.discount %],[% r.ecost_tax_included | $Price %],[% r.ecost_tax_excluded | $Price %],"[% r.notes %]",[% r.entrydate %],"[% r.booksellername %]","[% r.bookselleraddress %]","[% r.booksellerpostal %]",[% r.contractnumber %],"[% r.contractname %]","[% r.basketgroupdeliveryplace %]","[% r.basketgroupbillingplace %]","[% r.basketdeliveryplace %]","[% r.basketbillingplace %]" +[%- INCLUDE empty_line.inc -%] +[%- END -%] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/lateorders.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/lateorders.tt index 34b22266f2..2234688610 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/lateorders.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/lateorders.tt @@ -1,6 +1,8 @@ [%- USE KohaDates -%] -[%- INCLUDE csv_headers/acqui/lateorders.tt +%] -[% FOREACH o IN orders %] -"[% o.orderdate %] ([% o.latesince %] days)","[% o.estimateddeliverydate | $KohaDates %]","[% o.supplier (o.supplierid) %]","[% o.title %] [% IF o.author %]Author: [% o.author %].[% END %][% IF o.publisher %]Published by: [% o.publisher %].[% END %]","[% o.unitpricesupplier %] x [% o.quantity_to_receive %] = [% o.subtotal %] ([% o.budget %])","[% o.basketname %] ([% o.basketno %])","[% o.claims_count %]","[% o.claimed_date +%]" -[% END +%] +[%- INCLUDE csv_headers/acqui/lateorders.tt -%] +[%- INCLUDE empty_line.inc -%] +[%- FOREACH o IN orders -%] +"[% o.orderdate %] ([% o.latesince %] days)","[% o.estimateddeliverydate | $KohaDates %]","[% o.supplier (o.supplierid) %]","[% o.title %] [% IF o.author %]Author: [% o.author %].[% END %][% IF o.publisher %]Published by: [% o.publisher %].[% END %]","[% o.unitpricesupplier %] x [% o.quantity_to_receive %] = [% o.subtotal %] ([% o.budget %])","[% o.basketname %] ([% o.basketno %])","[% o.claims_count %]","[% o.claimed_date %]" +[%- INCLUDE empty_line.inc -%] +[%- END -%] ,,Total orders in late, [% orders.size %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/itemsearch_csv.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/itemsearch_csv.tt index 9ac5047641..6b58dc7b02 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/itemsearch_csv.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/itemsearch_csv.tt @@ -1,4 +1,6 @@ -[% INCLUDE csv_headers/catalogue/itemsearch.tt %] -[% FOREACH item IN results -%] - [% INCLUDE 'catalogue/itemsearch_item.csv.inc' item = item +%] +[%- INCLUDE csv_headers/catalogue/itemsearch.tt -%] +[%- INCLUDE empty_line.inc -%] +[%- FOREACH item IN results -%] + [%- INCLUDE 'catalogue/itemsearch_item.csv.inc' item = item -%] + [%- INCLUDE empty_line.inc -%] [%- END -%] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/reports/csv/cash_register_stats.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/reports/csv/cash_register_stats.tt index 923a3240fc..7453b84bb3 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/reports/csv/cash_register_stats.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/reports/csv/cash_register_stats.tt @@ -1,7 +1,9 @@ -[% USE ItemTypes %] -[%- INCLUDE csv_headers/reports/cash_register_stats.tt %] -[%- FOREACH row IN rows # Avoid line breaks in directives because of translatability %] - [%- FOREACH field IN row; field IF !loop.last; ItemTypes.GetDescription(field) IF loop.last; sep IF !loop.last; END %] -[% END -%] -TOTAL -[%- FOREACH field IN total; field; sep IF !loop.last; END %] +[%- USE ItemTypes -%] +[%- INCLUDE csv_headers/reports/cash_register_stats.tt -%] +[%- INCLUDE empty_line.inc -%] +[%- FOREACH row IN rows -%] + [%- FOREACH field IN row; field IF !loop.last; ItemTypes.GetDescription(field) IF loop.last; sep IF !loop.last; END -%] + [%- INCLUDE empty_line.inc %] +[%- END -%] +TOTAL[%- INCLUDE empty_line.inc -%] +[%- FOREACH field IN total; field; sep IF !loop.last; END -%] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/reports/csv/orders_by_budget.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/reports/csv/orders_by_budget.tt index a2b5fd3ef2..6541e28b2a 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/reports/csv/orders_by_budget.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/reports/csv/orders_by_budget.tt @@ -1,6 +1,8 @@ -[% INCLUDE csv_headers/reports/orders_by_budget.tt %] -[%- FOREACH row IN rows # Avoid line breaks in directives because of translatability %] - [%- FOREACH field IN row; field; sep IF !loop.last; END %] -[% END -%] -TOTAL -[%- FOREACH field IN totalrow; field; sep IF !loop.last; END %] +[%- INCLUDE csv/headers/reports/orders_by_budget.tt -%] +[%- INCLUDE empty_line.inc -%] +[%- FOREACH row IN rows -%] + [%- FOREACH field IN row; field; sep IF !loop.last; END -%] + [%- INCLUDE empty_line.inc -%] +[%- END -%] +TOTAL[%- INCLUDE empty_line.inc -%] +[%- FOREACH field IN totalrow; field; sep IF !loop.last; END -%] -- 2.11.0