From 8449b8b1fee9d8923deb04c849c5c7fe17e78a32 Mon Sep 17 00:00:00 2001 From: Jonathan Druart Date: Mon, 12 Jun 2017 17:04:47 -0300 Subject: [PATCH] Bug 18331: Fix CSV export (once and for all!) Content-Type: text/plain; charset=utf-8 I am going to try and explain the problem here in order to make it a have-to-read for next changes. There are several things to test when and most of the time we break something when we try to fix something else. You have to test different CSV export, not only one. There are not all processed the same way. For instance acqui/csv/basket.tt does not contain any strings to translate and the translate script will copy the original file as it. But reports/csv/orders_by_budget.tt will not! Indeed it contains "TOTAL" which will be translated. The generated tt file for the translated language will remove all the carriage returns! That's why we cannot rely on TT for newlines (contrary to bug 16914 assumption). There are two possible methods: 1/ Fix the translate script => Hum... nope 2/ Remove all carriage returns and make them explicit by including an file that only contains 1 carriage return This second method is implemented in this patch. How it works: Use the PRE_CHOMP "[%-" and POST_CHOMP "-%]" (http://www.template-toolkit.org/docs/manual/Config.html#section_PRE_CHOMP_POST_CHOMP) to remove all the newlines that could be added by TT in the original (en) files. Then include the new_line.inc to add a new line. That way original and translated files will behave the same way. The BLOCK in the csv_headers avoid to have the newlines added, in any cases. For instance: by default we will have an empty line at the end of the headers, but the translated headers will not have it. Test plan: At least 2 signoffs will be needed, please test carefully! You will need to generate CSV with at least 2 entries! Test with the non-translated interface (en) and with the language you want. To test with a translated language you will *have to*: - cd misc/translator - perl translate update LANG - manually edit the po file, remove the fuzzy tags, correctly translate the string (do not forget the additional %s in the headers) - perl translate install LANG 1/ Export basket and basketgroup as CSV 2/ Export late orders as CSV 3/ Export items (from the item search) as CSV 4/ Export the 2 reports "Orders by fund" and "Cash register" as CSV Signed-off-by: Lee Jamison Signed-off-by: Josef Moravec Signed-off-by: Katrin Fischer Signed-off-by: Marcel de Rooy --- .../prog/en/includes/csv_headers/acqui/basket.tt | 2 +- .../prog/en/includes/csv_headers/acqui/basketgroup.tt | 2 +- .../prog/en/includes/csv_headers/acqui/lateorders.tt | 2 +- .../prog/en/includes/csv_headers/catalogue/itemsearch.tt | 2 +- .../includes/csv_headers/reports/cash_register_stats.tt | 2 +- .../en/includes/csv_headers/reports/orders_by_budget.tt | 2 +- koha-tmpl/intranet-tmpl/prog/en/includes/empty_line.inc | 0 .../intranet-tmpl/prog/en/modules/acqui/csv/basket.tt | 6 ++++-- .../prog/en/modules/acqui/csv/basketgroup.tt | 10 ++++++---- .../prog/en/modules/acqui/csv/lateorders.tt | 10 ++++++---- .../prog/en/modules/catalogue/itemsearch_csv.tt | 8 +++++--- .../prog/en/modules/reports/csv/cash_register_stats.tt | 16 +++++++++------- .../prog/en/modules/reports/csv/orders_by_budget.tt | 14 ++++++++------ 13 files changed, 44 insertions(+), 32 deletions(-) create mode 100644 koha-tmpl/intranet-tmpl/prog/en/includes/empty_line.inc diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/basket.tt b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/basket.tt index a2086f6..60740d2 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/basket.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/basket.tt @@ -1 +1 @@ -Contract name,Order number,Entry date,ISBN,Author,Title,Publication year,Publisher,Collection title,Note for vendor,Quantity,RRP,Delivery place,Billing place +[%- BLOCK -%]Contract name,Order number,Entry date,ISBN,Author,Title,Publication year,Publisher,Collection title,Note for vendor,Quantity,RRP,Delivery place,Billing place[%- END -%] diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/basketgroup.tt b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/basketgroup.tt index d28d8f8..2f583b9 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/basketgroup.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/basketgroup.tt @@ -1 +1 @@ -Account number,Basket name,Order number,Author,Title,Publisher,Publication year,Collection title,ISBN,Quantity,RRP tax included, RRP tax excluded,Discount,Estimated cost tax included,Estimated cost tax excluded,Note for vendor,Entry date,Bookseller name,Bookseller physical address,Bookseller postal address,Contract number,Contract name,Basket group delivery place,Basket group billing place,Basket delivery place,Basket billing place +[%- BLOCK -%]Account number,Basket name,Order number,Author,Title,Publisher,Publication year,Collection title,ISBN,Quantity,RRP tax included, RRP tax excluded,Discount,Estimated cost tax included,Estimated cost tax excluded,Note for vendor,Entry date,Bookseller name,Bookseller physical address,Bookseller postal address,Contract number,Contract name,Basket group delivery place,Basket group billing place,Basket delivery place,Basket billing place[%- END -%] diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/lateorders.tt b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/lateorders.tt index a235269..a853d5f 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/lateorders.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/acqui/lateorders.tt @@ -1 +1 @@ -ORDER DATE,ESTIMATED DELIVERY DATE,VENDOR,INFORMATION,TOTAL COST,BASKET,CLAIMS COUNT,CLAIMED DATE +[%- BLOCK -%]ORDER DATE,ESTIMATED DELIVERY DATE,VENDOR,INFORMATION,TOTAL COST,BASKET,CLAIMS COUNT,CLAIMED DATE[%- END -%] diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/catalogue/itemsearch.tt b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/catalogue/itemsearch.tt index bac4b14..8add507 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/catalogue/itemsearch.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/catalogue/itemsearch.tt @@ -1 +1 @@ -Title, Publication date, Publisher, Collection, Barcode, Call number, Home library, Current location, Shelving location, Inventory number, Status, Checkouts +[%- BLOCK -%]Title, Publication date, Publisher, Collection, Barcode, Call number, Home library, Current location, Shelving location, Inventory number, Status, Checkouts[%- END -%] diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/reports/cash_register_stats.tt b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/reports/cash_register_stats.tt index 0746a69..11ec56b 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/reports/cash_register_stats.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/reports/cash_register_stats.tt @@ -1 +1 @@ -mfirstname[% sep %]card number[% sep %]firstname[% sep %]branchname[% sep %]date[% sep %]accounttype[% sep %]amount[% sep %]title[% sep %]barcode[% sep %]itype +[%- BLOCK -%]mfirstname[% sep %]card number[% sep %]firstname[% sep %]branchname[% sep %]date[% sep %]accounttype[% sep %]amount[% sep %]title[% sep %]barcode[% sep %]itype[%- END -%] diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/reports/orders_by_budget.tt b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/reports/orders_by_budget.tt index 2acc795..2494d4e 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/reports/orders_by_budget.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/csv_headers/reports/orders_by_budget.tt @@ -1 +1 @@ -Fund[% sep %]"Basket num"[% sep %]"Basket name"[% sep %]"Authorised by"[% sep %]"Biblio number"[% sep %]Title[% sep %]Currency[% sep %]"Vendor price"[% sep %]RRP[% sep %]"Budgeted cost"[% sep %]Quantity[% sep %]"Total RRP"[% sep %]"Total cost"[% sep %]"Entry date"[% sep %]"Date received"[% sep %]"Internal note"[% sep %]"Vendor note" +[%- BLOCK -%]Fund[% sep %]"Basket num"[% sep %]"Basket name"[% sep %]"Authorised by"[% sep %]"Biblio number"[% sep %]Title[% sep %]Currency[% sep %]"Vendor price"[% sep %]RRP[% sep %]"Budgeted cost"[% sep %]Quantity[% sep %]"Total RRP"[% sep %]"Total cost"[% sep %]"Entry date"[% sep %]"Date received"[% sep %]"Internal note"[% sep %]"Vendor note"[%- END -%] diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/empty_line.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/empty_line.inc new file mode 100644 index 0000000..e69de29 diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/basket.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/basket.tt index 6ee81aa..a353e59 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/basket.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/basket.tt @@ -1,4 +1,6 @@ -[% INCLUDE csv_headers/acqui/basket.tt %] +[%- INCLUDE csv_headers/acqui/basket.tt -%] +[% INCLUDE empty_line.inc -%] [%- FOREACH r IN rows -%] "[% r.contractname %]",[% r.ordernumber %],[% r.entrydate %],[% r.isbn %],"[% r.author %]","[% r.title %]",[% r.publicationyear %],"[% r.publishercode %]","[% r.collectiontitle %]","[% r.notes %]",[% r.quantity %],[% r.rrp %],"[% r.deliveryplace %]","[% r.billingplace %]" -[% END -%] +[%- INCLUDE empty_line.inc %] +[%- END -%] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/basketgroup.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/basketgroup.tt index e314194..b421ed2 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/basketgroup.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/basketgroup.tt @@ -1,4 +1,6 @@ -[% USE Price %] -[% INCLUDE csv_headers/acqui/basketgroup.tt %] -[% FOREACH r IN rows %][% r.clientnumber %],"[% r.basketname %]",[% r.ordernumber %],"[% r.author %]","[% r.title %]","[% r.publishercode %]",[% r.publicationyear %],"[% r.collectiontitle %]",[% r.isbn %],[% r.quantity %],[% r.rrp_tax_included | $Price %],[% r.rrp_tax_excluded | $Price %],[% r.discount %],[% r.ecost_tax_included | $Price %],[% r.ecost_tax_excluded | $Price %],"[% r.notes %]",[% r.entrydate %],"[% r.booksellername %]","[% r.bookselleraddress %]","[% r.booksellerpostal %]",[% r.contractnumber %],"[% r.contractname %]","[% r.basketgroupdeliveryplace %]","[% r.basketgroupbillingplace %]","[% r.basketdeliveryplace %]","[% r.basketbillingplace %]" -[% END %] +[%- USE Price -%] +[%- INCLUDE csv_headers/acqui/basketgroup.tt -%] +[%- INCLUDE empty_line.inc -%] +[%- FOREACH r IN rows -%][% r.clientnumber %],"[% r.basketname %]",[% r.ordernumber %],"[% r.author %]","[% r.title %]","[% r.publishercode %]",[% r.publicationyear %],"[% r.collectiontitle %]",[% r.isbn %],[% r.quantity %],[% r.rrp_tax_included | $Price %],[% r.rrp_tax_excluded | $Price %],[% r.discount %],[% r.ecost_tax_included | $Price %],[% r.ecost_tax_excluded | $Price %],"[% r.notes %]",[% r.entrydate %],"[% r.booksellername %]","[% r.bookselleraddress %]","[% r.booksellerpostal %]",[% r.contractnumber %],"[% r.contractname %]","[% r.basketgroupdeliveryplace %]","[% r.basketgroupbillingplace %]","[% r.basketdeliveryplace %]","[% r.basketbillingplace %]" +[%- INCLUDE empty_line.inc -%] +[%- END -%] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/lateorders.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/lateorders.tt index 34b2226..2234688 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/lateorders.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/acqui/csv/lateorders.tt @@ -1,6 +1,8 @@ [%- USE KohaDates -%] -[%- INCLUDE csv_headers/acqui/lateorders.tt +%] -[% FOREACH o IN orders %] -"[% o.orderdate %] ([% o.latesince %] days)","[% o.estimateddeliverydate | $KohaDates %]","[% o.supplier (o.supplierid) %]","[% o.title %] [% IF o.author %]Author: [% o.author %].[% END %][% IF o.publisher %]Published by: [% o.publisher %].[% END %]","[% o.unitpricesupplier %] x [% o.quantity_to_receive %] = [% o.subtotal %] ([% o.budget %])","[% o.basketname %] ([% o.basketno %])","[% o.claims_count %]","[% o.claimed_date +%]" -[% END +%] +[%- INCLUDE csv_headers/acqui/lateorders.tt -%] +[%- INCLUDE empty_line.inc -%] +[%- FOREACH o IN orders -%] +"[% o.orderdate %] ([% o.latesince %] days)","[% o.estimateddeliverydate | $KohaDates %]","[% o.supplier (o.supplierid) %]","[% o.title %] [% IF o.author %]Author: [% o.author %].[% END %][% IF o.publisher %]Published by: [% o.publisher %].[% END %]","[% o.unitpricesupplier %] x [% o.quantity_to_receive %] = [% o.subtotal %] ([% o.budget %])","[% o.basketname %] ([% o.basketno %])","[% o.claims_count %]","[% o.claimed_date %]" +[%- INCLUDE empty_line.inc -%] +[%- END -%] ,,Total orders in late, [% orders.size %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/itemsearch_csv.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/itemsearch_csv.tt index 9ac5047..6b58dc7 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/itemsearch_csv.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/catalogue/itemsearch_csv.tt @@ -1,4 +1,6 @@ -[% INCLUDE csv_headers/catalogue/itemsearch.tt %] -[% FOREACH item IN results -%] - [% INCLUDE 'catalogue/itemsearch_item.csv.inc' item = item +%] +[%- INCLUDE csv_headers/catalogue/itemsearch.tt -%] +[%- INCLUDE empty_line.inc -%] +[%- FOREACH item IN results -%] + [%- INCLUDE 'catalogue/itemsearch_item.csv.inc' item = item -%] + [%- INCLUDE empty_line.inc -%] [%- END -%] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/reports/csv/cash_register_stats.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/reports/csv/cash_register_stats.tt index 923a324..7453b84 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/reports/csv/cash_register_stats.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/reports/csv/cash_register_stats.tt @@ -1,7 +1,9 @@ -[% USE ItemTypes %] -[%- INCLUDE csv_headers/reports/cash_register_stats.tt %] -[%- FOREACH row IN rows # Avoid line breaks in directives because of translatability %] - [%- FOREACH field IN row; field IF !loop.last; ItemTypes.GetDescription(field) IF loop.last; sep IF !loop.last; END %] -[% END -%] -TOTAL -[%- FOREACH field IN total; field; sep IF !loop.last; END %] +[%- USE ItemTypes -%] +[%- INCLUDE csv_headers/reports/cash_register_stats.tt -%] +[%- INCLUDE empty_line.inc -%] +[%- FOREACH row IN rows -%] + [%- FOREACH field IN row; field IF !loop.last; ItemTypes.GetDescription(field) IF loop.last; sep IF !loop.last; END -%] + [%- INCLUDE empty_line.inc %] +[%- END -%] +TOTAL[%- INCLUDE empty_line.inc -%] +[%- FOREACH field IN total; field; sep IF !loop.last; END -%] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/reports/csv/orders_by_budget.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/reports/csv/orders_by_budget.tt index a2b5fd3..6541e28 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/reports/csv/orders_by_budget.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/reports/csv/orders_by_budget.tt @@ -1,6 +1,8 @@ -[% INCLUDE csv_headers/reports/orders_by_budget.tt %] -[%- FOREACH row IN rows # Avoid line breaks in directives because of translatability %] - [%- FOREACH field IN row; field; sep IF !loop.last; END %] -[% END -%] -TOTAL -[%- FOREACH field IN totalrow; field; sep IF !loop.last; END %] +[%- INCLUDE csv/headers/reports/orders_by_budget.tt -%] +[%- INCLUDE empty_line.inc -%] +[%- FOREACH row IN rows -%] + [%- FOREACH field IN row; field; sep IF !loop.last; END -%] + [%- INCLUDE empty_line.inc -%] +[%- END -%] +TOTAL[%- INCLUDE empty_line.inc -%] +[%- FOREACH field IN totalrow; field; sep IF !loop.last; END -%] -- 2.1.4