From 99e45488162febee65316916858db4a65725992e Mon Sep 17 00:00:00 2001
From: Matt Blenkinsop <matt.blenkinsop@ptfs-europe.com>
Date: Tue, 21 May 2024 09:37:52 +0000
Subject: [PATCH] Bug 36831: Add a method to determine the file delimiter

This patch adds a method to determine whether a file is a CSV or TSV file
---
 Koha/BackgroundJob/ImportKBARTFile.pm | 59 +++++++++++++++++++++++++--
 1 file changed, 55 insertions(+), 4 deletions(-)

diff --git a/Koha/BackgroundJob/ImportKBARTFile.pm b/Koha/BackgroundJob/ImportKBARTFile.pm
index d17dcf502f..070ca0837d 100644
--- a/Koha/BackgroundJob/ImportKBARTFile.pm
+++ b/Koha/BackgroundJob/ImportKBARTFile.pm
@@ -218,7 +218,8 @@ sub format_title {
 
 =head3 read_file
 
-Reads a file to provide report headers and lines to be processed
+Reads a file to provide report headers and lines to be processed.
+Automatically detects whether the file is TSV or CSV based on the first line
 
 =cut
 
@@ -226,14 +227,15 @@ sub read_file {
     my ($file) = @_;
 
     my $file_content = defined( $file->{file_content} ) ? decode_base64( $file->{file_content} ) : "";
-    my $delimiter    = $file->{filename} =~ /\.tsv$/    ? "\t"                                   : ",";
-    my $quote_char   = $file->{filename} =~ /\.tsv$/    ? "\""                                   : "\"";
+    my $delimiter    = identify_delimiter($file);
+
+    return ( undef, undef, "unknown_delimiter" ) unless $delimiter;
 
     open my $fh, "<", \$file_content or die;
     my $csv = Text::CSV_XS->new(
         {
             sep_char           => $delimiter,
-            quote_char         => $quote_char,
+            quote_char         => "\"",
             binary             => 1,
             allow_loose_quotes => 1
         }
@@ -251,6 +253,8 @@ sub read_file {
     return ( $column_headers, $lines, $error );
 }
 
+
+
 =head3 create_title_hash_from_line_data
 
 Takes a line and creates a hash of the values mapped to the column headings
@@ -466,4 +470,51 @@ sub rescue_EBSCO_files {
     return $column_headers;
 }
 
+=head3 identify_delimiter
+
+Identifies the delimiter used in the KBART file. Checks first for a TSV file as this is what the KBART standard specifies.
+If the TSV file is not detected then it checks for a CSV file.
+Returns the delimiter required to parse the file correctly
+
+=cut
+
+sub identify_delimiter {
+    my ($file) = @_;
+
+    my $file_content = defined( $file->{file_content} ) ? decode_base64( $file->{file_content} ) : "";
+    open my $tsv_fh, "<", \$file_content or die;
+
+    # Check for TSV
+    my $tsv = Text::CSV_XS->new(
+        {
+            sep_char           => "\t",
+            quote_char         => "\"",
+            binary             => 1,
+            allow_loose_quotes => 1
+        }
+    );
+    my $check_tsv = $tsv->getline($tsv_fh);
+    close($tsv_fh);
+
+    #If the sep_char is wrong then it will read the first row as one string
+    return "\t" if scalar(@$check_tsv) > 1;
+
+    open my $csv_fh, "<", \$file_content or die;
+
+    # Check for CSV
+    my $csv = Text::CSV_XS->new(
+        {
+            sep_char           => ",",
+            quote_char         => "\"",
+            binary             => 1,
+            allow_loose_quotes => 1
+        }
+    );
+    my $check_csv = $csv->getline($csv_fh);
+    close($csv_fh);
+
+    return ',' if scalar(@$check_csv) > 1;
+    return 0;
+}
+
 1;
-- 
2.37.1 (Apple Git-137.1)