Attachment 166989 Details for Bug 36831 – Bug 36831: Add a method to determine the file delimiter

[patch] Bug 36831: Add a method to determine the file delimiter

Bug-36831-Add-a-method-to-determine-the-file-delim.patch (text/plain), 3.38 KB, created by David Nind on 2024-05-21 22:06:57 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: David Nind

Created: 2024-05-21 22:06:57 UTC

Size: 3.38 KB

patch

obsolete

>From a481499326275a816f62af2f310ae5651dd3f9ba Mon Sep 17 00:00:00 2001
>From: Matt Blenkinsop <matt.blenkinsop@ptfs-europe.com>
>Date: Tue, 21 May 2024 09:37:52 +0000
>Subject: [PATCH] Bug 36831: Add a method to determine the file delimiter
>
>This patch adds a method to determine whether a file is a CSV or TSV file
>
>Signed-off-by: David Nind <david@davidnind.com>
>---
> Koha/BackgroundJob/ImportKBARTFile.pm | 59 +++++++++++++++++++++++++--
> 1 file changed, 55 insertions(+), 4 deletions(-)
>
>diff --git a/Koha/BackgroundJob/ImportKBARTFile.pm b/Koha/BackgroundJob/ImportKBARTFile.pm
>index d17dcf502f..070ca0837d 100644
>--- a/Koha/BackgroundJob/ImportKBARTFile.pm
>+++ b/Koha/BackgroundJob/ImportKBARTFile.pm
>@@ -218,7 +218,8 @@ sub format_title {
> 
> =head3 read_file
> 
>-Reads a file to provide report headers and lines to be processed
>+Reads a file to provide report headers and lines to be processed.
>+Automatically detects whether the file is TSV or CSV based on the first line
> 
> =cut
> 
>@@ -226,14 +227,15 @@ sub read_file {
>     my ($file) = @_;
> 
>     my $file_content = defined( $file->{file_content} ) ? decode_base64( $file->{file_content} ) : "";
>-    my $delimiter    = $file->{filename} =~ /\.tsv$/    ? "\t"                                   : ",";
>-    my $quote_char   = $file->{filename} =~ /\.tsv$/    ? "\""                                   : "\"";
>+    my $delimiter    = identify_delimiter($file);
>+
>+    return ( undef, undef, "unknown_delimiter" ) unless $delimiter;
> 
>     open my $fh, "<", \$file_content or die;
>     my $csv = Text::CSV_XS->new(
>         {
>             sep_char           => $delimiter,
>-            quote_char         => $quote_char,
>+            quote_char         => "\"",
>             binary             => 1,
>             allow_loose_quotes => 1
>         }
>@@ -251,6 +253,8 @@ sub read_file {
>     return ( $column_headers, $lines, $error );
> }
> 
>+
>+
> =head3 create_title_hash_from_line_data
> 
> Takes a line and creates a hash of the values mapped to the column headings
>@@ -466,4 +470,51 @@ sub rescue_EBSCO_files {
>     return $column_headers;
> }
> 
>+=head3 identify_delimiter
>+
>+Identifies the delimiter used in the KBART file. Checks first for a TSV file as this is what the KBART standard specifies.
>+If the TSV file is not detected then it checks for a CSV file.
>+Returns the delimiter required to parse the file correctly
>+
>+=cut
>+
>+sub identify_delimiter {
>+    my ($file) = @_;
>+
>+    my $file_content = defined( $file->{file_content} ) ? decode_base64( $file->{file_content} ) : "";
>+    open my $tsv_fh, "<", \$file_content or die;
>+
>+    # Check for TSV
>+    my $tsv = Text::CSV_XS->new(
>+        {
>+            sep_char           => "\t",
>+            quote_char         => "\"",
>+            binary             => 1,
>+            allow_loose_quotes => 1
>+        }
>+    );
>+    my $check_tsv = $tsv->getline($tsv_fh);
>+    close($tsv_fh);
>+
>+    #If the sep_char is wrong then it will read the first row as one string
>+    return "\t" if scalar(@$check_tsv) > 1;
>+
>+    open my $csv_fh, "<", \$file_content or die;
>+
>+    # Check for CSV
>+    my $csv = Text::CSV_XS->new(
>+        {
>+            sep_char           => ",",
>+            quote_char         => "\"",
>+            binary             => 1,
>+            allow_loose_quotes => 1
>+        }
>+    );
>+    my $check_csv = $csv->getline($csv_fh);
>+    close($csv_fh);
>+
>+    return ',' if scalar(@$check_csv) > 1;
>+    return 0;
>+}
>+
> 1;
>-- 
>2.39.2

Actions: View | Diff | Splinter Review

Attachments on bug 36831: 166958 | 166959 | 166960 | 166961 | 166989 | 166990 | 166991 | 168431 | 168432 | 168433 | 168434 | 170549 | 170550 | 170551 | 170552 | 170562 | 170563 | 170564 | 170565