Attachment 166433 Details for Bug 34788 – Bug 34788: Add an API endpoint to import a KBART file

[patch] Bug 34788: Add an API endpoint to import a KBART file

Bug-34788-Add-an-API-endpoint-to-import-a-KBART-fi.patch (text/plain), 7.77 KB, created by Nick Clemens (kidclamp) on 2024-05-08 19:36:29 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Nick Clemens (kidclamp)

Created: 2024-05-08 19:36:29 UTC

Size: 7.77 KB

patch

obsolete

>From d9327d0d6d1180acade95243a90aed9b48ccf3bf Mon Sep 17 00:00:00 2001
>From: Matt Blenkinsop <matt.blenkinsop@ptfs-europe.com>
>Date: Thu, 14 Sep 2023 12:34:06 +0000
>Subject: [PATCH] Bug 34788: Add an API endpoint to import a KBART file
>
>This patch adds the endpoint needed to queue an import_from_kbart_file background job
>
>Signed-off-by: Clemens Tubach <clemens.tubach@kit.edu>
>Signed-off-by: Nick Clemens <nick@bywatersolutions.com>
>---
> Koha/REST/V1/ERM/EHoldings/Titles.pm          | 13 ++++
> Koha/REST/V1/ERM/EHoldings/Titles/Local.pm    | 74 +++++++++++++++++++
> .../swagger/paths/erm_eholdings_titles.yaml   | 70 ++++++++++++++++++
> api/v1/swagger/swagger.yaml                   |  2 +
> 4 files changed, 159 insertions(+)
>
>diff --git a/Koha/REST/V1/ERM/EHoldings/Titles.pm b/Koha/REST/V1/ERM/EHoldings/Titles.pm
>index 8340cee7f1c..f28b1c992ff 100644
>--- a/Koha/REST/V1/ERM/EHoldings/Titles.pm
>+++ b/Koha/REST/V1/ERM/EHoldings/Titles.pm
>@@ -122,4 +122,17 @@ sub import_from_list {
> 
>     return Koha::REST::V1::ERM::EHoldings::Titles::Local::import_from_list($c);
> }
>+
>+=head3 import_from_kbart_file
>+
>+Controller function that handles importing a kbart file
>+
>+=cut
>+
>+sub import_from_kbart_file {
>+    my $c = shift->openapi->valid_input or return;
>+
>+    return Koha::REST::V1::ERM::EHoldings::Titles::Local::import_from_kbart_file($c);
>+}
>+
> 1;
>diff --git a/Koha/REST/V1/ERM/EHoldings/Titles/Local.pm b/Koha/REST/V1/ERM/EHoldings/Titles/Local.pm
>index d8d16741be6..3144bc510dc 100644
>--- a/Koha/REST/V1/ERM/EHoldings/Titles/Local.pm
>+++ b/Koha/REST/V1/ERM/EHoldings/Titles/Local.pm
>@@ -21,9 +21,12 @@ use Mojo::Base 'Mojolicious::Controller';
> 
> use Koha::ERM::EHoldings::Titles;
> use Koha::BackgroundJob::CreateEHoldingsFromBiblios;
>+use Koha::BackgroundJob::ImportKBARTFile;
> 
> use Scalar::Util qw( blessed );
> use Try::Tiny qw( catch try );
>+use MIME::Base64 qw( decode_base64 encode_base64 );
>+use POSIX qw( floor );
> 
> =head1 API
> 
>@@ -260,4 +263,75 @@ sub import_from_list {
>     };
> }
> 
>+
>+=head3 import_from_kbart_file
>+
>+=cut
>+
>+sub import_from_kbart_file {
>+    my $c = shift or return;
>+
>+    my $file = $c->req->json;
>+
>+    return try {
>+        my @job_ids;
>+        my @invalid_columns;
>+        my $max_allowed_packet = C4::Context->dbh->selectrow_array(q{SELECT @@max_allowed_packet});
>+        my $file_content       = defined( $file->{file_content} ) ? decode_base64( $file->{file_content} ) : "";
>+        $file_content =~ s/\n/\r/g;
>+        my @lines          = split /\r/, $file_content;
>+        my @column_headers = split /\t/, $lines[0];
>+        shift @lines;    # Remove headers row
>+        my @remove_null_lines = grep $_ ne '', @lines;
>+
>+        # Check that the column headers in the file match the standardised KBART phase II columns
>+        # If not, return a warning before the job is queued
>+        my @valid_headers = Koha::BackgroundJob::ImportKBARTFile::get_valid_headers();
>+        foreach my $header (@column_headers) {
>+            if ( !grep { $_ eq $header } @valid_headers ) {
>+                push @invalid_columns, $header;
>+            }
>+        }
>+        return $c->render(
>+            status  => 201,
>+            openapi => { invalid_columns => \@invalid_columns, valid_columns => \@valid_headers }
>+        ) if scalar(@invalid_columns) > 0;
>+
>+        my $file_size = length($file_content);
>+
>+        # If the file is too large, we can break the file into smaller chunks and enqueue one job per chunk
>+        if ( $file_size > $max_allowed_packet ) {
>+
>+            my $max_number_of_lines = Koha::BackgroundJob::ImportKBARTFile::calculate_chunked_file_size(
>+                $file_size, $max_allowed_packet,
>+                scalar(@remove_null_lines)
>+            );
>+            my @chunked_files;
>+            push @chunked_files, [ splice @remove_null_lines, 0, $max_number_of_lines ] while @remove_null_lines;
>+
>+            foreach my $chunk (@chunked_files) {
>+                unshift( @{$chunk}, join( "\t", @column_headers ) );
>+                my $chunked_file = {
>+                    filename     => $file->{filename},
>+                    file_content => encode_base64( join( "\r", @{$chunk} ) )
>+                };
>+                my $params = { file => $chunked_file };
>+                my $job_id = Koha::BackgroundJob::ImportKBARTFile->new->enqueue($params);
>+                push @job_ids, $job_id;
>+            }
>+        } else {
>+            my $params = { file => $file };
>+            my $job_id = Koha::BackgroundJob::ImportKBARTFile->new->enqueue($params);
>+            push @job_ids, $job_id;
>+        }
>+
>+        return $c->render(
>+            status  => 201,
>+            openapi => { job_ids => \@job_ids }
>+        );
>+    } catch {
>+        $c->unhandled_exception($_);
>+    };
>+}
>+
> 1;
>diff --git a/api/v1/swagger/paths/erm_eholdings_titles.yaml b/api/v1/swagger/paths/erm_eholdings_titles.yaml
>index d86e2b39143..a72ef34bd04 100644
>--- a/api/v1/swagger/paths/erm_eholdings_titles.yaml
>+++ b/api/v1/swagger/paths/erm_eholdings_titles.yaml
>@@ -493,3 +493,73 @@
>     x-koha-authorization:
>       permissions:
>         erm: 1
>+/erm/eholdings/local/titles/import_kbart:
>+  post:
>+    x-mojo-to: ERM::EHoldings::Titles#import_from_kbart_file
>+    operationId: importErmEHoldingsTitlesFromKbart
>+    tags:
>+      - erm_eholdings_titles
>+    summary: Import local titles from a KBART file
>+    consumes:
>+      - application/json
>+    produces:
>+      - application/json
>+    parameters:
>+      - description: The file to import
>+        in: body
>+        name: body
>+        required: true
>+        schema:
>+          type: object
>+          properties:
>+            filename:
>+              type: string
>+            file_content:
>+              type: string
>+          additionalProperties: false
>+    responses:
>+      201:
>+        description: Successfully enqueued the import job
>+        schema:
>+          type: object
>+          properties:
>+            job_ids:
>+              type: array
>+            invalid_columns:
>+              type: array
>+            valid_columns:
>+              type: array
>+          additionalProperties: false
>+      400:
>+        description: Bad parameter
>+        schema:
>+          $ref: "../swagger.yaml#/definitions/error"
>+      401:
>+        description: Authentication required
>+        schema:
>+          $ref: "../swagger.yaml#/definitions/error"
>+      403:
>+        description: Access forbidden
>+        schema:
>+          $ref: "../swagger.yaml#/definitions/error"
>+      404:
>+        description: Ressource not found
>+        schema:
>+          $ref: "../swagger.yaml#/definitions/error"
>+      409:
>+        description: Conflict in creating resource
>+        schema:
>+          $ref: "../swagger.yaml#/definitions/error"
>+      500:
>+        description: |-
>+          Internal server error. Possible `error_code` attribute values:
>+          * `internal_server_error`
>+        schema:
>+          $ref: "../swagger.yaml#/definitions/error"
>+      503:
>+        description: Under maintenance
>+        schema:
>+          $ref: "../swagger.yaml#/definitions/error"
>+    x-koha-authorization:
>+      permissions:
>+        erm: 1
>diff --git a/api/v1/swagger/swagger.yaml b/api/v1/swagger/swagger.yaml
>index 75517313c25..7352b1c2401 100644
>--- a/api/v1/swagger/swagger.yaml
>+++ b/api/v1/swagger/swagger.yaml
>@@ -321,6 +321,8 @@ paths:
>     $ref: "./paths/erm_eholdings_titles.yaml#/~1erm~1eholdings~1{provider}~1titles"
>   /erm/eholdings/local/titles/import:
>     $ref: ./paths/erm_eholdings_titles.yaml#/~1erm~1eholdings~1local~1titles~1import
>+  /erm/eholdings/local/titles/import_kbart:
>+    $ref: ./paths/erm_eholdings_titles.yaml#/~1erm~1eholdings~1local~1titles~1import_kbart
>   "/erm/eholdings/{provider}/titles/{title_id}":
>     $ref: "./paths/erm_eholdings_titles.yaml#/~1erm~1eholdings~1{provider}~1titles~1{title_id}"
>   "/erm/eholdings/{provider}/titles/{title_id}/resources":
>-- 
>2.39.2

Actions: View | Diff | Splinter Review

Attachments on bug 34788: 161379 | 161380 | 161381 | 161382 | 161383 | 161384 | 161391 | 165189 | 165190 | 165191 | 165192 | 165193 | 165194 | 165195 | 165196 | 165197 | 165837 | 165887 | 165888 | 165889 | 165890 | 165891 | 165892 | 165893 | 165894 | 166420 | 166433 | 166434 | 166435 | 166436 | 166437 | 166438 | 166439 | 166440 | 166441 | 166571 | 166645