From bda97eaa340c238ebec960acfd7da2d0e2e8dc37 Mon Sep 17 00:00:00 2001 From: Matt Blenkinsop Date: Thu, 14 Sep 2023 12:34:06 +0000 Subject: [PATCH] Bug 34788: Add an API endpoint to import a KBART file This patch adds the endpoint needed to queue an import_from_kbart_file background job --- Koha/REST/V1/ERM/EHoldings/Titles.pm | 13 ++++ Koha/REST/V1/ERM/EHoldings/Titles/Local.pm | 74 +++++++++++++++++++ .../swagger/paths/erm_eholdings_titles.yaml | 70 ++++++++++++++++++ api/v1/swagger/swagger.yaml | 2 + 4 files changed, 159 insertions(+) diff --git a/Koha/REST/V1/ERM/EHoldings/Titles.pm b/Koha/REST/V1/ERM/EHoldings/Titles.pm index 8340cee7f1c..f28b1c992ff 100644 --- a/Koha/REST/V1/ERM/EHoldings/Titles.pm +++ b/Koha/REST/V1/ERM/EHoldings/Titles.pm @@ -122,4 +122,17 @@ sub import_from_list { return Koha::REST::V1::ERM::EHoldings::Titles::Local::import_from_list($c); } + +=head3 import_from_kbart_file + +Controller function that handles importing a kbart file + +=cut + +sub import_from_kbart_file { + my $c = shift->openapi->valid_input or return; + + return Koha::REST::V1::ERM::EHoldings::Titles::Local::import_from_kbart_file($c); +} + 1; diff --git a/Koha/REST/V1/ERM/EHoldings/Titles/Local.pm b/Koha/REST/V1/ERM/EHoldings/Titles/Local.pm index 0aab8af8659..d135f0ac330 100644 --- a/Koha/REST/V1/ERM/EHoldings/Titles/Local.pm +++ b/Koha/REST/V1/ERM/EHoldings/Titles/Local.pm @@ -21,9 +21,12 @@ use Mojo::Base 'Mojolicious::Controller'; use Koha::ERM::EHoldings::Titles; use Koha::BackgroundJob::CreateEHoldingsFromBiblios; +use Koha::BackgroundJob::ImportKBARTFile; use Scalar::Util qw( blessed ); use Try::Tiny qw( catch try ); +use MIME::Base64 qw( decode_base64 encode_base64 ); +use POSIX qw( floor ); =head1 API @@ -275,4 +278,75 @@ sub import_from_list { }; } + +=head3 import_from_kbart_file + +=cut + +sub import_from_kbart_file { + my $c = shift or return; + + my $file = $c->req->json; + + return try { + my @job_ids; + my @invalid_columns; + my $max_allowed_packet = C4::Context->dbh->selectrow_array(q{SELECT @@max_allowed_packet}); + my $file_content = defined( $file->{file_content} ) ? decode_base64( $file->{file_content} ) : ""; + $file_content =~ s/\n/\r/g; + my @lines = split /\r/, $file_content; + my @column_headers = split /\t/, $lines[0]; + shift @lines; # Remove headers row + my @remove_null_lines = grep $_ ne '', @lines; + + # Check that the column headers in the file match the standardised KBART phase II columns + # If not, return a warning before the job is queued + my @valid_headers = Koha::BackgroundJob::ImportKBARTFile::get_valid_headers(); + foreach my $header (@column_headers) { + if ( !grep { $_ eq $header } @valid_headers ) { + push @invalid_columns, $header; + } + } + return $c->render( + status => 201, + openapi => { invalid_columns => \@invalid_columns, valid_columns => \@valid_headers } + ) if scalar(@invalid_columns) > 0; + + my $file_size = length($file_content); + + # If the file is too large, we can break the file into smaller chunks and enqueue one job per chunk + if ( $file_size > $max_allowed_packet ) { + + my $max_number_of_lines = Koha::BackgroundJob::ImportKBARTFile::calculate_chunked_file_size( + $file_size, $max_allowed_packet, + scalar(@remove_null_lines) + ); + my @chunked_files; + push @chunked_files, [ splice @remove_null_lines, 0, $max_number_of_lines ] while @remove_null_lines; + + foreach my $chunk (@chunked_files) { + unshift( @{$chunk}, join( "\t", @column_headers ) ); + my $chunked_file = { + filename => $file->{filename}, + file_content => encode_base64( join( "\r", @{$chunk} ) ) + }; + my $params = { file => $chunked_file }; + my $job_id = Koha::BackgroundJob::ImportKBARTFile->new->enqueue($params); + push @job_ids, $job_id; + } + } else { + my $params = { file => $file }; + my $job_id = Koha::BackgroundJob::ImportKBARTFile->new->enqueue($params); + push @job_ids, $job_id; + } + + return $c->render( + status => 201, + openapi => { job_ids => \@job_ids } + ); + } catch { + $c->unhandled_exception($_); + }; +} + 1; diff --git a/api/v1/swagger/paths/erm_eholdings_titles.yaml b/api/v1/swagger/paths/erm_eholdings_titles.yaml index d86e2b39143..a72ef34bd04 100644 --- a/api/v1/swagger/paths/erm_eholdings_titles.yaml +++ b/api/v1/swagger/paths/erm_eholdings_titles.yaml @@ -493,3 +493,73 @@ x-koha-authorization: permissions: erm: 1 +/erm/eholdings/local/titles/import_kbart: + post: + x-mojo-to: ERM::EHoldings::Titles#import_from_kbart_file + operationId: importErmEHoldingsTitlesFromKbart + tags: + - erm_eholdings_titles + summary: Import local titles from a KBART file + consumes: + - application/json + produces: + - application/json + parameters: + - description: The file to import + in: body + name: body + required: true + schema: + type: object + properties: + filename: + type: string + file_content: + type: string + additionalProperties: false + responses: + 201: + description: Successfully enqueued the import job + schema: + type: object + properties: + job_ids: + type: array + invalid_columns: + type: array + valid_columns: + type: array + additionalProperties: false + 400: + description: Bad parameter + schema: + $ref: "../swagger.yaml#/definitions/error" + 401: + description: Authentication required + schema: + $ref: "../swagger.yaml#/definitions/error" + 403: + description: Access forbidden + schema: + $ref: "../swagger.yaml#/definitions/error" + 404: + description: Ressource not found + schema: + $ref: "../swagger.yaml#/definitions/error" + 409: + description: Conflict in creating resource + schema: + $ref: "../swagger.yaml#/definitions/error" + 500: + description: |- + Internal server error. Possible `error_code` attribute values: + * `internal_server_error` + schema: + $ref: "../swagger.yaml#/definitions/error" + 503: + description: Under maintenance + schema: + $ref: "../swagger.yaml#/definitions/error" + x-koha-authorization: + permissions: + erm: 1 diff --git a/api/v1/swagger/swagger.yaml b/api/v1/swagger/swagger.yaml index 227921d7957..d2c2d0bf062 100644 --- a/api/v1/swagger/swagger.yaml +++ b/api/v1/swagger/swagger.yaml @@ -301,6 +301,8 @@ paths: $ref: "./paths/erm_eholdings_titles.yaml#/~1erm~1eholdings~1{provider}~1titles" /erm/eholdings/local/titles/import: $ref: ./paths/erm_eholdings_titles.yaml#/~1erm~1eholdings~1local~1titles~1import + /erm/eholdings/local/titles/import_kbart: + $ref: ./paths/erm_eholdings_titles.yaml#/~1erm~1eholdings~1local~1titles~1import_kbart "/erm/eholdings/{provider}/titles/{title_id}": $ref: "./paths/erm_eholdings_titles.yaml#/~1erm~1eholdings~1{provider}~1titles~1{title_id}" "/erm/eholdings/{provider}/titles/{title_id}/resources": -- 2.37.1 (Apple Git-137.1)