From d5afb802b906251a613e428d6d75e20801c028e4 Mon Sep 17 00:00:00 2001 From: Lari Taskula Date: Fri, 19 Feb 2021 11:52:48 +0000 Subject: [PATCH] Bug 20028: Add Patron Export REST controller This patch adds a REST controller for patron data exporting. GET /api/v1/patrons/51/export GET /api/v1/public/patrons/51/export Supports pagination parameters "_per_page" and "_page". Response is an array that contains "_per_page" amount of objects, each with the following format: { "data": {the object, e.g. patron} "type": DBIx source name, e.g. "Borrower" } Example response: [ { "data":{ "address":null, ..., "patron_id":1, ... }, "type":"Borrower" }, { "data":{ "borrower_message_preference_id":24, "borrowernumber":1, ..., "wants_digest":1 }, "type":"BorrowerMessagePreference" }, { "data":{ "auto_renew":false, ..., "checkout_id":97, "due_date":"2021-02-19T23:59:00+00:00", ..., "patron_id":1, ... }, "type":"Issue" }, { "data":{ "auto_renew":false, ..., "checkout_id":42, "due_date":"2021-01-19T23:59:00+00:00", ..., "patron_id":1, ... }, "type":"OldIssue" } ] To test: 1. prove t/db_dependent/api/v1/patrons_export.t Sponsored-by: The National Library of Finland --- Koha/REST/V1/Patrons/Export.pm | 225 ++++++++++++++++++ t/db_dependent/api/v1/patrons_export.t | 309 +++++++++++++++++++++++++ 2 files changed, 534 insertions(+) create mode 100644 Koha/REST/V1/Patrons/Export.pm create mode 100755 t/db_dependent/api/v1/patrons_export.t diff --git a/Koha/REST/V1/Patrons/Export.pm b/Koha/REST/V1/Patrons/Export.pm new file mode 100644 index 0000000000..618071fea4 --- /dev/null +++ b/Koha/REST/V1/Patrons/Export.pm @@ -0,0 +1,225 @@ +package Koha::REST::V1::Patrons::Export; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; + +use Mojo::Base 'Mojolicious::Controller'; + +use Koha::Patrons; + +use Scalar::Util qw(blessed); +use Try::Tiny; + +=head1 NAME + +Koha::REST::V1::Patrons::Export + +=head1 API + +=head2 Methods + +=head3 get + +Controller method that gets patron's related data, permission driven + +=cut + +sub get { + + my $c = shift->openapi->valid_input or return; + + my $patron = Koha::Patrons->find( $c->validation->param('patron_id') ); + my $body = $c->validation->param('body'); + + return try { + + unless ($patron) { + return $c->render( status => 404, openapi => { + error => "Patron not found." + } ); + } + + unless ( C4::Context->preference('AllowGDPRPatronExport') ) { + return $c->render( + status => 403, + openapi => { error => "Configuration prevents patron data export" } + ); + } + + my $args = $c->validation->output; + my ( $filtered_params, $reserved_params, $path_params ) = $c->extract_reserved_params($args); + + # If no pagination parameters are passed, default + $reserved_params->{_per_page} //= C4::Context->preference('RESTdefaultPageSize'); + $reserved_params->{_page} //= 1; + + my $export = $patron->export; + my ( $export_formatted, $total, $base_total ) = _format_export( + $export, $reserved_params->{_per_page}, $reserved_params->{_page} + ); + + $c->add_pagination_headers( + { + total => $total, + base_total => $base_total, + params => $args, + } + ); + + return $c->render( status => 200, openapi => $export_formatted ); + } + catch { + $c->unhandled_exception($_); + }; +} + +=head3 get_public + +Controller method that gets patron's related data, for unprivileged users + +=cut + +sub get_public { + + my $c = shift->openapi->valid_input or return; + + my $body = $c->validation->param('body'); + my $patron_id = $c->validation->param('patron_id'); + + return try { + + unless ( C4::Context->preference('AllowGDPRPatronExport') ) { + return $c->render( + status => 403, + openapi => { error => "Configuration prevents patron data export" } + ); + } + + my $patron = $c->stash('koha.user'); + + unless ( $patron->borrowernumber == $patron_id ) { + return $c->render( + status => 403, + openapi => { + error => "Accessing other patron's data is forbidden" + } + ); + } + + my $args = $c->validation->output; + my ( $filtered_params, $reserved_params, $path_params ) = $c->extract_reserved_params($args); + + # If no pagination parameters are passed, default + $reserved_params->{_per_page} //= C4::Context->preference('RESTdefaultPageSize'); + $reserved_params->{_page} //= 1; + + my $export = $patron->export; + my ( $export_formatted, $total, $base_total ) = _format_export( + $export, $reserved_params->{_per_page}, $reserved_params->{_page} + ); + + $c->add_pagination_headers( + { + total => $total, + base_total => $base_total, + params => $args, + } + ); + + return $c->render( status => 200, openapi => $export_formatted ); + } + catch { + $c->unhandled_exception($_); + }; +} + +sub _format_export { + my ($export, $per_page, $page) = @_; + + my $export_formatted = []; + my $total = 0; + my $skip = $per_page * ( $page-1 ); + foreach my $source ( sort keys %$export ) { + if ( $total >= $per_page ) { + last; + } + + if ( $source eq 'Borrower' ) { + if ( $skip > 0 ) { + $skip--; + next; + } + + push @$export_formatted, _format_item( { + source => $source, + data => $export->{$source}->to_api, + } ); + $total++; + next; + } + + while ( my $row = $export->{$source}->next ) { + if ( $skip > 0 ) { + $skip--; + next; + } + if ( $total >= $per_page ) { + last; + } + my $data; + if ( $row->can('to_api') ) { + $data = $row->to_api; + } elsif ( $row->can('unblessed') ) { + $data = $row->unblessed; + } else { + $data = { $row->get_columns }; + } + + push @$export_formatted, _format_item( { + source => $source, + data => $data, + } ); + $total++; + } + } + + my $base_total = 0; + foreach my $source ( keys %$export ) { + if ( $source eq 'Borrower' ) { + $base_total++; + } else { + $base_total = $base_total + $export->{$source}->count; + } + } + + return ( $export_formatted, $total, $base_total ); +} + +=head3 _format + +=cut + +sub _format_item { + my ($params) = @_; + + return { + type => $params->{'source'}, + data => $params->{'data'}, + }; +} + +1; diff --git a/t/db_dependent/api/v1/patrons_export.t b/t/db_dependent/api/v1/patrons_export.t new file mode 100755 index 0000000000..0105221ac1 --- /dev/null +++ b/t/db_dependent/api/v1/patrons_export.t @@ -0,0 +1,309 @@ +#!/usr/bin/perl + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; + +use Test::More tests => 2; + +use Test::Mojo; + +use t::lib::TestBuilder; +use t::lib::Mocks; + +my $schema = Koha::Database->new->schema; +my $builder = t::lib::TestBuilder->new; + +t::lib::Mocks::mock_preference( 'RESTBasicAuth', 1 ); + +# this is recommended because generation of test data may take a long time and +# Mojolicious would otherwise die with "Premature connection close" +$ENV{MOJO_INACTIVITY_TIMEOUT} = 120; + +my $t = Test::Mojo->new('Koha::REST::V1'); + +subtest 'get() tests' => sub { + + plan tests => 8; + + $schema->storage->txn_begin; + unauthorized_access_tests('GET', -1, undef); + $schema->storage->txn_rollback; + + $schema->storage->txn_begin; + + my $generated = generate_test_data(); + my $patron = $generated->{patron}; + my $test_objects = $generated->{test_objects}; + + my $password = '12345'; + my $librarian = $builder->build_object({ + class => 'Koha::Patrons', + value => { flags => 2**4 } # borrowers flag = 4 + }); + $librarian->set_password( { password => $password, skip_validation => 1 } ); + my $userid = $librarian->userid; + + t::lib::Mocks::mock_preference( 'AllowGDPRPatronExport', 1 ); + t::lib::Mocks::mock_preference( 'RESTdefaultPageSize', 20 ); + + $t->get_ok("//$userid:$password@/api/v1/patrons/" . $patron->id + . '/export') + ->status_is(200) + ->json_is($test_objects); + + subtest 'test pagination' => sub { + + plan tests => 15; + + t::lib::Mocks::mock_preference( 'RESTdefaultPageSize', 1 ); + $t->get_ok("//$userid:$password@/api/v1/patrons/" . $patron->id + . '/export') + ->status_is(200) + ->json_is([$test_objects->[0]]); + $t->get_ok("//$userid:$password@/api/v1/patrons/" . $patron->id + . '/export?_page=2') + ->status_is(200) + ->json_is([$test_objects->[1]]); + $t->get_ok("//$userid:$password@/api/v1/patrons/" . $patron->id + . '/export?_page=3') + ->status_is(200) + ->json_is([$test_objects->[2]]); + $t->get_ok("//$userid:$password@/api/v1/patrons/" . $patron->id + . '/export?_per_page=2&_page=1') + ->status_is(200) + ->json_is([$test_objects->[0],$test_objects->[1]]); + $t->get_ok("//$userid:$password@/api/v1/patrons/" . $patron->id + . '/export?_per_page=2&_page=2') + ->status_is(200) + ->json_is([$test_objects->[2],$test_objects->[3]]); + + }; + + t::lib::Mocks::mock_preference( 'AllowGDPRPatronExport', 0 ); + $t->get_ok("//$userid:$password@/api/v1/patrons/" + . $patron->id . '/export') + ->status_is(403) + ->json_is('/error', 'Configuration prevents patron data export'); + + $schema->storage->txn_rollback; + +}; + +subtest 'get_public() tests' => sub { + + plan tests => 11; + + $schema->storage->txn_begin; + unauthorized_access_tests('GET', -1, undef, 1); + $schema->storage->txn_rollback; + + $schema->storage->txn_begin; + + my $generated = generate_test_data(); + my $patron = $generated->{patron}; + my $test_objects = $generated->{test_objects}; + my $userid = $patron->userid; + my $password = $patron->{cleartext_password}; + + t::lib::Mocks::mock_preference( 'AllowGDPRPatronExport', 1 ); + t::lib::Mocks::mock_preference( 'RESTdefaultPageSize', 20 ); + + $t->get_ok("//$userid:$password@/api/v1/public/patrons/" + . ($patron->id-1) . '/export') + ->status_is(403) + ->json_is('/error', "Accessing other patron's data is forbidden"); + + $t->get_ok("//$userid:$password@/api/v1/public/patrons/" + . $patron->id . '/export') + ->status_is(200) + ->json_is($test_objects); + + subtest 'test pagination' => sub { + + plan tests => 15; + + t::lib::Mocks::mock_preference( 'RESTdefaultPageSize', 1 ); + $t->get_ok("//$userid:$password@/api/v1/public/patrons/" . $patron->id + . '/export') + ->status_is(200) + ->json_is([$test_objects->[0]]); + $t->get_ok("//$userid:$password@/api/v1/public/patrons/" . $patron->id + . '/export?_page=2') + ->status_is(200) + ->json_is([$test_objects->[1]]); + $t->get_ok("//$userid:$password@/api/v1/public/patrons/" . $patron->id + . '/export?_page=3') + ->status_is(200) + ->json_is([$test_objects->[2]]); + $t->get_ok("//$userid:$password@/api/v1/public/patrons/" . $patron->id + . '/export?_per_page=2&_page=1') + ->status_is(200) + ->json_is([$test_objects->[0],$test_objects->[1]]); + $t->get_ok("//$userid:$password@/api/v1/public/patrons/" . $patron->id + . '/export?_per_page=2&_page=2') + ->status_is(200) + ->json_is([$test_objects->[2],$test_objects->[3]]); + + }; + + t::lib::Mocks::mock_preference( 'AllowGDPRPatronExport', 0 ); + $t->get_ok("//$userid:$password@/api/v1/public/patrons/" + . $patron->id . '/export') + ->status_is(403) + ->json_is('/error', 'Configuration prevents patron data export'); + + $schema->storage->txn_rollback; + +}; + +sub generate_test_data { + my @related_sources = _get_related_sources(); + my $patron = $builder->build_object( + { + class => 'Koha::Patrons', + } + ); + + my $password = '12345'; + $patron->set_password( { password => $password, skip_validation => 1 } ); + $patron->discard_changes; + $patron->{cleartext_password} = $password; + my $test_objects = { + 'Borrower' => $patron->to_api, + }; + + my $limit_data = 5; # build this many random test objects + my $generated_data = 0; + + my $result_source = Koha::Patron->new->_result()->result_source; + foreach my $rel ( Koha::Patron->new->_result()->relationships() ) { + if ($generated_data >= $limit_data) { + last; + } + + my $related_source = $result_source->related_source( $rel ); + my $source_name = $related_source->source_name; + + my $info = $result_source->relationship_info( $rel ); + + # We are not interested in the "belongs_to" relationship of borrowers. + # These are tables like branches, categories and sms_provider. + if ( $info->{'attrs'}->{'is_depends_on'} ) { + next; + } + + ( my $rel_col = (keys %{$info->{'cond'}})[0] ) =~ s/^foreign\.//; + + # Generate test data into related tables + my $built; + if ( $related_source->result_class->can('koha_objects_class') ) { + $built = $builder->build_object( + { + class => $related_source->result_class->koha_objects_class, + value => { $rel_col => $patron->borrowernumber } + } + ); + if ( $built->can('to_api') ) { + $built = $built->to_api; + } elsif ( $built->can('unblessed') ) { + $built = $built->unblessed; + } + } else { + $built = $builder->build( + { + source => $source_name, + value => { $rel_col => $patron->borrowernumber } + } + ); + } + + $test_objects->{$source_name} = [] unless $test_objects->{$source_name}; + push @{ $test_objects->{$related_source->source_name} }, $built; + $generated_data++; + } + + my $test_objects_formatted = []; + foreach my $source ( sort keys %$test_objects ) { + if ( $source eq 'Borrower' ) { + push @$test_objects_formatted, { + data => $test_objects->{$source}, + type => $source, + }; + next; + } + + foreach my $item ( @{ $test_objects->{$source} } ) { + push @$test_objects_formatted, { + data => $item, + type => $source, + } + } + } + + return { + patron => $patron, + test_objects => $test_objects_formatted + }; +} + +# Centralized tests for 401s and 403s assuming the endpoint requires +# borrowers flag for access +sub unauthorized_access_tests { + my ($verb, $patron_id, $json, $public) = @_; + + my $endpoint = '/api/v1/' . ( $public ? 'public/' : '' ) . 'patrons'; + $endpoint .= ($patron_id) ? "/$patron_id/export" : ''; + + subtest 'unauthorized access tests' => sub { + plan tests => 5; + + my $verb_ok = lc($verb) . '_ok'; + + $t->$verb_ok($endpoint => json => $json) + ->status_is(401); + + my $unauthorized_patron = $builder->build_object( + { + class => 'Koha::Patrons', + value => { flags => 0 } + } + ); + my $password = "12345"; + $unauthorized_patron->set_password( + { password => $password, skip_validation => 1 } ); + my $unauth_userid = $unauthorized_patron->userid; + + $t->$verb_ok( "//$unauth_userid:$password\@$endpoint" => json => $json ) + ->status_is(403) + ->json_has('/required_permissions'); + }; +} + +sub _get_related_sources { + my $sources = {}; + my $res_source = Koha::Patron->new->_result()->result_source; + foreach my $rel ( Koha::Patron->new->_result()->relationships() ) { + my $related_source = $res_source->related_source($rel); + my $info = $res_source->relationship_info( $rel ); + next if $info->{'attrs'}->{'is_depends_on'}; + next if $sources->{$related_source->source_name}; + $sources->{$related_source->source_name} = 1; + } + $sources->{'Borrower'} = 1; # add Borrower itself + my @sorted = sort keys %$sources; + return @sorted; +} -- 2.25.1