From 579df7c57288db8ef725ef16d06281f6537b85d5 Mon Sep 17 00:00:00 2001 From: Lari Taskula Date: Tue, 15 Oct 2024 16:07:17 +0000 Subject: [PATCH] Bug 32440: Support purging statistics by type in cleanup_database.pl Cleanup_database can delete all statistics entries more than X days old. If one is using pseudonymization to create pseudonymized_transactions data, then one may wish to use that cleanup_database function to delete statistics entries that have been duplicated in pseudonymized_transactions. However, not all types of transactions in statistics are duplicated in pseudonymized transactions. Pseudonymized_transactions currently only includes checkouts, returns, and renewals. This patch adds two additional parameters to cleanup_database.pl: 1. --statistics-type Defines the types of statistics to purge. Will purge all types if parameter is omitted. Repeatable. 2. --statistics-type-pseudo Grabs values from @Koha::Statistic::pseudonymization_types. At the time of writing this patch, they are: renew, issue, return and onsite_checkout To test: 1. prove t/db_dependent/Koha/Statistic.t 2. Before applying this patch: 3. Create some statistics entry by checking out, renewing and checking in items. 4. perl misc/cronjobs/cleanup_database.pl --statistics 1 --verbose Observe: Purging statistics older than 1 days n statistics would have been removed Where n is the amount of statistics rows matching your test environment 5. Apply patch 6. Repeat step 4, observe same result 7. perl misc/cronjobs/cleanup_database.pl --statistics 1 --statistics-type-pseudo --verbose Observe: Purging statistics older than 1 days with types "onsite_checkout,renew,return,issue". n statistics would have been removed Where n is the amount of statistics rows matching your test environment (order of listed types does not matter) 8. perl misc/cronjobs/cleanup_database.pl --statistics 1 --statistics-type test1 --statistics-type test2 --verbose Observe: Purging statistics older than 1 days with types "test1,test2". n statistics would have been removed Where n is the amount of statistics rows matching your test environment (order of listed types does not matter) 9. perl misc/cronjobs/cleanup_database.pl --statistics 1 --statistics-type-pseudo --statistics-type test1 --verbose Observe: Purging statistics older than 1 days with types "test1,onsite_checkout,renew,return,issue". n statistics would have been removed Where n is the amount of statistics rows matching your test environment (order of listed types does not matter) 10. Try previous cleanup_database.pl commands with --confirm flag and make sure correct rows are purged --- Koha/Statistic.pm | 10 +++++++++- misc/cronjobs/cleanup_database.pl | 33 +++++++++++++++++++++++++------ t/db_dependent/Koha/Statistic.t | 7 +++---- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/Koha/Statistic.pm b/Koha/Statistic.pm index 4e981c31c2..4abc6ae18c 100644 --- a/Koha/Statistic.pm +++ b/Koha/Statistic.pm @@ -149,7 +149,7 @@ Generate a pesudonymized version of the statistic. sub pseudonymize { my ($self) = @_; - return unless ( $self->borrowernumber && grep { $_ eq $self->type } qw(renew issue return onsite_checkout) ); + return unless ( $self->borrowernumber && grep { $_ eq $self->type } @Koha::Statistic::pseudonymization_types ); # FIXME When getting the object from svc/renewal we get a DateTime object # normally we just fetch from DB to clear this, but statistics has no primary key @@ -161,6 +161,14 @@ sub pseudonymize { } +=head2 Pseudonymization types + +=head3 @pseudonymization_types + +=cut + +our @pseudonymization_types = qw(renew issue return onsite_checkout); + =head2 Internal methods =head3 _type diff --git a/misc/cronjobs/cleanup_database.pl b/misc/cronjobs/cleanup_database.pl index 0193018129..35d71f59a4 100755 --- a/misc/cronjobs/cleanup_database.pl +++ b/misc/cronjobs/cleanup_database.pl @@ -53,11 +53,12 @@ use Koha::Old::Patrons; use Koha::Patron::Debarments qw( DelDebarment ); use Koha::Patron::Messages; use Koha::PseudonymizedTransactions; +use Koha::Statistics; use Koha::UploadedFiles; sub usage { print STDERR < \$uploads_missing, 'oauth-tokens' => \$oauth_tokens, 'statistics:i' => \$pStatistics, + 'statistics-type:s' => \@statistics_types, + 'statistics-type-pseudo:i' => \$statistics_type_pseudo, 'deleted-catalog:i' => \$pDeletedCatalog, 'deleted-patrons:i' => \$pDeletedPatrons, 'old-issues:i' => \$pOldIssues, @@ -233,6 +241,9 @@ $jobs_days = DEFAULT_JOBS_PURGEDAYS if defined($jobs_days) @jobs_types = (DEFAULT_JOBS_PURGETYPES) if $jobs_days && @jobs_types == 0; $edifact_msg_days = DEFAULT_EDIFACT_MSG_PURGEDAYS if defined($edifact_msg_days) && $edifact_msg_days == 0; +@statistics_types = (@statistics_types, @Koha::Statistic::pseudonymization_types) if defined($statistics_type_pseudo); +@statistics_types = keys %{{ map { $_, 1 } @statistics_types }}; # remove duplicate types + # Choose the number of days at which to purge unaccepted list invites: # - DAYS defined in the list-invites parameter is prioritised first # - PurgeListShareInvitesOlderThan system preference is prioritised second @@ -585,11 +596,21 @@ if ($oauth_tokens) { } if ($pStatistics) { - print "Purging statistics older than $pStatistics days.\n" if $verbose; - my $statistics = - Koha::Statistics->filter_by_last_update( { timestamp_column_name => 'datetime', days => $pStatistics } ); - my $count = $statistics->count; - $statistics->delete if $confirm; + print "Purging statistics older than $pStatistics days" if $verbose; + my $statistics_search_params = {}; + + if ( scalar(@statistics_types) ) { + print " with types \"".join(',', @statistics_types)."\".\n" if $verbose; + $statistics_search_params->{type} = \@statistics_types; + + } + print "\n" if $verbose; + + my $statistics = Koha::Statistics->search( $statistics_search_params ); + my $filtered_statistics = + $statistics->filter_by_last_update( { timestamp_column_name => 'datetime', days => $pStatistics } ); + my $count = $filtered_statistics->count; + $filtered_statistics->delete if $confirm; if ($verbose) { say $confirm ? sprintf( "Done with purging %d statistics", $count ) diff --git a/t/db_dependent/Koha/Statistic.t b/t/db_dependent/Koha/Statistic.t index 000ecb2133..b8b67f5d8d 100755 --- a/t/db_dependent/Koha/Statistic.t +++ b/t/db_dependent/Koha/Statistic.t @@ -61,8 +61,7 @@ subtest 'new() tests' => sub { subtest 'pseudonymize() tests' => sub { - my @pseudonymization_types = qw(renew issue return onsite_checkout); - plan tests => scalar(@pseudonymization_types) + 2; + plan tests => scalar(@Koha::Statistic::pseudonymization_types) + 2; $schema->storage->txn_begin; @@ -73,10 +72,10 @@ subtest 'pseudonymize() tests' => sub { my @statistics = (); - ok( scalar(@pseudonymization_types) > 0, 'some pseudonymization_types are defined' ); + ok( scalar(@Koha::Statistic::pseudonymization_types) > 0, 'some pseudonymization_types are defined' ); my $sub_days = 0; # TestBuilder does not handle many rows of Koha::Statistics very intuitively - foreach my $type (@pseudonymization_types) { + foreach my $type (@Koha::Statistic::pseudonymization_types) { push( @statistics, $builder->build_object( -- 2.34.1