From 4b0d96ccefe7f545dbcfb9fc2446eda87569fc2c Mon Sep 17 00:00:00 2001 From: Marcel de Rooy Date: Sun, 19 Mar 2017 17:25:01 +0100 Subject: [PATCH] Bug 18300: Delete missing upload records Content-Type: text/plain; charset=utf-8 If you deleted files from the upload directories manually, or you rebooted with files in the temporary upload folder, or for some other reason have records without a file, you may want to cleanup your records in the uploaded_files table. This patch adds the method delete_missing to Koha::UploadedFiles. It also supports a keep_record parameter to do a dry run (count the missing files first). Also, we add an option --uploads-missing to cleanup_database. If you add the flag 1 after this option, you will delete missing files. If you add the flag 0 or only use the option, you will count missing files. A subtest is added to Upload.t for delete_missing tests. Test plan: [1] Run t/db_dependent/Upload.t [2] Upload a file and delete the file manually. [3] Run cleanup_database.pl --uploads-missing It should report at least one missing file now. Check that the record has not been deleted. [4] Run cleanup_database.pl --uploads-missing 1 It should report that it removed at least one file. Check that the record is gone. Signed-off-by: Marcel de Rooy Signed-off-by: Mirko Tietgen --- Koha/UploadedFiles.pm | 30 ++++++++++++++++++++++++++++++ misc/cronjobs/cleanup_database.pl | 17 ++++++++++++++++- t/db_dependent/Upload.t | 21 +++++++++++++++++++-- 3 files changed, 65 insertions(+), 3 deletions(-) diff --git a/Koha/UploadedFiles.pm b/Koha/UploadedFiles.pm index b64df0a..61e9702 100644 --- a/Koha/UploadedFiles.pm +++ b/Koha/UploadedFiles.pm @@ -105,6 +105,36 @@ sub delete_temporary { })->delete; } +=head3 delete_missing + + $cnt = Koha::UploadedFiles->delete_missing(); + + $cnt = Koha::UploadedFiles->delete_missing({ keep_record => 1 }); + +Deletes all records where the actual file is not found. + +Supports a keep_record hash parameter to do a check only. + +Returns the number of missing files (and/or deleted records). + +=cut + +sub delete_missing { + my ( $self, $params ) = @_; + $self = Koha::UploadedFiles->new if !ref($self); # handle class call + my $cnt = 0; + while( my $row = $self->next ) { + if( my $file = $row->full_path ) { + next if -e $file; + # We are passing keep_file since we already know that the file + # is missing and we do not want to see the warning + $row->delete({ keep_file => 1 }) if !$params->{keep_record}; + $cnt++; + } + } + return $cnt; +} + =head3 search_term Search_term allows you to pass a term to search in filename and hashvalue. diff --git a/misc/cronjobs/cleanup_database.pl b/misc/cronjobs/cleanup_database.pl index b39d9a7..03c382d 100755 --- a/misc/cronjobs/cleanup_database.pl +++ b/misc/cronjobs/cleanup_database.pl @@ -44,7 +44,7 @@ use Koha::UploadedFiles; sub usage { print STDERR < \$help, @@ -129,6 +131,7 @@ GetOptions( 'unique-holidays:i' => \$special_holidays_days, 'temp-uploads' => \$temp_uploads, 'temp-uploads-days:i' => \$temp_uploads_days, + 'uploads-missing:i' => \$uploads_missing, ) || usage(1); # Use default values @@ -161,6 +164,7 @@ unless ( $sessions || $pUnvSelfReg || $special_holidays_days || $temp_uploads + || defined $uploads_missing ) { print "You did not specify any cleanup work for the script to do.\n\n"; usage(1); @@ -321,6 +325,17 @@ if( $temp_uploads ) { print "Done purging temporary uploads.\n" if $verbose; } +if( defined $uploads_missing ) { + print "Looking for missing uploads\n" if $verbose; + my $keep = $uploads_missing == 1 ? 0 : 1; + my $count = Koha::UploadedFiles->delete_missing({ keep_record => $keep }); + if( $keep ) { + print "Counted $count missing uploaded files\n"; + } else { + print "Removed $count records for missing uploads\n"; + } +} + exit(0); sub RemoveOldSessions { diff --git a/t/db_dependent/Upload.t b/t/db_dependent/Upload.t index 6af3fce..8d91053 100644 --- a/t/db_dependent/Upload.t +++ b/t/db_dependent/Upload.t @@ -2,7 +2,7 @@ use Modern::Perl; use File::Temp qw/ tempdir /; -use Test::More tests => 11; +use Test::More tests => 12; use Test::Warn; use Test::MockModule; @@ -18,7 +18,7 @@ use Koha::Uploader; my $schema = Koha::Database->new->schema; $schema->storage->txn_begin; -my $builder = t::lib::TestBuilder->new; +our $builder = t::lib::TestBuilder->new; our $current_upload = 0; our $uploads = [ @@ -192,6 +192,23 @@ subtest 'Test delete via UploadedFile as well as UploadedFiles' => sub { # NOTE: Koha::Object->delete does not return 0E0 (yet?) }; +subtest 'Test delete_missing' => sub { + plan tests => 4; + + # If we add files via TestBuilder, they do not exist + my $upload01 = $builder->build({ source => 'UploadedFile' }); + my $upload02 = $builder->build({ source => 'UploadedFile' }); + # dry run first + my $deleted = Koha::UploadedFiles->delete_missing({ keep_record => 1 }); + is( $deleted, 2, 'Expect two missing files' ); + isnt( Koha::UploadedFiles->find( $upload01->{id} ), undef, 'Not deleted' ); + $deleted = Koha::UploadedFiles->delete_missing; + is( $deleted, 2, 'Deleted two missing files' ); + is( Koha::UploadedFiles->search({ + id => [ $upload01->{id}, $upload02->{id} ], + })->count, 0, 'Records are gone' ); +}; + subtest 'Call search_term with[out] private flag' => sub { plan tests => 3; -- 2.1.4