From 179a1ffc010b989c6f877c89aa9b072dfdcf9321 Mon Sep 17 00:00:00 2001 From: Marcel de Rooy Date: Sun, 19 Mar 2017 17:25:01 +0100 Subject: [PATCH] Bug 18300: Delete missing upload records If you deleted files from the upload directories manually, or you rebooted with files in the temporary upload folder, or for some other reason have records without a file, you may want to cleanup your records in the uploaded_files table. This patch adds the method delete_missing to Koha::UploadedFiles. It also supports a keep_record parameter to do a dry run (count the missing files first). Also, we add an option --uploads-missing to cleanup_database. If you add the flag 1 after this option, you will delete missing files. If you add the flag 0 or only use the option, you will count missing files. A subtest is added to Upload.t for delete_missing tests. Test plan: [1] Run t/db_dependent/Upload.t [2] Upload a file and delete the file manually. [3] Run cleanup_database.pl --uploads-missing It should report at least one missing file now. Check that the record has not been deleted. [4] Run cleanup_database.pl --uploads-missing 1 It should report that it removed at least one file. Check that the record is gone. Signed-off-by: Marcel de Rooy Signed-off-by: Mirko Tietgen Signed-off-by: Jonathan Druart --- Koha/UploadedFiles.pm | 30 ++++++++++++++++++++++++++++++ misc/cronjobs/cleanup_database.pl | 17 ++++++++++++++++- t/db_dependent/Upload.t | 21 +++++++++++++++++++-- 3 files changed, 65 insertions(+), 3 deletions(-) diff --git a/Koha/UploadedFiles.pm b/Koha/UploadedFiles.pm index b64df0a..61e9702 100644 --- a/Koha/UploadedFiles.pm +++ b/Koha/UploadedFiles.pm @@ -105,6 +105,36 @@ sub delete_temporary { })->delete; } +=head3 delete_missing + + $cnt = Koha::UploadedFiles->delete_missing(); + + $cnt = Koha::UploadedFiles->delete_missing({ keep_record => 1 }); + +Deletes all records where the actual file is not found. + +Supports a keep_record hash parameter to do a check only. + +Returns the number of missing files (and/or deleted records). + +=cut + +sub delete_missing { + my ( $self, $params ) = @_; + $self = Koha::UploadedFiles->new if !ref($self); # handle class call + my $cnt = 0; + while( my $row = $self->next ) { + if( my $file = $row->full_path ) { + next if -e $file; + # We are passing keep_file since we already know that the file + # is missing and we do not want to see the warning + $row->delete({ keep_file => 1 }) if !$params->{keep_record}; + $cnt++; + } + } + return $cnt; +} + =head3 search_term Search_term allows you to pass a term to search in filename and hashvalue. diff --git a/misc/cronjobs/cleanup_database.pl b/misc/cronjobs/cleanup_database.pl index b39d9a7..03c382d 100755 --- a/misc/cronjobs/cleanup_database.pl +++ b/misc/cronjobs/cleanup_database.pl @@ -44,7 +44,7 @@ use Koha::UploadedFiles; sub usage { print STDERR < \$help, @@ -129,6 +131,7 @@ GetOptions( 'unique-holidays:i' => \$special_holidays_days, 'temp-uploads' => \$temp_uploads, 'temp-uploads-days:i' => \$temp_uploads_days, + 'uploads-missing:i' => \$uploads_missing, ) || usage(1); # Use default values @@ -161,6 +164,7 @@ unless ( $sessions || $pUnvSelfReg || $special_holidays_days || $temp_uploads + || defined $uploads_missing ) { print "You did not specify any cleanup work for the script to do.\n\n"; usage(1); @@ -321,6 +325,17 @@ if( $temp_uploads ) { print "Done purging temporary uploads.\n" if $verbose; } +if( defined $uploads_missing ) { + print "Looking for missing uploads\n" if $verbose; + my $keep = $uploads_missing == 1 ? 0 : 1; + my $count = Koha::UploadedFiles->delete_missing({ keep_record => $keep }); + if( $keep ) { + print "Counted $count missing uploaded files\n"; + } else { + print "Removed $count records for missing uploads\n"; + } +} + exit(0); sub RemoveOldSessions { diff --git a/t/db_dependent/Upload.t b/t/db_dependent/Upload.t index 8f6add2..0489fee 100644 --- a/t/db_dependent/Upload.t +++ b/t/db_dependent/Upload.t @@ -2,7 +2,7 @@ use Modern::Perl; use File::Temp qw/ tempdir /; -use Test::More tests => 11; +use Test::More tests => 12; use Test::Warn; use Test::MockModule; @@ -18,7 +18,7 @@ use Koha::Uploader; my $schema = Koha::Database->new->schema; $schema->storage->txn_begin; -my $builder = t::lib::TestBuilder->new; +our $builder = t::lib::TestBuilder->new; our $current_upload = 0; our $uploads = [ @@ -192,6 +192,23 @@ subtest 'Test delete via UploadedFile as well as UploadedFiles' => sub { # NOTE: Koha::Object->delete does not return 0E0 (yet?) }; +subtest 'Test delete_missing' => sub { + plan tests => 4; + + # If we add files via TestBuilder, they do not exist + my $upload01 = $builder->build({ source => 'UploadedFile' }); + my $upload02 = $builder->build({ source => 'UploadedFile' }); + # dry run first + my $deleted = Koha::UploadedFiles->delete_missing({ keep_record => 1 }); + is( $deleted, 2, 'Expect two missing files' ); + isnt( Koha::UploadedFiles->find( $upload01->{id} ), undef, 'Not deleted' ); + $deleted = Koha::UploadedFiles->delete_missing; + is( $deleted, 2, 'Deleted two missing files' ); + is( Koha::UploadedFiles->search({ + id => [ $upload01->{id}, $upload02->{id} ], + })->count, 0, 'Records are gone' ); +}; + subtest 'Call search_term with[out] private flag' => sub { plan tests => 3; -- 2.9.3