@@ -, +, @@ been applied to a database --- C4/Installer/PerlDependencies.pm | 10 + Koha/AtomicUpdate.pm | 136 ++++++++ Koha/AtomicUpdater.pm | 349 +++++++++++++++++++++ Koha/Schema/Result/Atomicupdate.pm | 93 ++++++ installer/data/mysql/atomicupdate.pl | 148 +++++++++ .../mysql/atomicupdate/Bug14698-AtomicUpdater.pl | 36 +++ installer/data/mysql/kohastructure.sql | 14 + t/db_dependent/Koha/AtomicUpdater.t | 274 ++++++++++++++++ t/lib/TestObjects/AtomicUpdateFactory.pm | 105 +++++++ t/lib/TestObjects/ObjectFactory.pm | 5 + t/lib/TestObjects/objectFactories.t | 50 +++ 11 files changed, 1220 insertions(+) create mode 100644 Koha/AtomicUpdate.pm create mode 100644 Koha/AtomicUpdater.pm create mode 100644 Koha/Schema/Result/Atomicupdate.pm create mode 100644 installer/data/mysql/atomicupdate.pl create mode 100644 installer/data/mysql/atomicupdate/Bug14698-AtomicUpdater.pl create mode 100644 t/db_dependent/Koha/AtomicUpdater.t create mode 100644 t/lib/TestObjects/AtomicUpdateFactory.pm --- a/C4/Installer/PerlDependencies.pm +++ a/C4/Installer/PerlDependencies.pm @@ -217,6 +217,16 @@ our $PERL_DEPS = { 'min_ver' => '0.45', # Also needed for our use of PDF::Reuse }, + 'Data::Format::Pretty::Console' => { + 'usage' => 'Core', + 'required' => '1', + 'min_ver' => '0.34', + }, + 'Git' => { + 'usage' => 'AtomicUpdater', + 'required' => '1', + 'min_ver' => '0.41', + }, 'DateTime' => { 'usage' => 'Core', 'required' => '1', --- a/Koha/AtomicUpdate.pm +++ a/Koha/AtomicUpdate.pm @@ -0,0 +1,136 @@ +package Koha::AtomicUpdate; + +# Copyright Open Source Freedom Fighters +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use Carp; +use File::Basename; + +use Koha::Database; + +use base qw(Koha::Object); + +use Koha::Exception::BadParameter; + +sub type { + return 'Atomicupdate'; +} + +=head @allowedIssueIdentifierPrefixes +Define the prefixes you want to attach to your atomicupdate filenames here. +This could be a syspref or in KOHA_CONF, but it is rather easy to just add more +generally used issue number prefixes here. +Nobody wants more sysprefs. +=cut + +my @allowedIssueIdentifierPrefixes = ( + 'Bug', + '#', +); + +=head new + + my $atomicUpdate = Koha::AtomicUpdate->new({filename => 'Bug54321-FixItPlease.pl'}); + +Creates a Koha::AtomicUpdate-object from the given parameters-HASH +@PARAM1 HASHRef of object parameters: + 'filename' => MANDATORY, The filename of the atomicupdate-script without the path-component. + 'issue_id' => OPTIONAL, the desired issue_id. It is better to let the module + find this from the filename, but is useful for testing purposes. +@RETURNS Koha::AtomicUpdate-object +@THROWS Koha::Exception::Parse from getIssueIdentifier() +@THROWS Koha::Exception::File from _validateFilename(); +=cut + +sub new { + my ($class, $params) = @_; + $class->_validateParams($params); + + my $self = {}; + bless($self, $class); + $self->set($params); + return $self; +} + +sub _validateParams { + my ($class, $params) = @_; + + my @mandatoryParams = ('filename'); + foreach my $mp (@mandatoryParams) { + Koha::Exception::BadParameter->throw( + error => "$class->_validateParams():> Param '$mp' must be given.") + unless($params->{$mp}); + } + $params->{filename} = $class->_validateFilename($params->{filename}); + + $params->{issue_id} = $class->getIssueIdentifier($params->{issue_id} || $params->{filename}); +} + +=head _validateFilename + +Makes sure the given file is a valid AtomicUpdate-script. +Currently simply checks for naming convention and file suffix. + +NAMING CONVENTION: + Filename must contain one of the unique issue identifier prefixes from this + list @allowedIssueIdentifierPrefixes immediately followed by the numeric + id of the issue, optionally separated by any of the following [ :-] + Eg. Bug-45453, #102, #:53 + +@PARAM1 String, filename of validatable file, excluding path. +@RETURNS String, the koha.atomicupdates.filename if the given file is considered a well formed update script. + Removes the full path if present and returns only the filename component. + +@THROWS Koha::Exception::File, if the given file doesn't have a proper naming convention + +=cut + +sub _validateFilename { + my ($self, $fileName) = @_; + + Koha::Exception::File->throw(error => __PACKAGE__."->_validateFilename():> Filename '$fileName' has unknown suffix") + unless $fileName =~ /\.(sql|perl|pl)$/; #skip other files + + $fileName = File::Basename::basename($fileName); + + return $fileName; +} + +=head getIssueIdentifier + +Extracts the unique issue identifier from the atomicupdate DB upgrade script. + +@PARAM1 String, filename of validatable file, excluding path, or Git commit title, + or something else to parse. +@RETURNS String, The unique issue identifier + +@THROWS Koha::Exception::Parse, if the unique identifier couldn't be parsed. +=cut + +sub getIssueIdentifier { + my ($self, $fileName) = @_; + + foreach my $prefix (@allowedIssueIdentifierPrefixes) { + if ($fileName =~ m/$prefix[-: _]*?(\d+)/i) { + return ucfirst("$prefix$1"); + } + } + Koha::Exception::Parse->throw(error => __PACKAGE__."->getIssueIdentifier($fileName):> couldn't parse the unique issue identifier from filename using allowed prefixes '@allowedIssueIdentifierPrefixes'"); +} + +1; --- a/Koha/AtomicUpdater.pm +++ a/Koha/AtomicUpdater.pm @@ -0,0 +1,349 @@ +package Koha::AtomicUpdater; + +# Copyright Open Source Freedom Fighters +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use Carp; +use Scalar::Util qw(blessed); +use Try::Tiny; +use Data::Format::Pretty::Console qw(format_pretty); +use Git; + +use Koha::Database; +use Koha::Cache; +use Koha::AtomicUpdate; + +use base qw(Koha::Objects); + +use Koha::Exception::File; +use Koha::Exception::Parse; +use Koha::Exception::BadParameter; + +sub type { + return 'Atomicupdate'; +} + +sub object_class { + return 'Koha::AtomicUpdate'; +} + +sub _get_castable_unique_columns { + return ['atomicupdate_id']; +} + +my $updateOrderFilename = '_updateorder'; + +sub new { + my ($class, $params) = @_; + + my $cache = Koha::Cache->new(); + my $self = $cache->get_from_cache('Koha::AtomicUpdater') || {}; + bless($self, $class); + + $self->{verbose} = $params->{verbose} || $self->{verbose} || 0; + $self->{scriptDir} = $params->{scriptDir} || $self->{scriptDir} || C4::Context->config('intranetdir') . '/installer/data/mysql/atomicupdate/'; + $self->{gitRepo} = $params->{gitRepo} || $self->{gitRepo} || $ENV{KOHA_PATH}; + + return $self; +} + +=head getAtomicUpdates + + my $atomicUpdates = $atomicUpdater->getAtomicUpdates(); + +Gets all the AtomicUpdate-objects in the DB. This result should be Koha::Cached. +@RETURNS HASHRef of Koha::AtomicUpdate-objects, keyed with the issue_id +=cut + +sub getAtomicUpdates { + my ($self) = @_; + + my @au = $self->search({}); + my %au; #HASHify the AtomicUpdate-objects for easy searching. + foreach my $au (@au) { + $au{$au->issue_id} = $au; + } + return \%au; +} + +sub addAtomicUpdate { + my ($self, $params) = @_; + print "Adding atomicupdate '".$params->{issue_id}."'\n" if $self->{verbose} > 2; + + my $atomicupdate = Koha::AtomicUpdate->new($params); + $atomicupdate->store(); + $atomicupdate = $self->find({issue_id => $atomicupdate->issue_id}); + return $atomicupdate; +} + +sub removeAtomicUpdate { + my ($self, $issueId) = @_; + print "Deleting atomicupdate '$issueId'\n" if $self->{verbose} > 2; + + my $atomicupdate = $self->find({issue_id => $issueId}); + if ($atomicupdate) { + $atomicupdate->delete; + print "Deleted atomicupdate '$issueId'\n" if $self->{verbose} > 2; + } + else { + Koha::Exception::BadParameter->throw(error => __PACKAGE__."->removeIssueFromLog():> No such Issue '$issueId' stored to the atomicupdates-table"); + } +} + +sub listToConsole { + my ($self) = @_; + my @stringBuilder; + + my @atomicupdates = $self->search({}); + foreach my $au (@atomicupdates) { + push @stringBuilder, $au->unblessed(); + } + return Data::Format::Pretty::Console::format_pretty(\@stringBuilder); +} + +sub listPendingToConsole { + my ($self) = @_; + my @stringBuilder; + + my $atomicUpdates = $self->getPendingAtomicUpdates(); + foreach my $key (sort keys %$atomicUpdates) { + my $au = $atomicUpdates->{$key}; + push @stringBuilder, $au->unblessed(); + } + return Data::Format::Pretty::Console::format_pretty(\@stringBuilder); +} + +sub getPendingAtomicUpdates { + my ($self) = @_; + + my %pendingAtomicUpdates; + my $atomicupdateFiles = $self->_getValidAtomicUpdateScripts(); + my $atomicUpdatesDeployed = $self->getAtomicUpdates(); + foreach my $key (keys(%$atomicupdateFiles)) { + my $au = $atomicupdateFiles->{$key}; + unless ($atomicUpdatesDeployed->{$au->issue_id}) { + #This script hasn't been deployed. + $pendingAtomicUpdates{$au->issue_id} = $au; + } + } + return \%pendingAtomicUpdates; +} + +=head applyAtomicUpdates + + my $atomicUpdater = Koha::AtomicUpdater->new(); + my $appliedAtomicupdates = $atomicUpdater->applyAtomicUpdates(); + +Checks the atomicupdates/-directory for any not-applied update scripts and +runs them in the order specified in the _updateorder-file in atomicupdate/-directory. + +@RETURNS ARRAYRef of Koha::AtomicUpdate-objects deployed on this run +=cut + +sub applyAtomicUpdates { + my ($self) = @_; + + my %appliedUpdates; + + my $atomicUpdates = $self->getPendingAtomicUpdates(); + my $updateOrder = $self->getUpdateOrder(); + foreach my $issueId ( @$updateOrder ) { + my $atomicUpdate = $atomicUpdates->{$issueId}; + next unless $atomicUpdate; #Not each ordered Git commit necessarily have a atomicupdate-script. + + my $filename = $atomicUpdate->filename; + print "Applying file '$filename'\n" if $self->{verbose} > 2; + + if ( $filename =~ /\.sql$/ ) { + my $installer = C4::Installer->new(); + my $rv = $installer->load_sql( $self->{scriptDir}.'/'.$filename ) ? 0 : 1; + } elsif ( $filename =~ /\.(perl|pl)$/ ) { + do $self->{scriptDir}.'/'.$filename; + } + + $atomicUpdate->store(); + $appliedUpdates{$issueId} = $atomicUpdate; + print "File '$filename' applied\n" if $self->{verbose} > 2; + } + + #Check that we have actually applied all the updates. + my $stillPendingAtomicUpdates = $self->getPendingAtomicUpdates(); + if (scalar(%$stillPendingAtomicUpdates)) { + my @issueIds = sort keys %$stillPendingAtomicUpdates; + print "Warning! After upgrade, the following atomicupdates are still pending '@issueIds'\n Try rebuilding the atomicupdate-scripts update order from the original Git repository.\n"; + } + + return \%appliedUpdates; +} + +=head _getValidAtomicUpdateScripts + +@RETURNS HASHRef of Koha::AtomicUpdate-objects, of all the files + in the atomicupdates/-directory that can be considered valid. + Validity is currently conforming to the naming convention. + Keys are the issue_id of atomicupdate-scripts + Eg. {'Bug8584' => Koha::AtomicUpdate, + ... + } +=cut + +sub _getValidAtomicUpdateScripts { + my ($self) = @_; + + my %atomicUpdates; + opendir( my $dirh, $self->{scriptDir} ); + foreach my $file ( sort readdir $dirh ) { + print "Looking at file $file\n" if $self->{verbose} > 2; + + my $atomicUpdate; + try { + $atomicUpdate = Koha::AtomicUpdate->new({filename => $file}); + } catch { + if (blessed($_)) { + if ($_->isa('Koha::Exception::File')) { + #We can ignore filename validation issues, since the directory has + #loads of other types of files as well. Like README . .. + } + else { + $_->rethrow(); + } + } + else { + die $_; #Rethrow the unknown Exception + } + }; + next unless $atomicUpdate; + + $atomicUpdates{$atomicUpdate->issue_id} = $atomicUpdate; + } + return \%atomicUpdates; +} + +=head getUpdateOrder + + $atomicUpdater->getUpdateOrder(); + +@RETURNS ARRAYRef of Strings, IssueIds ordered from the earliest to the newest. +=cut + +sub getUpdateOrder { + my ($self) = @_; + + my $updateOrderFilepath = $self->{scriptDir}."/$updateOrderFilename"; + open(my $FH, "<:encoding(UTF-8)", $updateOrderFilepath) or die "Koha::AtomicUpdater->_saveAsUpdateOrder():> Couldn't open the updateOrderFile for reading\n$!\n"; + my @updateOrder = map {chomp($_); $_;} <$FH>; + close $FH; + return \@updateOrder; +} + +=head + + my $issueIdOrder = Koha::AtomicUpdater->buildUpdateOrderFromGit(10000); + +Creates a update order file '_updateorder' for atomicupdates to know which updates come before which. +This is a simple way to make sure the atomicupdates are applied in the correct order. +The update order file is by default in your $KOHA_PATH/installer/data/mysql/atomicupdate/_updateorder + +This requires a Git repository to be in the $ENV{KOHA_PATH} to be effective. + +@PARAM1 Integer, How many Git commits to include to the update order file, + 10000 is a good default. +@RETURNS ARRAYRef of Strings, The update order of atomicupdates from oldest to newest. +=cut + +sub buildUpdateOrderFromGit { + my ($self, $gitCommitsCount) = @_; + + my %orderedCommits; #Store the commits we have ordered here, so we don't reorder any followups. + my @orderedCommits; + + my $i = 0; #Index of array where we push issue_ids + my $commits = $self->_getGitCommits($gitCommitsCount); + foreach my $commit (reverse @$commits) { + + my ($commitHash, $commitTitle) = $self->_parseGitOneliner($commit); + unless ($commitHash && $commitTitle) { + next(); + } + + my $issueId; + try { + $issueId = Koha::AtomicUpdate->getIssueIdentifier($commitTitle); + } catch { + if (blessed($_)) { + if($_->isa('Koha::Exception::Parse')) { + #Silently ignore parsing errors + print "Koha::AtomicUpdater->buildUpdateOrderFromGit():> Couldn't parse issue_id from Git commit title '$commitTitle'.\n" + if $self->{verbose} > 1; + } + else { + $_->rethrow(); + } + } + else { + die $_; + } + }; + next unless $issueId; + + if ($orderedCommits{ $issueId }) { + next(); + } + else { + $orderedCommits{ $issueId } = $issueId; + $orderedCommits[$i] = $issueId; + $i++; + } + } + + $self->_saveAsUpdateOrder(\@orderedCommits); + return \@orderedCommits; +} + +sub _getGitCommits { + my ($self, $count) = @_; + my $repo = Git->repository(Directory => $self->{gitRepo}); + + #We can read and print 10000 git commits in less than three seconds :) good Git! + my @commits = $repo->command('show', '--pretty=oneline', '--no-patch', '-'.$count); + return \@commits; +} + +sub _parseGitOneliner { + my ($self, $gitLiner) = @_; + + my ($commitHash, $commitTitle) = ($1, $2) if $gitLiner =~ /^(\w{40}) (.+)$/; + unless ($commitHash && $commitTitle) { + print "Koha::AtomicUpdater->parseGitOneliner():> Couldn't parse Git commit '$gitLiner' to hash and title.\n" + if $self->{verbose} > 1; + return(); + } + return ($commitHash, $commitTitle); +} + +sub _saveAsUpdateOrder { + my ($self, $orderedUpdates) = @_; + + my $updateOrderFilepath = $self->{scriptDir}."/$updateOrderFilename"; + my $text = join("\n", @$orderedUpdates); + open(my $FH, ">:encoding(UTF-8)", $updateOrderFilepath) or die "Koha::AtomicUpdater->_saveAsUpdateOrder():> Couldn't open the updateOrderFile for writing\n$!\n"; + print $FH $text; + close $FH; +} + +1; --- a/Koha/Schema/Result/Atomicupdate.pm +++ a/Koha/Schema/Result/Atomicupdate.pm @@ -0,0 +1,93 @@ +use utf8; +package Koha::Schema::Result::Atomicupdate; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +=head1 NAME + +Koha::Schema::Result::Atomicupdate + +=cut + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +=head1 TABLE: C + +=cut + +__PACKAGE__->table("atomicupdates"); + +=head1 ACCESSORS + +=head2 atomicupdate_id + + data_type: 'integer' + extra: {unsigned => 1} + is_auto_increment: 1 + is_nullable: 0 + +=head2 issue_id + + data_type: 'varchar' + is_nullable: 0 + size: 20 + +=head2 filename + + data_type: 'varchar' + is_nullable: 0 + size: 30 + +=head2 modification_time + + data_type: 'timestamp' + datetime_undef_if_invalid: 1 + default_value: current_timestamp + is_nullable: 0 + +=cut + +__PACKAGE__->add_columns( + "atomicupdate_id", + { + data_type => "integer", + extra => { unsigned => 1 }, + is_auto_increment => 1, + is_nullable => 0, + }, + "issue_id", + { data_type => "varchar", is_nullable => 0, size => 20 }, + "filename", + { data_type => "varchar", is_nullable => 0, size => 30 }, + "modification_time", + { + data_type => "timestamp", + datetime_undef_if_invalid => 1, + default_value => \"current_timestamp", + is_nullable => 0, + }, +); + +=head1 PRIMARY KEY + +=over 4 + +=item * L + +=back + +=cut + +__PACKAGE__->set_primary_key("atomicupdate_id"); + + +# Created by DBIx::Class::Schema::Loader v0.07039 @ 2015-08-20 16:04:49 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:vECF28CFdwiSozjA4WL7DA + + +# You can replace this text with custom code or comments, and it will be preserved on regeneration +1; --- a/installer/data/mysql/atomicupdate.pl +++ a/installer/data/mysql/atomicupdate.pl @@ -0,0 +1,148 @@ +#!/usr/bin/perl +# +# Copyright Vaara-kirjastot 2015 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +use Modern::Perl; +use Getopt::Long; + +use C4::Context; + +use Koha::AtomicUpdater; + +my $verbose = 0; +my $help = 0; +my $apply = 0; +my $remove = ''; +my $insert = ''; +my $list = 0; +my $pending = 0; +my $directory = ''; +my $git = ''; + +GetOptions( + 'v|verbose:i' => \$verbose, + 'h|help' => \$help, + 'a|apply' => \$apply, + 'd|directory:s' => \$directory, + 'r|remove:s' => \$remove, + 'i|insert:s' => \$insert, + 'l|list' => \$list, + 'p|pending' => \$pending, + 'g|git:s' => \$git, +); + +my $usage = << 'ENDUSAGE'; + +Runs all the not-yet-applied atomicupdate-scripts and sql in the +atomicupdates-directory, in the order specified by the _updateorder-file. + +This script uses koha.atomicupdates-table to see if the update has already been +applied. + +Also acts as a gateway to CRUD the koha.database_updates-table. + + -v --verbose Integer, 1 is not so verbose, 3 is maximally verbose. + -h --help Flag, This nice help! + -a --apply Flag, Apply all the pending atomicupdates from the + atomicupdates-directory. + -d --directory Path, From which directory to look for atomicupdate-scripts. + Defaults to '$KOHA_PATH/installer/data/mysql/atomicupdate/' + -r --remove String, Remove the upgrade entry from koha.database_updates + eg. --remove "Bug71337" + -i --insert Path, Add an upgrade log entry for the given atomicupdate-file. + Useful to revert an accidental --remove -operation or for + testing. + eg. -i installer/data/mysql/atomicupdate/Bug5453-Example.pl + -l --list Flag, List all entries in the koha.database_updates-table. + This typically means all applied atomicupdates. + -p --pending Flag, List all pending atomicupdates from the + atomicupdates-directory. + -g --git Path, Build the update order from the Git repository given, + or default to the Git repository in $KOHA_PATH. + Eg. --git 1, to build with default values, or + --git /tmp/kohaclone/ to look for another repository + +EXAMPLES: + + atomicupdate.pl -g 1 -a + +Looks for the Git repository in $KOHA_PATH, parses the issue/commit identifiers +from the top 10000 commits and generates the _updateorder-file to tell in which +order the atomicupdates-scripts are executed. +Then applies all pending atomicupdate-scripts in the order (oldest to newest) +presented in the Git repository. + + + atomicupdate --apply -d /home/koha/kohaclone/installer/data/mysql/atomicupdate/ + +Applies all pending atomicupdate-scripts from the given directory. If the file +'_updateorder' is not present, it must be first generated, for example with the +--git 1 argument. + +UPDATEORDER: + +When deploying more than one atomicupdate, it is imperative to know in which order +the updates are applied. Atomicupdates can easily depend on each other and fail in +very strange and hard-to-debug -ways if the prerequisite modifications are not +in effect. +The correct update order is defined in the atomicupdates/_updateorder-file. This is +a simple list of issue/commit identifiers, eg. + + Bug5454 + Bug12432 + Bug3218 + #45 + +This file is most easily generated directly from the original Git repository, since +the order in which the Commits have been introduced most definetely is the order +they should be applied. +When deploying the atomicupdates to production environments without the +Git repository, the _updateorder file must be copied along the atomicupdate-scripts. + +P.S. Remember to put atomicupdate/_updateorder to your .gitignore + +ENDUSAGE + +if ( $help ) { + print $usage; + exit; +} + +my $atomicupdater = Koha::AtomicUpdater->new({verbose => $verbose, + scriptDir => $directory, + gitRepo => (length($git) == 1) ? '' : $git}); + +if ($git) { + $atomicupdater->buildUpdateOrderFromGit(10000); +} +if ($remove) { + $atomicupdater->removeAtomicUpdate($remove); +} +if ($insert) { + $atomicupdater->addAtomicUpdate({filename => $insert}); +} +if ($list) { + print $atomicupdater->listToConsole(); +} +if ($pending) { + print $atomicupdater->listPendingToConsole(); +} +if ($apply) { + $atomicupdater->applyAtomicUpdates(); +} --- a/installer/data/mysql/atomicupdate/Bug14698-AtomicUpdater.pl +++ a/installer/data/mysql/atomicupdate/Bug14698-AtomicUpdater.pl @@ -0,0 +1,36 @@ +#!/usr/bin/perl + +# Copyright Open Source Freedom Fighters +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use C4::Context; + +my $dbh = C4::Context->dbh(); + +$dbh->do(" +CREATE TABLE `atomicupdates` ( + `atomicupdate_id` int(11) unsigned NOT NULL auto_increment, + `issue_id` varchar(20) NOT NULL, + `filename` varchar(30) NOT NULL, + `modification_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`atomicupdate_id`), + UNIQUE KEY `origincode` (`issue_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; +"); +$dbh->do("INSERT INTO atomicupdates (issue_id, filename) VALUES ('Bug14698', 'Bug14698-AtomicUpdater.pl')"); + +print "Upgrade to Bug 14698 - AtomicUpdater - Keeps track of which updates have been applied to a database done\n"; --- a/installer/data/mysql/kohastructure.sql +++ a/installer/data/mysql/kohastructure.sql @@ -1784,6 +1784,20 @@ CREATE TABLE `printers_profile` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; -- +-- Table structure for table `atomicupdates` +-- + +DROP TABLE IF EXISTS `atomicupdates`; +CREATE TABLE `atomicupdates` ( + `atomicupdate_id` int(11) unsigned NOT NULL auto_increment, + `issue_id` varchar(20) NOT NULL, + `filename` varchar(128) NOT NULL, + `modification_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`atomicupdate_id`), + UNIQUE KEY `atomic_issue_id` (`issue_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; + +-- -- Table structure for table `repeatable_holidays` -- --- a/t/db_dependent/Koha/AtomicUpdater.t +++ a/t/db_dependent/Koha/AtomicUpdater.t @@ -0,0 +1,274 @@ +#!/usr/bin/perl + +# Copyright 2015 Open Source Freedom Fighters +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; +use Test::More; +use Try::Tiny; +use Encode; + +use t::lib::TestObjects::ObjectFactory; +use t::lib::TestObjects::AtomicUpdateFactory; +use t::lib::TestObjects::FileFactory; +use Koha::AtomicUpdater; + +my $testContext = {}; +my $atomicupdates = t::lib::TestObjects::AtomicUpdateFactory->createTestGroup([ + {issue_id => 'Bug12', + filename => 'Bug12-WatchExMachinaYoullLikeIt.pl'}, + {issue_id => 'Bug14', + filename => 'Bug14-ReturnOfZorro.perl'}, + {issue_id => '#14', + filename => '#14-RobotronInDanger.sql'}, + {issue_id => '#15', + filename => '#15-ILikedPrometheusButAlienWasBetter.pl'}, + ], undef, $testContext); + +#Make sure we get the correct update order, otherwise we get unpredictable results. +{ #Overload existing subroutines to provide a Mock implementation + no warnings 'redefine'; + package Koha::AtomicUpdater; + sub _getGitCommits { #instead of requiring a Git repository, we just mock the input. + return [#Newest commit + '2e8a39762b506738195f21c8ff67e4e7bfe6d7ab #:-55 : Fiftyfive', + '2e8a39762b506738195f21c8ff67e4e7bfe6d7ab #54 - KohaCon in Finland next year', + 'b447b595acacb0c4823582acf9d8a08902118e59 #53 - Place to be.pl', + '2e8a39762b506738195f21c8ff67e4e7bfe6d7ab bug 112 - Lapinlahden linnut', + '5ac7101d4071fe11f7a5d1445bb97ed1a603a9b5 Bug:-911 - What are you going to do?', + '1d54601b9cac0bd75ee97e071cf52ed49daef8bd #911 - Who are you going to call', + '1d54601b9cac0bd75ee97e071cf52ed49daef8bd bug 30 - Feature Yes yes', + '5ac7101d4071fe11f7a5d1445bb97ed1a603a9b5 #-29 - Bug squashable', + '2e8a39762b506738195f21c8ff67e4e7bfe6d7ab Bug :- 28 - Feature Squash', + 'b447b595acacb0c4823582acf9d8a08902118e59 BUG 27 - Bug help', + #Oldest commit + ]; + } +} + +subtest "Create update order from Git repository" => \&createUpdateOrderFromGit; +sub createUpdateOrderFromGit { + eval { + #Create the _updateorder-file to a temp directory and prepare it for autocleanup. + my $files = t::lib::TestObjects::FileFactory->createTestGroup([ + { filepath => 'atomicupdate/', + filename => '_updateorder', + content => '',}, + ], + undef, undef, $testContext); + #Instantiate the AtomicUpdater to operate on a temp directory. + my $atomicUpdater = Koha::AtomicUpdater->new({ + scriptDir => $files->{'_updateorder'}->dirname(), + }); + + #Start real testing. + my $issueIds = $atomicUpdater->buildUpdateOrderFromGit(4); + + is($issueIds->[0], + 'Bug27', + "First atomicupdate to deploy"); + is($issueIds->[1], + 'Bug28', + "Second atomicupdate to deploy"); + is($issueIds->[2], + '#29', + "Third atomicupdate to deploy"); + is($issueIds->[3], + 'Bug30', + "Last atomicupdate to deploy"); + + #Testing file access + $issueIds = $atomicUpdater->getUpdateOrder(); + is($issueIds->[0], + 'Bug27', + "First atomicupdate to deploy, from _updateorder"); + is($issueIds->[1], + 'Bug28', + "Second atomicupdate to deploy, from _updateorder"); + is($issueIds->[2], + '#29', + "Third atomicupdate to deploy, from _updateorder"); + is($issueIds->[3], + 'Bug30', + "Last atomicupdate to deploy, from _updateorder"); + }; + if ($@) { + ok(0, $@); + } +} + + + +subtest "List all deployed atomicupdates" => \&listAtomicUpdates; +sub listAtomicUpdates { + eval { + my $atomicUpdater = Koha::AtomicUpdater->new(); + my $text = $atomicUpdater->listToConsole(); + print $text; + + ok($text =~ m/Bug12-WatchExMachinaYoullLik/, + "Bug12-WatchExMachinaYoullLikeIt"); + ok($text =~ m/Bug14-ReturnOfZorro.perl/, + "Bug14-ReturnOfZorro"); + ok($text =~ m/#14-RobotronInDanger.sql/, + "#14-RobotronInDanger"); + ok($text =~ m/#15-ILikedPrometheusButAli/, + "#15-ILikedPrometheusButAlienWasBetter"); + + }; + if ($@) { + ok(0, $@); + } +} + +subtest "Delete an atomicupdate entry" => \&deleteAtomicupdate; +sub deleteAtomicupdate { + eval { + my $atomicUpdater = Koha::AtomicUpdater->new(); + my $atomicupdate = $atomicUpdater->cast($atomicupdates->{Bug12}->id); + ok($atomicupdate, + "AtomicUpdate '".$atomicupdates->{Bug12}->issue_id."' exists prior to deletion"); + + $atomicUpdater->removeAtomicUpdate($atomicupdate->issue_id); + $atomicupdate = $atomicUpdater->find($atomicupdates->{Bug12}->id); + ok(not($atomicupdate), + "AtomicUpdate '".$atomicupdates->{Bug12}->issue_id."' deleted"); + + }; + if ($@) { + ok(0, $@); + } +} + +subtest "Insert an atomicupdate entry" => \&insertAtomicupdate; +sub insertAtomicupdate { + eval { + my $atomicUpdater = Koha::AtomicUpdater->new(); + my $subtestContext = {}; + my $atomicupdates = t::lib::TestObjects::AtomicUpdateFactory->createTestGroup([ + {issue_id => 'Bug15', + filename => 'Bug15-Inserted.pl'}, + ], undef, $subtestContext, $testContext); + my $atomicupdate = $atomicUpdater->find({issue_id => 'Bug15'}); + ok($atomicupdate, + "Bug15-Inserted.pl"); + + t::lib::TestObjects::ObjectFactory->tearDownTestContext($subtestContext); + + $atomicupdate = $atomicUpdater->find({issue_id => 'Bug15'}); + ok(not($atomicupdate), + "Bug15-Inserted.pl deleted"); + }; + if ($@) { + ok(0, $@); + } +} + +subtest "List pending atomicupdates" => \&listPendingAtomicupdates; +sub listPendingAtomicupdates { + my ($atomicUpdater, $files, $text, $atomicupdates); + my $subtestContext = {}; + eval { + ##Test adding update scripts and deploy them, confirm that no pending scripts detected + $files = t::lib::TestObjects::FileFactory->createTestGroup([ + { filepath => 'atomicupdate/', + filename => '#911-WhoYouGonnaCall.pl', + content => '$ENV{ATOMICUPDATE_TESTS} = 1;',}, + { filepath => 'atomicupdate/', + filename => 'Bug911-WhatchaGonnaDo.pl', + content => '$ENV{ATOMICUPDATE_TESTS}++;',}, + { filepath => 'atomicupdate/', + filename => 'Bug112-LapinlahdenLinnut.pl', + content => '$ENV{ATOMICUPDATE_TESTS}++;',}, + ], + undef, $subtestContext, $testContext); + $atomicUpdater = Koha::AtomicUpdater->new({ + scriptDir => $files->{'#911-WhoYouGonnaCall.pl'}->dirname() + }); + + $text = $atomicUpdater->listPendingToConsole(); + print $text; + + ok($text =~ m/#911-WhoYouGonnaCall.pl/, + "#911-WhoYouGonnaCall is pending"); + ok($text =~ m/Bug911-WhatchaGonnaDo.pl/, + "Bug911-WhatchaGonnaDo is pending"); + ok($text =~ m/Bug112-LapinlahdenLinnut.pl/, + 'Bug112-LapinlahdenLinnut is pending'); + + $atomicupdates = $atomicUpdater->applyAtomicUpdates(); + t::lib::TestObjects::AtomicUpdateFactory->addToContext($atomicupdates, undef, $subtestContext, $testContext); #Keep track of changes + + is($atomicupdates->{'#911'}->issue_id, + '#911', + "#911-WhoYouGonnaCall.pl deployed"); + is($atomicupdates->{'Bug112'}->issue_id, + 'Bug112', + 'Bug112-LapinlahdenLinnut.pl deployed'); + is($atomicupdates->{'Bug911'}->issue_id, + 'Bug911', + "Bug911-WhatchaGonnaDo.pl deployed"); + + ##Test adding scripts to the atomicupdates directory and how we deal with such change. + $files = t::lib::TestObjects::FileFactory->createTestGroup([ + { filepath => 'atomicupdate/', + filename => '#53-PlaceToBe.pl', + content => '$ENV{ATOMICUPDATE_TESTS}++;',}, + { filepath => 'atomicupdate/', + filename => '#54-KohaConInFinlandNextYear.pl', + content => '$ENV{ATOMICUPDATE_TESTS}++;',}, + { filepath => 'atomicupdate/', + filename => '#55-Fiftyfive.pl', + content => '$ENV{ATOMICUPDATE_TESTS}++;',}, + ], + undef, $subtestContext, $testContext); + + $text = $atomicUpdater->listPendingToConsole(); + print $text; + + ok($text =~ m/#53-PlaceToBe.pl/, + "#53-PlaceToBe.pl is pending"); + ok($text =~ m/#54-KohaConInFinlandNextYear.pl/, + "#54-KohaConInFinlandNextYear.pl is pending"); + ok($text =~ m/#55-Fiftyfive.pl/u, + '#55-Fiftyfive.pl'); + + $atomicupdates = $atomicUpdater->applyAtomicUpdates(); + t::lib::TestObjects::AtomicUpdateFactory->addToContext($atomicupdates, undef, $subtestContext, $testContext); #Keep track of changes + + is($atomicupdates->{'#53'}->issue_id, + '#53', + "#53-PlaceToBe.pl deployed"); + is($atomicupdates->{'#54'}->issue_id, + '#54', + '#54-KohaConInFinlandNextYear.pl deployed'); + is($atomicupdates->{'#55'}->issue_id, + '#55', + "#55-Fiftyfive.pl deployed"); + + is($ENV{ATOMICUPDATE_TESTS}, + 6, + "All configured AtomicUpdates deployed"); + }; + if ($@) { + ok(0, $@); + } + t::lib::TestObjects::AtomicUpdateFactory->tearDownTestContext($subtestContext); +} + +t::lib::TestObjects::ObjectFactory->tearDownTestContext($testContext); +done_testing; --- a/t/lib/TestObjects/AtomicUpdateFactory.pm +++ a/t/lib/TestObjects/AtomicUpdateFactory.pm @@ -0,0 +1,105 @@ +package t::lib::TestObjects::AtomicUpdateFactory; + +# Copyright Vaara-kirjastot 2015 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +use Modern::Perl; +use Carp; + +use Koha::AtomicUpdater; +use Koha::Database; + +use Koha::Exception::UnknownProgramState; + +use base qw(t::lib::TestObjects::ObjectFactory); + +sub getDefaultHashKey { + return 'issue_id'; +} +sub getObjectType { + return 'Koha::AtomicUpdate'; +} + +=head t::lib::TestObjects::createTestGroup + + my $atomicupdates = t::lib::TestObjects::AtomicUpdateFactory->createTestGroup([ + {'issue_id' => 'Bug3432', + 'filename' => 'Bug3432-RavingRabbitsMayhem', + 'modification_time' => '2015-01-02 15:59:32', + }, + ], undef, $testContext1, $testContext2, $testContext3); + +Calls Koha::AtomicUpdater to add a Koha::AtomicUpdate object to DB. + +The HASH is keyed with the 'koha.atomicupdates.issue_id', or the given $hashKey. + +There is a duplication check to first look for atomicupdate-rows with the same 'issue_id'. +If a matching atomicupdate is found, then we use the existing Record instead of adding a new one. + +@RETURNS HASHRef of Koha::AtomicUpdate-objects + +See t::lib::TestObjects::ObjectFactory for more documentation +=cut + +sub handleTestObject { + my ($class, $object, $stashes) = @_; + + ##First see if the given Record already exists in the DB. For testing purposes we use the isbn as the UNIQUE identifier. + my $atomicupdate = Koha::AtomicUpdater->find({issue_id => $object->{issue_id}}); + unless ($atomicupdate) { + my $atomicupdater = Koha::AtomicUpdater->new(); + $atomicupdate = $atomicupdater->addAtomicUpdate($object); + } + + Koha::Exception::UnknownProgramState->throw(error => "$class->handleTestObject():> Cannot create a new object\n$@\n") + unless $atomicupdate; + + return $atomicupdate; +} + +=head validateAndPopulateDefaultValues +@OVERLOAD + +Validates given Object parameters and makes sure that critical fields are given +and populates defaults for missing values. +=cut + +sub validateAndPopulateDefaultValues { + my ($self, $object, $hashKey) = @_; + + $object->{issue_id} = 'BugRancidacid' unless $object->{issue_id}; + $object->{filename} = 'BugRancidacid-LaboratoryExperimentsGoneSour' unless $object->{filename}; + + $self->SUPER::validateAndPopulateDefaultValues($object, $hashKey); +} + +sub deleteTestGroup { + my ($class, $objects) = @_; + + while( my ($key, $object) = each %$objects) { + my $atomicupdate = Koha::AtomicUpdater->cast($object); + eval { + $atomicupdate->delete(); + }; + if ($@) { + warn "$class->deleteTestGroup():> Error hapened: $@\n"; + } + } +} + +1; --- a/t/lib/TestObjects/ObjectFactory.pm +++ a/t/lib/TestObjects/ObjectFactory.pm @@ -166,6 +166,11 @@ sub tearDownTestContext { t::lib::TestObjects::BiblioFactory->deleteTestGroup($stash->{biblio}); delete $stash->{biblio}; } + if ($stash->{atomicupdate}) { + require t::lib::TestObjects::AtomicUpdateFactory; + t::lib::TestObjects::AtomicUpdateFactory->deleteTestGroup($stash->{atomicupdate}); + delete $stash->{atomicupdate}; + } if ($stash->{borrower}) { require t::lib::TestObjects::BorrowerFactory; t::lib::TestObjects::BorrowerFactory->deleteTestGroup($stash->{borrower}); --- a/t/lib/TestObjects/objectFactories.t +++ a/t/lib/TestObjects/objectFactories.t @@ -30,6 +30,8 @@ use t::lib::TestObjects::BorrowerFactory; use Koha::Borrowers; use t::lib::TestObjects::ItemFactory; use Koha::Items; +use t::lib::TestObjects::AtomicUpdateFactory; +use Koha::AtomicUpdater; use t::lib::TestObjects::BiblioFactory; use Koha::Biblios; use t::lib::TestObjects::CheckoutFactory; @@ -115,6 +117,7 @@ sub testSerialFactory { my ($subscriptions, $subscription, $frequency, $numberpattern, $biblio, $sameBiblio, $borrower, $bookseller, $items, $serials); my $subtestContext = {}; my $dontDeleteTestContext = {}; + eval { ##Create and delete $subscriptions = t::lib::TestObjects::Serial::SubscriptionFactory->createTestGroup([ {internalnotes => 'TESTDEFAULTS', @@ -219,6 +222,10 @@ sub testSerialFactory { ok(defined($borrower), "Attached Borrower not deleted."); $bookseller = Koha::Acquisition::Booksellers->find( $bookseller->id ); ok(defined($bookseller), "Attached Bookseller not deleted."); + }; + if ($@) { + ok(0, $@); + } t::lib::TestObjects::ObjectFactory->tearDownTestContext($dontDeleteTestContext); }; @@ -229,6 +236,7 @@ subtest 't::lib::TestObjects::Acquisition' => \&testAcquisitionFactories; sub testAcquisitionFactories { my ($booksellers, $bookseller, $contacts, $contact); my $subtestContext = {}; + eval { ##Create and delete $booksellers = t::lib::TestObjects::Acquisition::BooksellerFactory->createTestGroup([{}], undef, $subtestContext); $bookseller = Koha::Acquisition::Booksellers->find({name => 'Bookselling Vendor'}); @@ -256,6 +264,11 @@ sub testAcquisitionFactories { ok(not(defined($contact)), "Contact 'Hippocrates' deleted."); $bookseller = Koha::Acquisition::Booksellers->find({name => 'Bookselling Vendor'}); ok(not(defined($bookseller)), "Bookseller 'Bookselling Vendor' deleted."); + }; + if ($@) { + ok(0, $@); + } + t::lib::TestObjects::ObjectFactory->tearDownTestContext($subtestContext); }; @@ -472,6 +485,43 @@ sub testLetterTemplateFactory { +########## AtomicUpdateFactory subtests ########## +subtest 't::lib::TestObjects::AtomicUpdateFactory' => \&testAtomicUpdateFactory; +sub testAtomicUpdateFactory { + my ($atomicUpdater, $atomicupdate); + my $subtestContext = {}; + ##Create and Delete using dependencies in the $testContext instantiated in previous subtests. + my $atomicupdates = t::lib::TestObjects::AtomicUpdateFactory->createTestGroup([ + {'issue_id' => 'Bug10', + 'filename' => 'Bug10-RavingRabbitsMayhem.pl', + 'modification_time' => '2015-01-02 15:59:32',}, + {'issue_id' => 'Bug11', + 'filename' => 'Bug11-RancidSausages.perl', + 'modification_time' => '2015-01-02 15:59:33',}, + ], + undef, $subtestContext); + $atomicUpdater = Koha::AtomicUpdater->new(); + $atomicupdate = $atomicUpdater->find({issue_id => $atomicupdates->{Bug10}->issue_id}); + is($atomicupdate->issue_id, + 'Bug10', + "Bug10-RavingRabbitsMayhem created"); + $atomicupdate = $atomicUpdater->find({issue_id => $atomicupdates->{Bug11}->issue_id}); + is($atomicupdate->issue_id, + 'Bug11', + "Bug11-RancidSausages created"); + + t::lib::TestObjects::ObjectFactory->tearDownTestContext($subtestContext); + + $atomicupdate = $atomicUpdater->find({issue_id => $atomicupdates->{Bug10}->issue_id}); + ok(not($atomicupdate), + "Bug10-RavingRabbitsMayhem deleted"); + $atomicupdate = $atomicUpdater->find({issue_id => $atomicupdates->{Bug11}->issue_id}); + ok(not($atomicupdate), + "Bug11-RancidSausages created"); +}; + + + ########## SystemPreferenceFactory subtests ########## subtest 't::lib::TestObjects::SystemPreferenceFactory' => \&testSystemPreferenceFactory; sub testSystemPreferenceFactory { --