From fbcd15ac98c987639a7335fed98d96ca99f8cd7a Mon Sep 17 00:00:00 2001 From: Olli-Antti Kivilahti Date: Thu, 20 Aug 2015 14:10:02 +0300 Subject: [PATCH] Bug 14698 - AtomicUpdater - Keeps track of which updates have been applied to a database When deploying production databases to test environments, it is imperative to keep track of which database changes have been applied. When one is running, in production, features that require DB changes, and want to add more features which need more DB changes, keeping track of which updates have been applied is challenging. After several cycles of upgrade-deploy, with multiple atomicupdate-scripts piling to the atomicupdates/-directory, it is impossible to easily distinguish which updates have been applied and which are not. Rerunning the same update scripts again and again causes lots of noise in the upgrade log and hides real issues from being detected. Also repeatedly running upgrade scripts might cause bad side effects which are potentially hard to repair. This feature adds a script atomicupdate.pl which runs all atomicupdate/* scripts and .sqls and logs the event to the koha.atomicupdates-table. On subsequent runs of atomicupdate.pl, the already deployed upgrades are skipped, greatly reducing the unnecessary clutter and risk of upgrading. Also the order in which the atomicupdates need to be applied is important. This _updateorder is automatically generatable from the original Git repository. You can also remove existing log entries to allow rerunning the same upgrade script again, list all applied upgrades and show pending updates. Unit tests included. --- C4/Installer/PerlDependencies.pm | 5 + Koha/AtomicUpdate.pm | 136 ++++++++ Koha/AtomicUpdater.pm | 349 +++++++++++++++++++++ Koha/Schema/Result/Atomicupdate.pm | 93 ++++++ installer/data/mysql/atomicupdate.pl | 148 +++++++++ .../mysql/atomicupdate/Bug14698-AtomicUpdater.pl | 36 +++ installer/data/mysql/kohastructure.sql | 14 + t/db_dependent/Koha/AtomicUpdater.t | 274 ++++++++++++++++ t/lib/TestObjects/AtomicUpdateFactory.pm | 105 +++++++ t/lib/TestObjects/ObjectFactory.pm | 5 + t/lib/TestObjects/objectFactories.t | 39 +++ 11 files changed, 1204 insertions(+) create mode 100644 Koha/AtomicUpdate.pm create mode 100644 Koha/AtomicUpdater.pm create mode 100644 Koha/Schema/Result/Atomicupdate.pm create mode 100644 installer/data/mysql/atomicupdate.pl create mode 100644 installer/data/mysql/atomicupdate/Bug14698-AtomicUpdater.pl create mode 100644 t/db_dependent/Koha/AtomicUpdater.t create mode 100644 t/lib/TestObjects/AtomicUpdateFactory.pm diff --git a/C4/Installer/PerlDependencies.pm b/C4/Installer/PerlDependencies.pm index 201b49e..c33f20d 100644 --- a/C4/Installer/PerlDependencies.pm +++ b/C4/Installer/PerlDependencies.pm @@ -217,6 +217,11 @@ our $PERL_DEPS = { 'min_ver' => '0.45', # Also needed for our use of PDF::Reuse }, + 'Data::Format::Pretty::Console' => { + 'usage' => 'Core', + 'required' => '1', + 'min_ver' => '0.34', + }, 'DateTime' => { 'usage' => 'Core', 'required' => '1', diff --git a/Koha/AtomicUpdate.pm b/Koha/AtomicUpdate.pm new file mode 100644 index 0000000..3242aeb --- /dev/null +++ b/Koha/AtomicUpdate.pm @@ -0,0 +1,136 @@ +package Koha::AtomicUpdate; + +# Copyright Open Source Freedom Fighters +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use Carp; +use File::Basename; + +use Koha::Database; + +use base qw(Koha::Object); + +use Koha::Exception::BadParameter; + +sub type { + return 'Atomicupdate'; +} + +=head @allowedIssueIdentifierPrefixes +Define the prefixes you want to attach to your atomicupdate filenames here. +This could be a syspref or in KOHA_CONF, but it is rather easy to just add more +generally used issue number prefixes here. +Nobody wants more sysprefs. +=cut + +my @allowedIssueIdentifierPrefixes = ( + 'Bug', + '#', +); + +=head new + + my $atomicUpdate = Koha::AtomicUpdate->new({filename => 'Bug54321-FixItPlease.pl'}); + +Creates a Koha::AtomicUpdate-object from the given parameters-HASH +@PARAM1 HASHRef of object parameters: + 'filename' => MANDATORY, The filename of the atomicupdate-script without the path-component. + 'issue_id' => OPTIONAL, the desired issue_id. It is better to let the module + find this from the filename, but is useful for testing purposes. +@RETURNS Koha::AtomicUpdate-object +@THROWS Koha::Exception::Parse from getIssueIdentifier() +@THROWS Koha::Exception::File from _validateFilename(); +=cut + +sub new { + my ($class, $params) = @_; + $class->_validateParams($params); + + my $self = {}; + bless($self, $class); + $self->set($params); + return $self; +} + +sub _validateParams { + my ($class, $params) = @_; + + my @mandatoryParams = ('filename'); + foreach my $mp (@mandatoryParams) { + Koha::Exception::BadParameter->throw( + error => "$class->_validateParams():> Param '$mp' must be given.") + unless($params->{$mp}); + } + $params->{filename} = $class->_validateFilename($params->{filename}); + + $params->{issue_id} = $class->getIssueIdentifier($params->{issue_id} || $params->{filename}); +} + +=head _validateFilename + +Makes sure the given file is a valid AtomicUpdate-script. +Currently simply checks for naming convention and file suffix. + +NAMING CONVENTION: + Filename must contain one of the unique issue identifier prefixes from this + list @allowedIssueIdentifierPrefixes immediately followed by the numeric + id of the issue, optionally separated by any of the following [ :-] + Eg. Bug-45453, #102, #:53 + +@PARAM1 String, filename of validatable file, excluding path. +@RETURNS String, the koha.atomicupdates.filename if the given file is considered a well formed update script. + Removes the full path if present and returns only the filename component. + +@THROWS Koha::Exception::File, if the given file doesn't have a proper naming convention + +=cut + +sub _validateFilename { + my ($self, $fileName) = @_; + + Koha::Exception::File->throw(error => __PACKAGE__."->_validateFilename():> Filename '$fileName' has unknown suffix") + unless $fileName =~ /\.(sql|perl|pl)$/; #skip other files + + $fileName = File::Basename::basename($fileName); + + return $fileName; +} + +=head getIssueIdentifier + +Extracts the unique issue identifier from the atomicupdate DB upgrade script. + +@PARAM1 String, filename of validatable file, excluding path, or Git commit title, + or something else to parse. +@RETURNS String, The unique issue identifier + +@THROWS Koha::Exception::Parse, if the unique identifier couldn't be parsed. +=cut + +sub getIssueIdentifier { + my ($self, $fileName) = @_; + + foreach my $prefix (@allowedIssueIdentifierPrefixes) { + if ($fileName =~ m/$prefix[ -:]*?(\d+)/i) { + return ucfirst("$prefix$1"); + } + } + Koha::Exception::Parse->throw(error => __PACKAGE__."->getIssueIdentifier($fileName):> couldn't parse the unique issue identifier from filename using allowed prefixes '@allowedIssueIdentifierPrefixes'"); +} + +1; diff --git a/Koha/AtomicUpdater.pm b/Koha/AtomicUpdater.pm new file mode 100644 index 0000000..d76c851 --- /dev/null +++ b/Koha/AtomicUpdater.pm @@ -0,0 +1,349 @@ +package Koha::AtomicUpdater; + +# Copyright Open Source Freedom Fighters +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use Carp; +use Scalar::Util qw(blessed); +use Try::Tiny; +use Data::Format::Pretty::Console qw(format_pretty); +use Git; + +use Koha::Database; +use Koha::Cache; +use Koha::AtomicUpdate; + +use base qw(Koha::Objects); + +use Koha::Exception::File; +use Koha::Exception::Parse; +use Koha::Exception::BadParameter; + +sub type { + return 'Atomicupdate'; +} + +sub object_class { + return 'Koha::AtomicUpdate'; +} + +sub _get_castable_unique_columns { + return ['atomicupdate_id']; +} + +my $updateOrderFilename = '_updateorder'; + +sub new { + my ($class, $params) = @_; + + my $cache = Koha::Cache->new(); + my $self = $cache->get_from_cache('Koha::AtomicUpdater') || {}; + bless($self, $class); + + $self->{verbose} = $params->{verbose} || $self->{verbose} || 0; + $self->{scriptDir} = $params->{scriptDir} || $self->{scriptDir} || C4::Context->config('intranetdir') . '/installer/data/mysql/atomicupdate/'; + $self->{gitRepo} = $params->{gitRepo} || $self->{gitRepo} || $ENV{KOHA_PATH}; + + return $self; +} + +=head getAtomicUpdates + + my $atomicUpdates = $atomicUpdater->getAtomicUpdates(); + +Gets all the AtomicUpdate-objects in the DB. This result should be Koha::Cached. +@RETURNS HASHRef of Koha::AtomicUpdate-objects, keyed with the issue_id +=cut + +sub getAtomicUpdates { + my ($self) = @_; + + my @au = $self->search({}); + my %au; #HASHify the AtomicUpdate-objects for easy searching. + foreach my $au (@au) { + $au{$au->issue_id} = $au; + } + return \%au; +} + +sub addAtomicUpdate { + my ($self, $params) = @_; + print "Adding atomicupdate '".$params->{issue_id}."'\n" if $self->{verbose} > 2; + + my $atomicupdate = Koha::AtomicUpdate->new($params); + $atomicupdate->store(); + $atomicupdate = $self->find({issue_id => $atomicupdate->issue_id}); + return $atomicupdate; +} + +sub removeAtomicUpdate { + my ($self, $issueId) = @_; + print "Deleting atomicupdate '$issueId'\n" if $self->{verbose} > 2; + + my $atomicupdate = $self->find({issue_id => $issueId}); + if ($atomicupdate) { + $atomicupdate->delete; + print "Deleted atomicupdate '$issueId'\n" if $self->{verbose} > 2; + } + else { + Koha::Exception::BadParameter->throw(error => __PACKAGE__."->removeIssueFromLog():> No such Issue '$issueId' stored to the atomicupdates-table"); + } +} + +sub listToConsole { + my ($self) = @_; + my @stringBuilder; + + my @atomicupdates = $self->search({}); + foreach my $au (@atomicupdates) { + push @stringBuilder, $au->unblessed(); + } + return Data::Format::Pretty::Console::format_pretty(\@stringBuilder); +} + +sub listPendingToConsole { + my ($self) = @_; + my @stringBuilder; + + my $atomicUpdates = $self->getPendingAtomicUpdates(); + foreach my $key (sort keys %$atomicUpdates) { + my $au = $atomicUpdates->{$key}; + push @stringBuilder, $au->unblessed(); + } + return Data::Format::Pretty::Console::format_pretty(\@stringBuilder); +} + +sub getPendingAtomicUpdates { + my ($self) = @_; + + my %pendingAtomicUpdates; + my $atomicupdateFiles = $self->_getValidAtomicUpdateScripts(); + my $atomicUpdatesDeployed = $self->getAtomicUpdates(); + foreach my $key (keys(%$atomicupdateFiles)) { + my $au = $atomicupdateFiles->{$key}; + unless ($atomicUpdatesDeployed->{$au->issue_id}) { + #This script hasn't been deployed. + $pendingAtomicUpdates{$au->issue_id} = $au; + } + } + return \%pendingAtomicUpdates; +} + +=head applyAtomicUpdates + + my $atomicUpdater = Koha::AtomicUpdater->new(); + my $appliedAtomicupdates = $atomicUpdater->applyAtomicUpdates(); + +Checks the atomicupdates/-directory for any not-applied update scripts and +runs them in the order specified in the _updateorder-file in atomicupdate/-directory. + +@RETURNS ARRAYRef of Koha::AtomicUpdate-objects deployed on this run +=cut + +sub applyAtomicUpdates { + my ($self) = @_; + + my %appliedUpdates; + + my $atomicUpdates = $self->getPendingAtomicUpdates(); + my $updateOrder = $self->getUpdateOrder(); + foreach my $issueId ( @$updateOrder ) { + my $atomicUpdate = $atomicUpdates->{$issueId}; + next unless $atomicUpdate; #Not each ordered Git commit necessarily have a atomicupdate-script. + + my $filename = $atomicUpdate->filename; + print "Looking at file '$filename'\n" if $self->{verbose} > 2; + + if ( $filename =~ /\.sql$/ ) { + my $installer = C4::Installer->new(); + my $rv = $installer->load_sql( $self->{scriptDir}.'/'.$filename ) ? 0 : 1; + } elsif ( $filename =~ /\.(perl|pl)$/ ) { + do $self->{scriptDir}.'/'.$filename; + } + + $atomicUpdate->store(); + $appliedUpdates{$issueId} = $atomicUpdate; + print "File '$filename' applied" if $self->{verbose} > 2; + } + + #Check that we have actually applied all the updates. + my $stillPendingAtomicUpdates = $self->getPendingAtomicUpdates(); + if (scalar(%$stillPendingAtomicUpdates)) { + my @issueIds = sort keys %$stillPendingAtomicUpdates; + print "Warning! After upgrade, the following atomicupdates are still pending '@issueIds'\n Try rebuilding the atomicupdate-scripts update order from the original Git repository.\n"; + } + + return \%appliedUpdates; +} + +=head _getValidAtomicUpdateScripts + +@RETURNS HASHRef of Koha::AtomicUpdate-objects, of all the files + in the atomicupdates/-directory that can be considered valid. + Validity is currently conforming to the naming convention. + Keys are the issue_id of atomicupdate-scripts + Eg. {'Bug8584' => Koha::AtomicUpdate, + ... + } +=cut + +sub _getValidAtomicUpdateScripts { + my ($self) = @_; + + my %atomicUpdates; + opendir( my $dirh, $self->{scriptDir} ); + foreach my $file ( sort readdir $dirh ) { + print "Looking at file $file\n" if $self->{verbose} > 2; + + my $atomicUpdate; + try { + $atomicUpdate = Koha::AtomicUpdate->new({filename => $file}); + } catch { + if (blessed($_)) { + if ($_->isa('Koha::Exception::File')) { + #We can ignore filename validation issues, since the directory has + #loads of other types of files as well. Like README . .. + } + else { + $_->rethrow(); + } + } + else { + die $_; #Rethrow the unknown Exception + } + }; + next unless $atomicUpdate; + + $atomicUpdates{$atomicUpdate->issue_id} = $atomicUpdate; + } + return \%atomicUpdates; +} + +=head getUpdateOrder + + $atomicUpdater->getUpdateOrder(); + +@RETURNS ARRAYRef of Strings, IssueIds ordered from the earliest to the newest. +=cut + +sub getUpdateOrder { + my ($self) = @_; + + my $updateOrderFilepath = $self->{scriptDir}."/$updateOrderFilename"; + open(my $FH, "<:encoding(UTF-8)", $updateOrderFilepath) or die "Koha::AtomicUpdater->_saveAsUpdateOrder():> Couldn't open the updateOrderFile for reading\n$!\n"; + my @updateOrder = map {chomp($_); $_;} <$FH>; + close $FH; + return \@updateOrder; +} + +=head + + my $issueIdOrder = Koha::AtomicUpdater->buildUpdateOrderFromGit(10000); + +Creates a update order file '_updateorder' for atomicupdates to know which updates come before which. +This is a simple way to make sure the atomicupdates are applied in the correct order. +The update order file is by default in your $KOHA_PATH/installer/data/mysql/atomicupdate/_updateorder + +This requires a Git repository to be in the $ENV{KOHA_PATH} to be effective. + +@PARAM1 Integer, How many Git commits to include to the update order file, + 10000 is a good default. +@RETURNS ARRAYRef of Strings, The update order of atomicupdates from oldest to newest. +=cut + +sub buildUpdateOrderFromGit { + my ($self, $gitCommitsCount) = @_; + + my %orderedCommits; #Store the commits we have ordered here, so we don't reorder any followups. + my @orderedCommits; + + my $i = 0; #Index of array where we push issue_ids + my $commits = $self->_getGitCommits($gitCommitsCount); + foreach my $commit (reverse @$commits) { + + my ($commitHash, $commitTitle) = $self->_parseGitOneliner($commit); + unless ($commitHash && $commitTitle) { + next(); + } + + my $issueId; + try { + $issueId = Koha::AtomicUpdate->getIssueIdentifier($commitTitle); + } catch { + if (blessed($_)) { + if($_->isa('Koha::Exception::Parse')) { + #Silently ignore parsing errors + print "Koha::AtomicUpdater->buildUpdateOrderFromGit():> Couldn't parse issue_id from Git commit title '$commitTitle'.\n" + if $self->{verbose} > 1; + } + else { + $_->rethrow(); + } + } + else { + die $_; + } + }; + next unless $issueId; + + if ($orderedCommits{ $issueId }) { + next(); + } + else { + $orderedCommits{ $issueId } = $issueId; + $orderedCommits[$i] = $issueId; + $i++; + } + } + + $self->_saveAsUpdateOrder(\@orderedCommits); + return \@orderedCommits; +} + +sub _getGitCommits { + my ($self, $count) = @_; + my $repo = Git->repository(Directory => $self->{gitRepo}); + + #We can read and print 10000 git commits in less than three seconds :) good Git! + my @commits = $repo->command('show', '--pretty=oneline', '--no-patch', '-'.$count); + return \@commits; +} + +sub _parseGitOneliner { + my ($self, $gitLiner) = @_; + + my ($commitHash, $commitTitle) = ($1, $2) if $gitLiner =~ /^(\w{40}) (.+)$/; + unless ($commitHash && $commitTitle) { + print "Koha::AtomicUpdater->parseGitOneliner():> Couldn't parse Git commit '$gitLiner' to hash and title.\n" + if $self->{verbose} > 1; + return(); + } + return ($commitHash, $commitTitle); +} + +sub _saveAsUpdateOrder { + my ($self, $orderedUpdates) = @_; + + my $updateOrderFilepath = $self->{scriptDir}."/$updateOrderFilename"; + my $text = join("\n", @$orderedUpdates); + open(my $FH, ">:encoding(UTF-8)", $updateOrderFilepath) or die "Koha::AtomicUpdater->_saveAsUpdateOrder():> Couldn't open the updateOrderFile for writing\n$!\n"; + print $FH $text; + close $FH; +} + +1; diff --git a/Koha/Schema/Result/Atomicupdate.pm b/Koha/Schema/Result/Atomicupdate.pm new file mode 100644 index 0000000..419bb65 --- /dev/null +++ b/Koha/Schema/Result/Atomicupdate.pm @@ -0,0 +1,93 @@ +use utf8; +package Koha::Schema::Result::Atomicupdate; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +=head1 NAME + +Koha::Schema::Result::Atomicupdate + +=cut + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +=head1 TABLE: C + +=cut + +__PACKAGE__->table("atomicupdates"); + +=head1 ACCESSORS + +=head2 atomicupdate_id + + data_type: 'integer' + extra: {unsigned => 1} + is_auto_increment: 1 + is_nullable: 0 + +=head2 issue_id + + data_type: 'varchar' + is_nullable: 0 + size: 20 + +=head2 filename + + data_type: 'varchar' + is_nullable: 0 + size: 30 + +=head2 modification_time + + data_type: 'timestamp' + datetime_undef_if_invalid: 1 + default_value: current_timestamp + is_nullable: 0 + +=cut + +__PACKAGE__->add_columns( + "atomicupdate_id", + { + data_type => "integer", + extra => { unsigned => 1 }, + is_auto_increment => 1, + is_nullable => 0, + }, + "issue_id", + { data_type => "varchar", is_nullable => 0, size => 20 }, + "filename", + { data_type => "varchar", is_nullable => 0, size => 30 }, + "modification_time", + { + data_type => "timestamp", + datetime_undef_if_invalid => 1, + default_value => \"current_timestamp", + is_nullable => 0, + }, +); + +=head1 PRIMARY KEY + +=over 4 + +=item * L + +=back + +=cut + +__PACKAGE__->set_primary_key("atomicupdate_id"); + + +# Created by DBIx::Class::Schema::Loader v0.07039 @ 2015-08-20 16:04:49 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:vECF28CFdwiSozjA4WL7DA + + +# You can replace this text with custom code or comments, and it will be preserved on regeneration +1; diff --git a/installer/data/mysql/atomicupdate.pl b/installer/data/mysql/atomicupdate.pl new file mode 100644 index 0000000..2401947 --- /dev/null +++ b/installer/data/mysql/atomicupdate.pl @@ -0,0 +1,148 @@ +#!/usr/bin/perl +# +# Copyright Vaara-kirjastot 2015 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +use Modern::Perl; +use Getopt::Long; + +use C4::Context; + +use Koha::AtomicUpdater; + +my $verbose = 0; +my $help = 0; +my $apply = 0; +my $remove = ''; +my $insert = ''; +my $list = 0; +my $pending = 0; +my $directory = ''; +my $git = ''; + +GetOptions( + 'v|verbose:i' => \$verbose, + 'h|help' => \$help, + 'a|apply' => \$apply, + 'd|directory:s' => \$directory, + 'r|remove:s' => \$remove, + 'i|insert:s' => \$insert, + 'l|list' => \$list, + 'p|pending' => \$pending, + 'g|git:s' => \$git, +); + +my $usage = << 'ENDUSAGE'; + +Runs all the not-yet-applied atomicupdate-scripts and sql in the +atomicupdates-directory, in the order specified by the _updateorder-file. + +This script uses koha.atomicupdates-table to see if the update has already been +applied. + +Also acts as a gateway to CRUD the koha.database_updates-table. + + -v --verbose Integer, 1 is not so verbose, 3 is maximally verbose. + -h --help Flag, This nice help! + -a --apply Flag, Apply all the pending atomicupdates from the + atomicupdates-directory. + -d --directory Path, From which directory to look for atomicupdate-scripts. + Defaults to '$KOHA_PATH/installer/data/mysql/atomicupdate/' + -r --remove String, Remove the upgrade entry from koha.database_updates + eg. --remove "Bug71337" + -i --insert Path, Add an upgrade log entry for the given atomicupdate-file. + Useful to revert an accidental --remove -operation or for + testing. + eg. -i installer/data/mysql/atomicupdate/Bug5453-Example.pl + -l --list Flag, List all entries in the koha.database_updates-table. + This typically means all applied atomicupdates. + -p --pending Flag, List all pending atomicupdates from the + atomicupdates-directory. + -g --git Path, Build the update order from the Git repository given, + or default to the Git repository in $KOHA_PATH. + Eg. --git 1, to build with default values, or + --git /tmp/kohaclone/ to look for another repository + +EXAMPLES: + + atomicupdate.pl -g 1 -a + +Looks for the Git repository in $KOHA_PATH, parses the issue/commit identifiers +from the top 10000 commits and generates the _updateorder-file to tell in which +order the atomicupdates-scripts are executed. +Then applies all pending atomicupdate-scripts in the order (oldest to newest) +presented in the Git repository. + + + atomicupdate --apply -d /home/koha/kohaclone/installer/data/mysql/atomicupdate/ + +Applies all pending atomicupdate-scripts from the given directory. If the file +'_updateorder' is not present, it must be first generated, for example with the +--git 1 argument. + +UPDATEORDER: + +When deploying more than one atomicupdate, it is imperative to know in which order +the updates are applied. Atomicupdates can easily depend on each other and fail in +very strange and hard-to-debug -ways if the prerequisite modifications are not +in effect. +The correct update order is defined in the atomicupdates/_updateorder-file. This is +a simple list of issue/commit identifiers, eg. + + Bug5454 + Bug12432 + Bug3218 + #45 + +This file is most easily generated directly from the original Git repository, since +the order in which the Commits have been introduced most definetely is the order +they should be applied. +When deploying the atomicupdates to production environments without the +Git repository, the _updateorder file must be copied along the atomicupdate-scripts. + +P.S. Remember to put atomicupdate/_updateorder to your .gitignore + +ENDUSAGE + +if ( $help ) { + print $usage; + exit; +} + +my $atomicupdater = Koha::AtomicUpdater->new({verbose => $verbose, + scriptDir => $directory, + gitRepo => (length($git) == 1) ? '' : $git}); + +if ($git) { + $atomicupdater->buildUpdateOrderFromGit(10000); +} +if ($remove) { + $atomicupdater->removeAtomicUpdate($remove); +} +if ($insert) { + $atomicupdater->addAtomicUpdate({filename => $insert}); +} +if ($list) { + print $atomicupdater->listToConsole(); +} +if ($pending) { + print $atomicupdater->listPendingToConsole(); +} +if ($apply) { + $atomicupdater->applyAtomicUpdates(); +} diff --git a/installer/data/mysql/atomicupdate/Bug14698-AtomicUpdater.pl b/installer/data/mysql/atomicupdate/Bug14698-AtomicUpdater.pl new file mode 100644 index 0000000..177f150 --- /dev/null +++ b/installer/data/mysql/atomicupdate/Bug14698-AtomicUpdater.pl @@ -0,0 +1,36 @@ +#!/usr/bin/perl + +# Copyright Open Source Freedom Fighters +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use C4::Context; + +my $dbh = C4::Context->dbh(); + +$dbh->do(" +CREATE TABLE `atomicupdates` ( + `atomicupdate_id` int(11) unsigned NOT NULL auto_increment, + `issue_id` varchar(20) NOT NULL, + `filename` varchar(30) NOT NULL, + `modification_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`atomicupdate_id`), + UNIQUE KEY `origincode` (`issue_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; +"); +$dbh->do("INSERT INTO atomicupdates (issue_id, filename) VALUES ('Bug14698', 'Bug14698-AtomicUpdater.pl')"); + +print "Upgrade to Bug 14698 - AtomicUpdater - Keeps track of which updates have been applied to a database done\n"; diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql index 13b6a6a..67a3125 100644 --- a/installer/data/mysql/kohastructure.sql +++ b/installer/data/mysql/kohastructure.sql @@ -1784,6 +1784,20 @@ CREATE TABLE `printers_profile` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; -- +-- Table structure for table `atomicupdates` +-- + +DROP TABLE IF EXISTS `atomicupdates`; +CREATE TABLE `atomicupdates` ( + `atomicupdate_id` int(11) unsigned NOT NULL auto_increment, + `issue_id` varchar(20) NOT NULL, + `filename` varchar(128) NOT NULL, + `modification_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`atomicupdate_id`), + UNIQUE KEY `atomic_issue_id` (`issue_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; + +-- -- Table structure for table `repeatable_holidays` -- diff --git a/t/db_dependent/Koha/AtomicUpdater.t b/t/db_dependent/Koha/AtomicUpdater.t new file mode 100644 index 0000000..381f379 --- /dev/null +++ b/t/db_dependent/Koha/AtomicUpdater.t @@ -0,0 +1,274 @@ +#!/usr/bin/perl + +# Copyright 2015 Open Source Freedom Fighters +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; +use Test::More; +use Try::Tiny; +use Encode; + +use t::lib::TestObjects::ObjectFactory; +use t::lib::TestObjects::AtomicUpdateFactory; +use t::lib::TestObjects::FileFactory; +use Koha::AtomicUpdater; + +my $testContext = {}; +my $atomicupdates = t::lib::TestObjects::AtomicUpdateFactory->createTestGroup([ + {issue_id => 'Bug12', + filename => 'Bug12-WatchExMachinaYoullLikeIt.pl'}, + {issue_id => 'Bug14', + filename => 'Bug14-ReturnOfZorro.perl'}, + {issue_id => '#14', + filename => '#14-RobotronInDanger.sql'}, + {issue_id => '#15', + filename => '#15-ILikedPrometheusButAlienWasBetter.pl'}, + ], undef, $testContext); + +#Make sure we get the correct update order, otherwise we get unpredictable results. +{ #Overload existing subroutines to provide a Mock implementation + no warnings 'redefine'; + package Koha::AtomicUpdater; + sub _getGitCommits { #instead of requiring a Git repository, we just mock the input. + return [#Newest commit + '2e8a39762b506738195f21c8ff67e4e7bfe6d7ab #:-55 : Fiftyfive', + '2e8a39762b506738195f21c8ff67e4e7bfe6d7ab #54 - KohaCon in Finland next year', + 'b447b595acacb0c4823582acf9d8a08902118e59 #53 - Place to be.pl', + '2e8a39762b506738195f21c8ff67e4e7bfe6d7ab bug 112 - Lapinlahden linnut', + '5ac7101d4071fe11f7a5d1445bb97ed1a603a9b5 Bug:-911 - What are you going to do?', + '1d54601b9cac0bd75ee97e071cf52ed49daef8bd #911 - Who are you going to call', + '1d54601b9cac0bd75ee97e071cf52ed49daef8bd bug 30 - Feature Yes yes', + '5ac7101d4071fe11f7a5d1445bb97ed1a603a9b5 #-29 - Bug squashable', + '2e8a39762b506738195f21c8ff67e4e7bfe6d7ab Bug :- 28 - Feature Squash', + 'b447b595acacb0c4823582acf9d8a08902118e59 BUG 27 - Bug help', + #Oldest commit + ]; + } +} + +subtest "Create update order from Git repository" => \&createUpdateOrderFromGit; +sub createUpdateOrderFromGit { + eval { + #Create the _updateorder-file to a temp directory and prepare it for autocleanup. + my $files = t::lib::TestObjects::FileFactory->createTestGroup([ + { filepath => 'atomicupdate/', + filename => '_updateorder', + content => '',}, + ], + undef, undef, $testContext); + #Instantiate the AtomicUpdater to operate on a temp directory. + my $atomicUpdater = Koha::AtomicUpdater->new({ + scriptDir => $files->{'_updateorder'}->dirname(), + }); + + #Start real testing. + my $issueIds = $atomicUpdater->buildUpdateOrderFromGit(4); + + is($issueIds->[0], + 'Bug27', + "First atomicupdate to deploy"); + is($issueIds->[1], + 'Bug28', + "Second atomicupdate to deploy"); + is($issueIds->[2], + '#29', + "Third atomicupdate to deploy"); + is($issueIds->[3], + 'Bug30', + "Last atomicupdate to deploy"); + + #Testing file access + $issueIds = $atomicUpdater->getUpdateOrder(); + is($issueIds->[0], + 'Bug27', + "First atomicupdate to deploy, from _updateorder"); + is($issueIds->[1], + 'Bug28', + "Second atomicupdate to deploy, from _updateorder"); + is($issueIds->[2], + '#29', + "Third atomicupdate to deploy, from _updateorder"); + is($issueIds->[3], + 'Bug30', + "Last atomicupdate to deploy, from _updateorder"); + }; + if ($@) { + ok(0, $@); + } +} + + + +subtest "List all deployed atomicupdates" => \&listAtomicUpdates; +sub listAtomicUpdates { + eval { + my $atomicUpdater = Koha::AtomicUpdater->new(); + my $text = $atomicUpdater->listToConsole(); + print $text; + + ok($text =~ m/Bug12-WatchExMachinaYoullLik/, + "Bug12-WatchExMachinaYoullLikeIt"); + ok($text =~ m/Bug14-ReturnOfZorro.perl/, + "Bug14-ReturnOfZorro"); + ok($text =~ m/#14-RobotronInDanger.sql/, + "#14-RobotronInDanger"); + ok($text =~ m/#15-ILikedPrometheusButAli/, + "#15-ILikedPrometheusButAlienWasBetter"); + + }; + if ($@) { + ok(0, $@); + } +} + +subtest "Delete an atomicupdate entry" => \&deleteAtomicupdate; +sub deleteAtomicupdate { + eval { + my $atomicUpdater = Koha::AtomicUpdater->new(); + my $atomicupdate = $atomicUpdater->cast($atomicupdates->{Bug12}->id); + ok($atomicupdate, + "AtomicUpdate '".$atomicupdates->{Bug12}->issue_id."' exists prior to deletion"); + + $atomicUpdater->removeAtomicUpdate($atomicupdate->issue_id); + $atomicupdate = $atomicUpdater->find($atomicupdates->{Bug12}->id); + ok(not($atomicupdate), + "AtomicUpdate '".$atomicupdates->{Bug12}->issue_id."' deleted"); + + }; + if ($@) { + ok(0, $@); + } +} + +subtest "Insert an atomicupdate entry" => \&insertAtomicupdate; +sub insertAtomicupdate { + eval { + my $atomicUpdater = Koha::AtomicUpdater->new(); + my $subtestContext = {}; + my $atomicupdates = t::lib::TestObjects::AtomicUpdateFactory->createTestGroup([ + {issue_id => 'Bug15', + filename => 'Bug15-Inserted.pl'}, + ], undef, $subtestContext, $testContext); + my $atomicupdate = $atomicUpdater->find({issue_id => 'Bug15'}); + ok($atomicupdate, + "Bug15-Inserted.pl"); + + t::lib::TestObjects::ObjectFactory->tearDownTestContext($subtestContext); + + $atomicupdate = $atomicUpdater->find({issue_id => 'Bug15'}); + ok(not($atomicupdate), + "Bug15-Inserted.pl deleted"); + }; + if ($@) { + ok(0, $@); + } +} + +subtest "List pending atomicupdates" => \&listPendingAtomicupdates; +sub listPendingAtomicupdates { + my ($atomicUpdater, $files, $text, $atomicupdates); + my $subtestContext = {}; + eval { + ##Test adding update scripts and deploy them, confirm that no pending scripts detected + $files = t::lib::TestObjects::FileFactory->createTestGroup([ + { filepath => 'atomicupdate/', + filename => '#911-WhoYouGonnaCall.pl', + content => '$ENV{ATOMICUPDATE_TESTS} = 1;',}, + { filepath => 'atomicupdate/', + filename => 'Bug911-WhatchaGonnaDo.pl', + content => '$ENV{ATOMICUPDATE_TESTS}++;',}, + { filepath => 'atomicupdate/', + filename => 'Bug112-LapinlahdenLinnut.pl', + content => '$ENV{ATOMICUPDATE_TESTS}++;',}, + ], + undef, $subtestContext, $testContext); + $atomicUpdater = Koha::AtomicUpdater->new({ + scriptDir => $files->{'#911-WhoYouGonnaCall.pl'}->dirname() + }); + + $text = $atomicUpdater->listPendingToConsole(); + print $text; + + ok($text =~ m/#911-WhoYouGonnaCall.pl/, + "#911-WhoYouGonnaCall is pending"); + ok($text =~ m/Bug911-WhatchaGonnaDo.pl/, + "Bug911-WhatchaGonnaDo is pending"); + ok($text =~ m/Bug112-LapinlahdenLinnut.pl/, + 'Bug112-LapinlahdenLinnut is pending'); + + $atomicupdates = $atomicUpdater->applyAtomicUpdates(); + t::lib::TestObjects::AtomicUpdateFactory->addToContext($atomicupdates, undef, $subtestContext, $testContext); #Keep track of changes + + is($atomicupdates->{'#911'}->issue_id, + '#911', + "#911-WhoYouGonnaCall.pl deployed"); + is($atomicupdates->{'Bug112'}->issue_id, + 'Bug112', + 'Bug112-LapinlahdenLinnut.pl deployed'); + is($atomicupdates->{'Bug911'}->issue_id, + 'Bug911', + "Bug911-WhatchaGonnaDo.pl deployed"); + + ##Test adding scripts to the atomicupdates directory and how we deal with such change. + $files = t::lib::TestObjects::FileFactory->createTestGroup([ + { filepath => 'atomicupdate/', + filename => '#53-PlaceToBe.pl', + content => '$ENV{ATOMICUPDATE_TESTS}++;',}, + { filepath => 'atomicupdate/', + filename => '#54-KohaConInFinlandNextYear.pl', + content => '$ENV{ATOMICUPDATE_TESTS}++;',}, + { filepath => 'atomicupdate/', + filename => '#55-Fiftyfive.pl', + content => '$ENV{ATOMICUPDATE_TESTS}++;',}, + ], + undef, $subtestContext, $testContext); + + $text = $atomicUpdater->listPendingToConsole(); + print $text; + + ok($text =~ m/#53-PlaceToBe.pl/, + "#53-PlaceToBe.pl is pending"); + ok($text =~ m/#54-KohaConInFinlandNextYear.pl/, + "#54-KohaConInFinlandNextYear.pl is pending"); + ok($text =~ m/#55-Fiftyfive.pl/u, + '#55-Fiftyfive.pl'); + + $atomicupdates = $atomicUpdater->applyAtomicUpdates(); + t::lib::TestObjects::AtomicUpdateFactory->addToContext($atomicupdates, undef, $subtestContext, $testContext); #Keep track of changes + + is($atomicupdates->{'#53'}->issue_id, + '#53', + "#53-PlaceToBe.pl deployed"); + is($atomicupdates->{'#54'}->issue_id, + '#54', + '#54-KohaConInFinlandNextYear.pl deployed'); + is($atomicupdates->{'#55'}->issue_id, + '#55', + "#55-Fiftyfive.pl deployed"); + + is($ENV{ATOMICUPDATE_TESTS}, + 6, + "All configured AtomicUpdates deployed"); + }; + if ($@) { + ok(0, $@); + } + t::lib::TestObjects::AtomicUpdateFactory->tearDownTestContext($subtestContext); +} + +t::lib::TestObjects::ObjectFactory->tearDownTestContext($testContext); +done_testing; \ No newline at end of file diff --git a/t/lib/TestObjects/AtomicUpdateFactory.pm b/t/lib/TestObjects/AtomicUpdateFactory.pm new file mode 100644 index 0000000..4aff6f1 --- /dev/null +++ b/t/lib/TestObjects/AtomicUpdateFactory.pm @@ -0,0 +1,105 @@ +package t::lib::TestObjects::AtomicUpdateFactory; + +# Copyright Vaara-kirjastot 2015 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +use Modern::Perl; +use Carp; + +use Koha::AtomicUpdater; +use Koha::Database; + +use Koha::Exception::UnknownProgramState; + +use base qw(t::lib::TestObjects::ObjectFactory); + +sub getDefaultHashKey { + return 'issue_id'; +} +sub getObjectType { + return 'Koha::AtomicUpdate'; +} + +=head t::lib::TestObjects::createTestGroup + + my $atomicupdates = t::lib::TestObjects::AtomicUpdateFactory->createTestGroup([ + {'issue_id' => 'Bug3432', + 'filename' => 'Bug3432-RavingRabbitsMayhem', + 'modification_time' => '2015-01-02 15:59:32', + }, + ], undef, $testContext1, $testContext2, $testContext3); + +Calls Koha::AtomicUpdater to add a Koha::AtomicUpdate object to DB. + +The HASH is keyed with the 'koha.atomicupdates.issue_id', or the given $hashKey. + +There is a duplication check to first look for atomicupdate-rows with the same 'issue_id'. +If a matching atomicupdate is found, then we use the existing Record instead of adding a new one. + +@RETURNS HASHRef of Koha::AtomicUpdate-objects + +See t::lib::TestObjects::ObjectFactory for more documentation +=cut + +sub handleTestObject { + my ($class, $object, $stashes) = @_; + + ##First see if the given Record already exists in the DB. For testing purposes we use the isbn as the UNIQUE identifier. + my $atomicupdate = Koha::AtomicUpdater->find({issue_id => $object->{issue_id}}); + unless ($atomicupdate) { + my $atomicupdater = Koha::AtomicUpdater->new(); + $atomicupdate = $atomicupdater->addAtomicUpdate($object); + } + + Koha::Exception::UnknownProgramState->throw(error => "$class->handleTestObject():> Cannot create a new object\n$@\n") + unless $atomicupdate; + + return $atomicupdate; +} + +=head validateAndPopulateDefaultValues +@OVERLOAD + +Validates given Object parameters and makes sure that critical fields are given +and populates defaults for missing values. +=cut + +sub validateAndPopulateDefaultValues { + my ($self, $object, $hashKey) = @_; + + $object->{issue_id} = 'BugRancidacid' unless $object->{issue_id}; + $object->{filename} = 'BugRancidacid-LaboratoryExperimentsGoneSour' unless $object->{filename}; + + $self->SUPER::validateAndPopulateDefaultValues($object, $hashKey); +} + +sub deleteTestGroup { + my ($class, $objects) = @_; + + while( my ($key, $object) = each %$objects) { + my $atomicupdate = Koha::AtomicUpdater->cast($object); + eval { + $atomicupdate->delete(); + }; + if ($@) { + warn "$class->deleteTestGroup():> Error hapened: $@\n"; + } + } +} + +1; diff --git a/t/lib/TestObjects/ObjectFactory.pm b/t/lib/TestObjects/ObjectFactory.pm index e4f6441..fb6de71 100644 --- a/t/lib/TestObjects/ObjectFactory.pm +++ b/t/lib/TestObjects/ObjectFactory.pm @@ -166,6 +166,11 @@ sub tearDownTestContext { t::lib::TestObjects::BiblioFactory->deleteTestGroup($stash->{biblio}); delete $stash->{biblio}; } + if ($stash->{atomicupdate}) { + require t::lib::TestObjects::AtomicUpdateFactory; + t::lib::TestObjects::AtomicUpdateFactory->deleteTestGroup($stash->{atomicupdate}); + delete $stash->{atomicupdate}; + } if ($stash->{borrower}) { require t::lib::TestObjects::BorrowerFactory; t::lib::TestObjects::BorrowerFactory->deleteTestGroup($stash->{borrower}); diff --git a/t/lib/TestObjects/objectFactories.t b/t/lib/TestObjects/objectFactories.t index c6668b9..7a4ef65 100644 --- a/t/lib/TestObjects/objectFactories.t +++ b/t/lib/TestObjects/objectFactories.t @@ -30,6 +30,8 @@ use t::lib::TestObjects::BorrowerFactory; use Koha::Borrowers; use t::lib::TestObjects::ItemFactory; use Koha::Items; +use t::lib::TestObjects::AtomicUpdateFactory; +use Koha::AtomicUpdater; use t::lib::TestObjects::BiblioFactory; use Koha::Biblios; use t::lib::TestObjects::CheckoutFactory; @@ -472,6 +474,43 @@ sub testLetterTemplateFactory { +########## AtomicUpdateFactory subtests ########## +subtest 't::lib::TestObjects::AtomicUpdateFactory' => \&testAtomicUpdateFactory; +sub testAtomicUpdateFactory { + my ($atomicUpdater, $atomicupdate); + my $subtestContext = {}; + ##Create and Delete using dependencies in the $testContext instantiated in previous subtests. + my $atomicupdates = t::lib::TestObjects::AtomicUpdateFactory->createTestGroup([ + {'issue_id' => 'Bug10', + 'filename' => 'Bug10-RavingRabbitsMayhem', + 'modification_time' => '2015-01-02 15:59:32',}, + {'issue_id' => 'Bug11', + 'filename' => 'Bug11-RancidSausages', + 'modification_time' => '2015-01-02 15:59:33',}, + ], + undef, $subtestContext); + $atomicUpdater = Koha::AtomicUpdater->new(); + $atomicupdate = $atomicUpdater->find({issue_id => $atomicupdates->{Bug10}->issue_id}); + is($atomicupdate->issue_id, + 'Bug10', + "Bug10-RavingRabbitsMayhem created"); + $atomicupdate = $atomicUpdater->find({issue_id => $atomicupdates->{Bug11}->issue_id}); + is($atomicupdate->issue_id, + 'Bug11', + "Bug11-RancidSausages created"); + + t::lib::TestObjects::ObjectFactory->tearDownTestContext($subtestContext); + + $atomicupdate = $atomicUpdater->find({issue_id => $atomicupdates->{Bug10}->issue_id}); + ok(not($atomicupdate), + "Bug10-RavingRabbitsMayhem deleted"); + $atomicupdate = $atomicUpdater->find({issue_id => $atomicupdates->{Bug11}->issue_id}); + ok(not($atomicupdate), + "Bug11-RancidSausages created"); +}; + + + ########## SystemPreferenceFactory subtests ########## subtest 't::lib::TestObjects::SystemPreferenceFactory' => \&testSystemPreferenceFactory; sub testSystemPreferenceFactory { -- 1.9.1