From b10ee1eac7d38ec1c152a623eff066b68647dba0 Mon Sep 17 00:00:00 2001 From: Olli-Antti Kivilahti Date: Wed, 10 Sep 2014 17:24:47 +0300 Subject: [PATCH] Cronjob monitoring script, nagios3 compatible After having issues with cronjobs not always running or getting regression issues, I decided to give a try to create a cronjobs monitoring solution. This patch introduces a wrapper script to log and time cronjobs. The wrapper script's log output is used to analyze whther cronjobs ran and if there were any errors. Also cronjobs timings can be validated to prevent nasty errors with for. ex. expiring waiting holds one day too early/late. This check can also be integrated to nagios! Also provided is a scaffold to start introducing more nagios-checks, with example command and service configurations. This patch introduces a koha_cronjobs-script to read the /var/spool/cron/crontabs/koha and analysz its contents. Then one can validate whether the cronjob timings are right (this is not mentioned in the manual at all) and what is right is (currently) based on my honest feelings. Then each cronjobs frequency is determined and the cronjobs log output is checked based on the script parameters. Currently the cronjob frequencies are split to minutely, hourly, daily, etc. Koha_cronjobs-script is meant to be ran on those intervals to catch errors ASAP. --- misc/cronjobtimer.sh | 55 +++++++ misc/nagios/README | 3 + misc/nagios/commands.cfg | 6 + misc/nagios/koha_cronjobs | 378 +++++++++++++++++++++++++++++++++++++++++++++ misc/nagios/services.cfg | 47 ++++++ 5 files changed, 489 insertions(+) create mode 100755 misc/cronjobtimer.sh create mode 100644 misc/nagios/README create mode 100644 misc/nagios/commands.cfg create mode 100755 misc/nagios/koha_cronjobs create mode 100644 misc/nagios/services.cfg diff --git a/misc/cronjobtimer.sh b/misc/cronjobtimer.sh new file mode 100755 index 0000000..f779b3f --- /dev/null +++ b/misc/cronjobtimer.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +## SYNOPSIS ## +# This script takes another script and measures it's runtime, appending it to the same log that the script's stderr and stdout go. +# First argument is the script name, relative to this scripts location. +# All arguments are passed directly to the first argument script. +# +# This script also locks the given script to prevent cronjobs from going to a repeating loop. +# + +logdirectory=$(grep -Po '(?<=).*?(?=)' $KOHA_CONF)/ +logfile=$1".log" +cronjobdir=$(echo "$logfile" | grep -Po '.*?(?=/)') +logpath=$logdirectory$logfile +croncommand=$KOHA_PATH"/misc/$@" +lockfile=$KOHA_PATH"/misc/$1.lock" +disableCronjobsFlag=$KOHA_PATH"/misc/cronjobs/disableCronjobs.flag" + +if [ ! -d "$logdirectory/$cronjobdir" ]; then + #Make sure that the cronjob folders exists + mkdir -p "$logdirectory/$cronjobdir" +fi + + +starttime=$(date +%s) +startMsg='Start: '$(date --date="@$starttime" "+%Y-%m-%d %H:%M:%S") +printf "$startMsg\n" >> $logpath + +#Check if cronjobs are disabled by the preproduction to production migration process +if [ -e $disableCronjobsFlag ]; then + echo "Disabled by disableCronjobsFlag" >> $logpath + exit 0; +fi + +#Lock the cronjob we are running! +if [ -e $lockfile ]; then + echo "Lockfile present" >> $logpath + exit 0; +fi +touch $lockfile + + +starttime=$(date +%s) + + +$croncommand 2&>> $logpath + + +endtime=$(date +%s) +runtime=$((endtime - starttime)) +timelog='End: '$(date --date="@$endtime" "+%Y-%m-%d %H:%M:%S")"\n"'Runtime: '$(($runtime/60/60))':'$(($runtime/60%60))':'$(($runtime%60)) + +printf "$timelog\n" >> $logpath + +rm $lockfile diff --git a/misc/nagios/README b/misc/nagios/README new file mode 100644 index 0000000..91b0b0d --- /dev/null +++ b/misc/nagios/README @@ -0,0 +1,3 @@ +nagios3 compatible monitoring scripts are found here. + +See commands.cfg and services.cfg for usage examples diff --git a/misc/nagios/commands.cfg b/misc/nagios/commands.cfg new file mode 100644 index 0000000..736530d --- /dev/null +++ b/misc/nagios/commands.cfg @@ -0,0 +1,6 @@ +#Link the koha_cronjobs to the nagios3 plugins directory symbolically. +# ln -s /home/koha/kohaclone/misc/nagios/koha_cronjobs /usr/lib/nagios/plugins/koha_cronjobs +define command{ + command_name ssh_koha_cronjobs + command_line /usr/lib/nagios/plugins/check_by_ssh -H '$HOSTADDRESS$' -C "/usr/lib/nagios/plugins/koha_cronjobs -c $ARG1$" + } diff --git a/misc/nagios/koha_cronjobs b/misc/nagios/koha_cronjobs new file mode 100755 index 0000000..b98735a --- /dev/null +++ b/misc/nagios/koha_cronjobs @@ -0,0 +1,378 @@ +#!/usr/bin/perl + +use Modern::Perl; + +## SYNOPSIS +# +# +# + +use File::Basename; +use Getopt::Long qw(:config no_ignore_case); + +my $kohaLogDirectory = "/home/koha/koha-dev/var/log/"; +my $help = 0; +my $verbose = 0; +my $cursive = 0; +my $validateCrontabConfiguration = 0; +my $checkMinutelyCronjobs = 0; +my $checkHourlyCronjobs = 0; +my $checkDailyCronjobs = 0; +my $checkWeeklyCronjobs = 0; +my $checkMonthlyCronjobs = 0; +my $checkAnnualCronjobs = 0; +my $debugLogwarn = ''; #Sets the logwarn flag -z active to analyze the whole logfile, instead of just the recent changes. + +GetOptions( + 'h|help|?' => \$help, + 'V|validate' => \$validateCrontabConfiguration, + 'H|hourly' => \$checkHourlyCronjobs, + 'm|minutely' => \$checkMinutelyCronjobs, + 'D|daily' => \$checkDailyCronjobs, + 'W|weekly' => \$checkWeeklyCronjobs, + 'M|monthly' => \$checkMonthlyCronjobs, + 'A|annually' => \$checkAnnualCronjobs, + 'v|verbose=i' => \$verbose, + 'l|logdir=s' => \$kohaLogDirectory, + 'c|cursive' => \$cursive, + 'd|debuglogwarn' => \$debugLogwarn, +); + +$debugLogwarn = ($debugLogwarn eq '1') ? ' -z ' : ''; + +my $usage = << 'ENDUSAGE'; + +REQUIREMENTS: +install logwarn from https://code.google.com/p/logwarn/ +Read rights to the crontab file of the crontab user. + +SUMMARY: +This script scans the log output of all the defined Koha cronjobs and prints to screen if any new +errors are found or if the scripts haven't ran for some reason. +It is important to understand logwarn's method of scanning log files: +Logwarn, every time it is invoked, remembers the last position of the logfile it is reading. Running +it more often than the cronjob, causes this script to warn, because it thinks the cronjob failed to run. +Thus from the checking point of view it is important to configure the different analysis frequencies. + +It is safe to analyze following script types regularly at the following intervals + --minutely scripts once an hour + --hourly and daily scripts once a day + --weekly scripts once a week + --monthly scripts once a month + --annually scripts once a year + +LIMITATIONS: +-It is important to note that if a cronjob hasn't ran atleast once before this monitoring script is triggered, + this script will think that there is an error in running that cronjob. +-This script might not catch "flapping" cronjobs. Cronjobs which sometimes run and sometimes don't. +-This script makes logwarn look for error markers. Not all error markers have been discovered and thus logwarn + might not catch all errors. + + +This script has the following parameters : + + -h --help this message + + -V --validate Validate the crontab configuration at /var/spool/cron/crontabs/koha. + Script tells you if the timing is not optimal or if you have missing + cronjob definitions. This feature requires read access to the crontab file. + Validation rules are defined in this script and can be changed to suit one's needs. + + -A --annually Analyze annually ran cronjobs that are ran few times a year. + + -M --monthly Analyze monthly cronjobs that are ran few times or once a month. + + -W --weekly Analyze weekly cronjobs. + + -D --daily Analyze cronjobs that are ran daily. + + -H --hourly Analyze hourly cronjobs, this means all cronjobs that are ran many times a day. + + -m --minutely Analyze minutely cronjobs, this means cronjobs that are ran many times per hour, or once a hour. + + -l --logdir Change the directory to look for Koha cronjob logs. + Defaults to /home/koha/koha-dev/var/log/ + + -c --cursive Output only very shy indicators of success and/or failure. + Meant to be used from a remote monitoring tool. + + -v --verbose show verbose information about script internals on few levels. + --verbose 1, prints basic verbose output + --verbose 2, also prints more detailed internal behaviour + + -d --debuglogwarn Sets the logwarn flag -p active to analyze the whole logfile, + instead of just the recent changes. + +ENDUSAGE + +die $usage if $help; + +if ( not($validateCrontabConfiguration) && not($checkMinutelyCronjobs) && not($checkHourlyCronjobs) + && not($checkDailyCronjobs) && not($checkWeeklyCronjobs) && not($checkMonthlyCronjobs) + && not($checkAnnualCronjobs) + ) { + die "\nYou must define atleast one of the following operations! -V -D -H -M -W\n\n$usage"; +} + + +## SET CRONJOB VALIDATION RULES, MARKERS AND EXPECTED VALUES +my $cronjobs = { + "advance_notices.pl" => { + "recommendedCronschedule" => "25 0 * * *", + "errorMarkers" => [ "",""], + }, + "cart_to_shelf.pl" => { + "recommendedCronschedule" => "15 0 * * *", + "errorMarkers" => [ "",""], + }, + "check-url-quick.pl" => { + "recommendedCronschedule" => "0 0 2 * *", + "errorMarkers" => [ "",""], + }, + "cleanup_database.pl" => { + "recommendedCronschedule" => "0 6 * * 7", + "errorMarkers" => [ "",""], + }, + "cloud-kw.pl" => { + "recommendedCronschedule" => "30 1 1 * *", + "errorMarkers" => [ "",""], + }, + "create_koc_db.pl" => { + "recommendedCronschedule" => "45 0 * * *", + "errorMarkers" => [ "",""], + }, + "delete_expired_opac_registrations.pl" => { + "recommendedCronschedule" => "0 0 * * *", + "errorMarkers" => [ "",""], + }, + "fines.pl" => { + "recommendedCronschedule" => "30 00 * * *", + "errorMarkers" => [ "",""], + }, + "gather_print_notices.pl" => { + "recommendedCronschedule" => "50 23 * * *", + "errorMarkers" => [ "",""], + }, + "j2a.pl" => { + "recommendedCronschedule" => "0 0 * * *", + "errorMarkers" => [ "",""], + }, + "overdue_notices.pl" => { + "recommendedCronschedule" => "35 0 * * *", + "errorMarkers" => [ "",""], + }, + "process_message_queue.pl" => { + "recommendedCronschedule" => "* * * * *", + "errorMarkers" => [ "",""], + }, + "pullSelectionListsFromBookvendors.pl" => { + "recommendedCronschedule" => "20 0 * * *", + "errorMarkers" => [ "",""], + }, + "purge_suggestions.pl" => { + "recommendedCronschedule" => "0 0 15 */6 *", + "errorMarkers" => [ "",""], + }, + "removeShortLoanStatus.pl" => { + "recommendedCronschedule" => "0 0 * * 7", + "errorMarkers" => [ "",""], + }, + "send_overdue_mail.pl" => { + "recommendedCronschedule" => "5 1 * * 2", + "errorMarkers" => [ "",""], + }, + "serialsUpdate.pl" => { + "recommendedCronschedule" => "20 0 * * *", + "errorMarkers" => [ "",""], + }, + "smsalertnumberFromPhone.pl" => { + "recommendedCronschedule" => "*/30 * * * *", + "errorMarkers" => [ "",""], + }, + "update_totalissues.pl" => { + "recommendedCronschedule" => "10 1 * * *", + "errorMarkers" => [ "",""], + }, + "auto_unsuspend_holds.pl" => { + "recommendedCronschedule" => "4 0 * * *", + "errorMarkers" => [ "",""], + }, + "build_holds_queue.pl" => { + "recommendedCronschedule" => "0 * * * *", + "errorMarkers" => [ "",""], + }, + "cancel_expired_holds.pl" => { + "recommendedCronschedule" => "5 0 * * *", + "errorMarkers" => [ "",""], + }, +}; + + +my @failedCronjobs; #This is needed to communicate cursively only the failed cronjobs, with no extra log output. + + +main(); #Friends don't let friends write unscoped script monsters :) + + +sub main { + open(my $KOHACRON, "<:encoding(utf8)", "/var/spool/cron/crontabs/koha") or die "Failed to open crontab file for reading: ".$!; + + while (<$KOHACRON>) { + +#next unless $_ =~ 'process_message_queue.pl'; + my $cronEntry = $_; chomp $cronEntry; + my ($schedule, $m, $h, $dom, $mon, $dow); + my ($jobTimingCategory); + my ($name, $path, $suffix); + next() if $cronEntry =~ /^#/; #Skip comment lines + next() if $cronEntry =~ /^\s*$/; #Skip empty lines + + print "\nStarting processing this cron entry:\n$cronEntry\n" if ( $verbose > 1 && not($cursive) ); + + my $manyTimes = '^\*/?[0-9]*$'; + my $fewTimesOrOnce = '^\*?/?[0-9]+'; + my $once = '^[0-9]+$'; + ##Extract the schedule portion + if ($cronEntry =~ s|^(([*/0-9]{1,4})\s+([*/0-9]{1,4})\s+([*/0-9]{1,4})\s+([*/0-9]{1,4})\s+([*/0-9]{1,4}))\s+||) { + $schedule = $1; $m = $2; $h = $3; $dom = $4; $mon = $5; $dow = $6; + + if ($mon =~ m|$fewTimesOrOnce|) { + $jobTimingCategory = 'annally'; + } + elsif ($dom =~ m|$fewTimesOrOnce|) { + $jobTimingCategory = 'Monthly'; #Monthly is capitalized to separate it from minutely + } + elsif ($dow =~ m|$fewTimesOrOnce|) { + $jobTimingCategory = 'weekly'; + } + elsif ($m =~ m|$once| && $h =~ m|$once|) { + $jobTimingCategory = 'daily'; + } + elsif ($h =~ m|$manyTimes| && $m =~ m|$once|) { + $jobTimingCategory = 'hourly'; + } + elsif ( $m =~ m|$manyTimes| ) { + $jobTimingCategory = 'minutely'; + } + } + else { + print "Couldn't parse the cronjob scheduling portion of $cronEntry\n"; + push @failedCronjobs, $cronEntry; + next(); + } + print "This is a $jobTimingCategory cronjob\n" if ( $verbose > 1 && not($cursive) ); + ##Extract the basename portion to match cronjob definitions. + #Doesn't work with cronjobs with a space ' ' in them! + if ($cronEntry =~ /\$KOHA_CRONJOB_TRIGGER\s+([^# ]+)/) { + ($name, $path, $suffix) = File::Basename::fileparse( $1 ); + chomp $name; + } + else { + print "Couldn't get the command portion of cronjob $cronEntry\n"; + push @failedCronjobs, $cronEntry; + next(); + } + + validateCronjob($schedule, $name) if $validateCrontabConfiguration; + + if ($checkMinutelyCronjobs && $jobTimingCategory =~ /^m/) { + checkCronjob($name, $path); + } + elsif ($checkHourlyCronjobs && $jobTimingCategory =~ /^h/) { + checkCronjob($name, $path); + } + elsif ($checkDailyCronjobs && $jobTimingCategory =~ /^d/) { + checkCronjob($name, $path); + } + elsif ($checkWeeklyCronjobs && $jobTimingCategory =~ /^w/) { + checkCronjob($name, $path); + } + elsif ($checkMonthlyCronjobs && $jobTimingCategory =~ /^M/) { #Monthly is capitalized so it wont get mixed with 'minutely' + checkCronjob($name, $path); + } + elsif ($checkAnnualCronjobs && $jobTimingCategory =~ /^a/) { + checkCronjob($name, $path); + } + else { + print "Doing nothing because not of the requested timing frequencey\n" if ($verbose > 1 && not($cursive)); + } + } + + #Check for any missing cronjobs + if ($validateCrontabConfiguration) { + foreach my $job (sort keys %$cronjobs) { + print "$job is not defined in the crontab!\n" unless exists $cronjobs->{$job}->{found}; + } + } + + #Print a report of failed cronjobs + if ($cursive) { + if (scalar(@failedCronjobs)) { + print "@failedCronjobs"; + exit 2; #Signal Nagios to critical this! + } + else { + print "OK!"; + exit 0; #Signal Nagios to exit OK! + } + } +} + +sub validateCronjob { + my ($cronjobSchedule, $cronjobName) = @_; + + my $cronjobRecommendation = $cronjobs->{$cronjobName}; + + return unless $cronjobRecommendation; + + if ($cronjobSchedule eq $cronjobRecommendation->{recommendedCronschedule}) { + print "Validated $cronjobName\n" if $verbose; + } + else { + print "Not recommended cronjob schedule for $cronjobName. This is recommended:\n".$cronjobRecommendation->{recommendedCronschedule}."\n"; + } + $cronjobRecommendation->{found} = 1; +} + +sub checkCronjob { + my ($cronjob, $path) = @_; + + if ($cronjobs->{$cronjob}->{checked}) { + #Don't check twice, because the first run already read all the fresh log entries and subsequent reads will fail! + print "Trying to run this $cronjob many times, skipping." if ( $verbose > 1 && not($cursive) ); + return; + } + $cronjobs->{$cronjob}->{checked} = 1; + + #Check if we even tried to run the cronjob? +# my $date = `date --date @\$((\$(date +%s)-3600)) -Idate`; +# chomp $date; + my $logwarnCmd = "logwarn -d /var/lib/nagios/.logwarn/ -p $debugLogwarn $kohaLogDirectory$path$cronjob.log 'Start: ' 2>&1 "; + open( my $LOGWARN_DIDITRUN, "$logwarnCmd |" ) or warn "Can't run '$logwarnCmd\n$!\n'"; + my $errors = 0; + my $lastRuntime = 0; + + while (<$LOGWARN_DIDITRUN>) { + $lastRuntime = $_; + } + if ($lastRuntime) { + print "Ran OK+ $cronjob :> $lastRuntime\n" if ( $verbose && not($cursive) ); + } + else { + print "Run FAILed! $cronjob\n" if ( not($cursive)); + $errors++; + } + close($LOGWARN_DIDITRUN); + + #Check if we get errors from the cronjob + $logwarnCmd = "logwarn -d /var/lib/nagios/.logwarn/ -p $debugLogwarn $kohaLogDirectory$path$cronjob.log '^BEGIN failed\-\-' '^syntax error at' '^Global symbol' '^DBD\:\:' 'Permission denied' '^Use of uninitialized value' '^Disabled by disableCronjobsFlag' '^Lockfile present' "; + open( my $LOGWARN_ERRORS, "$logwarnCmd |" ) or warn "Can't run '$logwarnCmd\n$!\n'"; + while (<$LOGWARN_ERRORS>) { + print "Error on $cronjob :> $_\n" if (not($cursive)); + $errors++; + last(); + } + close($LOGWARN_ERRORS); + + push @failedCronjobs, $cronjob if $errors; +} diff --git a/misc/nagios/services.cfg b/misc/nagios/services.cfg new file mode 100644 index 0000000..af48825 --- /dev/null +++ b/misc/nagios/services.cfg @@ -0,0 +1,47 @@ +#Host definition for the Koha cronjobs server +define host{ + use generic-host ; Name of host template to use + host_name aino.vaarakirjastot.fi + alias kohautility + address 10.110.249.16 + } + +#Make hourly checks for hourly and minutely cronjobs +define service{ + use generic-service + host_name aino.vaarakirjastot.fi + service_description Cronjobs Hourly + check_command ssh_koha_cronjobs!-m -H! + normal_check_interval 65 + max_check_attempts 1 + } + +#Check daily cronjobs +define service{ + use generic-service + host_name aino.vaarakirjastot.fi + service_description Cronjobs Daily + check_command ssh_koha_cronjobs!-D! + normal_check_interval 1450 + max_check_attempts 1 + } + +#Check weekly cronjobs +define service{ + use generic-service + host_name aino.vaarakirjastot.fi + service_description Cronjobs Weekly + check_command ssh_koha_cronjobs!-W! + normal_check_interval 10090 + max_check_attempts 1 + } + +#Check monthly cronjobs +define service{ + use generic-service + host_name aino.vaarakirjastot.fi + service_description Cronjobs Monthly + check_command ssh_koha_cronjobs!-M! + normal_check_interval 44650 + max_check_attempts 1 + } -- 1.7.9.5