Bugzilla – Attachment 31509 Details for
Bug 12902
Cronjob monitoring script, nagios3 compatible
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
initial patch
cronjobsandnagios.patch (text/plain), 20.36 KB, created by
Olli-Antti Kivilahti
on 2014-09-10 14:41:57 UTC
(
hide
)
Description:
initial patch
Filename:
MIME Type:
Creator:
Olli-Antti Kivilahti
Created:
2014-09-10 14:41:57 UTC
Size:
20.36 KB
patch
obsolete
>From b10ee1eac7d38ec1c152a623eff066b68647dba0 Mon Sep 17 00:00:00 2001 >From: Olli-Antti Kivilahti <olli-antti.kivilahti@jns.fi> >Date: Wed, 10 Sep 2014 17:24:47 +0300 >Subject: [PATCH] Cronjob monitoring script, nagios3 compatible > >After having issues with cronjobs not always running or getting regression issues, I decided to give a try >to create a cronjobs monitoring solution. > >This patch introduces a wrapper script to log and time cronjobs. > >The wrapper script's log output is used to analyze whther cronjobs ran and if there were any errors. >Also cronjobs timings can be validated to prevent nasty errors with for. ex. expiring waiting holds one day too >early/late. >This check can also be integrated to nagios! > >Also provided is a scaffold to start introducing more nagios-checks, with example command and service >configurations. > >This patch introduces a koha_cronjobs-script to read the /var/spool/cron/crontabs/koha and analysz its contents. >Then one can validate whether the cronjob timings are right (this is not mentioned in the manual at all) and what >is right is (currently) based on my honest feelings. >Then each cronjobs frequency is determined and the cronjobs log output is checked based on the script parameters. >Currently the cronjob frequencies are split to minutely, hourly, daily, etc. >Koha_cronjobs-script is meant to be ran on those intervals to catch errors ASAP. >--- > misc/cronjobtimer.sh | 55 +++++++ > misc/nagios/README | 3 + > misc/nagios/commands.cfg | 6 + > misc/nagios/koha_cronjobs | 378 +++++++++++++++++++++++++++++++++++++++++++++ > misc/nagios/services.cfg | 47 ++++++ > 5 files changed, 489 insertions(+) > create mode 100755 misc/cronjobtimer.sh > create mode 100644 misc/nagios/README > create mode 100644 misc/nagios/commands.cfg > create mode 100755 misc/nagios/koha_cronjobs > create mode 100644 misc/nagios/services.cfg > >diff --git a/misc/cronjobtimer.sh b/misc/cronjobtimer.sh >new file mode 100755 >index 0000000..f779b3f >--- /dev/null >+++ b/misc/cronjobtimer.sh >@@ -0,0 +1,55 @@ >+#!/bin/bash >+ >+## SYNOPSIS ## >+# This script takes another script and measures it's runtime, appending it to the same log that the script's stderr and stdout go. >+# First argument is the script name, relative to this scripts location. >+# All arguments are passed directly to the first argument script. >+# >+# This script also locks the given script to prevent cronjobs from going to a repeating loop. >+# >+ >+logdirectory=$(grep -Po '(?<=<logdir>).*?(?=</logdir>)' $KOHA_CONF)/ >+logfile=$1".log" >+cronjobdir=$(echo "$logfile" | grep -Po '.*?(?=/)') >+logpath=$logdirectory$logfile >+croncommand=$KOHA_PATH"/misc/$@" >+lockfile=$KOHA_PATH"/misc/$1.lock" >+disableCronjobsFlag=$KOHA_PATH"/misc/cronjobs/disableCronjobs.flag" >+ >+if [ ! -d "$logdirectory/$cronjobdir" ]; then >+ #Make sure that the cronjob folders exists >+ mkdir -p "$logdirectory/$cronjobdir" >+fi >+ >+ >+starttime=$(date +%s) >+startMsg='Start: '$(date --date="@$starttime" "+%Y-%m-%d %H:%M:%S") >+printf "$startMsg\n" >> $logpath >+ >+#Check if cronjobs are disabled by the preproduction to production migration process >+if [ -e $disableCronjobsFlag ]; then >+ echo "Disabled by disableCronjobsFlag" >> $logpath >+ exit 0; >+fi >+ >+#Lock the cronjob we are running! >+if [ -e $lockfile ]; then >+ echo "Lockfile present" >> $logpath >+ exit 0; >+fi >+touch $lockfile >+ >+ >+starttime=$(date +%s) >+ >+ >+$croncommand 2&>> $logpath >+ >+ >+endtime=$(date +%s) >+runtime=$((endtime - starttime)) >+timelog='End: '$(date --date="@$endtime" "+%Y-%m-%d %H:%M:%S")"\n"'Runtime: '$(($runtime/60/60))':'$(($runtime/60%60))':'$(($runtime%60)) >+ >+printf "$timelog\n" >> $logpath >+ >+rm $lockfile >diff --git a/misc/nagios/README b/misc/nagios/README >new file mode 100644 >index 0000000..91b0b0d >--- /dev/null >+++ b/misc/nagios/README >@@ -0,0 +1,3 @@ >+nagios3 compatible monitoring scripts are found here. >+ >+See commands.cfg and services.cfg for usage examples >diff --git a/misc/nagios/commands.cfg b/misc/nagios/commands.cfg >new file mode 100644 >index 0000000..736530d >--- /dev/null >+++ b/misc/nagios/commands.cfg >@@ -0,0 +1,6 @@ >+#Link the koha_cronjobs to the nagios3 plugins directory symbolically. >+# ln -s /home/koha/kohaclone/misc/nagios/koha_cronjobs /usr/lib/nagios/plugins/koha_cronjobs >+define command{ >+ command_name ssh_koha_cronjobs >+ command_line /usr/lib/nagios/plugins/check_by_ssh -H '$HOSTADDRESS$' -C "/usr/lib/nagios/plugins/koha_cronjobs -c $ARG1$" >+ } >diff --git a/misc/nagios/koha_cronjobs b/misc/nagios/koha_cronjobs >new file mode 100755 >index 0000000..b98735a >--- /dev/null >+++ b/misc/nagios/koha_cronjobs >@@ -0,0 +1,378 @@ >+#!/usr/bin/perl >+ >+use Modern::Perl; >+ >+## SYNOPSIS >+# >+# >+# >+ >+use File::Basename; >+use Getopt::Long qw(:config no_ignore_case); >+ >+my $kohaLogDirectory = "/home/koha/koha-dev/var/log/"; >+my $help = 0; >+my $verbose = 0; >+my $cursive = 0; >+my $validateCrontabConfiguration = 0; >+my $checkMinutelyCronjobs = 0; >+my $checkHourlyCronjobs = 0; >+my $checkDailyCronjobs = 0; >+my $checkWeeklyCronjobs = 0; >+my $checkMonthlyCronjobs = 0; >+my $checkAnnualCronjobs = 0; >+my $debugLogwarn = ''; #Sets the logwarn flag -z active to analyze the whole logfile, instead of just the recent changes. >+ >+GetOptions( >+ 'h|help|?' => \$help, >+ 'V|validate' => \$validateCrontabConfiguration, >+ 'H|hourly' => \$checkHourlyCronjobs, >+ 'm|minutely' => \$checkMinutelyCronjobs, >+ 'D|daily' => \$checkDailyCronjobs, >+ 'W|weekly' => \$checkWeeklyCronjobs, >+ 'M|monthly' => \$checkMonthlyCronjobs, >+ 'A|annually' => \$checkAnnualCronjobs, >+ 'v|verbose=i' => \$verbose, >+ 'l|logdir=s' => \$kohaLogDirectory, >+ 'c|cursive' => \$cursive, >+ 'd|debuglogwarn' => \$debugLogwarn, >+); >+ >+$debugLogwarn = ($debugLogwarn eq '1') ? ' -z ' : ''; >+ >+my $usage = << 'ENDUSAGE'; >+ >+REQUIREMENTS: >+install logwarn from https://code.google.com/p/logwarn/ >+Read rights to the crontab file of the crontab user. >+ >+SUMMARY: >+This script scans the log output of all the defined Koha cronjobs and prints to screen if any new >+errors are found or if the scripts haven't ran for some reason. >+It is important to understand logwarn's method of scanning log files: >+Logwarn, every time it is invoked, remembers the last position of the logfile it is reading. Running >+it more often than the cronjob, causes this script to warn, because it thinks the cronjob failed to run. >+Thus from the checking point of view it is important to configure the different analysis frequencies. >+ >+It is safe to analyze following script types regularly at the following intervals >+ --minutely scripts once an hour >+ --hourly and daily scripts once a day >+ --weekly scripts once a week >+ --monthly scripts once a month >+ --annually scripts once a year >+ >+LIMITATIONS: >+-It is important to note that if a cronjob hasn't ran atleast once before this monitoring script is triggered, >+ this script will think that there is an error in running that cronjob. >+-This script might not catch "flapping" cronjobs. Cronjobs which sometimes run and sometimes don't. >+-This script makes logwarn look for error markers. Not all error markers have been discovered and thus logwarn >+ might not catch all errors. >+ >+ >+This script has the following parameters : >+ >+ -h --help this message >+ >+ -V --validate Validate the crontab configuration at /var/spool/cron/crontabs/koha. >+ Script tells you if the timing is not optimal or if you have missing >+ cronjob definitions. This feature requires read access to the crontab file. >+ Validation rules are defined in this script and can be changed to suit one's needs. >+ >+ -A --annually Analyze annually ran cronjobs that are ran few times a year. >+ >+ -M --monthly Analyze monthly cronjobs that are ran few times or once a month. >+ >+ -W --weekly Analyze weekly cronjobs. >+ >+ -D --daily Analyze cronjobs that are ran daily. >+ >+ -H --hourly Analyze hourly cronjobs, this means all cronjobs that are ran many times a day. >+ >+ -m --minutely Analyze minutely cronjobs, this means cronjobs that are ran many times per hour, or once a hour. >+ >+ -l --logdir Change the directory to look for Koha cronjob logs. >+ Defaults to /home/koha/koha-dev/var/log/ >+ >+ -c --cursive Output only very shy indicators of success and/or failure. >+ Meant to be used from a remote monitoring tool. >+ >+ -v --verbose show verbose information about script internals on few levels. >+ --verbose 1, prints basic verbose output >+ --verbose 2, also prints more detailed internal behaviour >+ >+ -d --debuglogwarn Sets the logwarn flag -p active to analyze the whole logfile, >+ instead of just the recent changes. >+ >+ENDUSAGE >+ >+die $usage if $help; >+ >+if ( not($validateCrontabConfiguration) && not($checkMinutelyCronjobs) && not($checkHourlyCronjobs) >+ && not($checkDailyCronjobs) && not($checkWeeklyCronjobs) && not($checkMonthlyCronjobs) >+ && not($checkAnnualCronjobs) >+ ) { >+ die "\nYou must define atleast one of the following operations! -V -D -H -M -W\n\n$usage"; >+} >+ >+ >+## SET CRONJOB VALIDATION RULES, MARKERS AND EXPECTED VALUES >+my $cronjobs = { >+ "advance_notices.pl" => { >+ "recommendedCronschedule" => "25 0 * * *", >+ "errorMarkers" => [ "",""], >+ }, >+ "cart_to_shelf.pl" => { >+ "recommendedCronschedule" => "15 0 * * *", >+ "errorMarkers" => [ "",""], >+ }, >+ "check-url-quick.pl" => { >+ "recommendedCronschedule" => "0 0 2 * *", >+ "errorMarkers" => [ "",""], >+ }, >+ "cleanup_database.pl" => { >+ "recommendedCronschedule" => "0 6 * * 7", >+ "errorMarkers" => [ "",""], >+ }, >+ "cloud-kw.pl" => { >+ "recommendedCronschedule" => "30 1 1 * *", >+ "errorMarkers" => [ "",""], >+ }, >+ "create_koc_db.pl" => { >+ "recommendedCronschedule" => "45 0 * * *", >+ "errorMarkers" => [ "",""], >+ }, >+ "delete_expired_opac_registrations.pl" => { >+ "recommendedCronschedule" => "0 0 * * *", >+ "errorMarkers" => [ "",""], >+ }, >+ "fines.pl" => { >+ "recommendedCronschedule" => "30 00 * * *", >+ "errorMarkers" => [ "",""], >+ }, >+ "gather_print_notices.pl" => { >+ "recommendedCronschedule" => "50 23 * * *", >+ "errorMarkers" => [ "",""], >+ }, >+ "j2a.pl" => { >+ "recommendedCronschedule" => "0 0 * * *", >+ "errorMarkers" => [ "",""], >+ }, >+ "overdue_notices.pl" => { >+ "recommendedCronschedule" => "35 0 * * *", >+ "errorMarkers" => [ "",""], >+ }, >+ "process_message_queue.pl" => { >+ "recommendedCronschedule" => "* * * * *", >+ "errorMarkers" => [ "",""], >+ }, >+ "pullSelectionListsFromBookvendors.pl" => { >+ "recommendedCronschedule" => "20 0 * * *", >+ "errorMarkers" => [ "",""], >+ }, >+ "purge_suggestions.pl" => { >+ "recommendedCronschedule" => "0 0 15 */6 *", >+ "errorMarkers" => [ "",""], >+ }, >+ "removeShortLoanStatus.pl" => { >+ "recommendedCronschedule" => "0 0 * * 7", >+ "errorMarkers" => [ "",""], >+ }, >+ "send_overdue_mail.pl" => { >+ "recommendedCronschedule" => "5 1 * * 2", >+ "errorMarkers" => [ "",""], >+ }, >+ "serialsUpdate.pl" => { >+ "recommendedCronschedule" => "20 0 * * *", >+ "errorMarkers" => [ "",""], >+ }, >+ "smsalertnumberFromPhone.pl" => { >+ "recommendedCronschedule" => "*/30 * * * *", >+ "errorMarkers" => [ "",""], >+ }, >+ "update_totalissues.pl" => { >+ "recommendedCronschedule" => "10 1 * * *", >+ "errorMarkers" => [ "",""], >+ }, >+ "auto_unsuspend_holds.pl" => { >+ "recommendedCronschedule" => "4 0 * * *", >+ "errorMarkers" => [ "",""], >+ }, >+ "build_holds_queue.pl" => { >+ "recommendedCronschedule" => "0 * * * *", >+ "errorMarkers" => [ "",""], >+ }, >+ "cancel_expired_holds.pl" => { >+ "recommendedCronschedule" => "5 0 * * *", >+ "errorMarkers" => [ "",""], >+ }, >+}; >+ >+ >+my @failedCronjobs; #This is needed to communicate cursively only the failed cronjobs, with no extra log output. >+ >+ >+main(); #Friends don't let friends write unscoped script monsters :) >+ >+ >+sub main { >+ open(my $KOHACRON, "<:encoding(utf8)", "/var/spool/cron/crontabs/koha") or die "Failed to open crontab file for reading: ".$!; >+ >+ while (<$KOHACRON>) { >+ >+#next unless $_ =~ 'process_message_queue.pl'; >+ my $cronEntry = $_; chomp $cronEntry; >+ my ($schedule, $m, $h, $dom, $mon, $dow); >+ my ($jobTimingCategory); >+ my ($name, $path, $suffix); >+ next() if $cronEntry =~ /^#/; #Skip comment lines >+ next() if $cronEntry =~ /^\s*$/; #Skip empty lines >+ >+ print "\nStarting processing this cron entry:\n$cronEntry\n" if ( $verbose > 1 && not($cursive) ); >+ >+ my $manyTimes = '^\*/?[0-9]*$'; >+ my $fewTimesOrOnce = '^\*?/?[0-9]+'; >+ my $once = '^[0-9]+$'; >+ ##Extract the schedule portion >+ if ($cronEntry =~ s|^(([*/0-9]{1,4})\s+([*/0-9]{1,4})\s+([*/0-9]{1,4})\s+([*/0-9]{1,4})\s+([*/0-9]{1,4}))\s+||) { >+ $schedule = $1; $m = $2; $h = $3; $dom = $4; $mon = $5; $dow = $6; >+ >+ if ($mon =~ m|$fewTimesOrOnce|) { >+ $jobTimingCategory = 'annally'; >+ } >+ elsif ($dom =~ m|$fewTimesOrOnce|) { >+ $jobTimingCategory = 'Monthly'; #Monthly is capitalized to separate it from minutely >+ } >+ elsif ($dow =~ m|$fewTimesOrOnce|) { >+ $jobTimingCategory = 'weekly'; >+ } >+ elsif ($m =~ m|$once| && $h =~ m|$once|) { >+ $jobTimingCategory = 'daily'; >+ } >+ elsif ($h =~ m|$manyTimes| && $m =~ m|$once|) { >+ $jobTimingCategory = 'hourly'; >+ } >+ elsif ( $m =~ m|$manyTimes| ) { >+ $jobTimingCategory = 'minutely'; >+ } >+ } >+ else { >+ print "Couldn't parse the cronjob scheduling portion of $cronEntry\n"; >+ push @failedCronjobs, $cronEntry; >+ next(); >+ } >+ print "This is a $jobTimingCategory cronjob\n" if ( $verbose > 1 && not($cursive) ); >+ ##Extract the basename portion to match cronjob definitions. >+ #Doesn't work with cronjobs with a space ' ' in them! >+ if ($cronEntry =~ /\$KOHA_CRONJOB_TRIGGER\s+([^# ]+)/) { >+ ($name, $path, $suffix) = File::Basename::fileparse( $1 ); >+ chomp $name; >+ } >+ else { >+ print "Couldn't get the command portion of cronjob $cronEntry\n"; >+ push @failedCronjobs, $cronEntry; >+ next(); >+ } >+ >+ validateCronjob($schedule, $name) if $validateCrontabConfiguration; >+ >+ if ($checkMinutelyCronjobs && $jobTimingCategory =~ /^m/) { >+ checkCronjob($name, $path); >+ } >+ elsif ($checkHourlyCronjobs && $jobTimingCategory =~ /^h/) { >+ checkCronjob($name, $path); >+ } >+ elsif ($checkDailyCronjobs && $jobTimingCategory =~ /^d/) { >+ checkCronjob($name, $path); >+ } >+ elsif ($checkWeeklyCronjobs && $jobTimingCategory =~ /^w/) { >+ checkCronjob($name, $path); >+ } >+ elsif ($checkMonthlyCronjobs && $jobTimingCategory =~ /^M/) { #Monthly is capitalized so it wont get mixed with 'minutely' >+ checkCronjob($name, $path); >+ } >+ elsif ($checkAnnualCronjobs && $jobTimingCategory =~ /^a/) { >+ checkCronjob($name, $path); >+ } >+ else { >+ print "Doing nothing because not of the requested timing frequencey\n" if ($verbose > 1 && not($cursive)); >+ } >+ } >+ >+ #Check for any missing cronjobs >+ if ($validateCrontabConfiguration) { >+ foreach my $job (sort keys %$cronjobs) { >+ print "$job is not defined in the crontab!\n" unless exists $cronjobs->{$job}->{found}; >+ } >+ } >+ >+ #Print a report of failed cronjobs >+ if ($cursive) { >+ if (scalar(@failedCronjobs)) { >+ print "@failedCronjobs"; >+ exit 2; #Signal Nagios to critical this! >+ } >+ else { >+ print "OK!"; >+ exit 0; #Signal Nagios to exit OK! >+ } >+ } >+} >+ >+sub validateCronjob { >+ my ($cronjobSchedule, $cronjobName) = @_; >+ >+ my $cronjobRecommendation = $cronjobs->{$cronjobName}; >+ >+ return unless $cronjobRecommendation; >+ >+ if ($cronjobSchedule eq $cronjobRecommendation->{recommendedCronschedule}) { >+ print "Validated $cronjobName\n" if $verbose; >+ } >+ else { >+ print "Not recommended cronjob schedule for $cronjobName. This is recommended:\n".$cronjobRecommendation->{recommendedCronschedule}."\n"; >+ } >+ $cronjobRecommendation->{found} = 1; >+} >+ >+sub checkCronjob { >+ my ($cronjob, $path) = @_; >+ >+ if ($cronjobs->{$cronjob}->{checked}) { >+ #Don't check twice, because the first run already read all the fresh log entries and subsequent reads will fail! >+ print "Trying to run this $cronjob many times, skipping." if ( $verbose > 1 && not($cursive) ); >+ return; >+ } >+ $cronjobs->{$cronjob}->{checked} = 1; >+ >+ #Check if we even tried to run the cronjob? >+# my $date = `date --date @\$((\$(date +%s)-3600)) -Idate`; >+# chomp $date; >+ my $logwarnCmd = "logwarn -d /var/lib/nagios/.logwarn/ -p $debugLogwarn $kohaLogDirectory$path$cronjob.log 'Start: ' 2>&1 "; >+ open( my $LOGWARN_DIDITRUN, "$logwarnCmd |" ) or warn "Can't run '$logwarnCmd\n$!\n'"; >+ my $errors = 0; >+ my $lastRuntime = 0; >+ >+ while (<$LOGWARN_DIDITRUN>) { >+ $lastRuntime = $_; >+ } >+ if ($lastRuntime) { >+ print "Ran OK+ $cronjob :> $lastRuntime\n" if ( $verbose && not($cursive) ); >+ } >+ else { >+ print "Run FAILed! $cronjob\n" if ( not($cursive)); >+ $errors++; >+ } >+ close($LOGWARN_DIDITRUN); >+ >+ #Check if we get errors from the cronjob >+ $logwarnCmd = "logwarn -d /var/lib/nagios/.logwarn/ -p $debugLogwarn $kohaLogDirectory$path$cronjob.log '^BEGIN failed\-\-' '^syntax error at' '^Global symbol' '^DBD\:\:' 'Permission denied' '^Use of uninitialized value' '^Disabled by disableCronjobsFlag' '^Lockfile present' "; >+ open( my $LOGWARN_ERRORS, "$logwarnCmd |" ) or warn "Can't run '$logwarnCmd\n$!\n'"; >+ while (<$LOGWARN_ERRORS>) { >+ print "Error on $cronjob :> $_\n" if (not($cursive)); >+ $errors++; >+ last(); >+ } >+ close($LOGWARN_ERRORS); >+ >+ push @failedCronjobs, $cronjob if $errors; >+} >diff --git a/misc/nagios/services.cfg b/misc/nagios/services.cfg >new file mode 100644 >index 0000000..af48825 >--- /dev/null >+++ b/misc/nagios/services.cfg >@@ -0,0 +1,47 @@ >+#Host definition for the Koha cronjobs server >+define host{ >+ use generic-host ; Name of host template to use >+ host_name aino.vaarakirjastot.fi >+ alias kohautility >+ address 10.110.249.16 >+ } >+ >+#Make hourly checks for hourly and minutely cronjobs >+define service{ >+ use generic-service >+ host_name aino.vaarakirjastot.fi >+ service_description Cronjobs Hourly >+ check_command ssh_koha_cronjobs!-m -H! >+ normal_check_interval 65 >+ max_check_attempts 1 >+ } >+ >+#Check daily cronjobs >+define service{ >+ use generic-service >+ host_name aino.vaarakirjastot.fi >+ service_description Cronjobs Daily >+ check_command ssh_koha_cronjobs!-D! >+ normal_check_interval 1450 >+ max_check_attempts 1 >+ } >+ >+#Check weekly cronjobs >+define service{ >+ use generic-service >+ host_name aino.vaarakirjastot.fi >+ service_description Cronjobs Weekly >+ check_command ssh_koha_cronjobs!-W! >+ normal_check_interval 10090 >+ max_check_attempts 1 >+ } >+ >+#Check monthly cronjobs >+define service{ >+ use generic-service >+ host_name aino.vaarakirjastot.fi >+ service_description Cronjobs Monthly >+ check_command ssh_koha_cronjobs!-M! >+ normal_check_interval 44650 >+ max_check_attempts 1 >+ } >-- >1.7.9.5 >
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 12902
: 31509