View | Details | Raw Unified | Return to bug 12902
Collapse All | Expand All

(-)a/misc/cronjobtimer.sh (+55 lines)
Line 0 Link Here
1
#!/bin/bash
2
3
## SYNOPSIS ##
4
# This script takes another script and measures it's runtime, appending it to the same log that the script's stderr and stdout go.
5
# First argument is the script name, relative to this scripts location.
6
# All arguments are passed directly to the first argument script.
7
#
8
# This script also locks the given script to prevent cronjobs from going to a repeating loop.
9
#
10
11
logdirectory=$(grep -Po '(?<=<logdir>).*?(?=</logdir>)' $KOHA_CONF)/
12
logfile=$1".log"
13
cronjobdir=$(echo "$logfile" | grep -Po '.*?(?=/)')
14
logpath=$logdirectory$logfile
15
croncommand=$KOHA_PATH"/misc/$@"
16
lockfile=$KOHA_PATH"/misc/$1.lock"
17
disableCronjobsFlag=$KOHA_PATH"/misc/cronjobs/disableCronjobs.flag"
18
19
if [ ! -d "$logdirectory/$cronjobdir" ]; then
20
        #Make sure that the cronjob folders exists
21
        mkdir -p "$logdirectory/$cronjobdir"
22
fi
23
24
25
starttime=$(date +%s)
26
startMsg='Start: '$(date --date="@$starttime" "+%Y-%m-%d %H:%M:%S")
27
printf "$startMsg\n" >> $logpath
28
29
#Check if cronjobs are disabled by the preproduction to production migration process
30
if [ -e $disableCronjobsFlag ]; then
31
       echo "Disabled by disableCronjobsFlag" >> $logpath
32
        exit 0;
33
fi
34
35
#Lock the cronjob we are running!
36
if [ -e $lockfile ]; then
37
       echo "Lockfile present" >> $logpath
38
       exit 0;
39
fi
40
touch $lockfile
41
42
43
starttime=$(date +%s)
44
45
46
$croncommand 2&>> $logpath
47
48
49
endtime=$(date +%s)
50
runtime=$((endtime - starttime))
51
timelog='End: '$(date --date="@$endtime" "+%Y-%m-%d %H:%M:%S")"\n"'Runtime: '$(($runtime/60/60))':'$(($runtime/60%60))':'$(($runtime%60))
52
53
printf "$timelog\n" >> $logpath
54
55
rm $lockfile
(-)a/misc/nagios/README (+3 lines)
Line 0 Link Here
1
nagios3 compatible monitoring scripts are found here.
2
3
See commands.cfg and services.cfg for usage examples
(-)a/misc/nagios/commands.cfg (+6 lines)
Line 0 Link Here
1
#Link the koha_cronjobs to the nagios3 plugins directory symbolically.
2
# ln -s /home/koha/kohaclone/misc/nagios/koha_cronjobs /usr/lib/nagios/plugins/koha_cronjobs
3
define command{
4
        command_name    ssh_koha_cronjobs
5
        command_line    /usr/lib/nagios/plugins/check_by_ssh -H '$HOSTADDRESS$' -C "/usr/lib/nagios/plugins/koha_cronjobs -c $ARG1$"
6
        }
(-)a/misc/nagios/koha_cronjobs (+378 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
use Modern::Perl;
4
5
## SYNOPSIS
6
#
7
#
8
#
9
10
use File::Basename;
11
use Getopt::Long qw(:config no_ignore_case);
12
13
my $kohaLogDirectory = "/home/koha/koha-dev/var/log/";
14
my $help = 0;
15
my $verbose = 0;
16
my $cursive = 0;
17
my $validateCrontabConfiguration = 0;
18
my $checkMinutelyCronjobs = 0;
19
my $checkHourlyCronjobs = 0;
20
my $checkDailyCronjobs = 0;
21
my $checkWeeklyCronjobs = 0;
22
my $checkMonthlyCronjobs = 0;
23
my $checkAnnualCronjobs = 0;
24
my $debugLogwarn = ''; #Sets the logwarn flag -z active to analyze the whole logfile, instead of just the recent changes.
25
26
GetOptions(
27
    'h|help|?'     => \$help,
28
    'V|validate'   => \$validateCrontabConfiguration,
29
    'H|hourly'     => \$checkHourlyCronjobs,
30
    'm|minutely'   => \$checkMinutelyCronjobs,
31
    'D|daily'      => \$checkDailyCronjobs,
32
    'W|weekly'     => \$checkWeeklyCronjobs,
33
    'M|monthly'    => \$checkMonthlyCronjobs,
34
    'A|annually'   => \$checkAnnualCronjobs,
35
    'v|verbose=i'  => \$verbose,
36
    'l|logdir=s'   => \$kohaLogDirectory,
37
    'c|cursive'    => \$cursive,
38
    'd|debuglogwarn' => \$debugLogwarn,
39
);
40
41
$debugLogwarn = ($debugLogwarn eq '1') ? ' -z ' : '';
42
43
my $usage = << 'ENDUSAGE';
44
45
REQUIREMENTS:
46
install logwarn from https://code.google.com/p/logwarn/
47
Read rights to the crontab file of the crontab user.
48
49
SUMMARY:
50
This script scans the log output of all the defined Koha cronjobs and prints to screen if any new
51
errors are found or if the scripts haven't ran for some reason.
52
It is important to understand logwarn's method of scanning log files:
53
Logwarn, every time it is invoked, remembers the last position of the logfile it is reading. Running
54
it more often than the cronjob, causes this script to warn, because it thinks the cronjob failed to run.
55
Thus from the checking point of view it is important to configure the different analysis frequencies.
56
57
It is safe to analyze following script types regularly at the following intervals
58
  --minutely scripts once an hour
59
  --hourly and daily scripts once a day
60
  --weekly scripts once a week
61
  --monthly scripts once a month
62
  --annually scripts once a year
63
64
LIMITATIONS:
65
-It is important to note that if a cronjob hasn't ran atleast once before this monitoring script is triggered,
66
 this script will think that there is an error in running that cronjob.
67
-This script might not catch "flapping" cronjobs. Cronjobs which sometimes run and sometimes don't.
68
-This script makes logwarn look for error markers. Not all error markers have been discovered and thus logwarn
69
 might not catch all errors.
70
71
72
This script has the following parameters :
73
74
    -h --help         this message
75
76
    -V --validate     Validate the crontab configuration at /var/spool/cron/crontabs/koha.
77
                      Script tells you if the timing is not optimal or if you have missing
78
                      cronjob definitions. This feature requires read access to the crontab file.
79
                      Validation rules are defined in this script and can be changed to suit one's needs.
80
81
    -A --annually     Analyze annually ran cronjobs that are ran few times a year.
82
83
    -M --monthly      Analyze monthly cronjobs that are ran few times or once a month.
84
85
    -W --weekly       Analyze weekly cronjobs.
86
87
    -D --daily        Analyze cronjobs that are ran daily.
88
89
    -H --hourly       Analyze hourly cronjobs, this means all cronjobs that are ran many times a day.
90
91
    -m --minutely     Analyze minutely cronjobs, this means cronjobs that are ran many times per hour, or once a hour.
92
93
    -l --logdir       Change the directory to look for Koha cronjob logs.
94
                      Defaults to /home/koha/koha-dev/var/log/
95
96
    -c --cursive      Output only very shy indicators of success and/or failure.
97
                      Meant to be used from a remote monitoring tool.
98
99
    -v --verbose      show verbose information about script internals on few levels.
100
                      --verbose 1, prints basic verbose output
101
                      --verbose 2, also prints more detailed internal behaviour
102
103
    -d --debuglogwarn Sets the logwarn flag -p active to analyze the whole logfile,
104
                      instead of just the recent changes.
105
106
ENDUSAGE
107
108
die $usage if $help;
109
110
if (   not($validateCrontabConfiguration) && not($checkMinutelyCronjobs) && not($checkHourlyCronjobs)
111
    && not($checkDailyCronjobs) && not($checkWeeklyCronjobs) && not($checkMonthlyCronjobs)
112
    && not($checkAnnualCronjobs)
113
   ) {
114
    die "\nYou must define atleast one of the following operations! -V -D -H -M -W\n\n$usage";
115
}
116
117
118
## SET CRONJOB VALIDATION RULES, MARKERS AND EXPECTED VALUES
119
my $cronjobs = {
120
    "advance_notices.pl" => {
121
        "recommendedCronschedule" => "25 0 * * *",
122
        "errorMarkers" => [ "",""],
123
    },
124
    "cart_to_shelf.pl" => {
125
        "recommendedCronschedule" => "15 0 * * *",
126
        "errorMarkers" => [ "",""],
127
    },
128
    "check-url-quick.pl" => {
129
        "recommendedCronschedule" => "0 0 2 * *",
130
        "errorMarkers" => [ "",""],
131
    },
132
    "cleanup_database.pl" => {
133
        "recommendedCronschedule" => "0 6 * * 7",
134
        "errorMarkers" => [ "",""],
135
    },
136
    "cloud-kw.pl" => {
137
        "recommendedCronschedule" => "30 1 1 * *",
138
        "errorMarkers" => [ "",""],
139
    },
140
    "create_koc_db.pl" => {
141
        "recommendedCronschedule" => "45 0 * * *",
142
        "errorMarkers" => [ "",""],
143
    },
144
    "delete_expired_opac_registrations.pl" => {
145
        "recommendedCronschedule" => "0 0 * * *",
146
        "errorMarkers" => [ "",""],
147
    },
148
    "fines.pl" => {
149
        "recommendedCronschedule" => "30 00 * * *",
150
        "errorMarkers" => [ "",""],
151
    },
152
    "gather_print_notices.pl" => {
153
        "recommendedCronschedule" => "50 23 * * *",
154
        "errorMarkers" => [ "",""],
155
    },
156
    "j2a.pl" => {
157
        "recommendedCronschedule" => "0 0 * * *",
158
        "errorMarkers" => [ "",""],
159
    },
160
    "overdue_notices.pl" => {
161
        "recommendedCronschedule" => "35 0 * * *",
162
        "errorMarkers" => [ "",""],
163
    },
164
    "process_message_queue.pl" => {
165
        "recommendedCronschedule" => "* * * * *",
166
        "errorMarkers" => [ "",""],
167
    },
168
    "pullSelectionListsFromBookvendors.pl" => {
169
        "recommendedCronschedule" => "20 0 * * *",
170
        "errorMarkers" => [ "",""],
171
    },
172
    "purge_suggestions.pl" => {
173
        "recommendedCronschedule" => "0 0 15 */6 *",
174
        "errorMarkers" => [ "",""],
175
    },
176
    "removeShortLoanStatus.pl" => {
177
        "recommendedCronschedule" => "0 0 * * 7",
178
        "errorMarkers" => [ "",""],
179
    },
180
    "send_overdue_mail.pl" => {
181
        "recommendedCronschedule" => "5 1 * * 2",
182
        "errorMarkers" => [ "",""],
183
    },
184
    "serialsUpdate.pl" => {
185
        "recommendedCronschedule" => "20 0 * * *",
186
        "errorMarkers" => [ "",""],
187
    },
188
    "smsalertnumberFromPhone.pl" => {
189
        "recommendedCronschedule" => "*/30 * * * *",
190
        "errorMarkers" => [ "",""],
191
    },
192
    "update_totalissues.pl" => {
193
        "recommendedCronschedule" => "10 1 * * *",
194
        "errorMarkers" => [ "",""],
195
    },
196
    "auto_unsuspend_holds.pl" => {
197
        "recommendedCronschedule" => "4 0 * * *",
198
        "errorMarkers" => [ "",""],
199
    },
200
    "build_holds_queue.pl" => {
201
        "recommendedCronschedule" => "0 * * * *",
202
        "errorMarkers" => [ "",""],
203
    },
204
    "cancel_expired_holds.pl" => {
205
        "recommendedCronschedule" => "5 0 * * *",
206
        "errorMarkers" => [ "",""],
207
    },
208
};
209
210
211
my @failedCronjobs; #This is needed to communicate cursively only the failed cronjobs, with no extra log output.
212
213
214
main(); #Friends don't let friends write unscoped script monsters :)
215
216
217
sub main {
218
    open(my $KOHACRON, "<:encoding(utf8)", "/var/spool/cron/crontabs/koha") or die "Failed to open crontab file for reading: ".$!;
219
220
    while (<$KOHACRON>) {
221
222
#next unless $_ =~ 'process_message_queue.pl';
223
        my $cronEntry = $_; chomp $cronEntry;
224
        my ($schedule, $m, $h, $dom, $mon, $dow);
225
        my ($jobTimingCategory);
226
        my ($name, $path, $suffix);
227
        next() if $cronEntry =~ /^#/; #Skip comment lines
228
        next() if $cronEntry =~ /^\s*$/; #Skip empty lines
229
230
        print "\nStarting processing this cron entry:\n$cronEntry\n" if ( $verbose > 1 && not($cursive) );
231
232
        my $manyTimes = '^\*/?[0-9]*$';
233
        my $fewTimesOrOnce  = '^\*?/?[0-9]+';
234
        my $once      = '^[0-9]+$';
235
        ##Extract the schedule portion
236
        if ($cronEntry =~ s|^(([*/0-9]{1,4})\s+([*/0-9]{1,4})\s+([*/0-9]{1,4})\s+([*/0-9]{1,4})\s+([*/0-9]{1,4}))\s+||) {
237
            $schedule = $1; $m = $2; $h = $3; $dom = $4; $mon = $5; $dow = $6;
238
239
            if ($mon =~ m|$fewTimesOrOnce|) {
240
                $jobTimingCategory = 'annally';
241
            }
242
            elsif ($dom =~ m|$fewTimesOrOnce|) {
243
                $jobTimingCategory = 'Monthly'; #Monthly is capitalized to separate it from minutely
244
            }
245
            elsif ($dow =~ m|$fewTimesOrOnce|) {
246
                $jobTimingCategory = 'weekly';
247
            }
248
            elsif ($m =~ m|$once| && $h =~ m|$once|) {
249
                $jobTimingCategory = 'daily';
250
            }
251
            elsif ($h =~ m|$manyTimes| && $m =~ m|$once|) {
252
                $jobTimingCategory = 'hourly';
253
            }
254
            elsif ( $m =~ m|$manyTimes| ) {
255
                $jobTimingCategory = 'minutely';
256
            }
257
        }
258
        else {
259
            print "Couldn't parse the cronjob scheduling portion of $cronEntry\n";
260
            push @failedCronjobs, $cronEntry;
261
            next();
262
        }
263
        print "This is a $jobTimingCategory cronjob\n" if ( $verbose > 1 && not($cursive) );
264
        ##Extract the basename portion to match cronjob definitions.
265
        #Doesn't work with cronjobs with a space ' ' in them!
266
        if ($cronEntry =~ /\$KOHA_CRONJOB_TRIGGER\s+([^# ]+)/) {
267
            ($name, $path, $suffix) = File::Basename::fileparse( $1 );
268
            chomp $name;
269
        }
270
        else {
271
            print "Couldn't get the command portion of cronjob $cronEntry\n";
272
            push @failedCronjobs, $cronEntry;
273
            next();
274
        }
275
276
        validateCronjob($schedule, $name) if $validateCrontabConfiguration;
277
278
        if ($checkMinutelyCronjobs && $jobTimingCategory =~ /^m/) {
279
            checkCronjob($name, $path);
280
        }
281
        elsif ($checkHourlyCronjobs && $jobTimingCategory =~ /^h/) {
282
            checkCronjob($name, $path);
283
        }
284
        elsif ($checkDailyCronjobs && $jobTimingCategory =~ /^d/) {
285
            checkCronjob($name, $path);
286
        }
287
        elsif ($checkWeeklyCronjobs && $jobTimingCategory =~ /^w/) {
288
            checkCronjob($name, $path);
289
        }
290
        elsif ($checkMonthlyCronjobs && $jobTimingCategory =~ /^M/) { #Monthly is capitalized so it wont get mixed with 'minutely'
291
            checkCronjob($name, $path);
292
        }
293
        elsif ($checkAnnualCronjobs && $jobTimingCategory =~ /^a/) {
294
            checkCronjob($name, $path);
295
        }
296
        else {
297
            print "Doing nothing because not of the requested timing frequencey\n" if ($verbose > 1 && not($cursive));
298
        }
299
    }
300
301
    #Check for any missing cronjobs
302
    if ($validateCrontabConfiguration) {
303
        foreach my $job (sort keys %$cronjobs) {
304
            print "$job is not defined in the crontab!\n" unless exists $cronjobs->{$job}->{found};
305
        }
306
    }
307
308
    #Print a report of failed cronjobs
309
    if ($cursive) {
310
        if (scalar(@failedCronjobs)) {
311
            print "@failedCronjobs";
312
            exit 2; #Signal Nagios to critical this!
313
        }
314
        else {
315
            print "OK!";
316
            exit 0; #Signal Nagios to exit OK!
317
        }
318
    }
319
}
320
321
sub validateCronjob {
322
    my ($cronjobSchedule, $cronjobName) = @_;
323
324
    my $cronjobRecommendation = $cronjobs->{$cronjobName};
325
326
    return unless $cronjobRecommendation;
327
328
    if ($cronjobSchedule eq $cronjobRecommendation->{recommendedCronschedule}) {
329
        print "Validated $cronjobName\n" if $verbose;
330
    }
331
    else {
332
        print "Not recommended cronjob schedule for $cronjobName. This is recommended:\n".$cronjobRecommendation->{recommendedCronschedule}."\n";
333
    }
334
    $cronjobRecommendation->{found} = 1;
335
}
336
337
sub checkCronjob {
338
    my ($cronjob, $path) = @_;
339
340
    if ($cronjobs->{$cronjob}->{checked}) {
341
        #Don't check twice, because the first run already read all the fresh log entries and subsequent reads will fail!
342
        print "Trying to run this $cronjob many times, skipping." if ( $verbose > 1 && not($cursive) );
343
        return;
344
    }
345
    $cronjobs->{$cronjob}->{checked} = 1;
346
347
    #Check if we even tried to run the cronjob?
348
#    my $date = `date --date @\$((\$(date +%s)-3600)) -Idate`;
349
#    chomp $date;
350
    my $logwarnCmd = "logwarn -d /var/lib/nagios/.logwarn/ -p $debugLogwarn $kohaLogDirectory$path$cronjob.log 'Start: ' 2>&1 ";
351
    open( my $LOGWARN_DIDITRUN, "$logwarnCmd |" ) or warn "Can't run '$logwarnCmd\n$!\n'";
352
    my $errors = 0;
353
    my $lastRuntime = 0;
354
355
    while (<$LOGWARN_DIDITRUN>) {
356
        $lastRuntime = $_;
357
    }
358
    if ($lastRuntime) {
359
        print "Ran OK+ $cronjob :> $lastRuntime\n" if ( $verbose && not($cursive) );
360
    }
361
    else {
362
        print "Run FAILed! $cronjob\n" if ( not($cursive));
363
        $errors++;
364
    }
365
    close($LOGWARN_DIDITRUN);
366
367
    #Check if we get errors from the cronjob
368
    $logwarnCmd = "logwarn -d /var/lib/nagios/.logwarn/ -p $debugLogwarn $kohaLogDirectory$path$cronjob.log '^BEGIN failed\-\-' '^syntax error at' '^Global symbol' '^DBD\:\:' 'Permission denied' '^Use of uninitialized value' '^Disabled by disableCronjobsFlag' '^Lockfile present' ";
369
    open( my $LOGWARN_ERRORS, "$logwarnCmd |" ) or warn "Can't run '$logwarnCmd\n$!\n'";
370
    while (<$LOGWARN_ERRORS>) {
371
        print "Error on $cronjob :> $_\n" if (not($cursive));
372
        $errors++;
373
        last();
374
    }
375
    close($LOGWARN_ERRORS);
376
377
    push @failedCronjobs, $cronjob if $errors;
378
}
(-)a/misc/nagios/services.cfg (-1 / +47 lines)
Line 0 Link Here
0
- 
1
#Host definition for the Koha cronjobs server
2
define host{
3
        use                     generic-host            ; Name of host template to use
4
        host_name               aino.vaarakirjastot.fi
5
        alias                   kohautility
6
        address                 10.110.249.16
7
        }
8
9
#Make hourly checks for hourly and minutely cronjobs
10
define service{
11
        use                     generic-service
12
        host_name               aino.vaarakirjastot.fi
13
        service_description     Cronjobs Hourly
14
        check_command           ssh_koha_cronjobs!-m -H!
15
                normal_check_interval           65
16
                max_check_attempts              1
17
        }
18
19
#Check daily cronjobs
20
define service{
21
        use                     generic-service
22
        host_name               aino.vaarakirjastot.fi
23
        service_description     Cronjobs Daily
24
        check_command           ssh_koha_cronjobs!-D!
25
                normal_check_interval           1450
26
                max_check_attempts              1
27
        }
28
29
#Check weekly cronjobs
30
define service{
31
        use                     generic-service
32
        host_name               aino.vaarakirjastot.fi
33
        service_description     Cronjobs Weekly
34
        check_command           ssh_koha_cronjobs!-W!
35
                normal_check_interval           10090
36
                max_check_attempts              1
37
        }
38
39
#Check monthly cronjobs
40
define service{
41
        use                     generic-service
42
        host_name               aino.vaarakirjastot.fi
43
        service_description     Cronjobs Monthly
44
        check_command           ssh_koha_cronjobs!-M!
45
                normal_check_interval           44650
46
                max_check_attempts              1
47
        }

Return to bug 12902