#!/usr/bin/perl
#
# mon - schedules service tests and triggers alerts upon failures
#
# Jim Trocki, trockij@transmeta.com
#
# $Id: mon,v 1.64 1998/12/14 17:24:22 trockij Exp $
#
# Copyright (C) 1998 Jim Trocki
#
#    This program is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation; either version 2 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
#
$RCSID='$Id: mon,v 1.64 1998/12/14 17:24:22 trockij Exp $';
$AUTHOR='trockij@transmeta.com';
use Getopt::Std;
use Text::ParseWords;
use POSIX;
use Fcntl;
use Socket;
use Time::Period;
use Sys::Hostname;
use Sys::Syslog;
use Time::HiRes qw(gettimeofday tv_interval);

getopts ("fhlSva:A:c:d:D:o:m:r:s:i:p:P:");

#
# definitions
#
$CF = $opt_c || (-d "/etc/mon" ? "/etc/mon" : "/etc") . "/mon.cf";
$SCRIPTDIR = "/usr/lib/mon/mon.d";
$ALERTDIR  = "/usr/lib/mon/alert.d";
$STATEDIR  = -d "/var/state/mon" ? "/var/state/mon"
	    : -d "/var/lib/mon" ? "/var/lib/mon"
	    : "/usr/lib/mon/state.d";
$AUTHFILE  = (-d "/etc/mon" ? "/etc/mon" : "/usr/lib/mon") . "/auth.cf";

$OCFILE    = "/usr/lib/mon/oncall.cf";
$PIDFILE   = (-d "/var/run/mon" ? "/var/run/mon"
		: -d "/var/run" ? "/var/run"
		: "/etc") . "/mon.pid";
$MAX_KEEP  = 100;
$SLEEPINT  = 1;
$CLIENT_TIMEOUT = 30;
$SERVPORT  = 32777;
$MAXPROCS  = 0;
$STOPPED   = 0;
$STOPPED_TIME = 0;
$SNMP = 0;
$HOSTNAME  = hostname;
$OS = `uname -s 2>/dev/null` || "Unknown";
chomp $OS;
$PWD = getcwd;
$CF = "$PWD/$CF" if ($CF !~ /^\//);

#
# flags
#
$FL_MONITOR = 1;
$FL_UPALERT = 2;
$FL_TRAP = 4;
$FL_TRAPTIMEOUT = 8;

#
# specific trap types
#
($TRAP_COLDSTART, $TRAP_WARMSTART, $TRAP_LINKDOWN, $TRAP_LINKUP,
    $TRAP_AUTHFAIL, $TRAP_EGPNEIGHBORLOSS, $TRAP_ENTERPRISE, $TRAP_HEARTBEAT) = (0..7);

($TRAP_MON, $TRAP_SNMPV1) = (0, 1);

#
# operational statuses
#
($STAT_FAIL, $STAT_OK, $STAT_COLDSTART, $STAT_WARMSTART, $STAT_LINKDOWN,
    $STAT_UNKNOWN, $STAT_TIMEOUT, $STAT_UNTESTED, $STAT_DEPEND, $STAT_WARN) = (0..9);

%OPSTAT = ("fail" => $STAT_FAIL, "ok" => $STAT_OK, "coldstart" => $STAT_COLDSTART,
	"warmstart" => $STAT_WARMSTART, "linkdown" => $STAT_LINKDOWN,
	"unknown" => $STAT_UNKNOWN, "timeout" => $STAT_TIMEOUT,
	"untested" => $STAT_UNTESTED, "dependency" => $STAT_DEPEND);

#
# fast lookup hashes for alerts and monitors
#
%MONITORHASH = ();
%ALERTHASH = ();

#
# these two things can be taken care of without
# initializing things further
#
if ($opt_v) {
    print "$RCSID\n";
    exit;
}

if ($opt_h) {
    &usage();
    exit;
}

#
# read config file
#
&read_cf ($CF) ||
    &die_die ("err", "could not open cf file: $CF: $!");

#
# cmdline args override config file
#
$ALERTDIR  = $opt_a if ($opt_a);
$AUTHFILE  = $opt_A if ($opt_A);
$STATEDIR  = $opt_D if ($opt_D);
$SLEEPINT  = $opt_i if ($opt_i);
$MAX_KEEP  = $opt_k if ($opt_k);
$MAXPROCS  = $opt_m if ($opt_m);
$PIDFILE   = $opt_P if defined($opt_P);	# allow empty pidfile
$SERVPORT  = $opt_p if ($opt_p);
$SCRIPTDIR = $opt_s if ($opt_s);
$OCFILE    = $opt_o if ($opt_o);
if ($opt_r) {
    die "bad randstart value\n"
	if (!defined(&dhmstos($opt_r)));
    $RANDSTART = &dhmstos($opt_r);
}
if ($opt_S) {
    $STOPPED = 1;
    $STOPPED_TIME = time;
}

#
# do some sanity checks on dirs
#
$SCRIPTDIR = "$PWD/$SCRIPTDIR" if ($SCRIPTDIR !~ /^\//);
$ALERTDIR = "$PWD/$ALERTDIR" if ($ALERTDIR !~ /^\//);
$STATEDIR = "$PWD/$STATEDIR" if ($STATEDIR !~ /^\//);
$AUTHFILE = "$PWD/$AUTHFILE" if ($AUTHFILE !~ /^\//);
$OCFILE = "$PWD/$OCFILE" if ($OCFILE !~ /^\//);
die "state dir $STATEDIR does not exist\n" if (!-d $STATEDIR);
die "auth file $AUTHFILE does not exist\n" if (!-e $AUTHFILE);
#die "oncall file $OCFILE does not exist\n" if (!-e $OCFILE);

#
# build lookup tables for alerts and monitors
#
&gen_scriptdir_hash();

#
# fork and become a daemon
#
$OS eq "Linux" && Sys::Syslog::setlogsock ('unix');
openlog ("mon", "cons,pid", "daemon");
&daemon() if ($opt_f);
if ($PIDFILE ne '' && open PID, ">$PIDFILE") {
    print PID "$$\n";
    close PID;
}

#
# load the auth control, oncall, bind, and listen
#
&load_auth(1);
%oncall = ();
#&load_oncall(1);

#
# init client interface
#   %clients is an I/O structure, indexed by the fd of the client
#   $numclients is the number of clients currently connected
#   $clientcount is used in &client_accept
#   $iovec is fd_set for clients and traps
#
%clients = ();
$numclients = 0;
$clientcount = 0;
$iovec = '';
&setup_server();

&set_last_test ();

#
# randomize startup checks if asked to
#
&randomize_startdelay()
    if ($RANDSTART);

@last_alerts = ();
@last_failures = ();
$procs = 0;				# number of outstanding procs
$i=0;					# loop iteration counter
$lasttm=time;				# the last time(2) the mon loop started
$fdset_rbits = $fdset_ebits = '';
%watch_disabled = ();
%alias = ();

$SIG{HUP} = \&reset;
$SIG{INT} = \&handle_sigterm;		# for interactive debugging
$SIG{TERM} = \&handle_sigterm;
$SIG{PIPE} = 'IGNORE';

#
# load previously saved state
#
&load_state ("disabled") if ($opt_l);

syslog ('info', "mon server started");

#
# main monitoring loop
#
for (;;) {
&debug (1, "$i" . ($STOPPED ? " (stopped)" : "") . "\n");
    $i++;
    $tm = time;

    #
    # step through the watch groups, decrementing and
    # handing expired timers
    #
    if (!$STOPPED) {
	foreach $group (keys %watch) {
	    #
	    # skip over disabled watch
	    #
	    next if ($watch_disabled{$group} == 1);

	    for ($service=0;$service<@{$watch{$group}};$service++) {

		$sref = \%{$watch{$group}[$service]};

		$t = $tm - $lasttm;
		$t = 1 if ($t <= 0);

		#
		# trap timer
		#
		if ($$sref{"traptimeout"}) {
		    $$sref{"_trap_timer"} -= $t;

		    if ($$sref{"_trap_timer"} <= 0 && $tm - $$sref{"_last_uptrap"} >
				$$sref{"traptimeout"}) {
			$$sref{"_trap_timer"} = $$sref{"traptimeout"};
			&handle_trap_timeout ($group, $service);
		    }
		}

		#
		# trap duration timer
		#
		if (defined ($$sref{"_trap_duration_timer"})) {
		    $$sref{"_trap_duration_timer"} -= $t;

		    if ($$sref{"_trap_duration_timer"} <= 0) {
		    	$$sref{"_op_status"} = $STAT_OK;
			undef $$sref{"_trap_duration_timer"};
		    }
		}

		#
		# polling monitor timer
		#
		if ($$sref{"interval"} && $$sref{"_timer"} <= 0 &&
			!$running{"$group/$service"}) {

		    if (($MAXPROCS && $procs < $MAXPROCS) || !$MAXPROCS) {
			&run_monitor();
		    } else {
			syslog ('info', "throttled at $procs processes");
		    }

		} else {
		    $$sref{"_timer"} -= $t;
		    if ($$sref{"_timer"} < 0) {
		    	$$sref{"_timer"} = 0;
		    }
		}
	    }
	}
    }

    $lasttm = time;

    #
    # collect any output from subprocs
    #
    &collect_output();

    #
    # clean up after exited processes, and trigger alerts
    # $sref is redefined after this point
    #
    &proc_cleanup();

    #
    # handle client, server, and trap I/O
    # this routine sleeps for $SLEEPINT if no I/O is ready
    #
    &handle_io();
}

&clean_up();
exit;

##############################################################################

#
# handle alert event
#
sub do_alert {
    my ($group, $service, $output, $retval, $flags) = @_;
    my ($tmnow, $fac, $args, $service_n, $i, $summary);
    my (@groupargs, $period, $last_alert, $alert, $alerts_sent);
    my ($sref, $pref, $range, @alerts);

    $sref = \%{$watch{$group}[$service]};

    $tmnow = time;
    $service_n = $watch{$group}[$service]{"service"};

    ($summary) = split("\n", $output);
    $summary = "(NO SUMMARY)" if ($summary =~ /^\s*$/m);

    $alerts_sent = 0;

    #
    # check each time period for pending alerts
    #
    foreach $period (keys %{$$sref{"periods"}}) {
	#
	# only send alerts that are in the proper period
	#
    	next if (!inPeriod ($tmnow, $period));

    	$pref = \%{$$sref{"periods"}{$period}};

	#
	# do this if we're not handling an upalert
	#
	if (!($flags & $FL_UPALERT)) {
	    #
	    # only alert once every "alertevery" seconds, unless
	    # output from monitor is different
	    #
	    if ($$pref{"alertevery"} != 0 &&
		    ($tmnow - $$pref{"_last_alert"} < $$pref{"alertevery"}) &&
			( $$pref{"_alertsum"} ? 
			    ($$sref{"_failure_output"} =~ /^$summary/ ) :
			    ($$sref{"_failure_output"} eq $output) ) ) {

		syslog ("info", "not alerting for failure of $group/" .
		    &getservbynum ($group, $service));
		next;
	    }

	    #
	    # handle alertafter conditions
	    #
	    if (defined ($$pref{"alertafter"})) {
		$$pref{"_failcount"}++;

		if ($tmnow - $$pref{'_1stfailtime'} <= $$pref{'alertafterival'}
		    && $$pref{"_failcount"} < $$pref{"alertafter"}) {
		    next;
		}

		#
		# start a new time interval
		#
		if ($tmnow - $$pref{'_1stfailtime'} > $$pref{'alertafterival'}) {
		    $$pref{"_failcount"} = 1;
		}

		if ($$pref{"_failcount"} == 1) {
		    $$pref{"_1stfailtime"} = $tmnow;
		}

		if ($$pref{"_failcount"} < $$pref{"alertafter"}) {
		    next;
		}
	    }
	}

	#
	# at this point, no alerts are blocked,
	# so send the alerts
	#

	#
	# trigger multiple alerts in this period
	#
	if ($flags & $FL_UPALERT) {
	    @alerts = @{$$pref{"upalerts"}};
	} else  {
	    @alerts = @{$$pref{"alerts"}};
	}

	for ($i=0;$i<@alerts;$i++) {

	    my $range;

	    if ($alerts[$i] =~ /^exit\s*=\s*((\d+|\d+-\d+))\s/i) {
		$range=$1;
		next if (!&inRange($retval, $range));
		($fac, $args) = (split (/\s+/, $alerts[$i], 3))[1,2];
	    } else {
		($fac, $args) = split (/\s+/, $alerts[$i], 2);
	    }
	    @groupargs = grep (!/^\*/, @{$groups{$group}});

	    if (!defined $ALERTHASH{$fac}) {
		syslog ('err', "no alert found while trying to run [$fac]");
&debug(3, "no alert found while trying to run [$fac]\n");
		next;
	    } else {
	    	$fac = $ALERTHASH{$fac};
	    }

	    #
	    # log why we are triggering an alert
	    #
	    if ($flags & $FL_UPALERT) {
		syslog ("alert", "calling upalert $fac for $group/$service_n" .
		    " ($fac,$args) $summary");
	    } elsif ($flags & $FL_TRAPTIMEOUT) {
		syslog ("alert", "calling trap timeout alert $fac for $group/$service_n" .
		    " ($fac,$args) $summary");
	    } elsif ($flags & $FL_TRAP) {
		syslog ("alert", "calling trap alert $fac for $group/$service_n" .
		    " ($fac,$args) $summary");
	    } else {
		syslog ("alert", "calling alert $fac for $group/$service_n" .
		    " ($fac,$args) $summary");
	    }

	    #
	    # set env variables to pass to the alert
	    #
	    foreach $v (keys %{$$sref{"ENV"}}) {
	    	$ENV{$v} = $$sref{"ENV"}{$v};
	    }
	    $ENV{"MON_LAST_SUMMARY"} = $$sref{"_last_summary"};
	    $ENV{"MON_LAST_OUTPUT"} = $$sref{"_last_output"};
	    $ENV{"MON_LAST_FAILURE"} = $$sref{"_last_failure"};
	    $ENV{"MON_FIRST_FAILURE"} = $$sref{"_first_failure"};
	    $ENV{"MON_LAST_SUCCESS"} = $$sref{"_last_success"};
	    $ENV{"MON_DESCRIPTION"} = $$sref{"description"};

	    if (!open (OUTF, "| $ALERTDIR/$fac -s '$service_n' " .
		    "-g '$group' -h '@groupargs' -t '$tmnow' " .
		    (($flags & $FL_UPALERT) ? "-u " : "") .
		    (($flags & $FL_TRAP) ? "-T " : "") .
		    (($flags & $FL_TRAPTIMEOUT) ? "-O " : "") .
		    "-l '$$pref{alertevery}' $args")) {
		syslog ('err', "could not open pipe to $ALERTDIR/$fac: $!");
	    } else {
		print OUTF $output;
		close (OUTF);
	    }
	    $alerts_sent++;
	}

	$$pref{"_last_alert"} = $tmnow;
    }

    return if (!$alerts_sent);

    #
    # tally this alert
    #
    $$sref{"_alert_count"}++;

    #
    # store this in the log
    #
    shift @last_alerts if (@last_alerts > $MAX_KEEP);
    if ($flags & $FL_UPALERT) {
	push @last_alerts, "upalert $group " . &getservbynum ($group, $service) .
	    " $tmnow $fac ($args) $summary";
    } else {
	push @last_alerts, "alert $group " . &getservbynum ($group, $service) .
	    " $tmnow $fac ($args) $summary";
    }
}



######################################################################
#
# walk through the watch list and reset the time
# the service was last called
#
sub set_last_test {
    my ($i, $k, $t);
    $t = time;
    foreach $k (keys %watch) {
    	for ($servnum=0; $servnum<@{$watch{$k}};$servnum++) {
	    $watch{$k}[$servnum]{"_timer"} = $watch{$k}[$servnum]{"interval"};
	    $watch{$k}[$servnum]{"_last_alert"} = 0;
	}
    }

}


######################################################################
#
# parse configuration file
#
# build the following data structures:
#
# %group
#       each element of %group is an array of hostnames
#       group records are terminated by a blank line in the
#       configuration file
# %watch{"group"}[service-number]{"variable"} = value
# %watch is a hash of arrays of hashes
#
# variables for internal use are:
#   $watch{"group"}[service-number]{"_timer"}, countdown timer between checks
#   $watch{"group"}[service-number]{"_last_alert"}, used by &do_alert
# 
#
sub read_cf {
    my ($CF) = @_;
    my ($l, $var, $watchgroup, $ingroup, $curgroup, $inwatch,
	$watchgroup, $servnum, $args, $hosts, %disabled, $h, $i,
	$aliasReading, $aliasGroup);

    #
    # parse configuration file
    #
    open (CFG, $CF) ||
	die "could not open cf file: $CF: $!\n";

    $servnum = 0;

    for (;;) {
    	last if (!defined ($l = <CFG>));
	next if $l =~ /^#/;
	chomp $l;
	$l =~ s/^\s*//;
	$l =~ s/\s*$//;

	#
	# variables than can be overriden by the command line
	#
	if ($l =~ /^alertdir\s*=\s*(\S+)/) {
	    $ALERTDIR = $1;
	    next;
	} elsif ($l =~ /^mondir\s*=\s*(\S+)/) {
	    $SCRIPTDIR = $1;
	    next;
	} elsif ($l =~ /^histlength\s*=\s*(\d+)/) {
	    $MAX_KEEP = $1;
	    next;
	} elsif ($l =~ /^serverport\s*=\s*(\d+)/) {
	    $SERVPORT = $1;
	    next;
	} elsif ($l =~ /^pidfile\s*=\s*(\S+)/) {
	    $PIDFILE = $1;
	    next;
	} elsif ($l =~ /^randstart\s*=\s*(\S+)/) {
	    $RANDSTART = &dhmstos($1);
	    die "cf error: bad syntax, line $.\n"
		if (!defined ($RANDSTART));
	    next;
	} elsif ($l =~ /^maxprocs\s*=\s*(\d+)/) {
	    $MAXPROCS = $1;
	    next;
	} elsif ($l =~ /^statedir\s*=\s*(\S+)/) {
	    $STATEDIR = $1;
	    next;
	} elsif ($l =~ /^authfile\s*=\s*(\S+)/) {
	    $AUTHFILE = $1;
	    next;
	} elsif ($l =~ /^ocfile\s*=\s*(\S+)/) {
	    $OCFILE = $1;
	    next;
	} elsif ($l =~ /^cltimeout\s*=\s*(\S+)/) {
	    $CLIENT_TIMEOUT = &dhmstos($1);
	    die "cf error: bad syntax, line $.\n"
		if (!defined ($CLIENT_TIMEOUT));
	    next;
	} elsif ($l =~ /use snmp/i) {
	    if (!defined (eval "use SNMP")) {
	    	die "perl error: cannot use SNMP: $@\n";
	    }
	    $SNMP = 1;
	}

	#
	# end of record
	#
	if ($l eq "") {
	    $ingroup    = 0;
	    $curgroup   = "";
	    $inwatch    = 0;
	    $watchgroup = "";
	    $servnum = 0;
	    $period = 0;
	    undef $aliasReading;
	    next;
	}

	#
	# group record
	#
	if ($l =~ /^hostgroup\s+([a-zA-Z0-9_.-]+)\s*(.*)/) {
	    $curgroup = $1;
	    $hosts = $2;
	    %disabled = ();
	    foreach $h (grep (/^\*/, @{$groups{$curgroup}})) {
		$h =~ s/^\*//;
		$disabled{$h} = 1;
	    }
	    @{$groups{$curgroup}} = split(/\s+/, $hosts);
	    #
	    # keep hosts which were previously disabled
	    #
	    for ($i=0;$i<@{$groups{$curgroup}};$i++) {
		$groups{$curgroup}[$i] = "*$groups{$curgroup}[$i]"
		    if ($disabled{$groups{$curgroup}[$i]});
	    }
	    $ingroup = 1;
	    next;
	
	} elsif ($ingroup) {
	    push (@{$groups{$curgroup}}, split(/\s+/, $l));
	    for ($i=0;$i<@{$groups{$curgroup}};$i++) {
		$groups{$curgroup}[$i] = "*$groups{$curgroup}[$i]"
		    if ($disabled{$groups{$curgroup}[$i]});
	    }
	    next;
	}
	
	#
	# alias record
	#
	if ($l =~ /^alias\s+([a-zA-Z0-9_.-]+)\s*$/) {
	    $aliasReading = 1;
	    $aliasGroup = $1;
	    next;

	} elsif ($aliasReading) {
	    if ($l =~ /\A(.*)\Z/) {
		push (@{$alias{$aliasGroup}}, $1);
		next;
	    }
	}

	#
	# watch record
	#
	if ($l =~ /^watch\s+([a-zA-Z0-9_.-]+)\s*/) {
	    $watchgroup = $1;
	    if (!defined ($groups{$watchgroup})) {
	    	@{$groups{$watchgroup}} = ($watchgroup);
	    }
	    die "cf error: watch already defined, line $.\n"
	    	if ($watch{$watchgroup});
	    $ingroup    = 0;
	    $curgroup   = "";
	    $servnum = 0;
	    $period = 0;
	    $inwatch = 1;
	    next;

	} elsif ($inwatch) {
	    #
	    # env variables
	    #
	    if ($l =~ /^([A-Z_][A-Z0-9_]*)=(.*)/) {
		die "cf error: environment variable defined without a service, line $.\n"
		    if ($servnum == 0);
		${$watch{$watchgroup}[$servnum-1]{"ENV"}}{$1} = $2;
	    	next;

	    #
	    # non-env variables
	    #
	    } else {
		$l =~ /^(\w+)\s*(.*)$/;
		$var = $1;
		$args = $2;
	    }

	    #
	    # service entry
	    #
	    if ($var eq "service") {
	    	$servnum++;
		$watch{$watchgroup}[$servnum-1]{"service"} = $args;
		$watch{$watchgroup}[$servnum-1]{"interval"} = 1800;
		$watch{$watchgroup}[$servnum-1]{"randskew"} = 0;
		$watch{$watchgroup}[$servnum-1]{"_op_status"} = $STAT_UNTESTED;
		$watch{$watchgroup}[$servnum-1]{"_last_op_status"} = $STAT_UNTESTED;
		$watch{$watchgroup}[$servnum-1]{"_ack"} = 0;
		$watch{$watchgroup}[$servnum-1]{"_ack_comment"} = '';
		$watch{$watchgroup}[$servnum-1]{"_failure_count"} = 0
		    if (!defined($watch{$watchgroup}[$servnum-1]{"_failure_count"}));
		$watch{$watchgroup}[$servnum-1]{"_start_of_monitor"} = time
		    if (!defined($watch{$watchgroup}[$servnum-1]{"_start_of_monitor"}));
		$watch{$watchgroup}[$servnum-1]{"_alert_count"} = 0
		    if (!defined($watch{$watchgroup}[$servnum-1]{"_alert_count"}));
		$watch{$watchgroup}[$servnum-1]{"_last_failure"} = 0
		    if (!defined($watch{$watchgroup}[$servnum-1]{"_last_failure"}));
		$watch{$watchgroup}[$servnum-1]{"_last_success"} = 0
		    if (!defined($watch{$watchgroup}[$servnum-1]{"_last_success"}));
		$watch{$watchgroup}[$servnum-1]{"_last_trap"} = 0
		    if (!defined($watch{$watchgroup}[$servnum-1]{"_last_trap"}));
		$watch{$watchgroup}[$servnum-1]{"_exitval"} = "undef"
		    if (!defined($watch{$watchgroup}[$servnum-1]{"_exitval"}));
		next;
	    }

	    if (!$servnum) {
	    	die "cf error: need to specify service in watch record, line $.\n";
	    }


	    #
	    # period definition
	    #
	    if ($var eq "period") {
		$period = 1;
	    	if (inPeriod (time, $args) == -1) {
		    die "cf error: malformed period, line $.\n";
		}
		$periodstr = $args;
		$watch{$watchgroup}[$servnum-1]{"periods"}{$periodstr}{"alertevery"} = 0;
		@{$watch{$watchgroup}[$servnum-1]{"periods"}{$periodstr}{"alerts"}} = ();
		next;
	    }

	    #
	    # alert
	    #
	    if ($var eq "alert" && !$period) {
	    	die "cf error: need to specify a period for alert, line $.\n";
	    } elsif ($var eq "upalert" && !$period) {
	    	die "cf error: need to specify a period for upalert, line $.\n";
	    } elsif ($var eq "alertevery" && !$period) {
	    	die "cf error: need to specify a period for alertevery, line $.\n";
	    } elsif ($var eq "alertafter" && !$period) {
	    	die "cf error: need to specify a period for alertafter, line $.\n";
	    }

	    #
	    # for each service there can be one or more alert periods
	    # this is stored as an array of hashes named
	    #     %{$watch{$watchgroup}[$servnum-1]{"periods"}}
	    # each index for this hash is something like "wd {Mon-Fri} hr {7am-11pm}"
	    # the value of the hash is an array containing the list of alert commands
	    # and arguments
	    #
	    if ($var eq "alert") {
	    	push @{$watch{$watchgroup}[$servnum-1]{"periods"}{$periodstr}{"alerts"}},
		    $args;
	    } elsif ($var eq "upalert") {
	    	$watch{$watchgroup}[$servnum-1]{"_upalert"} = 1;
	    	push @{$watch{$watchgroup}[$servnum-1]{"periods"}{$periodstr}{"upalerts"}},
		    $args;

	    #
	    # non-alert variables
	    #
	    } else {
		if ($var eq "interval") {
		    $args = &dhmstos ($args) ||
			die "cf error: invalid time interval, line $.\n";

		} elsif ($var eq "traptimeout") {
		    $args = &dhmstos ($args) ||
		    	die "cf error: invalid waitfortrap interval, line $.\n";
		    $watch{$watchgroup}[$servnum-1]{"_trap_timer"} = $args;

		} elsif ($var eq "trapduration") {
		    $args = &dhmstos ($args) ||
		    	die "cf error: invalid trapduration interval, line $.\n";

		} elsif ($var eq "randskew") {
		    $args = &dhmstos ($args) ||
			die "cf error: invalid random skew time, line $.\n";

		} elsif ($var eq "alertevery") {
                    my $summary_flag;
                    if ($args =~ /(\S+)(\s+)summary(\s*)$/i) {
                        $summary_flag = 1;
                        $args = $1;
                    } else {
                        $summary_flag = 0;
                    }
		    $args = &dhmstos ($args) ||
			die "cf error: invalid time interval, line $.\n";
		    $watch{$watchgroup}[$servnum-1]{"periods"}{$periodstr}{"alertevery"} =
		    	$args;
		    $watch{$watchgroup}[$servnum-1]{"periods"}{$periodstr}{"_alertsum"} =
                        $summary_flag;
		    next;

		} elsif ($var eq "alertafter") {
		    if ($args !~ /(\d+)\s+(\d+[hms])$/) {
		    	die "cf error: invalid interval specification, line $.\n";
		    }

		    if (($1 * $watch{$watchgroup}[$servnum-1]{"interval"}) >= &dhmstos($2)) {
		    	die "cf error:  interval & alertafter not sensible.\nNo alerts can be generated with those parameters, line $.\n";
		    }
		    $watch{$watchgroup}[$servnum-1]{"periods"}{$periodstr}{"alertafter"} =
		    	$1;
		    $watch{$watchgroup}[$servnum-1]{"periods"}{$periodstr}{"alertafterival"} =
		    	&dhmstos ($2);
		    $watch{$watchgroup}[$servnum-1]{"periods"}{$periodstr}{"_1stfailtime"} = 0;
		    $watch{$watchgroup}[$servnum-1]{"periods"}{$periodstr}{"_failcount"} = 0;

		} elsif ($var eq "upalertafter") {
		    $args = &dhmstos ($args) ||
			die "cf error: invalid upalertafter specification, line $.\n";
		}

		$watch{$watchgroup}[$servnum-1]{$var} = $args;
	    }

	    next;
	}
    }
    close (CFG);
    1;
}


######################################################################
#
# convert a string like "20m" into seconds
#
sub dhmstos {
    my ($str) = @_;
    my ($s);

    if ($str =~ /^\s*(\d+(?:\.\d+)?)([dhms])\s*$/i) {
	if ($2 eq "m") {
	    $s = $1 * 60;
	} elsif ($2 eq "h") {
	    $s = $1 * 60 * 60;
	} elsif ($2 eq "d") {
	    $s = $1 * 60 * 60 * 24;
	} else {
	    $s = $1;
	}
    } else {
    	return undef;
    }
    $s;
}


######################################################################
#
# reset the state of the server on SIGHUP, and reread config
# file.
#
sub reset {
    my ($keepstate) = @_;
    my ($pid, $group, $service);


    #
    # reap children that may be running
    #
    foreach $pid (keys %runningpid) {
	($group, $service) = split (/\//, $runningpid{$pid});
    	kill 15, $pid;
	waitpid ($pid, 0);
	syslog ('info', "reset killed child $pid, exit status $?");
	&remove_proc ($pid);
    }

    %watch = ();
    %groups = ();
    $procs = 0;
    syslog ('info', "resetting, and re-reading configuration $CF");
    &read_cf ($CF);
    &gen_scriptdir_hash();
    $lasttm=time; # the last time(2) the loop started
    $fdset_rbits = $fdset_ebits = '';
    &set_last_test ();
    &randomize_startdelay() if ($RANDSTART);
    &load_state ("disabled") if ($keepstate);
}

######################################################################
#
# remove a process from our state
#
sub remove_proc {
    my ($pid) = @_;

    return if (!defined $runningpid{$pid});

    vec ($fdset_rbits, fileno($fhandles{$runningpid{$pid}}), 1) = 0;
    close ($fhandles{$runningpid{$pid}});
    delete $fhandles{$runningpid{$pid}};
    delete $running{$runningpid{$pid}};
    delete $runningpid{$pid};
    $procs--;
}


######################################################################
#
# clean up before exiting
#
sub clean_up {
    unlink $PIDFILE unless $PIDFILE eq '';
}


######################################################################
#
# exit on SIGTERM
#
sub handle_sigterm {
    syslog ("info", "caught TERM signal, exiting");
    &clean_up();
    exit (1);
}


######################################################################
#
# setup server
#
sub setup_server {
    my ($proto, $fl);

    #
    # client server, such as moncmd
    #
    $proto = getprotobyname ('tcp');
    socket (SERVER, PF_INET, SOCK_STREAM, $proto) ||
    	&die_die ("err", "could not create TCP socket: $!");
    setsockopt (SERVER, SOL_SOCKET, SO_REUSEADDR, pack ("l", 1)) ||
    	&die_die ("err", "could not setsockopt: $!");
    bind (SERVER, sockaddr_in ($SERVPORT, INADDR_ANY)) ||
    	&die_die ("err", "could not bind TCP server port: $!");
    listen (SERVER, SOMAXCONN);

    #
    # remote monitor traps
    #
    $proto = getprotobyname ('udp');
    socket (TRAPSERVER, PF_INET, SOCK_DGRAM, $proto) ||
    	&die_die ("err", "could not create UDP socket: $!");
    bind (TRAPSERVER, sockaddr_in ($SERVPORT, INADDR_ANY)) ||
    	&die_die ("err", "could not bind UDP server port: $!");
#    $fl = fcntl (TRAPSERVER, F_GETFL, $fl)
    fcntl (TRAPSERVER, F_GETFL, $fl)
    	|| &die_die ("err", "could not get fd options: $!");
    $fl |= O_NONBLOCK;
    fcntl (TRAPSERVER, F_SETFL, $fl)
    	|| &die_die ("err", "could not set fd options: $!");
}


#
# set up a client connection if necessary
#
sub client_accept {
    my ($rin, $rout, $n, $fno, $sock, $port, $addr, $fl, $CLIENT);

    $CLIENT = "c" . $clientcount++;

    if (!defined ($sock = accept ($CLIENT, SERVER))) {
    	syslog ('err', "accept returned error: $!");
	return;
    }

&debug(1, "accepted client\n");
    $fno = fileno ($CLIENT);

    #
    # set socket to nonblocking
    #
    if (!defined ($fl = fcntl ($CLIENT, F_GETFL, $fl))) {
	syslog ("err", "could not get fd options for client: $!");
	close ($CLIENT);
	return;
    }

    $fl |= O_NONBLOCK;

    if (!defined (fcntl ($CLIENT, F_SETFL, $fl))) {
    	syslog ("err", "could not set fd options for client: $!");
	close ($CLIENT);
	return;
    }

    ($port, $addr) = unpack_sockaddr_in ($sock);
    syslog ('info', "client connection from " . inet_ntoa ($addr) .
	    ":" . $port);

    select ($CLIENT);
    $|=1;
    select (STDOUT);

    $clients{$fno}{"fhandle"} = $CLIENT;
    $clients{$fno}{"user"} = undef;		# username if authenticated
    $clients{$fno}{"timeout"} = $CLIENT_TIMEOUT;
    $clients{$fno}{"last_read"} = time;		# last time data was read
    $clients{$fno}{"buf"} = '';
    $numclients++;
}


#
# do all pending client commands
#
sub client_dopending {
    my ($cl, $cmd, $l);

    foreach $cl (keys %clients) {
    	if ($clients{$cl}{"buf"} =~ /^([^\r\n]+)[\r\n]+/s) {
	    $cmd = $1;
	    $l = length ($cmd);
	    $clients{$cl}{"buf"} =~ s/^[^\r\n]+[\r\n]+//s;
	    &client_command ($cl, $cmd);
	}
    }
}


#
# close a client connection
#
sub client_close {
    my ($cl, $reason) = @_;

    syslog ('info', "closing client $cl: $reason") if (defined $reason);
    close ($clients{$cl}{"fhandle"});
    delete $clients{$cl};
    vec ($iovec, $cl, 1) = 0;
    $numclients--;
}


######################################################################
#
# Handle a connection from a client
#
sub client_command {
    my ($cl, $l) = @_;
    my ($cmd, $args, $group, $service, $s, $sname, $stchanged);
    my ($var, $value, $msg, @l, $sock, $port, $addr, $sref, $auth, $fh);
    my ($user, $pass, @argsList, $comment);


    syslog ('info', "client command \"$l\"")
	if ($l !~ /^\s*login/i);

    $fh = $clients{$cl}{"fhandle"};

#    &sock_write ($fh, "220 $HOSTNAME mon server ready.\n");

    if ($l !~ /^(login|disable|enable|quit|list|set|get|
		    stop|start|loadstate|savestate|reset|
		    reload|term|test|servertime|ack)\s*(.*)?$/ix) {
	&sock_write ($fh, "520 invalid command\n");
	&client_close ($cl, "invalid command");
	return;
    }
    ($cmd, $args) = ("\L$1", $2);

    $stchanged = 0;

    #
    # quit command
    #
    if ($cmd eq "quit") {
	&sock_write ($fh, "220 quitting\n");
	&client_close ($cl);

    #
    # login
    #
    } elsif ($cmd eq "login") {
	($user, $pass) = split (/\s+/, $args, 2);
	if (!defined &auth ("unix", $user, $pass)) {
	    &sock_write ($fh,  "530 login unsuccessful\n");
	} else {
	    $clients{$cl}{"user"} = $user;
	    syslog ("info", "authenticated $user");
	    &sock_write ($fh,  "220 login accepted\n");
	}

    #
    # reset
    #
    } elsif ($cmd eq "reset" && &check_auth ($clients{$cl}{"user"}, $cmd)) {
	my ($keepstate);
	if ($args =~ /stopped/i) {
	    $STOPPED = 1;
	    $STOPPED_TIME = time;
	}
	if ($args =~ /keepstate/) {
	    $keepstate = 1;
	}
	&reset($keepstate);
	&sock_write ($fh,  "220 reset PID $$@$HOSTNAME\n");

    #
    # reload
    #
    } elsif ($cmd eq "reload" && &check_auth ($clients{$cl}{"user"}, $cmd)) {
	if (!defined &reload (split (/\s+/, $args))) {
	    &sock_write ($fh,  "520 unknown reload command\n");
	} else {
	    &sock_write ($fh,  "220 reload completed\n");
	}

    #
    # test
    #
    } elsif ($cmd eq "test" && &check_auth ($clients{$cl}{"user"}, $cmd))  {
	($group, $service) = split (/\s+/, $args);

	if (!defined (($watch{$group}[&getservbyname($group, $service)]))) {
	    &sock_write ($fh,  "$group,$service not defined\n");
	} else {
	    $watch{$group}[&getservbyname($group, $service)]{"_timer"} = 0;
	}

    #
    # load state
    #
    } elsif ($cmd eq "loadstate" && &check_auth ($clients{$cl}{"user"}, $cmd)) {
	foreach (split (/\s+/, $args)) {
	    &load_state ($_);
	}
	&sock_write ($fh,  "220 loadstate completed\n");

    #
    # save state
    #
    } elsif ($cmd eq "savestate" && &check_auth ($clients{$cl}{"user"}, $cmd)) {
	foreach (split (/\s+/, $args)) {
	    &save_state ($_);
	}
	&sock_write ($fh,  "220 savestate completed\n");

    #
    # term
    #
    } elsif ($cmd eq "term"  && &check_auth ($clients{$cl}{"user"}, $cmd)) {
	&sock_write ($fh,  "220 terminating server\n");
	&client_close ($fh, "terminated by user command");
	close (SERVER);
	syslog ("info", "terminating by user command");
	&clean_up();
	exit;

    #
    # stop testing
    #
    } elsif ($cmd eq "stop"&& &check_auth ($clients{$cl}{"user"}, $cmd)) {
	$STOPPED = 1;
	$STOPPED_TIME = time;
	&sock_write ($fh,  "220 stop completed\n");

    #
    # start testing
    #
    } elsif ($cmd eq "start" && &check_auth ($clients{$cl}{"user"}, $cmd)) {
	$STOPPED = 0;
	$STOPPED_TIME = 0;
	&sock_write ($fh,  "220 start completed\n");

    #
    # set
    #
    } elsif ($cmd eq "set" && &check_auth ($clients{$cl}{"user"}, $cmd)) {
	if ($args =~ /^maxkeep\s+(\d+)/) {
	    $MAX_KEEP = $1;
	    &sock_write ($fh,  "220 set completed\n");
	} else {
	    ($group, $service, $var, $value) = split (/\s+/, $args);
	    if (!defined ($watch{$group}[&getservbyname($group, $service)])) {
		&sock_write ($fh,  "$group,$service not defined\n");
	    } elsif ($var eq "opstatus") {
		if (!defined ($OPSTAT{$value})) {
		    &sock_write ($fh,  "520 undefined opstatus\n");
		} else {
		    $watch{$group}[&getservbyname($group, $service)]{"_op_status"} = $value;
		    &sock_write ($fh,  "220 set completed\n");
		}

	    } else {
		$watch{$group}[&getservbyname($group, $service)]{$var} = $value;
		&sock_write ($fh,  "$group $service $var = $value\n");
		&sock_write ($fh,  "220 set completed\n");
	    }
	}

    #
    # get
    #
    } elsif ($cmd eq "get" && &check_auth ($clients{$cl}{"user"}, $cmd)) {
	if ($args =~ /^maxkeep\s*$/) {
	    &sock_write ($fh,  "maxkeep = $MAX_KEEP\n");
	    &sock_write ($fh,  "220 set completed\n");
	} else {
	    ($group, $service, $var) = split (/\s+/, $args);
	    if (!defined ($watch{$group}[&getservbyname($group, $service)])) {
		&sock_write ($fh,  "520 $group,$service not defined\n");
	    } else {
		&sock_write ($fh,  "$group $service $var =
			$watch{$group}[&getservbyname($group, $service)]{$var}\n");
		&sock_write ($fh,  "220 get completed\n");
	    }
	}

    #
    # list
    #
    } elsif ($cmd eq "list" && &check_auth ($clients{$cl}{"user"}, $cmd)) {
	@argsList = split(/\s+/, $args);
	($cmd, $args) = split (/\s+/, $args);

	#
	# list service descriptions
	#
	if ($cmd eq "descriptions") {
	    foreach $group (keys %watch) {
		for ($service=0;$service<@{$watch{$group}};$service++) {
		    if ($watch{$group}[$service]{"description"} !~ /^\s*$/) {
			&sock_write ($fh,  "$group " . &getservbynum($group, $service) .
			    " $watch{$group}[$service]{'description'}\n");
		    }
		}
	    }
	    &sock_write ($fh,  "220 list descriptions completed\n");

	#
	# list group members
	#
	} elsif ($cmd eq "group") {
	    if ($groups{$args}) {
		&sock_write ($fh,  "hostgroup $args @{$groups{$args}}\n");
		&sock_write ($fh,  "220 list group completed\n");
	    } else {
		&sock_write ($fh,  "520 list group error, undefined group\n");
	    }

	#
	# list status of all services
	#
	} elsif ($cmd eq "opstatus") {
	    foreach $group (keys %watch) {
		for ($service=0;$service<@{$watch{$group}};$service++) {
		    $sref = \%{$watch{$group}[$service]};
		    my ($s) = $$sref{"_last_summary"};
		    $s =~ s/['"]/\\$&/g;
		    my $comment;
		    if ($$sref{_ack} == 1) {
			$comment = '"' . $$sref{"_ack_comment"} . '"';
		    } else {
		    	$comment = '""';
		    }

		    &sock_write ($fh,  "group=$group service=" . &getservbynum($group, $service) .
			" opstatus=$$sref{_op_status} exitval=$$sref{_exitval}" .
			" timer=$$sref{_timer}" .
			" last_success=$$sref{_last_success}" .
			" last_failure=$$sref{_last_failure}" .
			" last_trap=$$sref{_last_trap}" .
			" ack=$$sref{_ack}" .
			" ackcomment=$comment" .
			" last_summary=\"$s\"\n");
		}
	    }
	    &sock_write ($fh,  "220 list opstatus completed\n");

	#
	# list disabled hosts and services
	#
	} elsif ($cmd eq "disabled") {
	    foreach $group (keys %groups) {
		@l = grep (/^\*/, @{$groups{$group}});
		if (@l) {
		    grep (s/^\*//, @l);
		    &sock_write ($fh,  "group $group: @l\n");
		}
	    }
	    foreach $group (keys %watch) {
		if ($watch_disabled{$group} == 1) {
		    &sock_write ($fh,  "watch $group\n");
		}
		for ($service=0;$service<@{$watch{$group}};$service++) {
		    if ($watch{$group}[$service]{'disable'} == 1) {
			&sock_write ($fh,  "watch $group service " .
			    &getservbynum($group, $service) . "\n");
		    }
		}
	    }
	    &sock_write ($fh,  "220 list disabled completed\n");

	#
	# list last alert history
	#
	} elsif ($cmd eq "alerthist") {
	    &sock_write ($fh,  join ("\n", @last_alerts) . "\n") if @last_alerts;
	    &sock_write ($fh,  "220 list alerthist completed\n");

	#
	# list time of last failures for each service
	#
	} elsif ($cmd eq "failures") {
	    foreach $group (keys %watch) {
		for ($s=0;$s<@{$watch{$group}};$s++) {
		    $sref = \%{$watch{$group}[$s]};
		    next if ($$sref{"_op_status"} != $STAT_FAIL);
		    $$sref{"_failure_output"} =~ /^(.*)$/m;
		    &sock_write ($fh,  "$group " . &getservbynum($group, $s) .
			" $$sref{_last_failure} $$sref{_timer} failed $1\n");
		}
	    }
	    &sock_write ($fh,  "220 list failures completed\n");

	#
	# list the failure history
	#
	} elsif ($cmd eq "failurehist") {
	    &sock_write ($fh,  join ("\n", @last_failures) . "\n")
		if @last_failures;
	    &sock_write ($fh,  "220 list failurehist completed\n");

	#
	# list the time of last successes for each service
	#
	} elsif ($cmd eq "successes") {
	    foreach $group (keys %watch) {
		for ($s=0;$s<@{$watch{$group}};$s++) {
		    $sref = \%{$watch{$group}[$s]};
		    next if ($$sref{"_op_status"} == $STAT_OK);
		    $$sref{"_current_output"} =~ /^(.*)$/m;
		    &sock_write ($fh,  "$group " . &getservbynum($group, $s) .
			" $$sref{_last_success} $$sref{_timer} succeeded $1\n");
		}
	    }
	    &sock_write ($fh,  "220 list successes completed\n");

	#
	# list process IDs
	#
	} elsif ($cmd eq "pids") {
	    &sock_write ($fh,  "$$ server\n");
	    foreach $value (keys %runningpid) {
		($group, $s) = split (/\//, $runningpid{$value});
		&sock_write ($fh,  "$value $group $watch{$group}[$s]{service}\n");
	    }
	    &sock_write ($fh,  "220 list pids completed\n");

	#
	# list watch groups and services
	#
	} elsif ($cmd eq "watch") {
	    foreach $group (keys %watch) {
		for ($s=0;$s<@{$watch{$group}};$s++) {
		    if (!defined ($service=&getservbynum($group, $s))) {
			&sock_write ($fh,  "$group (undefined service)\n");
		    } else {
			&sock_write ($fh,  "$group $service\n");
		    }
		}
	    }
	    &sock_write ($fh,  "220 list watch completed\n");

	#
	# list server state
	#
	} elsif ($cmd eq "state") {
	    if ($STOPPED) {
		&sock_write ($fh,  "scheduler stopped since $STOPPED_TIME\n");
	    } else {
		&sock_write ($fh,  "scheduler running\n");
	    }
	    &sock_write ($fh,  "220 list state completed\n");
	
	#
	# list aliases
	#
	} elsif ($cmd eq "aliases") {
	    my (@listAliasesRequest) = @argsList;

	    shift (@listAliasesRequest);

	    # if no alias request, all alias are responded
	    unless (@listAliasesRequest) {
	    	@listAliasesRequest = keys (%alias);
	    }

	    foreach $alias (@listAliasesRequest){
	    	&sock_write ($fh, "alias $alias\n");
		foreach $value (@{$alias{$alias}}) {
		    &sock_write ($fh,  "$value\n");
		}
		&sock_write ($fh, "\n");
	    }
	    &sock_write ($fh,  "220 list aliases completed\n");
	
	#
	# list aliasgroups
	#
	} elsif ($cmd eq "aliasgroups") {
	    my (@listAliasesRequest);
	    @listAliasesRequest = keys (%alias);

	    &sock_write ($fh,  "@listAliasesRequest\n");
	    &sock_write ($fh,  "220 list aliasgroups completed\n");

	} else {
	    &sock_write ($fh,  "520 unknown list command\n");
	}


    #
    # acknowledge a failure
    #
    } elsif ($cmd eq "ack" && &check_auth ($clients{$cl}{"user"}, $cmd)) {
	($group, $service, $comment) = split (/\s+/, $args, 3);
	my ($svc);

	if (!defined ($watch{$group})) {
	    &sock_write ($fh,  "520 unknown group\n");
	} elsif (!defined ($svc = &getservbyname($group, $service))) {
	    &sock_write ($fh,  "520 unknown service\n");
	} elsif ($watch{$group}[$svc]{"_op_status"} == $STAT_OK ||
		$watch{$group}[$svc]{"_op_status"} == $STAT_UNTESTED ||
		$watch{$group}[$svc]{"_op_status"} == $STAT_DEPEND) {
	    &sock_write ($fh,  "520 service is in a non-failure state\n");
	} else {
	    $watch{$group}[$svc]{"_ack"} = 1;
	    $watch{$group}[$svc]{"_ack_comment"} = $comment;
	    &sock_write ($fh,  "220 ack completed\n");
	}

    #
    # disable watch, service or host
    #
    } elsif ($cmd eq "disable" && &check_auth ($clients{$cl}{"user"}, $cmd)) {
	($cmd, $args) = split (/\s+/, $args, 2);

	#
	# disable watch
	#
	if ($cmd eq "watch") {
	    if (!defined (&disen_watch($args, 0))) {
		&sock_write ($fh,  "520 disable error, unknown watch \"$args\"\n");
	    } else {
		$stchanged++;
		&sock_write ($fh,  "220 disable watch completed\n");
	    }

	#
	# disable service
	#
	} elsif ($cmd eq "service") {
	    ($group, $service) = split (/\s+/, $args, 2);

	    if (!defined (&disen_service ($group, $service, 0))) {
		&sock_write ($fh,  "520 disable error, unknown group or service\n");
	    } else {
		$stchanged++;
		&sock_write ($fh,  "220 disable service completed\n");
	    }

	#
	# disable host
	#
	} elsif ($cmd eq "host") {
	    foreach $var (split (/\s+/, $args)) {
		&disen_host ($var, 0);
	    }
		$stchanged++;
	    &sock_write ($fh,  "220 disable host completed\n");
	}

    #
    # enable watch, service or host
    #
    } elsif ($cmd eq "enable" && &check_auth ($clients{$cl}{"user"}, $cmd)) {
	($cmd, $args) = split (/\s+/, $args, 2);

	#
	# enable watch
	#
	if ($cmd eq "watch") {
	    if (!defined(&disen_watch($args, 1))) {
		&sock_write ($fh,  "520 enable error, unknown watch\n");
	    } else {
		$stchanged++;
		&sock_write ($fh,  "220 enable watch completed\n");
	    }


	#
	# enable service
	#
	} elsif ($cmd eq "service") {
	    ($group, $service) = split (/\s+/, $args, 2);

	    if (!defined (&disen_service ($group, $service, 1))) {
		&sock_write ($fh,  "520 enable error, unknown group\n");
	    } else {
		$stchanged++;
		&sock_write ($fh,  "220 enable completed\n");
	    }

	#
	# enable host
	#
	} elsif ($cmd eq "host") {
	    foreach $var (split (/\s+/, $args)) {
		&disen_host ($var, 1);
		$stchanged++;
	    }
	    &sock_write ($fh,  "220 enable completed\n");

	} else {
	    &sock_write ($fh,  "520 command could not be executed\n");
	}

    #
    # server time
    #
    } elsif ($cmd eq "servertime" && &check_auth ($clients{$cl}{"user"}, $cmd)) {
	&sock_write ($fh,  time, " ", scalar (localtime), "\n");
	&sock_write ($fh,  "220 servertime completed\n");

    } else {
	&sock_write ($fh,  "520 command could not be executed\n");
    }

    &save_state ("disabled") if ($stchanged);
}


######################################################################
#
# show usage
#
sub usage {
    print <<"EOF";
usage: mon [-a dir] [-c config] [-d] [-f] [-i secs] [-k num]
	[-m num] [-p num] [-P file] [-r num] [-s dir] 
       mon -v

  -a dir	alert script dir
  -c config	config file, defaults to "mon.cf"
  -d		debug
  -f		fork and become a daemon
  -i secs	sleep interval (seconds), defaults to 1
  -k num	keep history of last num events
  -m num	throttle at maximum number of monitor processes
  -o file       on-call schedule
  -p num	server listens on port num
  -P file	PID file
  -r num	randomize startup schedule
  -s dir	monitor script dir
  -v		print version

Report bugs to $AUTHOR
$RCSID
EOF
}


######################################################################
#
# look up a service by number, returning the service name
#
sub getservbynum {
    my ($group, $s) = @_;

    $watch{$group}[$s]{"service"};
}

######################################################################
#
# look up a service by name, returning the service number
#
sub getservbyname {
    my ($group, $service) = @_;
    my ($s);

    for ($s=0;$s<@{$watch{$group}};$s++) {
	if ($watch{$group}[$s]{"service"} eq $service) {
	    return $s;
	}
    }
    undef;
}


######################################################################
#
# become a daemon
#
sub daemon {
    if ($pid = fork()) {
	# the parent goes away all happy and stuff
    	exit (0);
    } elsif (!defined $pid) {
    	die "could not fork: $!\n";
    }

    setsid();
    chdir ('/');
    umask (022);

    if (!open (N, "+</dev/null")) {
    	syslog ("err", "could not open /dev/null: %m");
    	exit(1);
    }
    if (!open(STDOUT, ">&N") ||
        !open (STDIN, "<&N") ||
	!open (STDERR, ">&N")) {
    	syslog ("err", "could not redirect: %m");
	exit(1);
    }
    syslog ('info', "running as daemon");
}


######################################################################
#
# debug
#
sub debug {
    my ($level, @l) = @_;

    return if ($level > $opt_d);

    if ($opt_d && !$opt_f) {
    	print STDERR @l;
    } else {
    	syslog ('debug', join ('', @l));
    }
}


######################################################################
#
# die_die
#
sub die_die {
    my ($level, $msg) = @_;

    die "[$level] $msg\n" if ($opt_d);

    syslog ($level, "fatal, $msg");
    closelog();
    exit (1);
}


######################################################################
#
# handle cleanup of exited processes
# trigger alerts on failures (or send no alert if disabled)
# do some accounting
#
sub proc_cleanup {
    my ($summary, $tmnow);

    $tmnow = time;
    if (keys %running) {
	while (($p = waitpid (-1, &WNOHANG)) >0) {

	    ($group, $service) = split (/\//, $runningpid{$p});
	    $sref = \%{$watch{$group}[$service]};

	    #
	    # suck in any extra data
	    #
	    $fh = $fhandles{$runningpid{$p}};
	    while ($z = sysread ($fh, $buf, 8192)) {
		$ibufs{$runningpid{$p}} .= $buf;
	    }

	    $$sref{"_exitval"} = int($?>>8);
&debug (1, "PID $p ($runningpid{$p}) exited with [$$sref{'_exitval'}]\n");

	    #
	    # error exit value, handle alert
	    #
	    if ($?) {

		#
		# accounting
		#
		$$sref{"_failure_count"}++;
		$$sref{"_last_failure"} = $tmnow;
		if ($$sref{"_op_status"} == $STAT_OK ||
			$$sref{"_op_status"} == $STAT_UNKNOWN ||
			$$sref{"_op_status"} == $STAT_DEPEND ||
			$$sref{"_op_status"} == $STAT_UNTESTED) {
		    $$sref{"_first_failure"} = $tmnow;
		}
		$$sref{"_op_status"} = $STAT_FAIL;
		($summary) = split("\n", $ibufs{$runningpid{$p}});
		$summary = "(NO SUMMARY)" if ($summary =~ /^\s*$/m);
		$$sref{"_last_summary"} = $summary;
		shift @last_failures if (@last_failures > $MAX_KEEP);
		push @last_failures, "$group " . &getservbynum ($group, $service) .
		    " $tm $summary";
		&syslog ('crit', "failure for $last_failures[-1]");

		#
		# if the alarm is disabled, ignore it
		#
		if ($$sref{"disable"} == 1) {
		    syslog ("notice", "ignoring alert for $group.",
			    &getservbynum ($group, $service));

		#
		# no alerts for ack'd failures
		#
		} elsif ($$sref{"_ack"} == 1) {
		    syslog ("notice", "no alert for $group." .
			    &getservbynum ($group, $service) .
			    "because of ack'd failure");

		#
		# otherwise, trigger it
		#
		} else {
		    &do_alert ($group, $service, $ibufs{$runningpid{$p}}, $?>>8, $FL_MONITOR);
		}

		$$sref{"_failure_output"} = $ibufs{$runningpid{$p}};

	    #
	    # record the time of the last success
	    #
	    } else {

		#
		# if this service has just come back up and
		# we are paying attention to this event,
		# let someone know
		#
                if (defined ($$sref{"_op_status"}) &&
			$$sref{"_op_status"} == $STAT_FAIL) {
		    if (defined($$sref{"_upalert"}) && $tmnow - $$sref{"_first_failure"} >=
			    $$sref{"upalertafter"}) {
			&do_alert ($group, $service, $$sref{"_last_output"}, 0, $FL_UPALERT);
		    }

		    syslog ('info', "downtime for $group/" .
			getservbynum ($group, $service) . " is $$sref{'_last_downtime'} ");
		}

		$$sref{"_ack"} = 0;
		$$sref{"_ack_comment"} = '';
		$$sref{"_first_failure"} = 0;
		$$sref{"_last_failure"} = 0;

		#
		# reset the alertevery timer
		#
		foreach $period (keys %{$$sref{"periods"}}) {
		    $$sref{"periods"}{$period}{"_last_alert"} = 0;
		}

		$$sref{"_last_success"} = $tmnow;
		$$sref{"_op_status"} = $STAT_OK;
	    }

	    #
	    # save the output
	    #
	    $$sref{"_last_output"} = $ibufs{$runningpid{$p}};

	    &remove_proc ($p);
	}
    }
}


######################################################################
#
# collect output from running processes
#
sub collect_output {
    return if (!keys %running);
&debug (1, "things are running, checking for output\n");

    $nfound = select($rout=$fdset_rbits, undef, undef, 0);
&debug (1, "select returned $nfound file handles\n");
    if ($nfound) {
	#
	# look for the file descriptors that are readable,
	# and try to read as much as possible from them
	#
	foreach $k (keys %fhandles) {
	    $fh = $fhandles{$k};
	    if (vec ($rout, fileno($fh), 1) == 1) {
		$z = 0;
		while ($z = sysread ($fh, $buf, 8192)) {
		    $ibufs{$k} .= $buf;
&debug (1, "[$buf] from $fh\n");
		}

		#
		# ignore if EAGAIN, since we're nonblocking
		#
		if (!defined($z) && $! == &EAGAIN) {

		#
		# error on this descriptor
		#
		} elsif (!defined($z)) {
&debug (1, "error on $fh: $!\n");
		    &syslog ('err', "error on $fh: $!");
		    vec($fdset_rbits, fileno($fh), 1) = 0;
		} elsif ($z == 0 && $! == &EAGAIN) {
&debug (1, "EAGAIN on $fh\n");

		#
		# if EOF encountered, stop trying to
		# get input from this file descriptor
		#
		} elsif ($z == 0) {
&debug (1, "EOF on $fh\n");
		    vec($fdset_rbits, fileno($fh), 1) = 0;

		}
	    }
	}
    }
}




######################################################################
#
# handle forking a monitor process, and set up variables
#
sub run_monitor {
    my (@args, @groupargs, $pid, @ghosts, $monitor, $monitorargs);

    $fhandles{"$group/$service"} = "\U$group$service";

    ($monitor, $monitorargs) = ($$sref{monitor} =~ /^(\S+)(\s+(.*))?$/);
&debug (1, "LOOKING FOR [$monitor]\n");
    if (!defined $MONITORHASH{$monitor}) {
    	syslog ('err', "no monitor found while trying to run [$monitor]");
&debug(3, "no monitor found while trying to run [$monitor]\n");
	return undef;
    } else {
    	$monitor = $MONITORHASH{$monitor};
    }
    $monitor = "$monitor $monitorargs";

    #
    # if monitor ends with ";;", do not append groups
    # to command line
    #

    @ghosts = ();
    if ($monitor =~ s/;;\s*$//) {
	@args = quotewords ('\s+', 0, $monitor);
	@ghosts = (1);

    } else {
	@ghosts = grep (!/^\*/, @{$groups{$group}});
	@args = (quotewords ('\s+', 0, $monitor), @ghosts);
    }

    if (@ghosts == 0 && !defined ($$sref{"allow_empty_group"})) {
    	syslog ('err', "monitor for $group/" .
		&getservbynum ($group, &getservbyname ($group, $service)) .
		" not called because of no host arguments\n");

    } else {

	#
	# dependency check
	#
	if ($$sref{"depend"}) {
	    $status = &checkDepend ($group, $service);
	    &debug (2, "status = $status\n\n\n");
	    if (!defined ($status)) {
		syslog ('err', "eval error for checkDepend on $group " .
		    &getservbynum ($group, $service) . ": $@\n");
		return;
	    } elsif (!$status) {
	       if ($$sref{"_op_status"} != $STAT_FAIL &&
	       			$$sref{"_op_status"} != $STAT_DEPEND) {
		   $$sref{'_last_op_status'} = $$sref{'_op_status'};
		   $$sref{'_op_status'} = $STAT_DEPEND;
		}
	       return;
	    } else {
		$$sref{'_op_status'} = $$sref{'_last_op_status'};
	    }
	}


	$pid = open($fhandles{"$group/$service"}, '-|');
	if (!defined $pid) {
	    syslog ('err', "Could not fork\n");
	    delete $fhandles{"$group/$service"};
	    return 0;
	} elsif ($pid == 0) {
	    open(STDERR, '>&STDOUT')
		or syslog ('err', "Could not dup stderr\n");
	    open(STDIN, "</dev/null")
		or syslog ('err', "Could not connect stdin to /dev/null\n");
	    my $v;
	    foreach $v (keys %{$$sref{"ENV"}}) {
	    	$ENV{$v} = $$sref{"ENV"}{$v};
	    }
	    $ENV{"MON_LAST_SUMMARY"} = $$sref{"_last_summary"};
	    $ENV{"MON_LAST_OUTPUT"} = $$sref{"_last_output"};
	    $ENV{"MON_LAST_FAILURE"} = $$sref{"_last_failure"};
	    $ENV{"MON_FIRST_FAILURE"} = $$sref{"_first_failure"};
	    $ENV{"MON_LAST_SUCCESS"} = $$sref{"_last_success"};
	    exec @args or syslog ('err', "could not exec '@args': $!")
		&& exit(1);
	}

&debug (1, "watching file handle ", fileno ($fhandles{"$group/$service"}),
    " for $group/$service\n");

	#
	# set nonblocking I/O and setup bit vector for select(2)
	#
	$fl = fcntl $fhandles{"$group/$service"}, F_GETFL, $fl;
	$fl |= O_NONBLOCK;
	fcntl $fhandles{"$group/$service"}, F_SETFL, $fl;

	vec ($fdset_rbits,
	    fileno($fhandles{"$group/$service"}), 1) = 1;
	$fdset_ebits |= $fdset_rbits;

	#
	# note that this is running
	#
	$running{"$group/$service"} = 1;
	$runningpid{$pid} = "$group/$service";
	$ibufs{"$group/$service"} = "";
	$procs++;
    }

    #
    # set the countdown timer for this service
    #
    if ($$sref{"randskew"} != 0) {
    	$$sref{"_timer"} = $$sref{"interval"} +
	     (int (rand (2)) == 0 ? -int(rand($$sref{"randskew"}) + 1) :
	     	int(rand($$sref{"randskew"})+1));
    } else {
	$$sref{"_timer"} =
	    $$sref{"interval"};
    }
}


######################################################################
#
# randomize the delay before each test
# $opt{"randstart"} is seconds
#
sub randomize_startdelay {
    my ($group, $service);

    foreach $group (keys %watch) {
	for ($service=0;$service<@{$watch{$group}};$service++) {
            $watch{$group}[$service]{"_timer"} =
                int (rand ($RANDSTART));
        }
    }

}


#
# return 1 if $val is within $range,
# where $range = "number" or "number-number"
#
sub inRange {
    my ($val, $range) = @_;
    my ($retval);

    $retval = 0;
    if ($range =~ /^(\d+)$/ && $val == $1) {
        $retval = 1

    } elsif ($range =~ /^(\d+)\s*-\s*(\d+)$/ &&
	    ($val >= $1 && $val <= $2)) {
        $retval = 1
    }

    $retval;
}


#
# disable ($cmd==0) or enable a watch
#
sub disen_watch {
    my ($w, $cmd) = @_;

    return undef if (!defined ($watch{$w}));
    if ($cmd == 0) {
	$watch_disabled{$w} = 1;
    } else {
	$watch_disabled{$w} = 0;
    }
}


#
# disable ($cmd==0) or enable a service
#
sub disen_service {
    my ($g, $s, $cmd) = @_;
    my ($snum);

    return undef if (!defined ($watch{$g}));
    return undef if (!defined ($snum=&getservbyname ($g, $s)));
    if ($cmd == 0) {
	$watch{$g}[$snum]{"disable"} = 1;
    } else {
	$watch{$g}[$snum]{"disable"} = 0;
    }
}


#
# disable ($cmd==0) or enable a host
#
sub disen_host {
    my ($h, $cmd) = @_;
    my ($var, $g);

    foreach $g (keys %groups) {
	if ($cmd == 0) {
	    grep (s/^$h$/*$h/, @{$groups{$g}});
	} else {
	    grep (s/^\*$h$/$h/, @{$groups{$g}});
	}
    }
}


#
# save state
#
sub save_state {
    my (@states) = @_;
    my ($group, $service, @l, $state);

    foreach $state (@states) {
	if ($state eq "disabled") {
	    if (!open (STATE, ">$STATEDIR/disabled")) {
		syslog ("err", "could not write to state file: $!");
		next;
	    }

	    foreach $group (keys %groups) {
		@l = grep (/^\*/, @{$groups{$group}});
		if (@l) {
		    grep (s/^\*//, @l);
		    grep { print STATE "disable host $_\n" } @l;
		}
	    }
	    foreach $group (keys %watch) {
		if ($watch_disabled{$group} == 1) {
		    print STATE "disable watch $group\n";
		}
		for ($service=0;$service<@{$watch{$group}};$service++) {
		    if ($watch{$group}[$service]{'disable'} == 1) {
			print STATE "disable service $group " .
			    &getservbynum($group, $service) . "\n";
		    }
		}
	    }
	    close (STATE);

	} elsif ($state eq "opstatus") {
	    if (!open (STATE, ">$STATEDIR/opstatus")) {
		syslog ("err", "could not write to opstatus state file: $!");
		next;
	    }
	    foreach $group (keys %watch) {
	    	foreach $service (@{$watch{$group}}) {
		    print STATE "group=$group service=" .
		    	&getservbynum($group, $service) .
			" op_status=$watch{$group}[$service]{_op_status}" .
			" failure_count=$watch{$group}[$service]{_failure_count}" .
			" alert_count=\n";
		}
	    }
	    close (STATE);
	}
    }
}


#
# load state
#
sub load_state {
    my (@states) = @_;
    my ($l, $cmd, $args, $group, $service, $what, $state);

    foreach $state (@states) {
    	if ($state eq "disabled") {
	    if (!open (STATE, "$STATEDIR/disabled")) {
		syslog ("err", "could not read state file: $!");
		next;
	    }

	    while (defined ($l = <STATE>)) {
		chomp $l;
		($cmd, $what, $args) = split (/\s+/, $l, 3);

		next if ($cmd ne "disable");

		if ($what eq "host") {
		    &disen_host ($args);
		} elsif ($what eq "watch") {
		    syslog ("err", "undefined watch reading state file: $l")
			if (!defined &disen_watch ($args));
		} elsif ($what eq "service") {
		    ($group, $service) = split (/\s+/, $args, 2);
		    syslog ("err",
		    	"undefined group or service reading state file: $l")
			if (!defined &disen_service ($group, $service));
		}
	    }

	    syslog ("info", "state '$state' loaded");
	    close (STATE);
	}
    }
}


#
# authenticate a login
#
sub auth {
    my ($type, $user, $plaintext) = @_;
    my ($pass);

    (undef, $pass) = getpwnam($user);
    return undef
        if (!defined $pass);

    if ((crypt ($plaintext, $pass)) ne $pass) {
    	return undef;
    }
    return 1;
}


#
# load the table of who can do which commands
#
sub load_auth {
    my ($startup) = @_;
    my ($l, $cmd, $users, $u);

    %AUTHCMDS = ();
    if (!open (C, $AUTHFILE)) {
	&err_startup ( "could not open $AUTHFILE: $!");
	return undef;
    }

    while (defined ($l = <C>)) {
	next if ($l =~ /^\s*#/ || $l =~ /^\s*$/);
	chomp $l;
	$l =~ s/^\s*//;
	$l =~ s/\s*$//;
    	($cmd, $users) = split (/\s*:\s*/, $l, 2);
	if (!defined $users) {
	    &err_startup ($startup, "could not parse line $. of auth file\n");
	    next;
	}
	foreach $u (split (/\s*,\s*/, $users)) {
	    $AUTHCMDS{"\L$cmd"}{$u} = 1;
	}
    }
    close (C);
}


#
# return undef if $user isn't permitted to perform $cmd
#
sub check_auth {
    my ($user, $cmd) = @_;

    return 1 if ($AUTHCMDS{$cmd}{"all"});
    return 1 if (defined ($user) && $AUTHCMDS{$cmd}{$user});
    syslog ("err", "user '$user' tried '$cmd', not authenticated");
    return undef;
}


#
# reload things
#
sub reload {
    my (@what) = @_;

    for (@what) {
    	if ($_ eq "auth") {
	    &load_auth();
	} elsif ($_ eq "oncall") {
	    &load_oncall();

	} else {
	    return undef;
	}
    }

    return 1;
}


#
# (re)load the oncall schedule
#
sub load_oncall {
    my ($startup) = @_;
    my ($group, $service, $time, $who, %newoncall);

    if (!open (ONCALL, $OCFILE)) {
    	&err_startup ($startup, "could not open $OCFILE: $!");
	return undef;
    }

    %newoncall = ();
    while (<ONCALL>) {
    	next if (/^\s*$/ || /^\s*#/);
	chomp;
	if (!/^\s* ([a-zA-Z0-9_.-]+) \s+
		([a-zA-Z0-9_.-]+) \s+
		(\w{3} \s+ \d{1,2}:\d\d|default|none) \s+
		(.*) \s*$/xi) {

	    &err_startup ($startup,
	    	syslog ('err', "error in oncall configuration, line $."));
	    close (ONCALL);
	    return undef;
	}

	($group, $service, $time, $who) = ($1, $2, $3, $4);
	$group =~ tr/A-Z/a-z/;
	$service =~ tr/A-Z/a-z/;
	$time =~ tr/A-Z/a-z/;

	if (!defined($groups{$group})) {
	    &err_startup ($startup, "group $group in oncall line $. not defined in $CF");
	    close (ONCALL);
	    return undef;
	} elsif (!defined (&getservbyname($group, $service))) {
	    &err_startup ($startup, "service $service in oncall line $. not defined in $CF");
	    close (ONCALL);
	    return undef;
	}

	print "[$group] [$service] [$time] [$who]\n";
    }
    close (ONCALL);

    %oncall = %newoncall;
    1;
}


sub err_startup {
    my ($startup, $msg) = @_;

    if ($startup) {
    	die "$msg\n";
    } else {
    	syslog ('err', $msg);
    }
}


#
# handle a trap
#
sub handle_trap {
    my ($buf, $from) = @_;
    my ($sref, $time, $l, $lasttag);
    my ($port, $addr, $noalert, %trap);

    $time = time;
    $noalert = 0;
    %trap = ();
    undef $lasttag;

#
# MON-specific tags
# pro	protocol
# aut	auth
# typ	type (0=mon, 1=snmpv1)
# spc	specific type (TRAP_*)
# seq	sequence
# grp	group
# svc	service
# hst	host
# sta	status (opstatus)
# tsp	timestamp as time(2) value
# sum	summary output
# dtl	detail (terminated by \n.\n)
#
# SNMP-specific tags
# ent	enterprise OID
# agt	agent address
# gtp	generic trap type
# stp	enterprise-specific trap type
# tmp	sysUptime timestamp
# vbl	varbindlist (OID = value)
#

    foreach $l (split (/\n/, $buf)) {
    	if ($l =~ /^(\w+)=(.*)/) {
	    chomp $2;
	    $lasttag = $1;
	    $trap{$1} = $2;
	} elsif (defined $lasttag) {
	    $trap{$lasttag} .= "\n$l";
	} else {
	    syslog ('err', "unspecified tag in trap: $l");
	}
    }

    if ($trap{"typ"} == $TRAP_SNMPV1) {
	# do something here
    	return;
    }

    if (!defined $trap{"typ"} || !defined ($trap{"spc"})) {
	syslog ('err', "no trap type specified from " . inet_ntoa ($addr));
    	return undef;
    }

    ($port, $addr) = sockaddr_in ($from);

    if (!defined ($groups{$trap{"grp"}})) {
    	syslog ('err', "trap received for undefined group $trap{grp}");
&debug (1, "unknown group $trap{grp}\n");
	return;
    } elsif (!defined (&getservbyname($trap{"grp"}, $trap{"svc"}))) {
    	syslog ('err', "trap received for undefined service type $trap{grp}/$trap{svc}");
&debug (1, "unknown service [$trap{svc}]\n");
	return;
    }

    $sref = \%{$watch{$trap{"grp"}}[&getservbyname($trap{"grp"}, $trap{"svc"})]};
    $$sref{"_last_trap"} = $time;

    syslog ('info', "trap $trap{typ} $trap{spc} from " . inet_ntoa ($addr) .
    	" for $trap{grp} $trap{svc}, status $trap{sta}");

    if ($trap{"spc"} == $TRAP_COLDSTART) {
    	$$sref{"_op_status"} = $STAT_COLDSTART;
	$$sref{"_trap_duration_timer"} = $$sref{"trapduration"}
	    if ($$sref{"trapduration"});

    } elsif ($trap{"spc"} == $TRAP_WARMSTART) {
    	$$sref{"_op_status"} = $STAT_WARMSTART;
	$$sref{"_trap_duration_timer"} = $$sref{"trapduration"}
	    if ($$sref{"trapduration"});
	$$sref{"_last_uptrap"} = $time;

    } elsif ($trap{"spc"} == $TRAP_LINKDOWN) {
	$$sref{"_op_status"} = $STAT_LINKDOWN;
	$$sref{"_failure_count"}++;
	$$sref{"_first_failure"} = $tm if ($$sref{"_op_status"} != $STAT_FAIL);
	$$sref{"_trap_duration_timer"} = $$sref{"trapduration"}
	    if ($$sref{"trapduration"});

    } elsif ($trap{"spc"} == $TRAP_LINKUP) {
    	$$sref{"_op_status"} = $STAT_OK;
	$$sref{"_last_uptrap"} = $time;
	$$sref{"_trap_duration_timer"} = $$sref{"trapduration"}
	    if ($$sref{"trapduration"});

    } elsif ($trap{"spc"} == $TRAP_ENTERPRISE) {
    	$$sref{"_op_status"} = $STAT_UNKNOWN;
	$$sref{"_trap_duration_timer"} = $$sref{"trapduration"}
	    if ($$sref{"trapduration"});

    } elsif ($trap{"spc"} == $TRAP_HEARTBEAT) {
    	$$sref{"_op_status"} = $STAT_OK;
	$$sref{"_last_uptrap"} = $time;
	$noalert++;

    } else {
    	syslog ('err', "trap received from " . inet_ntoa ($addr) .
		" for undefined type $trap{typ} $trap{spc} $trap{grp}");
	return;
    }

    shift @last_failures if (@last_failures > $MAX_KEEP);
    push @last_failures, "$trap{grp} " . &getservbynum ($trap{"grp"}, $trap{"svc"}) .
	" $tm $trap{typ} $trap{spc} $trap{sum}";

    &do_alert ($trap{"grp"}, &getservbynum ($trap{"grp"}, $trap{"svc"}),
    	$trap{"sum"} . $trap{"dtl"}, $trap{"sta"}, $FL_TRAP) unless ($noalert);
}


#
# trap timeout
#
sub handle_trap_timeout {
    my ($group, $service) = @_;
    my ($tmnow);

    $tmnow = time;

&debug(1, "trap timeout for $group/$service\n");

    $$sref{"_failure_count"}++;
    $$sref{"_last_failure"} = $tmnow;
    $$sref{"_first_failure"} = $tmnow if ($$sref{"_op_status"} != $STAT_FAIL);
    $$sref{"_op_status"} = $STAT_FAIL;
    $$sref{"_last_summary"} = "trap timeout";
    shift @last_failures if (@last_failures > $MAX_KEEP);
    push @last_failures, "$group " . &getservbynum ($group, $service) .
	" $tm $summary";
    &syslog ('crit', "failure for $last_failures[-1]");

    &do_alert ($group, $service, undef, undef, $FL_TRAPTIMEOUT);
}


#
# dependency check
#
# return -1, if "infinite" loop type is "O"
# return undef, for unknown status of dependent service 
#      or a loop type of -O
# return 0zero, if dependent service failed
# return op_status > 0, if dependent service is successfull
#
sub checkDepend {
	my($group, $service, $depth) = @_;
	my($dsref, @dservices, $dservice, $dgroup, $dserviceNum, 
		$depend, $dstatus, $str, $i, $j, $dlastChecked, @traverse);

	$dsref =  \%{$watch{$group}[$service]};	
	$depend = $$dsref{'depend'};
	$dstatus = $$dsref{'_op_status'};
	chomp $depend;

	#
	# MAKE THIS IGNORE DISABLED SERVICES AND WATCHES
	#
	if($$dsref{'_last_success'} > $$dsref{'_last_failure'}) {
		$dlastChecked=$$dsref{'_last_success'};
	} else {
		$dlastChecked=$$dsref{'_last_failure'};
	}

	if(!$depth) {
		$depth = 0;
		undef @traverse;
	}
	push(@traverse, "$group.$service");	# -O, 

	#
	# we have reach a watch:service without any
	# dependencies. or a known op_status of a ---BUG--
	# service. return it.
	#
	if (!defined $depend || (
		($dstatus == $STAT_OK || $dstatus == $STAT_DEPEND ||
		 $dstatus == $STAT_UNKNOWN) && $depth > 0)) {
		return $dstatus
	}

       #
       # check for loops in the dependency
       #
       &debug(2, "Traverse: ");
       &debug(2, join(" ", @traverse),"\n");
       for($i=0; $i<@traverse; $i++) {
	   for($j=$i+1; $j<@traverse; $j++) {
	       &debug(2, "LOOP $i, $j: $traverse[$i] eq $traverse[$j]\n");
	       #
	       # is there a loop?
	       #
	       if($traverse[$i] eq $traverse[$j]) {
		   #
		   # -O loop
		   #
		   if ($j > 2 && $i > 0) {
		       &debug(2, "loop type: -O\n");
		       $$sref{'_op_status'} = $STAT_DEPEND       # unitialize op status for type -O
			       if ($$sref{"_op_status"} == $STAT_OK);
		       return undef;

		   #
		   # O loop
		   #
		   } else {
		       &debug(2, "loop type: O\n");
		       return -1;
		   }
	       }
	   }
       }




	&debug (2, "$depth group:service = $group:",&getservbynum($group, $service),"\n");
	&debug (2, "$depth depending = $depend\n");

#	@dservices = split(/[^\w\._\-:]+/, $depend);
	#
	# recursively evaluate the dependencies
	#
	@dservices = $depend =~ /[a-zA-Z0-9_.-]+:[a-zA-Z0-9_.-]+/g;
	foreach $str (@dservices) {
		my($sublastChecked, $subsref);
		$str =~ s/\s+//g;	
		next if($str =~ /^\d+$/);
		next if($str =~ /^$/);

		&debug (2, "$depth str=$str\n");

		($dgroup ,$dservice) = split(':', $str);

		($dserviceNum) = &getservbyname($dgroup, $dservice);
		$subsref =  \%{$watch{$dgroup}[$dservice]};

		if($$subsref{'_last_success'} > $$subsref{'_last_failure'}) {
			$sublastChecked=$$subsref{'_last_success'};
		} else {
			$sublastChecked=$$subsref{'_last_failure'};
		}

		#
		# do it recursively
		#
		$dstatus = &checkDepend($dgroup, $dserviceNum, ++$depth);
		&debug (2, "\tdstatus=$dstatus\n");

		#
		# this is bad,  break deadlock
		#
		return $dstatus if ($dstatus < 0);

		$dstatus='undef' if ($dstatus == $STAT_DEPEND ||
		    $dstatus == $STAT_UNTESTED || $dstatus == $STAT_UNKNOWN);

		#
		# is this an depency of A<-B<-C However 
		# Either A or B was not checked BEFORE
		# C. In this case we just set to undef
		# The better way is to for check
		# the other services. So we would
		# need to force check A and B.
		#
		$dstatus='undef' if ($dlastChecked >= $sublastChecked);	

		$depend =~ s/^${str}([^[\w\._\-:]*)/${dstatus}$1/g;		# head sub
	      	$depend =~ s/([^\w\._\-:]+)${str}([^[\w\._\-:]+)/$1${dstatus}$2/g;
		$depend =~ s/([^\w\._\-:]+)${str}$/$1${dstatus}/g;		# trail sub

		&debug (2, "$depth sub: $depend\n");

	}
	&debug (2, "$depth evaluating: --$depend--\n");
	&debug (2, "$depth value=",eval($depend),"\n");
	return eval($depend);
}


#
# (correctly) write to a socket
#
sub sock_write {
    my ($sock, $buf) = @_;
    my ($nleft, $nwritten);

    $nleft = length ($buf);
    while ($nleft) {
    	$nwritten = syswrite ($sock, $buf, $nleft);
	return undef if (!defined ($nwritten));
	$nleft -= $nwritten;
	$buf =~ s/^.{$nwritten}//s;
    }
}


#
# do I/O processing for traps and client connections
#
sub handle_io {
    my ($n, $cl, $from, $niovec, $trapbuf, $cl, $buf, $sleep, $tm0, $tm1);

    #
    # build iovec for server connections, traps, and clients
    #
    $iovec = $niovec = '';
    vec ($iovec, fileno (TRAPSERVER), 1) = 1;
    vec ($iovec, fileno (SERVER), 1) = 1;
    foreach $cl (keys %clients) {
	vec ($iovec, $cl, 1) = 1;
    }

    #
    # handle client I/O while there is some to handle
    #
    $sleep = $SLEEPINT;
    $tm0 = [gettimeofday];
    while ($n = select ($niovec = $iovec, undef, undef, $sleep)) {
	$tm1 = [gettimeofday];

	#
	# traps
	#
	if (vec ($niovec, fileno (TRAPSERVER), 1)) {
	    if (!defined ($from = recv (TRAPSERVER, $trapbuf, 65536, 0))) {
		syslog ('err', "error trying to recv a trap: $!");
	    } else {
		&handle_trap($trapbuf, $from);
	    }
	    next;

	#
	# client connections
	#
	} elsif (vec ($niovec, fileno (SERVER), 1)) {
	    &client_accept();
	}

	#
	# read data from clients if any exists
	#
	if ($numclients) {
	    foreach $cl (keys %clients) {
		next if (!vec ($niovec, $cl, 1));

		$buf = '';
		$n = sysread ($clients{$cl}{"fhandle"}, $buf, 8192);
		if ($n == 0 && $! != &EAGAIN) {
		    &client_close ($cl);
		} elsif (!defined $n) {
		    &client_close ($cl, "read error: $!");
		} else {
		    $clients{$cl}{"buf"} .= $buf;
		    $clients{$cl}{"timeout"} = $CLIENT_TIMEOUT;
		    $clients{$cl}{"last_read"} = time;
		}
	    }
	}

	#
	# execute client commands which have been read
	#
	&client_dopending() if ($numclients);
	last if (tv_interval ($tm0, $tm1) >= $SLEEPINT);
	$sleep = $SLEEPINT - tv_interval ($tm0, $tm1);
    }

    if (!defined ($n)) {
	    syslog ('err', "select returned an error for I/O loop: $!");
    }

    #
    # count down client inactivity timeouts and close expired connections
    #
    if ($numclients) {
	foreach $cl (keys %clients) {
	    $clients{$cl}{"timeout"} = time - $clients{$cl}{"last_read"};
	    if ($clients{$cl}{"timeout"} >= $CLIENT_TIMEOUT) {
		&client_close ($cl, "timeout after ${CLIENT_TIMEOUT}s");
	    }
	}
    }
}


#
# generate alert and monitor path hashes
#
sub gen_scriptdir_hash {
    my ($d, @scriptdirs, @alertdirs, $s, $group, $monitor, $period, $found);

    %MONITORHASH = ();
    %ALERTHASH = ();

    foreach $d (split (/\s*:\s*/, $SCRIPTDIR)) {
	if (-d "$d" && -x "$d") {
	    push (@scriptdirs, $d);
	} else {
	    syslog ('err', "scriptdir $d is not usable");
&debug (3, "scriptdir $d is not usable\n");
	}
    }
&debug (3, "scriptdirs=[@scriptdirs]\n");

    foreach $d (split (/\s*:\s*/, $ALERTDIR)) {
	if (-d $d && -x $d) {
	    push (@alertdirs, $d);
	} else {
	    syslog ('err', "alertdir $d is not usable");
&debug (3, "alertdir $d is not usable\n");
	}
    }
&debug (3, "alertdirs=[@alertdirs]\n");

    foreach $group (keys %watch) {
    	for ($s=0; $s<@{$watch{$group}}; $s++) {
	    next if (!defined $watch{$group}[$s]{"monitor"});
	    $monitor = (split (/\s+/, $watch{$group}[$s]{"monitor"}))[0];
	    $found = 0;
	    foreach (@scriptdirs) {
	    	if (-x "$_/$monitor") {
		    $MONITORHASH{$monitor} = "$_/$monitor";
		    $found++;
		    last;
		}
	    }
	    if (!$found) {
	    	syslog ('err', "$monitor not found in one of (\@scriptdirs)");
&debug (3, "$monitor not found in one of (@scriptdirs)\n");
	    }
	}
    }

    foreach $group (keys %watch) {
    	for ($s=0; $s<@{$watch{$group}}; $s++) {
	    foreach $period (keys %{$watch{$group}[$s]{"periods"}}) {
		foreach $alert (@{$watch{$group}[$s]{"periods"}{$period}{"alerts"}}) {
		    $alert =~ s/^(\S+).*/\1/;
		    $found = 0;
		    foreach (@alertdirs) {
			if (-x "$_/$alert") {
			    $ALERTHASH{$alert} = "$_/$alert";
			    $found++;
			}
		    }
		    if (!$found) {
			syslog ('err', "$alert not found in one of (\@alerttdirs)");
&debug (3, "$alert not found in one of (@alertdirs)\n");
		    }
		}
	    }
	}
    }

    foreach $monitor (keys %MONITORHASH) {
&debug(3, "monitor [$monitor] [$MONITORHASH{$monitor}]\n");
    }
    foreach $alert (keys %ALERTHASH) {
&debug(3, "alert [$alert] [$ALERTHASH{$alert}]\n");
    }
}
