#!/usr/bin/perl

# Uncomment to run in the foreground and spew debugging messages
#$DEBUG = 1;

use Getopt::Std;
use File::MultiTail;
use Mon::Client;
use Sys::Hostname;
use Proc::Daemon;

$Version = '0.4';

getopts("U:P:h:g:s:p:x:t:T:C");

unless(
       scalar @ARGV && defined $opt_p
       &&
       (
	defined $opt_C
	||
	( defined $opt_U 
	  && defined $opt_P
	  && defined $opt_h 
	  && defined $opt_g 
	  && defined $opt_s 
	  )
       )
      ) {
    print <<USAGE;
$0 Version $Version, written by Dan Urist <durist\@world.std.com>.

Watch for patterns in logfiles and send Mon traps.  This script daemonizes
itself, so it doesn't need to be backgrounded when it is started.

$0 -U user -P password -h monhost -g group -s service -p pattern_file [-x xpattern_file] [-t sleep] [-T throttle] logfile [logfile...]

$0 -C -p pattern_file [-x xpattern_file] logfile [logfile...]

-C               # Check mode; reports lines we would have trapped on and exits

-U user          # Mon trap user
-P password      # Mon trap user's password
-h monhost       # Host that runs mon
-g group         # Mon hostgroup for this trap
-s service       # Mon service name for this trap
-p pattern_file  # File containing perl regexps for error patterns, 
                 # one per line
-x xpattern_file # (optional) file containing list of patterns to exclude
                 # which would otherwise match
-t sleep         # (optional) Number of seconds to sleep between file checks; 
                 # default is 30.
-T throttle      # (optional) Number of minutes to wait between sending traps 
                 # for the same error match and filename. If another match is 
                 # seen within "throttle" minutes, the counter is reset; e.g. 
                 # if throttle is 5 minutes and we get a match every 4 minutes,
                 # only one trap will be sent. Use with caution!

WARNING: Extraneous whitespace and empty lines will be interpreted as patterns
in the pattern and xpattern files!

USAGE
exit;
}

# Become a daemon if we're not in debug mode
Proc::Daemon::Init unless $opt_C || $DEBUG;

my $INTERVAL = $opt_t || 30; # Time to sleep between file checks
my $THROTTLE = $opt_T * 60 if defined($opt_T);  # Time to wait between sending traps for the same file and error.
                                                # If not set, we send for everything. Note that we check based on
                                                # the time of our last check of the file, not on a timestamp in
                                                # the file, so $THROTTLE should be bigger than $opt_t
#
# Read in the patterns we're looking for
# There must be NO EXTRANEOUS WHITESPACE in this file!
# We do this instead of passing the pattern file to File::MultiTail
# (which can cleverly handle filenames!) because:
# 1) we need to extract the matched string for a summary error line
# 2) File::MultiTail ignores whitespace and comments (^#), which could
# be legitimate patterns. NB if we want to add patterns we need to restart
# this script! FIXME we could add a HUP handler...

# Make sure there's something in the pattern file
@_ = stat $opt_p;
die "$0: Error Pattern file is empty\n" unless $_[7] > 0;

open(PATTERNS, $opt_p)
  or die "$0: Could not open Error Pattern File \"$PATTERNS\"\n";

# We "or" the patterns together to speed things up a bit.
chomp(@_ = <PATTERNS>);
close PATTERNS;
$Errorpattern = join('|', @_);

$DEBUG && print "Error pattern is: ", $Errorpattern, "\n";

#
# Read in the excluded patterns
# There must be NO EXTRANEOUS WHITESPACE in this file!
#
my $Xpattern;
if(defined($opt_x)){
  # Make sure there's something in the Xpattern file
  @_ = stat $opt_x;
  if( $_[7] > 0 ){
    open(XPATTERNS, $opt_x) || die "Could not open Excluded Pattern File \"$opt_x\"\n";
    chomp(@_ = <XPATTERNS>);
    close XPATTERNS;
    $Xpattern = join('|', @_);
    $DEBUG && print "Exclude pattern is: ", $Xpattern, "\n";
  }
  else{
    warn "$0: Excluded Pattern file is empty; ignoring\n";
    undef $Xpattern;
  }
}

#
# Report what we would have trapped on
#
if ($opt_C) {
    my $lf;
    my $line;

    foreach $lf (@ARGV) {
	open (IN, $lf) || warn "$0: Could not open logfile $lf";
	my $n = 0;
	while ($line = <IN>) {
	    $n++;
	    print "$lf: $ln: $line"
		if $line =~ /$Errorpattern/ 
		    && ( !defined($Xpattern) || $line !~ /$Xpattern/ );
	} 
	close IN;
    }
    exit;
}

my $mon = new Mon::Client(
			  host => $opt_h,
			  username => $opt_U,
			  password => $opt_P
			 );
die "$0: new Mon::Client failed for host \"$opt_h\"\n" unless defined $mon;

#
# Open the files with File::MultiTail
#
my %MTargs = (
	      OutputPrefix => 'f',
	      Files => \@ARGV,
	      Pattern => [$Errorpattern],
	      NumLines => 0,
	      RemoveDuplicate => 1,
	      Function => \&wanted,
	     );
$MTargs{ExceptPattern} = [$Xpattern] if defined($Xpattern);

my $tail = new MultiTail(%MTargs);

my $Seen = {} if defined $THROTTLE;
my $Catch_Up = 0; # This little kludge is here because NumLines=>0 is still
                  # returning lines on first read
while(1){
  $tail->read;
  $Catch_Up = 1;
  sleep $INTERVAL;
}

sub wanted {
  return unless $Catch_Up;
  $DEBUG && print "Examining log files\n";
  my($arrayref) = @_;

  # We send a trap for each line that failed
  my $line;
  my $matched;
  my $file;
  my $now;
  foreach $line (@{$arrayref}){ # Lines do not appear to be in any order??
                                # Probably because File::MultiTail is using a hash to unique them; annoying

    $DEBUG && print "Examining matched line\n";

    # We have to do two expensive regexp matches to get the matched chunk and filenames; yuck.
    # It'd be nice if MultiTail gave us this back, maybe thru a hashref instead
    # of an arrayref. Oh well. Also note that the line has now been polluted with
    # the file name, potentially affecting the pattern match. MultiTail should REALLY
    # give us back a hashref, or better yet an object??
    $line =~ /($Errorpattern)/;
    $matched = $1;
    $line =~ /^(\S+)\s*:/;
    $file = $1;

    if( defined($THROTTLE) ){
      $now = time;

      # If we've already seen this error for this file and it's less than $THROTTLE,
      # skip it, but update the time. THROTTLE needs to be used with caution!
      if( exists $Seen->{$file}->{$matched} && ($now - $Seen->{$file}->{$matched} < $THROTTLE)){
	$DEBUG && print "Match was throttled; skipping\n";
	$Seen->{$file}->{$matched} = $now;
	next;
      }
      $Seen->{$file}->{$matched} = $now;
    }

    $DEBUG && print "Sending trap: Group $opt_g, Service $opt_s, for failure \"$line\"\n";
    $status = $mon->send_trap(
			      group           => $opt_g,
			      service         => $opt_s,
			      retval          => 1,
			      opstatus        => "fail",
			      summary         => hostname . ":" . $file . ":" . $matched,
			      detail          => hostname . ":" . $line
			     );
    warn "$0: Mon::Client::send_trap failed\n" unless defined($status);
  }
}