#!/bin/sh
#!/bin/sh
# Description: Basic events check
# Author: Swift Griggs <sgriggs@parsec.com>
# 
# [Log]
#   Apr-2021: Initial version written for demonstration during MEP webinar at PARSEC
#

# Do not depend on an upstream alerter.  Send directly from this script. 
# Set to "NO" for disable or enter your preferred email or whitespace
# delimited list to enable.
EMAIL_ALERTS=NO

# Without this on, the script likely will only output "OK"
VERBOSE=YES

# Cache files
UPTIME_CACHE=/var/tmp/uptime.cache
RING_BUFFER_CACHE=/var/tmp/ringbuffer.cache

# Globals in use
RETURN_VALUE=0
FIRST_RUN=0

# Send log messages and debug chatter where it's wanted
geko(){

  # We'll use 'logger' if it's found	  
  which logger >/dev/null 2>&1
  if [ $? -eq 0 ]; then
    # use the logger program for writing to syslog
    Log=logger
  else
    # essentially disable logging 
    Log="echo $1 >/dev/null"
  fi 
  

  if [ $VERBOSE = "YES" ]; then
    echo "$1"
    $Log $1
  else
    # Do not echo to the screen but log to syslog
    $Log $1
  fi 
}

# Make sure we have what we need to run the script
sanity_check(){

  # make sure we can create files in /tmp
  echo testing > /tmp/test.$$
  if [ $? -ne 0 ]; then
    # Something is wrong with /tmp
    geko "ERROR: Cannot create a test file in /tmp!"
    exit 2
  else
    rm /tmp/test.$$
  fi 

  # Mailx is needed for the email sending.  However, it's not fatal if we
  # don't have it.
  which mailx >/dev/null 2>&1
  if [ $? -ne 0 ]; then
    geko "WARNING: The 'mailx' program is not in your path. This script won't be able to send email alerts!"
  fi 

  # Be nice to have logger for syslog entries.
  which logger >/dev/null 2>&1
  if [ $? -ne 0 ]; then
    geko "WARNING: The 'logger' program is not in your path. It's used for syslog entries. Continuing."
  fi   
}


send_email_alert(){
  IS_EMAIL_ENABLED=`echo $EMAIL_ALERTS | grep '@' | wc -l`
  if [ $IS_EMAIL_ENABLED -ne 1 ]; then
    # Email feature is disabled. Do nothing.
    return 0
  fi 

  # Send the alert 
  which mailx >/dev/null 2>&1 
  if [ $? -eq 0 ]; then
    NOW=`date`
    MYNAME=`uname -n`
    # we have mailx to send a message. Use it. 
    echo $1 | mailx -s "Event Alert from $MYNAME detected at $NOW" $EMAIL_ALERTS
  fi 
}

# One very significant event is a reboot. Check to make sure
# we haven't rebooted since the last check. Remember that on
# most unix systems /tmp is cleared during reboot but /var/tmp 
# is left alone. 
check_uptime(){
  # Turns out uptime is 3.0 BSD and universally the same
  DAYS_UP=`uptime | awk '{print $3}'`

  OLD_DAYS_UP=0
  if [ -r "$UPTIME_CACHE" ] ; then
    OLD_DAYS_UP=`cat "$UPTIME_CACHE"`
  else
    # create the cache file. It doesn't exist
    echo $DAYS_UP > "$UPTIME_CACHE"
    FIRST_RUN=1
  fi
  
  if [ $OLD_DAYS_UP -gt $DAYS_UP ] && [ $FIRST_RUN = 0 ]; then
    geko "WARNING: Reboot event detected."
    send_email_alert "Reboot detected since last check."
    # reset the count back so the script won't re-alert
    echo $DAYS_UP > "$UPTIME_CACHE"    
    return 1
  else
    # Everything is fine. No reboots. Just return normally.
    geko "INFO: System has been up since last check. No reboots."
    return 0 
  fi   
}

ring_buffer_check(){
  MYOS=`uname -s`
  case $MYOS in
    FreeBSD)
      # FreeBSD's 'dmesg' works a bit differently
      dmesg -a | egrep -i 'error|warn|fatal|stopped|died|unexpect|lost' > /tmp/ring_buffer_messages.$$
      ;;
    *)
      # Everyone else doesn't need the -a flag
      dmesg | egrep -i 'error|warn|fatal|stopped|died|unexpect|lost' > /tmp/ring_buffer_messages.$$
      ;;
  esac
    
  
  if ! [ -r "$RING_BUFFER_CACHE" ] ; then
    # Write out the ring buffer cache entires we'd care about
    case $MYOS in
      FreeBSD)
        dmesg -a | egrep -i 'error|warn|fatal|stopped|died|unexpect|lost' > "$RING_BUFFER_CACHE"
        ;;
      *)
        dmesg | egrep -i 'error|warn|fatal|stopped|died|unexpect|lost' > "$RING_BUFFER_CACHE"
        ;;
     esac
  else
    FIRST_RUN=1
  fi
  
  # On a first-run they will always be the same
  DIFFERNCES=`diff /tmp/ring_buffer_messages.$$ "$RING_BUFFER_CACHE" | wc -l`
  diff /tmp/ring_buffer_messages.$$ "$RING_BUFFER_CACHE" > /tmp/new_messages.$$
  
  # cleanup the temp file by making it the new cache
  mv /tmp/ring_buffer_messages.$$ "$RING_BUFFER_CACHE"
  if [ $DIFFERNCES -ge 1 ]; then
    geko "WARNING: New warnings or errors in the kernel ring buffer"
    send_email_alert "WARNING: New warnings have accumulated in the system's kernel ring buffer."
    return 1
  else
    geko "INFO: The kernel ring buffer has no new errors or warnings."
    return 0  
  fi 
  rm /tmp/new_messages.$$
}


os_specific_events(){
  # Find a sane alert-default based on the number of CPUs
  MYOS=`uname -s`
  case $MYOS in
  Linux)
    ring_buffer_check
    ;;
  NetBSD)
    ring_buffer_check
    ;;
  OpenBSD)
    ring_buffer_check
    ;;
  FreeBSD)
    ring_buffer_check
    ;;
  SunOS)
    ring_buffer_check
    ;;
  AIX)
    # AIX has no real ring buffer to check. Bleh.
    ;;
  OSF1)
    # no dmesg on tru64, either
    ;;
  IRIX)
    # IRIX doesn't use it's ring buffer like others do
    ;;
  HP-UX)
    ring_buffer_check
    ;;
  OpenUNIX)
    ring_buffer_check
    ;;
  *)
    # can't do much if we don't know what OS we are on
    echo >/dev/null 
    ;;
  esac
}


# main()
check_uptime
os_specific_events

# Add a user-friendly console output one-liner.  No 'geko' here.  Some
# monitoring software likes *some* response to act as a sort of keepalive
# string or sanity check beyond the return value / exit code.
if [ $RETURN_VALUE -eq 0 ]; then
  echo "OK"
else
  if [ $FIRST_RUN -eq 1 ]; then
    # There read failures on bogus disk entries.
    # Do not squawk about that on a first-run.
    echo "OK"
  else
    echo "ERROR: One or more failures or runtime errors was detected."
  fi
fi

# Leave the caller with a numeric value. Plays nice with 
# monitoring software expecting a non-zero exit being in error
exit $RETURN_VALUE
