#!/bin/sh
# Description: Basic disk hardware check
# Author: Swift Griggs <sgriggs@parsec.com>
# 
# [Log]
#   Apr-2021: Initial version written for demonstration during MEP webinar at PARSEC
#

# Do not depend on an upstream alerter.  Send directly from this script. 
# Set to "NO" for disable or enter your preferred email or whitespace delimited list to enable.
EMAIL_ALERTS=NO

# Without this on, the script likely will only output "OK"
VERBOSE=YES
DISK_LIST_CACHE=/var/tmp/disk_health_check

# Globals in use
DISK_LIST=""
RETURN_VALUE=0
FIRST_RUN=0

# Send log messages and debug chatter where it's wanted
geko(){

  # We'll use 'logger' if it's found	  
  which logger >/dev/null 2>&1
  if [ $? -eq 0 ]; then
    # use the logger program for writing to syslog
    Log=logger
  else
    # essentially disable logging 
    Log="echo $1 >/dev/null"
  fi 
  

  if [ $VERBOSE = "YES" ]; then
    echo "$1"
    $Log $1
  else
    # Do not echo to the screen but log to syslog
    $Log $1
  fi 
}

# Make sure we have what we need to run the script
sanity_check(){

  # make sure we can create files in /tmp
  echo testing > /tmp/test.$$
  if [ $? -ne 0 ]; then
    # Something is wrong with /tmp
    geko "ERROR: Cannot create a test file in /tmp!"
    exit 2
  else
    rm /tmp/test.$$
  fi 

  # Mailx is needed for the email sending.  However, it's not fatal if we
  # don't have it.
  which mailx >/dev/null 2>&1
  if [ $? -ne 0 ]; then
    geko "WARNING: The 'mailx' program is not in your path. This script won't be able to send email alerts!"
  fi 

  # without dd we can't read test the disks fortunately, it's on nearly
  # every version of Unix.  Amusingly, Linux seems to be the only one that
  # likes to drop it from their distribution occasionally (folly).
  which dd >/dev/null 2>&1
  if [ $? -ne 0 ]; then
    geko "WARNING: The 'dd' program is not in your path. It's required for read testing. Fatal."
    exit 3
  fi   

  # Be nice to have logger for syslog entries.
  which logger >/dev/null 2>&1
  if [ $? -ne 0 ]; then
    geko "WARNING: The 'logger' program is not in your path. It's used for syslog entries. Continuing."
  fi   
}


show_disks_devices(){
  MYOS=`uname -s`
  # There is no one universal way to do this under Unix
  case $MYOS in
  Linux)
    # all raw disks, no partitions
    ls /dev/sd* | egrep -v '[0-9]'
    ;;
  SunOS)
    ls /dev/dsk/*s2
    ;;
  OSF1)
    ls /dev/disk/dsk*c
    ;;
  NetBSD)
    # Be aware. This isn't always the best way to go.
    # If you have events clobbering your ring-buffer it won't work
    # This is rare on BSD boxes but it does happen occasionally. 
    RING_BUFFER_LIST=`dmesg | egrep '^sd[0-9]+|^wd[0-9]' | grep at | awk '{print $1}' |sort -u`
    for NAME in $RING_BUFFER_LIST ; do 
      echo /dev/${NAME}d
    done
    ;;
  OpenBSD)
    RING_BUFFER_LIST=`dmesg | egrep '^sd[0-9]+|^wd[0-9]' | grep at | awk '{print $1}' |sort -u`
    for NAME in $RING_BUFFER_LIST ; do 
      echo /dev/${NAME}d
    done    
    ;;
  AIX)
    ls /dev/hdisk*
    ;;
  FreeBSD)
    ls /dev/ada* | grep -v p
    ;;   
  *)
    geko "ERROR: Your operating system is not supported by this script! Unable to gather initial disk list."
    exit 100
    ;;
  esac
}

send_email_alert(){
  IS_EMAIL_ENABLED=`echo $EMAIL_ALERTS | grep '@' | wc -l`
  if [ $IS_EMAIL_ENABLED -ne 1 ]; then
    # Email feature is disabled. Do nothing.
    return 0
  fi 
  # Send the alert 
  which mailx >/dev/null 2>&1 
  if [ $? -eq 0 ]; then
    # we have mailx to send a message. Use it. 
    echo $1 has failed a read test. | mailx -s "Disk Alert" $EMAIL_ALERTS
  fi 
}


# Either build an initial list of read-test-passing disks (first run) or 
# check the existing list for any failures of previously known-good disks. 
check_disks(){

  # We need a historical record of what disks were there before
  # So, the first run of the script is just going to report things
  # are fine. Subsequent runs can tell you if you lost devices. 
  # If new devices show up, no alerts will be generated. If they depart
  # you will generate an alert. 
  if ! [ -r $DISK_LIST_CACHE ]; then
    FIRST_RUN=1
    geko "WARNING: First run. Disk records must be built first."
    DISK_LIST=`show_disks_devices` # also trims whitespace & newlines
    for DISK in $DISK_LIST ; do 
      read_test_disk $DISK
      if [ $? -eq 0 ]; then
        # Lovely little breadcrumb for sysadmins later to find.
        geko "INFO: Disk $DISK passed it's initial read test. Including it in future checks."
        
        # The disk needs to pass at least one read test
        # before we consider it valid and begin additional
        # checks for it later on. In this case, it's valid.
        echo $DISK >> $DISK_LIST_CACHE
      fi 
    done

  else
    geko "INFO: Re-using previously known good disk list: $DISK_LIST_CACHE"
    # state variable. Assume all will pass
    FRESH_DISK_LIST=0
    
    # Just re-use the old, previously-checked list. 
    DISK_LIST=`cat $DISK_LIST_CACHE`
    for DISK in $DISK_LIST ; do 
      read_test_disk $DISK
      if [ $? -ne 0 ]; then 
        # This disk just failed it's read test. This disk used to work!
        geko "ERROR: Disk \"$DISK\" failed 10K read test."
        send_email_alert $DISK
        
        # Once a disk fails a read test, we really need to re-confidence all the
        # surviving disks on the next run. So, we'll remove the existing disk list
        # after this run reporting on each failure. 
        FRESH_DISK_LIST=1
      else
        geko "INFO: $DISK OK"
      fi
    done

    # Force a refresh if any disks failed.
    if [ $FRESH_DISK_LIST -ne 0 ]; then
      geko "WARNING: After a failure occured of at least one disk device, next run of this script will refresh the disk list."
      # Remove the cache file containing the disk list
      rm "$DISK_LIST_CACHE"
    fi
  fi
}

# Performs a non-destructive read test on the disk 
# passed as $1
read_test_disk(){
  # if we don't get an argument, that's a problem
  if [ "${1}x" = x ]; then
    geko "ERROR: runtime error in read_test_disk, no arguements. Exactly 1 is required."
    exit 1
  fi

  # Perform the 10 Kilobyte read test. DD was already checked for.
  dd if=$1 of=/dev/null bs=1024 count=10 >/dev/null 2>&1
  if [ $? -eq 0 ]; then
    # success.
    return 0
  else
    # Any failures result in a non-zero exit.
    RETURN_VALUE=5
    return 1
  fi 
}

# main()
check_disks

# Add a user-friendly console output one-liner.  No 'geko' here.  Some
# monitoring software likes *some* response to act as a sort of keepalive
# string or sanity check beyond the return value / exit code.
if [ $RETURN_VALUE -eq 0 ]; then
  echo "OK"
else
  if [ $FIRST_RUN -eq 1 ]; then
    # There read failures on bogus disk entries.
    # Do not squawk about that on a first-run.
    echo "OK"
  else
    echo "ERROR: One or more failures or runtime errors was detected."
  fi
fi

# Leave the caller with a numeric value. Plays nice with 
# monitoring software expecting a non-zero exit being in error
exit $RETURN_VALUE
