OwlCyberSecurity - MANAGER

Edit File: check_backup.sh

#!/bin/bash
# shellcheck disable=SC2009

# DEFF VARS
cp_backup_fstab_check=1
cp_backup_mount_check=1
cp_backup_rofs_check=1
cp_backup_cpbackup_age_check=0
cp_backup_mkfs_check=1
cp_backup_spaceCrit=5
wMINUTES=$((7200 * 60))
cMINUTES=$((11520 * 60))
while getopts w:c:htfmrkaS: opts; do
  case $opts in
    h) echo -e "
	Help:
	Usage: check_backup.sh [arg]...[arg]
	[list of args] 
	      -t # equal echo test  
	      -w 'min'  # equal wMINUTES=\$['min' * 60 ]   
	      -c 'min'  # equal cMINUTES=\$['min' * 60 ]   
	      -i # equal cp_backup_iscsid_check=0  
	      -s # equal cp_backup_iscsi_session_check=0  
	      -f # equal cp_backup_fstab_check=0  
	      -m # equal cp_backup_mount_check=0  
	      -r # equal cp_backup_rofs_check=0  
	      -a # equal cp_backup_cpbackup_age_check=0
	      -k # equal cp_backup_mkfs_check=0 
	      -S 'crit free space in %' # equal cp_backup_spaceCrit='crit free space in %'
	      -h # equal this message \n" && exit ;;
    t) echo test ;;
    w) wMINUTES=$((OPTARG * 60)) ;;
    c) cMINUTES=$((OPTARG * 60)) ;;
    f) cp_backup_fstab_check=0 ;;
    m) cp_backup_mount_check=0 ;;
    r) cp_backup_rofs_check=0 ;;
    a) cp_backup_cpbackup_age_check=0 ;;
    k) cp_backup_mkfs_check=0 ;;
    S) cp_backup_spaceCrit=${OPTARG} ;;
    *) ;;
  esac
done

# Check FSTAB
if ((cp_backup_fstab_check == 1)); then
  if ! grep -q -Po "(^\d.*/backup\s+nfs(:?\d)?\s+rw)" /etc/fstab; then
    status="[CRITICAL:]"
    statustxt="No active /backup partition in /etc/fstab"
    echo "${status} ${statustxt}"
    exit 2
  fi

# if some one performed MKFS
if ((cp_backup_mkfs_check == 1)); then
  if ps auxf | grep "[m]kfs.xfs" | grep -v -i "screen"; then
    status="[CRITICAL:]"
    statustxt="mkfs.xfs in progress\n"
    echo "${status} ${statustxt}"
    exit 2
  fi
fi

# check backup partition is mounted ?
if ((cp_backup_mount_check == 1)); then

if ! (grep -q "^/dev/sd[a-z]\{1,3\}1 /backup " /proc/mounts || grep -q -Po "(/backup\s+nfs(:?\d)?\s+rw)" /proc/mounts); then
    statustxt="/backup partition is not mounted"
    if ps auxf | grep "[f]sck" | grep -q -v -i "screen"; then
      statustxt+=" Fsck running !"
    fi
    status="[CRITICAL:]"
    echo "${status} ${statustxt}"
    exit 2
  fi
fi
#Check if backup running too long
if ps aux | grep "/usr/local/cpanel/scripts/[c]pbackup" -m 1; then
  STARTDATE=$(date -d "$(ps -eo pid,lstart,cmd | grep "/usr/local/cpanel/scripts/[c]pbackup" -m 1 | awk '{print $2" "$3" "$4" "$5" "$6}')" +"%s")
  RUNTIME=$(($(date +"%s") - STARTDATE))

if ((RUNTIME >= 432000)); then # "Cpbackup duration longer then 5 days"
    statustxt="cpbackup duration longer than 5 days"
    status="[CRITICAL:]"
    echo "${status} ${statustxt}"
    exit 2
  elif ((RUNTIME >= 259200)); then
    statustxt="cpbackup duration longer than 3 days"
    status="[WARNING:]"
    echo "${status} ${statustxt}"
    exit 1
  fi
fi

#check RO-FS /backup
#touch with sudo only! and check status
if ((cp_backup_rofs_check == 1)); then

if ! error=$(ls /backup/ro_backup_test.tmp 2>&1 >/dev/null); then
    status="[CRITICAL:]"
    statustxt="cannot access /backup: $error"
    echo "${status} ${statustxt}"
    exit 2
  fi

if ! sudo /bin/touch /backup/ro_backup_test.tmp; then
    status="[CRITICAL:]"
    statustxt="/backup partition is in RO-FS state"
    echo "${status} ${statustxt}"
    exit 2
  fi
fi

#check size /backup partition
if ((cp_backup_spaceCrit >= 1)); then
  cp_backup_spaceWarn=$((cp_backup_spaceCrit * 2))
  UsedSpace=$(df -h /backup | grep -E -o "[[:space:]].*[[:space:]].*[[:space:]].*[[:space:]]/backup" | awk '{print $4}' | awk -F "%" '{print $1}' | sort -u | head -n 1)
  FreeSpace=$((100 - UsedSpace))
  if ((FreeSpace <= cp_backup_spaceCrit)); then
    statustxt="free space on /backup partition less than $cp_backup_spaceCrit%"
    status="[CRITICAL:]"

echo "${status} ${statustxt}"
    exit 2
  elif ((FreeSpace > cp_backup_spaceCrit)) && ((FreeSpace <= cp_backup_spaceWarn)); then
    statustxt="free space on /backup partition less than $cp_backup_spaceWarn%"
    status="[WARNING:]"

echo "${status} ${statustxt}"
    exit 1

fi
fi

# check cp-bakup age

# check if process stuck
# usage: checkstuck [command] [timeout]
checkstuck() {
  ${1} >/dev/null 2>&1 &
  infinipid=$!
  timeout=${2}
  fuckup=$((SECONDS + timeout))
  while [[ $(pgrep -P "${infinipid}") ]]; do
    if ((SECONDS >= fuckup)); then
      kill -9 ${infinipid}
      echo "Stuck on '${1}' with Pid '${infinipid}' Please re-mount backup partition."
      exit 2
    fi
  done
}

#checkstuck infinit 2

if ((cp_backup_cpbackup_age_check == 1)); then

DATENOW=$(date +%s)

#get last shared user
  shared_user=$(/usr/sbin/whmapi1 listaccts want="user" | grep -v "has_backup:" | awk -F":" '{print $2 }' | grep -m 1 '[^[:blank:]]')

checkstuck "stat /backup/cpbackup/daily/$shared_user" 15 # check if backup partition stuck (timeout 15 sec)

LASTKNIGHT=$(tail -n 1 /backup/users_with_old_backups.tmp)
  LASTKNIGHT="${LASTKNIGHT:=$shared_user}"
  NOBACKUP=$(grep -E -v -c "^\s*$" /backup/users_without_backups.tmp)
  OLDUSERS=$(grep -E -v -c "^\s*$" /backup/users_with_old_backups.tmp)
  WHDATE=$(stat -c "%Y" /backup/cpbackup/daily/"${LASTKNIGHT}")
  #if [[ $(ps ax | grep "[/s]cripts/cpbackup") ]];
  if ps ax | grep -q "[/r]oot/bin/nc_cp_backup.sh"; then
    runBKP="[cpbackup in progress]"
    echo "$runBKP"
    exit 0
  fi

if ! [[ $WHDATE ]]; then
    statustxt="Can't access /backup/cpbackup/daily/${LASTKNIGHT} folder"
    status="[CRITICAL:]"
    echo "${status} ${statustxt}"
    exit 2
  fi
  DELTA=$((DATENOW - WHDATE))
  if ((DELTA > wMINUTES)) && ((DELTA < cMINUTES)); then
    statustxt="Backups older than $((DELTA / 60 / 60 / 24)) day(s) ${OLDUSERS} account(s) affected and ${NOBACKUP} account(s) without backups"
    if ((NOBACKUP > 0)); then
      status="[CRITICAL:]"
      echo "${status} ${statustxt}"
      exit 2
    else
      status="[WARNING:]"
      echo "${status} ${statustxt}"
      exit 1
    fi
  elif ((DELTA > cMINUTES)); then
    statustxt="Backups older than $((DELTA / 60 / 60 / 24)) day(s) ${OLDUSERS} account(s) affected and ${NOBACKUP} account(s) without backups"
    status="[CRITICAL:]"
    echo "${status} ${statustxt}"
    exit 2
  elif ((NOBACKUP > 0)); then
    statustxt="Backups up to date but - ${NOBACKUP} account(s) without backups"
    status="[CRITICAL:]"
    echo "${status} ${statustxt}"
    exit 2
  fi
fi

echo "[OK]"
exit 0