Current File : //lib64/nagios/plugins/nccustom/check_smart_logs.sh
#!/bin/bash
#####################################################
# check smartctl output for errors in selftest logs #
# #
# Created by Bogdan Kukharskiy #
# Namecheap #
#####################################################
# This script is for checking if there are errors or failures in smart selftest logs, with the possibility to exclude drives from check (by serial number)
#
# Usage: "check_smart_logs.sh [-c path_to_smartd.conf] [-x exclude drives (can be as list separated by ',')] [-s include short test] [-v verbose output]"
#
# Returns the nagios native status codes:
# Nagios Status
# 0 = OK (All drives have completed smartctl selftest without errors)
# 1 = WARNING (one or more drives have no self-tests have been logged)
# 2 = CRITICAL (one or more drives have failed self-tests)
# 3 = UNKNOWN (Wrong usage)
#
# Now with NVME support
#
declare -a EXCLUDE_ARRAY
RXSAS="Background long" #default regexp pattern to grep tests (SAS disks)
RXATA="Extended offline" #default regexp pattern to grep tests (ATA disks)
OKSAS="Completed|test in progress" #default regexp pattern for passed selftest (SAS disks) or test in progress
OKATA="Completed without error|routine in progress" #default regexp pattern for passed selftest (ATA disks) or test in progress
NVME_HEALTH="PASSED" #default regexp pattern for passed selftest (NVME disks)
NVME_PERCENT=98 #default percent of NVME usage (wear-out), it can be more than 100, up to 255
verbose=0
LOGTYPE="selftest" #default type of the log, in some cases it is needed to use 'xselftest'
## USAGE MESSAGE
usage() {
cat << EOF
usage: $0 options
Now with NVME support
This script is for checking if there are errors or failures in smart selftest logs, with the possibility to exclude drives from check (by serial number)
OPTIONS:
-h Show this message
-x exclude drives by serial number (can be as list separated by ',')
-c path_to_smartd.conf, default '/etc/smartmontools/smartd.conf' and '/etc/smartd.conf'
-s include short tests
-p percent of NVME usage (wear-out), it can be more than 100, up to 255 (default 98)
-l quantity ERR LOG ENTRIES
-v be more verbose
EOF
}
#Function for checking an existance in an array ($1) of an element ($2)
#'n' means NOT found in array, 'y' is for found)
function contains() {
local n=$#
local value=${!n}
for ((i=1;i < $#;i++)) {
if [ "${!i}" == "${value}" ]; then
echo "y"
return 0
fi
}
echo "n"
return 1
}
## FETCH ARGUMENTS
while getopts "hvsx:c:p:l:" OPTION; do
case "${OPTION}" in
h)
usage
exit 3
;;
x)
IFS=, read -r -a EXCLUDE_ARRAY <<<"${OPTARG}" unset IFS
;;
c)
SMARTDPATH=${OPTARG}
;;
p)
NVME_PERCENT=${OPTARG}
;;
l)
ERR_LOG_ENTRIES_param=${OPTARG}
;;
s)
RXSAS="${RXSAS}\|Background short"
RXATA="${RXATA}\|Short offline"
;;
v)
verbose=1
;;
\?) echo "No reasonable options found!"
exit 3
;;
esac
done
## CHECK ARGUMENTS
if [[ -z ${SMARTDPATH} ]]; then
if [ -f /etc/smartmontools/smartd.conf ]; then
SMARTDPATH="/etc/smartmontools/smartd.conf"
else
SMARTDPATH="/etc/smartd.conf"
fi
fi
if [[ ! -e ${SMARTDPATH} ]]; then
echo "Error! File '${SMARTDPATH}' does not exists"
exit 3
fi
if [[ ! -f ${SMARTDPATH} ]]; then
echo "Error! '${SMARTDPATH}' is not a file"
exit 3
fi
if [[ ! -r ${SMARTDPATH} ]]; then
echo "Error! File '${SMARTDPATH}' is not readable"
exit 3
fi
if [[ ${NVME_PERCENT} -lt 1 ]] || [[ ${NVME_PERCENT} -gt 255 ]]; then
echo "Error! Treshold for percent of NVME usage (wear-out) should be > 0 and < 255; ${NVME_PERCENT}"
exit 3
fi
## MAIN ROUTINE
#echo "-= DEBUG start =-"
#while read FLINE; do
# DRIVE=$(echo ${FLINE} | cut -d' ' -f1,2,3)
# if [[ ${DRIVE} == *"nvme"* ]]; then
# DRIVE=$(echo ${DRIVE} | cut -d' ' -f2)
# echo "NVME drive found:${DRIVE}:"
# echo -n "FOR ${DRIVE} : "
# SERIAL=$(sudo smartctl -i ${DRIVE} | grep -i "Serial number" | awk '{print$3}')
# echo ${SERIAL}
# fi
#done < ${SMARTDPATH}
#echo "EA: ${EXCLUDE_ARRAY[*]}"
#echo "RXATA: ${RXATA}"
#echo "RXSAS: ${RXSAS}"
#echo "-= DEBUG end =-"
declare -a WARNING_ARRAY
declare -a CRITICAL_ARRAY
while read -r FLINE; do
DRIVELINE=$(echo "${FLINE}" | cut -d' ' -f1,2,3)
if [[ ${DRIVELINE} == *"nvme"* ]]; then
DRIVELINE=$(echo "${DRIVELINE}" | cut -d' ' -f2)
fi
SERIAL=$(sudo smartctl -i ${DRIVELINE} | grep -i "Serial number" | awk '{print$3}')
if [[ $(contains "${EXCLUDE_ARRAY[@]}" "${SERIAL}") == "n" ]]; then #checking that drive serial number is NOT in excluded array
if [[ ${DRIVELINE} == *"nvme"* ]]; then
# NVME device found
PERCENT_USED=$(sudo smartctl -a ${DRIVELINE} | grep -i "Percentage Used" | awk '{print$3}' | cut -d"%" -f1)
OVERALL_HEALTH=$(sudo smartctl -H ${DRIVELINE} | grep "SMART overall-health self-assessment test result" | cut -d":" -f2 | tr -d " ")
ERR_LOG_ENTRIES=$(sudo smartctl -a ${DRIVELINE} | grep -i "Error Information Log Entries" | awk '{print$5}')
if [[ ${verbose} == 1 ]]; then
echo "FOR ${DRIVELINE} |Serial: ${SERIAL}| Health-check: ${OVERALL_HEALTH}| Wear-out,%: ${PERCENT_USED}| ErrLogEntries: ${ERR_LOG_ENTRIES}"
# if [[ ${ERR_LOG_ENTRIES} -nq 0 ]]; then
# echo "${ERR_LOG}"
fi
if [[ ${ERR_LOG_ENTRIES} -gt ${ERR_LOG_ENTRIES_param} ]] || ! [[ ${OVERALL_HEALTH} =~ ${NVME_HEALTH} ]]; then
CRITICAL_ARRAY+=("${SERIAL}") #adding serial to critical array
elif [[ ${PERCENT_USED} -gt ${NVME_PERCENT} ]]; then
WARNING_ARRAY+=("${SERIAL}") #adding serial to warning array
fi
# Non-NVME devices
else
#let's check if the device supports simple selftest log
if sudo smartctl -l selftest ${DRIVELINE} | grep -q -i "SMART Self-test Log not supported"; then
LOGTYPE="xselftest";
else
LOGTYPE="selftest";
fi
RESULTSTRATA=$(sudo smartctl -l ${LOGTYPE} ${DRIVELINE} | grep -e "${RXATA}" |head -n 1)
RESULTSTRSAS=$(sudo smartctl -l ${LOGTYPE} ${DRIVELINE} | grep -e "${RXSAS}" |head -n 1)
if [[ ${verbose} == 1 ]]; then
echo "FOR ${DRIVELINE} |${SERIAL}| : ${RESULTSTRATA} ${RESULTSTRSAS}"
fi
if [[ -z ${RESULTSTRATA} ]] && [[ -z ${RESULTSTRSAS} ]] ; then #no acceptable results were found
# echo "WARNING! For drive with serial ${SERIAL} no self-tests have been logged"
WARNING_ARRAY+=("${SERIAL}") #adding serial to warning array
elif ! [[ ${RESULTSTRATA} =~ ${OKATA} ]] && ! [[ ${RESULTSTRSAS} =~ ${OKSAS} ]]; then
CRITICAL_ARRAY+=("${SERIAL}") #adding serial to critical array
fi
fi
fi
done < ${SMARTDPATH}
if ! [[ ${#CRITICAL_ARRAY[@]} -eq 0 ]]; then
echo "CRITICAL! Following drives have failed SMART selftest: ${CRITICAL_ARRAY[*]}"
exit 2
elif ! [[ ${#WARNING_ARRAY[@]} -eq 0 ]]; then
echo "WARNING! Following drives have no SMART selftest logged or NVME devices close to end of usage: ${WARNING_ARRAY[*]}"
exit 1
else
echo "OK! All drives passed SMART selftest succesfully"
if ! [[ ${#EXCLUDE_ARRAY[@]} -eq 0 ]]; then
echo "Drives excluded: ${EXCLUDE_ARRAY[*]}"
fi
exit 0
fi