#!/bin/bash # Check status of a list of servers # # Any servers in urls that yield a bad response are collated and sent via email # to all recipients. # recipients are sourced from a separate file in the cwd; it is expected to # have a single email address per line. # # If WATCHDOG_LOG is set, all output will be redirected there. # # Dependencies # - curl: Polling the servers # - msmtp: Sending email; requires separate configuration # NOTE: Currently only useful for services running over HTTP #set -eux main() { local recipients_file="./recipients" local urls_file="./urls" # TODO: How to check these sensibly? #local services_file="./services" if [ ! -f ${recipients_file} ]; then echo "[ERROR] Please create a file '${recipients_file}'" exit 1 fi if [ ! -f ${urls_file} ]; then echo "[ERROR] Please create a file '${urls_file}'" exit 1 fi # See above TODO #if [ ! -f ${services_file} ]; then # echo "[ERROR] Please create a file '${services_file}'" # exit 1 #fi # Source array contents from files, ignoring commented lines local recipients=($(cat ${recipients_file} | sed 's/^#.*$//')) local urls=($(cat ${urls_file} | sed 's/^#.*$//')) # See above TODO #local services=($(cat ${services_file} | sed 's/^#.*$//')) declare -A FLAGGED if [ -n "${WATCHDOG_LOG}" ]; then rotate_log WATCHDOG_LOG # Redirect all output to WATCHDOG_LOG but preserve a link to stdout should we # still want to use it exec 3>&1 1>>${WATCHDOG_LOG} 2>&1 fi echo ">> $(date)" echo "== Checking:" for uri in "${urls[@]}"; do \ full_uri="https://${uri}" echo -n " ${full_uri} ... "; stat="$(curl -s -o /dev/null -w "%{http_code}%{stdout}" ${full_uri})" echo "${stat}" # All other HTTP responses should not technically be errors if ((${stat} >= 400 && ${stat} <= 599)); then FLAGGED[${full_uri}]=${stat} fi done ret=0 if ((${#FLAGGED[@]} > 0)); then echo "== Error(s) detected. Sending mail to:" for rcp in ${recipients[@]}; do echo " ${rcp}" done send_mail FLAGGED ret=1 else echo "== OK" fi exit ${ret} } rotate_log() { local -n log=$1 local max_log=4096 # 4M as we get size in K below touch ${log} local log_size=$(du -k "${log}" | cut -f 1) if ((${log_size} > ${max_log})); then # Clobber any previous backup mv -f ${log} ${log}.1 touch ${log} fi } send_mail() { local -n error_list=$1 local error_string="" for i in "${!error_list[@]}"; do # Use bash's awful syntax to escape any / in the server string code=${i//\//\\\/} server=${error_list[$i]} error_string+=" ${code}: ${server}\n" done cat mail_template \ | sed "s/{ERR_LIST}/${error_string}/g" \ | msmtp -a omn_server_watch ${recipients[@]} \ || echo "[ERROR] Failed to send mail!" } main