#!/bin/bash

#
# The goal of this script is to run nmon daily, and compress the 
# previous log files
#

# init some stuff
date=$(date +%Y%m%d)
## init ## where to output the data
logdir='/var/log/nmon'
if [[ ! -d ${logdir} ]]
 then
   mkdir -p ${logdir}
 fi
## init ## who to notify if nmon or logger cannot be found
email='traxler@lsu.edu'
## init ## how long to wait before killing current instances 10 minutes is 20 30 second intervals
((waitTime=20))
## init ## number of days of nmon data to keep 
((days=30))
## init ##figure out fqdn if we can
hostname=$(uname -n)
  ## make sure hostname is fqdn (why does this have to be this hard)
  hnlen=${#hostname}
  tmp=${hostname:nhlen-3:3}
  if [[ 'Xedu' != "X${tmp}" && 'Xorg' != "X${tmp}" ]]
   then
     hostname="${hostname}.hpc.lsu.edu"
   fi
## init ## try to find nmon command
CMD=$(which nmon)
if [[ 'X' == "X${CMD}" ]]
 then
   if [[ -x /usr/local/bin/nmon ]]
    then 
      CMD='/usr/local/bin/nmon'
    else
      echo "nmon not in path on host: ${hostname} " | mail -s "missing nmon" ${email}
      exit
    fi
 fi
## init ## try to find logger
LOGGER=$(which logger)
if [[ 'X' == "X${LOGGER}" ]]
 then
   echo "logger not in path on host: ${hostname} " | mail -s "missing logger" ${email}
   LOGGER='echo '
 fi
## init ## make sure logging directory exists
if [[ ! ${logdir} ]]
 then
   mkdir -p ${logdir}
 fi
## init ## set name of output file
outfile="${logdir}/$hostname.$date.nmon"

## init ## set interval to 15 minutes
((minuteInterval=15))
((interval=minuteInterval*60))

## init ## it is possible that nmon is already/still running
others=$(ps -ef | grep '[n]mon -F /var/log/nmon' | wc -l)
   ## wait 10 minutes, if still running , kill it and proceed
   ((cnt = 0))
   while ((0 < others))
    do
      sleep 30
      ((count = count + 1))
      others=$(ps -ef | grep '[n]mon -F /var/log/nmon' | wc -l)
      if ((waitTime < count))
       then
           if ((0 < others))
         then
           ## kill all instances that are running
           ps -ef | grep '[n]mon -F /var/log/nmon' | awk '{print $2}' | while read pid
            do
              kill -9 ${pid}
            done
           ${LOGGER} nmon "existing nmon process ${pid} killed"
         fi
       fi
    done
## init ## make sure this is Linux
os=$(uname -a | awk '{print $1}')

# Main section now -- clean up and start nmon
if [[ "${os}" = "Linux" ]]
 then
    ## figure out how many iterations left in today
    hour=$(date +"%-H")
    min=$(date +"%-M")
    ((count=((((24 - hour) * 60) - min) / minuteInterval) - 1))

    ## gzip all files int he directory before the new file is created.
    ### ls -1 ${logdir}/*.nmon | grep -v "${outfile}" | while read line
    ls -1 ${logdir}/*.nmon | while read line
     do
       rsync -4 ${line} rsync://nmon.hpc.lsu.edu/nmon-${hostname}/
       gzip -N -S .gz -9 -f -r ${line}    > /dev/null 2>/dev/null
     done    

    # actually start nmon
    ${CMD} -F ${outfile} -s${interval} -c${count}

    # purge old files
    ls -1 ${logdir}/*.nmon ${logdir}/*.nmon.gz | while read line
     do
       fileDate=$(stat --printf="%y" ${line} | cut -b1-10)
       cutoff=$(date -d "${days} days ago" "+%Y-%m-%d")
       if [[ "${cutoff}" > "${fileDate}" ]]
        then
          rm -f ${line}
        fi
     done    
    ## log to syslog that nmon was started
    ${LOGGER} nmon "nmon data collection started at: ${date} interval: ${interval} for count: ${count} times"
 fi