1 #!/usr/bin/env python
    2 '''
    3 Copyright (c) 2012-2014 Matthias Lee
    4 
    5 This program is free software: you can redistribute it and/or modify
    6 it under the terms of the GNU General Public License as published by
    7 the Free Software Foundation, either version 3 of the License, or
    8 (at your option) any later version.
    9 
   10 This program is distributed in the hope that it will be useful,
   11 but WITHOUT ANY WARRANTY; without even the implied warranty of
   12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   13 GNU General Public License for more details.
   14 
   15 You should have received a copy of the GNU General Public License
   16 along with this program.  If not, see <http://www.gnu.org/licenses/>.
   17 '''
   18 
   19 import os
   20 import sys
   21 from shutil import rmtree
   22 import argparse
   23 import logging as log
   24 
   25 import pyNmonParser
   26 import pyNmonPlotter
   27 import pyNmonReport
   28 
   29 class pyNmonAnalyzer:
   30 	# Holds final 2D arrays of each stat
   31 	processedData = {}
   32 	nmonParser = None
   33 
   34 	# Holds System Info gathered by nmon
   35 	sysInfo = []
   36 	bbbInfo = []
   37 	args = []
   38 
   39   # IWT 20150108 line below altered
   40 	stdReport = [('CPU_ALL', ['user', 'sys', 'wait'], 'stackedGraph: true, fillGraph: true'), ('DISKBUSY', ['sda1', 'sdb1'], ''), ('DISKBSIZE', ['sda1', 'sdb1']),('MEM', ['memtotal', 'active'], ''), ('NET', ['eth0'], '')]
   41 
   42 	def __init__(self, args=None, raw_args=None):
   43 		if args is None and raw_args is None:
   44 			log.error("args and rawargs cannot be None.")
   45 			sys.exit()
   46 		if args is None:
   47 			self.args = self.parseargs(raw_args)
   48 		else:
   49 			self.args = args
   50 
   51 		if self.args.defaultConf:
   52 			# write out default report and exit
   53 			log.warn("Note: writing default report config file to " + self.args.confFname)
   54 			self.saveReportConfig(self.stdReport)
   55 			sys.exit()
   56 
   57 		if self.args.buildReport:
   58 			# check whether specified report config exists
   59 			if os.path.exists("report.config") == False:
   60 				log.warn("looks like the specified config file(\""+self.args.confFname+"\") does not exist.")
   61 				ans = raw_input("\t Would you like us to write the default file out for you? [y/n]:")
   62 
   63 				if ans.strip().lower() == "y":
   64 					self.saveReportConfig(self.stdReport)
   65 					log.warn("Wrote default config to report.config.")
   66 					log.warn("Please adjust report.config to ensure the correct devices will be graphed.")
   67 				else:
   68 					log.warn("\nNOTE: you could try using the default config file with: -r report.config")
   69 				sys.exit()
   70 
   71 		# check ouput dir, if not create
   72 		if os.path.exists(self.args.outdir) and self.args.overwrite:
   73 			try:
   74 				rmtree(self.args.outdir)
   75 			except:
   76 				log.error("Removing old dir:",self.args.outdir)
   77 				sys.exit()
   78 
   79 		elif os.path.exists(self.args.outdir):
   80 			log.error("Results directory already exists, please remove or use '-x' to overwrite")
   81 			sys.exit()
   82 
   83 		# Create results path if not existing
   84 		try:
   85 			os.makedirs(self.args.outdir)
   86 		except:
   87 			log.error("Creating results dir:", self.args.outdir)
   88 			sys.exit()
   89 
   90 		# This is where the magic begins
   91 		self.nmonParser = pyNmonParser.pyNmonParser(self.args.input_file, self.args.outdir, self.args.overwrite)
   92 		self.processedData = self.nmonParser.parse()
   93 
   94 		if self.args.outputCSV or "inter" in self.args.reportType.lower():
   95 			log.info("Preparing CSV files..")
   96 			self.outputData("csv")
   97 		if self.args.buildReport:
   98 			if "stat" in self.args.reportType.lower():
   99 				log.info("Preparing static Report..")
  100 				self.buildReport()
  101 			elif "inter" in self.args.reportType.lower():
  102 				log.info("Preparing interactive Report..")
  103 				self.buildInteractiveReport(self.processedData, self.args.dygraphLoc)
  104 			else:
  105 				log.error("Report type: \"%s\" is not recognized" % self.args.reportType)
  106 				sys.exit()
  107 
  108 		log.info("All done, exiting.")
  109 
  110 	def parseargs(self, raw_args):
  111 		parser = argparse.ArgumentParser(description="nmonParser converts NMON monitor files into time-sorted CSV/Spreadsheets for easier analysis, without the use of the MS Excel Macro. Also included is an option to build an HTML report with graphs, which is configured through report.config.")
  112 		parser.add_argument("-x","--overwrite", action="store_true", dest="overwrite", help="overwrite existing results (Default: False)")
  113 		parser.add_argument("-d","--debug", action="store_true", dest="debug", help="debug? (Default: False)")
  114 		parser.add_argument("--force", action="store_true", dest="force", help="force using of config (Default: False)")
  115 		parser.add_argument("-i","--inputfile",dest="input_file", default="test.nmon", help="Input NMON file")
  116 		parser.add_argument("-o","--output", dest="outdir", default="./report/", help="Output dir for CSV (Default: ./report/)")
  117 		parser.add_argument("-c","--csv", action="store_true", dest="outputCSV", help="CSV output? (Default: False)")
  118 		parser.add_argument("-b","--buildReport", action="store_true", dest="buildReport", help="report output? (Default: False)")
  119 		parser.add_argument("-t","--reportType", dest="reportType", default="interactive", help="Should we be generating a \"static\" or \"interactive\" report (Default: interactive)")
  120 		parser.add_argument("-r","--reportConfig", dest="confFname", default="./report.config", help="Report config file, if none exists: we will write the default config file out (Default: ./report.config)")
  121 		parser.add_argument("--dygraphLocation", dest="dygraphLoc", default="http://dygraphs.com/dygraph-dev.js", help="Specify local or remote location of dygraphs library. This only applies to the interactive report. (Default: http://dygraphs.com/dygraph-dev.js)")
  122 		parser.add_argument("--defaultConfig", action="store_true", dest="defaultConf", help="Write out a default config file")
  123 		parser.add_argument("-l","--log",dest="logLevel", default="INFO", help="Logging verbosity, use DEBUG for more output and showing graphs (Default: INFO)")
  124 		args = parser.parse_args(raw_args)
  125 
  126 		if len(sys.argv) == 1:
  127 			# no arguments specified
  128 			parser.print_help()
  129 			sys.exit()
  130 
  131 		logLevel = getattr(log, args.logLevel.upper())
  132 		if logLevel is None:
  133 			print "ERROR: Invalid logLevel:", args.loglevel
  134 			sys.exit()
  135 		if args.debug:
  136 			log.basicConfig(level=logLevel, format='%(asctime)s - %(filename)s:%(lineno)d - %(levelname)s - %(message)s')
  137 		else:
  138 			log.basicConfig(level=logLevel, format='%(levelname)s - %(message)s')
  139 
  140 		return args
  141 
  142 	def saveReportConfig(self, reportConf, configFname="report.config"):
  143 		# TODO: add some error checking
  144 		f = open(configFname,"w")
  145 		header = '''
  146 # Plotting configuration file.
  147 # =====
  148 # Please edit this file carefully, generally the CPU and MEM options are left with
  149 # 	their defaults. For the static report, these have special under the hood calculations
  150 #   to give you the used memory vs total memory instead of free vs total.
  151 # For the Interactive report, the field names are used to pic out the right field from the CSV
  152 # files for plotting.
  153 #
  154 # Do adjust DISKBUSY and NET to plot the desired data
  155 #
  156 # Defaults for Linux Systems:
  157 # CPU_ALL=user,sys,wait{stackedGraph: true, fillGraph: true}
  158 # DISKBUSY=sda1,sdb1{}
  159 # MEM=memtotal,active{}
  160 # NET=eth0{}
  161 #
  162 # Defaults for AIX Systems
  163 # CPU_ALL=user,sys,wait{stackedGraph: true, fillGraph: true}
  164 # DISKBUSY=hdisk1,hdisk10{}
  165 # MEM=Real total(MB),Real free(MB){}
  166 # NET=en2{}
  167 
  168 '''
  169 		f.write(header)
  170 		for stat, fields, plotOpts in reportConf:
  171 			line = stat + "="
  172 			if len(fields) > 0:
  173 				line += ",".join(fields)
  174 			line += "{%s}\n" % plotOpts
  175 			f.write(line)
  176 		f.close()
  177 
  178 	def loadReportConfig(self, configFname="report.config"):
  179 		# TODO: add some error checking
  180 		f = open(configFname, "r")
  181 		reportConfig = []
  182 
  183 		# loop over all lines
  184 		for l in f:
  185 			l = l.strip()
  186 			stat=""
  187 			fields = []
  188 			# ignore lines beginning with #
  189 			if l[0:1] != "#":
  190 				bits = l.split("=")
  191 
  192 				# check whether we have the right number of elements
  193 				if len(bits) == 2:
  194 					# interactive/dygraph report options
  195 					optStart=-1
  196 					optEnd=-1
  197 					if ("{" in bits[1]) != ("}" in bits[1]):
  198 						log.error("Failed to parse, {..} mismatch")
  199 					elif "{" in bits[1] and "}" in bits[1]:
  200 						optStart=bits[1].find("{")+1
  201 						optEnd=bits[1].rfind("}")
  202 						plotOpts=bits[1][optStart:optEnd].strip()
  203 					else:
  204 						plotOpts = ""
  205 
  206 					stat = bits[0]
  207 					if bits[1] != "":
  208 						if optStart != -1:
  209 							fields = bits[1][:optStart-1].split(",")
  210 						else:
  211 							fields = bits[1].split(",")
  212 
  213 					if self.args.debug:
  214 						log.debug("%s %s" % (stat, fields))
  215 
  216 					# add to config
  217 					reportConfig.append((stat,fields,plotOpts))
  218 
  219 		f.close()
  220 		return reportConfig
  221 
  222 	def buildReport(self):
  223 		nmonPlotter = pyNmonPlotter.pyNmonPlotter(self.processedData, self.args.outdir, debug=self.args.debug)
  224 
  225 		# Note: CPU and MEM both have different logic currently, so they are just handed empty arrays []
  226 		#       For DISKBUSY and NET please do adjust the columns you'd like to plot
  227 
  228 		if os.path.exists(self.args.confFname):
  229 			reportConfig = self.loadReportConfig(configFname=self.args.confFname)
  230 		else:
  231 			log.error("something went wrong.. looks like %s is missing. run --defaultConfig to generate a template" % (self.args.confFname))
  232 			sys.exit()
  233 
  234 		if self.isAIX():
  235 			# check whether a Linux reportConfig is being used on an AIX nmon file
  236 			wrongConfig = False
  237 			indicators = {"DISKBUSY":"sd","NET":"eth","MEM":"memtotal"}
  238 			for cat,param,_ in reportConfig:
  239 				if cat in indicators and indicators[cat] in param:
  240 					wrongConfig=True
  241 
  242 			if wrongConfig:
  243 				if not self.args.force:
  244 					log.error("It looks like you might have the wrong settings in your report.config.")
  245 					log.error("From what we can see you have settings for a Linux system but an nmon file of an AIX system")
  246 					log.error("if you want to ignore this error, please use --force")
  247 					sys.exit()
  248 
  249 		# TODO implement plotting options
  250 		outFiles = nmonPlotter.plotStats(reportConfig, self.isAIX())
  251 
  252 		# Build HTML report
  253 		pyNmonReport.createReport(outFiles, self.args.outdir)
  254 
  255 	def isAIX(self):
  256 		#TODO: find better test to see if it is AIX
  257 		if "PROCAIO" in self.processedData:
  258 			return True
  259 		return False
  260 
  261 	def buildInteractiveReport(self, data, dygraphLoc):
  262 		# Note: CPU and MEM both have different logic currently, so they are just handed empty arrays []
  263 		#       For DISKBUSY and NET please do adjust the collumns you'd like to plot
  264 
  265 		if os.path.exists(self.args.confFname):
  266 			reportConfig = self.loadReportConfig(configFname=self.args.confFname)
  267 		else:
  268 			log.error("something went wrong.. looks like %s is missing. run --defaultConfig to generate a template" % (self.args.confFname))
  269 			sys.exit()
  270 
  271 		# Build interactive HTML report using dygraphs
  272 		pyNmonReport.createInteractiveReport(reportConfig, self.args.outdir, data=data, dygraphLoc=dygraphLoc)
  273 
  274 
  275 	def outputData(self, outputFormat):
  276 		self.nmonParser.output(outputFormat)
  277 
  278 if __name__ == "__main__":
  279 	_ = pyNmonAnalyzer(raw_args=sys.argv[1:])
  280