1 #!/usr/bin/env python 2 ''' 3 Copyright (c) 2012-2014 Matthias Lee 4 5 This program is free software: you can redistribute it and/or modify 6 it under the terms of the GNU General Public License as published by 7 the Free Software Foundation, either version 3 of the License, or 8 (at your option) any later version. 9 10 This program is distributed in the hope that it will be useful, 11 but WITHOUT ANY WARRANTY; without even the implied warranty of 12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 GNU General Public License for more details. 14 15 You should have received a copy of the GNU General Public License 16 along with this program. If not, see <http://www.gnu.org/licenses/>. 17 ''' 18 19 import os 20 import datetime 21 import numpy as np 22 import logging as log 23 24 import matplotlib as mpl 25 # If no display is attached it will fail to plot and save figures.. so lets check 26 # If we are now using the Agg backend, we cannot display to screen, so toggle "show" for debug 27 if 'DISPLAY' in os.environ.keys() and os.environ['DISPLAY'] != "": 28 try: 29 mpl.use("TkAgg") 30 AggOnly = False 31 except: 32 log.error('problem using TkAgg, check whether you have an attached display, else force mpl.use("Agg")') 33 else: 34 mpl.use("Agg") 35 AggOnly = True 36 log.info("Note: using failsafe backend, Agg") 37 38 import matplotlib.pyplot as plt 39 40 41 class pyNmonPlotter: 42 # Holds final 2D arrays of each stat 43 processedData = {} 44 45 def __init__(self, processedData, outdir="./data/", overwrite=False, debug=False): 46 # TODO: check input vars or "die" 47 self.imgPath = os.path.join(outdir,"img") 48 self.debug = debug 49 self.processedData = processedData 50 if not (os.path.exists(self.imgPath)): 51 try: 52 os.makedirs(self.imgPath) 53 except: 54 log.error("Creating results dir:",self.imgPath) 55 exit() 56 57 def plotStats(self, todoList, isAIX=False): 58 outFiles=[] 59 if len(todoList) <= 0: 60 log.error("Nothing to plot") 61 exit() 62 63 for stat, fields, plotOpts in todoList: 64 if "CPU" in stat: 65 # parse NMON date/timestamps and produce datetime objects 66 times = [datetime.datetime.strptime(d, "%d-%b-%Y %H:%M:%S") for d in self.processedData["CPU_ALL"][0][1:]] 67 values=[] 68 values.append((self.processedData["CPU_ALL"][1][1:],"usr")) 69 values.append((self.processedData["CPU_ALL"][2][1:],"sys")) 70 values.append((self.processedData["CPU_ALL"][3][1:],"wait")) 71 72 data=(times,values) 73 fname = self.plotStat(data, xlabel="Time", ylabel="CPU load (%)", title="CPU vs Time", isPrct=True, stacked=True) 74 outFiles.append(fname) 75 76 elif "DISKBUSY" in stat: 77 # parse NMON date/timestamps and produce datetime objects 78 times = [datetime.datetime.strptime(d, "%d-%b-%Y %H:%M:%S") for d in self.processedData["DISKBUSY"][0][1:]] 79 80 values=[] 81 for i in self.processedData["DISKBUSY"]: 82 colTitle = i[:1][0] 83 for col in fields: 84 if col in colTitle: 85 read = np.array([float(x) for x in i[1:]]) 86 values.append((read,colTitle)) 87 88 data=(times,values) 89 fname = self.plotStat(data, xlabel="Time", ylabel="Disk Busy (%)", title="Disk Busy vs Time", yrange=[0,105]) 90 outFiles.append(fname) 91 92 # 20150108 IWT added next section 93 elif "DISKREAD" in stat: 94 # parse NMON date/timestamps and produce datetime objects 95 times = [datetime.datetime.strptime(d, "%d-%b-%Y %H:%M:%S") for d in self.processedData["DISKREAD"][0][1:]] 96 97 values=[] 98 mx=0.0 99 for i in self.processedData["DISKREAD"]: 100 colTitle = i[:1][0] 101 for col in fields: 102 if col in colTitle: 103 read = np.array([float(x) for x in i[1:]]) 104 values.append((read,colTitle)) 105 if mx < max(read): 106 mx = max(read) 107 108 data=(times,values) 109 fname = self.plotStat(data, xlabel="Time", ylabel="Disk Read", title="Disk Read vs Time", yrange=[0,mx*1.2]) 110 outFiles.append(fname) 111 112 # 20150108 IWT added next section 113 elif "DISKWRITE" in stat: 114 # parse NMON date/timestamps and produce datetime objects 115 times = [datetime.datetime.strptime(d, "%d-%b-%Y %H:%M:%S") for d in self.processedData["DISKWRITE"][0][1:]] 116 117 values=[] 118 mx=0.0 119 for i in self.processedData["DISKWRITE"]: 120 colTitle = i[:1][0] 121 for col in fields: 122 if col in colTitle: 123 read = np.array([float(x) for x in i[1:]]) 124 values.append((read,colTitle)) 125 if mx < max(read): 126 mx = max(read) 127 128 data=(times,values) 129 fname = self.plotStat(data, xlabel="Time", ylabel="Disk Write", title="Disk Write vs Time", yrange=[0,mx*1.2]) 130 outFiles.append(fname) 131 132 # 20150108 IWT added next section 133 elif "DISKXFER" in stat: 134 # parse NMON date/timestamps and produce datetime objects 135 times = [datetime.datetime.strptime(d, "%d-%b-%Y %H:%M:%S") for d in self.processedData["DISKXFER"][0][1:]] 136 137 values=[] 138 mx=0.0 139 for i in self.processedData["DISKXFER"]: 140 colTitle = i[:1][0] 141 for col in fields: 142 if col in colTitle: 143 read = np.array([float(x) for x in i[1:]]) 144 values.append((read,colTitle)) 145 if mx < max(read): 146 mx = max(read) 147 148 data=(times,values) 149 fname = self.plotStat(data, xlabel="Time", ylabel="Disk Transfer", title="Disk Transfer vs Time", yrange=[0,mx*1.2]) 150 outFiles.append(fname) 151 152 # 20150108 IWT added next section 153 elif "DISKBSIZE" in stat: 154 # parse NMON date/timestamps and produce datetime objects 155 times = [datetime.datetime.strptime(d, "%d-%b-%Y %H:%M:%S") for d in self.processedData["DISKBSIZE"][0][1:]] 156 157 values=[] 158 mx=0.0 159 for i in self.processedData["DISKBSIZE"]: 160 colTitle = i[:1][0] 161 for col in fields: 162 if col in colTitle: 163 read = np.array([float(x) for x in i[1:]]) 164 values.append((read,colTitle)) 165 if mx < max(read): 166 mx = max(read) 167 168 data=(times,values) 169 fname = self.plotStat(data, xlabel="Time", ylabel="Disk Block Size", title="Disk Block Size vs Time", yrange=[0,mx*1.2]) 170 outFiles.append(fname) 171 172 elif "MEM" in stat: 173 # TODO: implement using Stacked graphs for this 174 # parse NMON date/timestamps and produce datetime objects 175 times = [datetime.datetime.strptime(d, "%d-%b-%Y %H:%M:%S") for d in self.processedData["MEM"][0][1:]] 176 values=[] 177 178 mem=np.array(self.processedData["MEM"]) 179 180 colidx = {"total":1,"free":5,"cache":10,"buffers":13} 181 if isAIX: 182 colidx={"total":5,"free":3} 183 184 # used = total - free - buffers - cache 185 total = np.array([float(x) for x in mem[colidx["total"]][1:]]) 186 free = np.array([float(x) for x in mem[colidx["free"]][1:]]) 187 188 if not isAIX: 189 cache = np.array([float(x) for x in mem[colidx["cache"]][1:]]) 190 buffers = np.array([float(x) for x in mem[colidx["buffers"]][1:]]) 191 192 used = total - free - cache - buffers 193 else: 194 used = total - free 195 196 values.append((used,"used mem")) 197 values.append((total,"total mem")) 198 199 data=(times,values) 200 fname = self.plotStat(data, xlabel="Time", ylabel="Memory in MB", title="Memory vs Time", isPrct=False, yrange=[0,max(total)*1.2]) 201 outFiles.append(fname) 202 203 # 20150108 IWT added next section 204 elif "NETPACKET" in stat: 205 # parse NMON date/timestamps and produce datetime objects 206 times = [datetime.datetime.strptime(d, "%d-%b-%Y %H:%M:%S") for d in self.processedData["NETPACKET"][0][1:]] 207 208 values=[] 209 210 read=np.array([]) 211 write=np.array([]) 212 for i in self.processedData["NETPACKET"]: 213 colTitle = i[:1][0] 214 for iface in fields: 215 if iface in colTitle and "read" in colTitle: 216 read = np.array([float(x) for x in i[1:]]) 217 values.append((read,colTitle)) 218 219 elif iface in colTitle and "write" in colTitle: 220 write = np.array([float(x) for x in i[1:]]) 221 values.append((write,colTitle)) 222 223 data=(times,values) 224 fname = self.plotStat(data, xlabel="Time", ylabel="Network Packets/s", title="Network Packets vs Time", yrange=[0,max(max(read),max(write))*1.2]) 225 outFiles.append(fname) 226 227 elif "NET" in stat: 228 # parse NMON date/timestamps and produce datetime objects 229 times = [datetime.datetime.strptime(d, "%d-%b-%Y %H:%M:%S") for d in self.processedData["NET"][0][1:]] 230 values=[] 231 232 read=np.array([]) 233 write=np.array([]) 234 for i in self.processedData["NET"]: 235 colTitle = i[:1][0] 236 for iface in fields: 237 if iface in colTitle and "read" in colTitle: 238 read = np.array([float(x) for x in i[1:]]) 239 values.append((read,colTitle)) 240 241 elif iface in colTitle and "write" in colTitle: 242 write = np.array([float(x) for x in i[1:]]) 243 values.append((write,colTitle)) 244 245 data=(times,values) 246 fname = self.plotStat(data, xlabel="Time", ylabel="Network KB/s", title="Net vs Time", yrange=[0,max(max(read),max(write))*1.2]) 247 outFiles.append(fname) 248 249 return outFiles 250 251 252 253 def plotStat(self, data, xlabel="time", ylabel="", title="title", isPrct=False, yrange=[0,105], stacked=False): 254 255 # figure dimensions 256 fig = plt.figure(figsize=(13,4), frameon=True) 257 # resizing to hack the legend in the right location 258 fig.subplots_adjust(right=.8) 259 ax = fig.add_subplot(1,1,1) 260 261 # retrieve timestamps and datapoints 262 times, values = data 263 264 if stacked: 265 # TODO: parameterize out so that it can be more versatile 266 a = np.array([float(x) for x in values[0][0]]) 267 b = np.array([float(x) for x in values[1][0]]) 268 c = np.array([float(x) for x in values[2][0]]) 269 y = np.row_stack((a,b,c)) 270 y_ax = np.cumsum(y, axis=0) 271 ax.fill_between(times, 0, y_ax[0,:], facecolor="green", label="usr") 272 ax.fill_between(times, y_ax[0,:], y_ax[1,:], facecolor="red", label="sys") 273 ax.fill_between(times, y_ax[1,:], y_ax[2,:], facecolor="blue", label="wait") 274 275 # hack for getting around missing legend 276 p1 = plt.Rectangle((0, 0), 1, 1, fc="g") 277 p2 = plt.Rectangle((0, 0), 1, 1, fc="r") 278 p3 = plt.Rectangle((0, 0), 1, 1, fc="b") 279 ax.legend([p1, p2, p3],["usr","sys","wait"], fancybox=True, loc='center left', bbox_to_anchor=(1, 0.5)) 280 281 else: 282 # plot 283 for v,label in values: 284 ax.plot_date(times, v, "-", label=label) 285 286 ax.legend(fancybox=True, loc='center left', bbox_to_anchor=(1, 0.5)) 287 288 289 # format axis 290 ax.xaxis.set_major_locator(mpl.ticker.MaxNLocator(10)) 291 ax.xaxis.set_major_formatter(mpl.dates.DateFormatter("%m-%d %H:%M:%S")) 292 ax.xaxis.set_minor_locator(mpl.ticker.MaxNLocator(100)) 293 ax.autoscale_view() 294 if isPrct: 295 ax.set_ylim([0,105]) 296 else: 297 ax.set_ylim(yrange) 298 ax.grid(True) 299 300 fig.autofmt_xdate() 301 ax.set_ylabel(ylabel) 302 ax.set_xlabel(xlabel) 303 if self.debug: 304 if not AggOnly: 305 plt.show() 306 else: 307 log.error("cant .show() when using the Agg backend") 308 309 outFilename = os.path.join(self.imgPath,title.replace (" ", "_")+".png") 310 plt.savefig(outFilename) 311 return outFilename 312