Script visualizing iozone -a output

IozoneResults.py

#!/usr/local/bin/python3

import re
import numpy as np
import pylab as plt

class IozoneResult(np.ndarray):

	def __new__(subtype, shape, dtype=float, buffer=None, offset=0, strides=None, order=None, name=None,fileSizes=None):
		obj = super(IozoneResult, subtype).__new__(subtype, shape, dtype, buffer, offset, strides, order)
		obj.name = name
		return obj
	
	def loadFile(self,fileName):
		self.name=fileName
		singleFile=open(fileName)
		startData=False
		j=0

		fileData=[]
		startData=False
		for line in singleFile.readlines():
			if not line.strip():
				continue
			if re.search("iozone test complete",line) != None:
				startData=False
			if startData:
				lineData=np.array([int(s) for s in line.split()])
				if fileData==[]:
					fileData=lineData
				else:
					fileData=np.dstack((fileData,lineData))
					
			if re.search("reclen",line) != None:
				startData=True
		fileData=fileData.transpose().squeeze()	
		self.resize(fileData.shape,refcheck=False)
		self[:]=fileData
		return self

	def loadNumpy(self,array,fileSizes=None):
		self.resize(array.shape,refcheck=False)
		self[:]=array
		if fileSizes is not None:
			self[:,0]=fileSizes
		return self

			

	def imshow(self,second=None,myCmap=None):
		fig = plt.figure()
		if second is None:
			minVal=self[:,3:].min()
			maxVal=self[:,3:].max()
			if minVal*maxVal<0:
				extreme=max(abs(minVal),abs(maxVal))
				minVal=extreme*(-1)
				maxVal=extreme
				print("EXTREME!!!")
	
			ax = fig.add_subplot(111)
			cax = IozoneResult.__imshow__(self,ax,fig,minVal,maxVal,cmap=myCmap)
		else:
			minVal=min((self[:,3:].min(),second[:,3:].min()))
			maxVal=max((self[:,3:].max(),second[:,3:].max()))
		
	
			ax = fig.add_subplot(121)
			cax = IozoneResult.__imshow__(self,ax,fig,minVal,maxVal,cmap=myCmap)
			ax=fig.add_subplot(122)
			cax = IozoneResult.__imshow__(second,ax,fig,minVal,maxVal,cmap=myCmap)


	def imshowlog10(self,second=None):
		fig = plt.figure()

		result=np.log10(self)
		self.loadNumpy(result)

		result=np.log10(second)
		second.loadNumpy(result)
		

		if second is None:
			ax = fig.add_subplot(111)
			cax = IozoneResult.__imshow__(self,ax,fig,self[:,3:].min(),self[:,3:].max())
		else:
			
			minVal=min((self[:,3:].min(),second[:,3:].min()))
			maxVal=max((self[:,3:].max(),second[:,3:].max()))
		
			ax = fig.add_subplot(121)
			cax = IozoneResult.__imshow__(self,ax,fig,minVal,maxVal)
			ax=fig.add_subplot(122)
			cax = IozoneResult.__imshow__(second,ax,fig,minVal,maxVal)




			

	@classmethod
	def __imshow__(myType,self,ax,fig,minVal,maxVal,cmap=None):
		######################################
		#Divide by 1024 - results in MB/s !!!#
		######################################
		cax = ax.matshow(self[:,3:]/1024,aspect='auto',vmin=minVal/1024,vmax=maxVal/1024,cmap=cmap)
		cbar=fig.colorbar(cax,label="MB/s")#,ticks=[3,6,9,12])
		#cbar.ax.set_yticklabels(["kB","MB","GB","TB"]);
		ax.set_xticks(np.arange(0,12))
		ax.xaxis.set_tick_params(labelsize=5)
		xLabels=["write","rewrite","read","reread","rand \n read"," rand\n write","bkwr \n read","rec\n rewrite","stride\n read","fwrite","frewrite","fread","freread"]
		ax.set_xticklabels(xLabels)
		ax.grid(color='w', linestyle='-', linewidth=2)
		ax.xaxis.grid(False)

		yticks=[]
		yLabels=[]
		yold=0
		for i in np.arange(0,self[:,0].size):
			y=self[i,0]
			if yold==y:
				continue
			else:
				yold=y
				print("i="+str(i))
				print("y"+str(y))
				yticks.append(i-0.5)	
				yLabels.append(y)
			

		ax.set_yticks(yticks)
		print(yticks)
		ax.set_yticklabels(yLabels)
		plt.title(self.name)
		plt.ylabel('file and record size')
		return cax
	
	



#A=IozoneResult([],fileName="lustre-2.10-a-1.out")
#print(type(A))
#B=A.loadData()
#print("Type of B"+str(type(B)))
#print(type(A))
#print(A.shape)
#A.imshow()


if __name__ == "__main__":
	import sys
	import glob

	if len(sys.argv)==1:
		print("Need at least one argumnet")
		sys.exit(1)
	elif len(sys.argv)==2:
		results=[]
		for name in glob.glob('./'+sys.argv[1]+'*'):
			print("Processing file:"+name)
			tmpData=IozoneResult([])
			tmpData.loadFile(name)
			if results==[]:
				results=np.empty_like(tmpData)
				results=tmpData
			else:
				results=np.dstack((results,tmpData))

		result=results.mean(axis=2)
		mean=IozoneResult([],name="Average "+sys.argv[1])
		mean.loadNumpy(result,fileSizes=tmpData[:,0])

		result=results.std(axis=2)
		stddev=IozoneResult([],name="Standard deviation")
		stddev.loadNumpy(result,fileSizes=tmpData[:,0])

		mean.imshow(stddev,myCmap=plt.cm.gist_ncar)
	
		rdev=IozoneResult([],name="Relative deviation "+sys.argv[1])
		rdev.loadNumpy(stddev/mean,fileSizes=tmpData[:,0])
		
		rdev.imshow()
	
		plt.show()
	elif len(sys.argv)==3:
		results=[]
		for name in glob.glob('./'+sys.argv[1]+'*'):
			print("Processing file:"+name)
			tmpData=IozoneResult([])
			tmpData.loadFile(name)
			if results==[]:
				results=np.empty_like(tmpData)
				results=tmpData
			else:
				results=np.dstack((results,tmpData))

		result=results.mean(axis=2)
		mean1=IozoneResult([],name="Average "+sys.argv[1])
		mean1.loadNumpy(result)
		stddev1=results.std(axis=2)

		results=[]
		for name in glob.glob('./'+sys.argv[2]+'*'):
			print("Processing file:"+name)
			tmpData=IozoneResult([])
			tmpData.loadFile(name)
			if results==[]:
				results=np.empty_like(tmpData)
				results=tmpData
			else:
				results=np.dstack((results,tmpData))

		result=results.mean(axis=2)
		mean2=IozoneResult([],name="Average "+sys.argv[2])
		mean2.loadNumpy(result)
		stddev2=results.std(axis=2)
		diffDev=stddev1+stddev2

		mean1.imshow(mean2)

		result=np.empty_like(mean1)
		result=np.abs(mean1-mean2)/(mean1+mean2)
		contrast=IozoneResult([],name="Contrast")
		contrast.loadNumpy(result,fileSizes=tmpData[:,0])

		result=(mean1-mean2)
		diff=IozoneResult([],name="Difference "+mean1.name+" - "+mean2.name)
		diff.loadNumpy(result,fileSizes=tmpData[:,0])
		diffstd=IozoneResult([],name="Difference deviation")
		diffstd.loadNumpy(diffDev,fileSizes=tmpData[:,0])
		
		contrast.imshow()
		diff.imshow(diffstd,myCmap=plt.cm.spectral)
		#contrast.imshow(diff)
		
		
	
		plt.show()
		
		

The gist used to prepare results for https://funinit.wordpress.com/2018/01/17/analysing-performance-of-shared-file-system-on-production-cluster/