matplotlibでwindowパフォーマンスログのCSVからグラフ描画する。

graphtest5.py

# -*- coding: cp932 -*-
import datetime as DT
import os, csv, re

# np.nan as NaN
import numpy as np

from matplotlib import pyplot as plt
from matplotlib import dates
from matplotlib.dates import date2num

import matplotlib.font_manager
prop = matplotlib.font_manager.FontProperties(fname=r'C:\Windows\Fonts\msgothic.ttc')

margin_1min = DT.timedelta(minutes=1)
margin_3min = DT.timedelta(minutes=3)
margin_5min = DT.timedelta(minutes=5)
margin_30sec = DT.timedelta(seconds=30)



basedir = r"D:\temp\perflogs"
files = ['SV1.csv', 'SV2.csv', 'SV3.csv']
#files = ['BKITSV_1340-1640.csv']

## コピーテスト結果のTSVファイル
testresult = "result.txt"
tsvfile = os.path.join(basedir, testresult)

def get_start_end(tsvfile):
    f = file(tsvfile, "rb")
    fcsv = csv.reader(f, dialect=csv.excel_tab)
    fields = fcsv.next()
    data_ = [elem for elem in fcsv]
    data = zip(*data_)
    f.close()

    index = sorted([int(i) for i in set(data[0])])

    startend = []
    for i in index:
        data = filter(lambda x:int(x[0]) == i, data_) #i回目のデータ群を抽出
        transposed = zip(*data) #リストを転置、transposed[n] で n番目カラムのデータ群
        timestart = sorted(transposed[2])[0] #最も早い start
        timeend   = sorted(transposed[3])[-1] #最も遅い end
        startend.append([timestart,timeend])

    return startend

ret = get_start_end(tsvfile)

startend = zip(*ret) #リストを転置

str_starts = startend[0] #start
str_ends = startend[1] #end

def filterrange(objlist, minval, maxval):
    return filter(lambda x:minval <= x and x <= maxval, objlist)
    


timestarts = []
for t in str_starts:
    dt = "2014/6/26 " + t
    timestarts.append(DT.datetime.strptime(dt, "%Y/%m/%d %H:%M:%S"))


timeends = []
for t in str_ends:
    dt = "2014/6/26 " + t
    timeends.append(DT.datetime.strptime(dt, "%Y/%m/%d %H:%M:%S"))

# 先頭に全体の開始・終了時間を加える。
timestarts = [timestarts[0]] + timestarts
timeends = [timeends[-1]] + timeends


import pylab
from math import ceil

#inch = (30,40),200
zoom = 2
inch, dpi = (8.3*zoom,11.7*zoom), 200

fig = pylab.figure(figsize=inch, dpi=dpi)


def ceil10(sample):
    for n in range(11):
        maxle = 10 ** n
        if sample < maxle:
            return maxle
    return sample


class perflog:
    def __init__(self):
        self.x = []
        self.y = []
        self.fields = []
        self.hostname = ""
        
    def create_from_csv(self, csvfile):
        f = file(csvfile, "rb")
        fcsv = csv.reader(f)
        tmp = fcsv.next()
        self.fields = [re.sub(r"\\\\[A-Z|a-z|0-9]+\\", "", fld) for fld in tmp]
        tmp2 = tmp[1]
        self.hostname = re.findall(r"\\\\([A-Z|a-z|0-9]+)\\", tmp2)[0]
        del tmp
        del tmp2
        for n,fld in enumerate(self.fields):
            print "\t", n, fld
            
        data_ = [elem for elem in fcsv]
        
        data = zip(*data_)
        f.close()

        self.x = np.array([DT.datetime.strptime(dt, "%m/%d/%Y %H:%M:%S.%f") for dt in data[0]])
        self.y.append(None)
        for n in range(1,len(self.fields[1:])+1):
            ytmp = np.array([None if elem == " " else float(elem) for elem in data[n]])
            self.y.append(ytmp)

series = []
for fname in files:
    csvfile = os.path.join(basedir, fname)
    print "source:", csvfile

    a_series = perflog()
    a_series.create_from_csv(csvfile)
    series.append(a_series)

a_fields = series[0].fields

# num of figure tiles
figs_x = 3
figs_y = int(ceil(len(a_fields[1:]) / float(figs_x)))

# fixed scale
#scales = [None, 10000000000L, 10000, 100000, 1, 1, 1, 1, 10000000, 100, 1000000, 100, 10, 100, 100, 100, 10000000000L, 10000, 1000000000, 100, 100000000]

scales0 = [(0, 0), (0, 2800000000.0), (0, 1600.0), (0, 42000.0), (-0.059999999999999998, 0.059999999999999998), (-0.059999999999999998, 0.059999999999999998), (-0.059999999999999998, 0.059999999999999998), (-0.059999999999999998, 0.059999999999999998), (0, 35000000.0), (0, 120.0), (0, 2500000.0), (0, 84.0), (0, 18.0), (0, 0.60000000000000009), (0, 25.0), (0, 40.0), (0, 2500000000.0), (0, 120000.0), (0, 200000000.0), (0, 38.0), (0, 50000000.0)]
scales2 = [(0, 0), (0, 2800000000.0), (0, 1600.0), (0, 42000.0), (-0.059999999999999998, 0.059999999999999998), (-0.059999999999999998, 0.059999999999999998), (-0.059999999999999998, 0.059999999999999998), (-0.059999999999999998, 0.059999999999999998), (0, 35000000.0), (0, 120.0), (0, 2500000.0), (0, 84.0), (0, 18.0), (0, 0.60000000000000009), (0, 25.0), (0, 40.0), (0, 2600000000.0), (0, 100000.0), (0, 200000000.0), (0, 38.0), (0, 50000000.0)]


# dynamic scale
#scales = [None for count in a_fields]
#scales0 = [(0,0) for count in a_fields]
#scales2 = [(0,0) for count in a_fields]

del a_fields

topvals = {}
#試行回数分だけ全体グラフから範囲を切り出す
for index in range(len(timestarts)):
    fig.clear()

    rangestart = timestarts[index]
    rangeend = timeends[index] 

    if index == 0:
        _RANGEMODE = "ALL"
        rangemin = timestarts[index] - margin_5min
        rangemax = timeends[index] + margin_5min
    else:
        _RANGEMODE = "narrow"
        rangemin = timestarts[index] - margin_30sec
        rangemax = timestarts[index] + margin_3min
    
    #系列数（サーバ台数）分繰り返す
    for a_series in series:
        x = a_series.x
        yy = a_series.y
        fields = a_series.fields
        hostname = a_series.hostname

        plotx = x[np.intersect1d(np.where(x >= rangemin)[0] , np.where(x <= rangemax)[0])]
        #各グラフにデータを描画
        for n in range(1,len(fields[1:])+1):
            y = yy[n]
            #print n,
            # subplot(max_fields_vertical, max_fields_horisontal, nuber:up->down & left->right)
            ax = pylab.subplot(figs_y, figs_x, n)
            #pylab.ylim(ymin = 0)

            # replace empty data " " as None
            #y = np.array([None if elem == " " else float(elem) for elem in data[n]])


            #開始時間 終了時間 の範囲で filter
            ploty = y[np.intersect1d(np.where(x >= rangemin)[0] , np.where(x <= rangemax)[0])]

            #ax.xaxis.set_major_locator(dates.HourLocator())
            hfmt = dates.DateFormatter('%H:%M:%S')
            ax.xaxis.set_major_formatter(hfmt)

            if _RANGEMODE == "ALL":
                scales = scales0
            else:
                scales = scales2

            if scales[n] != None:
                ymin, ymax = scales[n]
                if fields[n].find("%") >= 0:
                    ymax = 100
            else:
                if fields[n].find("%") >= 0:
                    ymax = 100
                else:
                    maxval = np.nan_to_num(y.astype(float)).max()
                    ymax = ceil10(maxval)
                    ymin = 0
                        
                scales[n] = topval
            
            ax.set_ylim(bottom = ymin, top=ymax)

            pylab.plot(plotx, ploty, label=hostname)
        pylab.legend(bbox_to_anchor=(1.7,1), fontsize="xx-large")
            #print hostname, n, unicode(fields[n],"cp932"), topval, maxval

        #pylab.savefig(os.path.join(basedir, "%02d_%s.png" % (index, hostname)), bbox_inches='tight')
        
    #各グラフのx軸に範囲を示す枠を描画
    for n in range(1,len(fields[1:])+1):
        ax = pylab.subplot(figs_y, figs_x, n)
        
        pylab.title(unicode(fields[n],"cp932"), fontproperties=prop)
        if _RANGEMODE == "ALL":
            #X軸に開始時間で線を引く
            for t_start, t_end in zip(timestarts[1:], timeends[1:]):
                #axvspan(開始, 終了, facecolor=[塗りつぶす色], alpha=[透過度])
                pylab.axvspan(t_start, t_end, facecolor='y', alpha=0.5)
            pylab.xticks(timestarts[1:])
            ax.set_xlim(rangemin, rangemax)

            cur_ymin, cur_ymax = ax.get_ylim()
            ymin, ymax = scales0[n]
            if ymin > cur_ymin:
                ymin = cur_ymin

            if ymax < cur_ymax:
                ymax = cur_ymax

            scales0[n] = (ymin, ymax)
        elif _RANGEMODE == "narrow":
            #X軸に開始時間で線を引く
            #axvspan(開始, 終了, facecolor=[塗りつぶす色], alpha=[透過度])
            pylab.axvspan(rangestart, rangeend, facecolor='y', alpha=0.5)
            #pylab.xticks(timestarts)
            ax.set_xlim(rangemin, rangemax)

            cur_ymin, cur_ymax = ax.get_ylim()
            ymin, ymax = scales2[n]
            if ymin > cur_ymin:
                ymin = cur_ymin

            if ymax < cur_ymax:
                ymax = cur_ymax

            scales2[n] = (ymin, ymax)

    fig.autofmt_xdate(rotation=90)
    

    index_underbar = os.path.basename(csvfile).find("_")
    #figfilename = "%02d_" % index + os.path.basename(csvfile)[:index_underbar] + "_%s-%s" % (rangestart.strftime("%H%M"), rangeend.strftime("%H%M")) + ".png"
    figfilename = "%02d_" % index + "servers" + "_%s-%s" % (rangestart.strftime("%H%M"), rangeend.strftime("%H%M")) + ".png"
    figfilepath = os.path.join(basedir, figfilename)

    #fig.suptitle(os.path.basename(figfilename))

    pylab.savefig(figfilepath, bbox_inches='tight')
    print figfilename
    #pylab.show()

print scales0
print scales2