Program to detect and correct spikes in RRDtool graph

RRDxportSpike.py

import math
import pickle
#import rrdtool
import subprocess
import os
import sys
import traceback
import xml.sax

import numpy

class RRDxportSpike(xml.sax.ContentHandler):

    #debug = True
    debug = False

    startE = None
    key = None
    value = None
    series = {}
    
    graphs_to_fix = []    

    def __init__(self):
        xml.sax.ContentHandler.__init__(self)

    def startElement(self, name, attrs):
        #print("startElement '" + name + "'")
        self.startE = name

    def endElement(self, name):
        #print("endElement '" + name + "'")
        if name == 'v':
            #print self.key, self.value, magnitude( float(self.value) )
            self.series[self.key] = float(self.value)

            self.key = None
            self.value = None

    def characters(self, content):
        #print("characters '" + content + "'")

        if self.startE == 't':
            self.key = content
        if self.startE == 'v':
            self.value = content

    def dump(self):

        std_dev = numpy.sqrt(numpy.var(self.series.values() ))
        print 'standard deviation=', std_dev

        #min = sys.maxint
        count = 0 
        for k in sorted(self.series.keys()):
            v = self.series[k]
            if v > (std_dev * 2):
                print k, v, "***"
                count = count + 1
            elif self.debug:
                print k, v
        
            if count > 1:
                print 'skipping...' 
                break


        if count == 1:
            return True

        return False    

        '''    
            if v < min:
                min = v
        for k in sorted(self.series.keys()):
            v = self.series[k] - min
            print "adj: ", k, v
        print 'min:', min
        '''

if __name__ == "__main__":

    try:
        g = RRDxportSpike()

        d = sys.argv[1]
        s = 1412985600 #sys.argv[2]
        e = 1421961522 #sys.argv[3]

        pickle_file = open(d + '_associations.pickle', 'rb')
        pickle_data = pickle.load(pickle_file)         
        pickle_len = len(pickle_data)

        print d, pickle_len
    
        graphs_to_fix = []

        #for i in range(0,100):
        for i in range(0, pickle_len):
            print pickle_data[i]
            metric = pickle_data[i][0]
            args = 'xport --start %d --end %d DEF:%d=%s.rrd:%d:AVERAGE XPORT:%d:%s' % (s,e,i,d,i,i,metric)
            print i, args
            arg1 = 'DEF:%d=%s.rrd:%d:AVERAGE' % (i,d,i)
            arg2 = 'XPORT:%d:%s' % (i,'foo')
            #subprocess.call(['/usr/bin/rrdtool','xport', '--start', str(s), '--end', str(e), arg1, arg2 ])
            
            p = subprocess.Popen(['/usr/bin/rrdtool','xport', '--start', str(s), '--end', str(e), arg1, arg2 ], stdout=subprocess.PIPE)
            #output, err = p.communicate()
            #print output
 
            try:           
                xml.sax.parse(p.stdout, g) 
                if g.dump():
                    graphs_to_fix.append(i)           
            except:
                print e

        for i in graphs_to_fix:
            print i, pickle_data[i]
            

    except:
        print traceback.format_exc()
        print sys.exc_info()[0]
        sys.exit(-1)