perfRegression.py

#!/usr/bin/python

# This is a variant of regression.py that repeats regression.py tests across revisions.
# Usage: ./develop/performance/perfRegression.py -a
# Tested with Python 2.6.1

import sys
import os
import subprocess
import optparse
import re
import copy
import json

def line():
    print "-------------------------------"

def targetRelative( filename ):
    return os.path.normpath(os.path.join(targetrepo, filename))

# In testing, it appears the current process environment can be altered by subprocess.Popen.
# So make a copy before any potential modification occurs.
globalEnv = copy.deepcopy( os.environ )
globalEnv["HGPLAIN"]="1" # Disable pager, etc in any following calls

def fullExec(cmd, cwd=None, env=None):
    proc = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,env=env if env else globalEnv)
    result = proc.wait()
    outstr, errstr = proc.communicate()
    return bool(result),outstr,errstr

targetrepo = "scratch/performance/repo"
outstats = "scratch/performance/result.json"
instats  = [outstats]
baserev = "12c627cd1ede"

srcrepo = os.path.join( os.path.join( os.path.dirname(__file__), ".."), ".." )
stddir  = "sample/test"
stdfile = "sample/test/regression.txt"

db = []

help  = "%prog -a\n"
help += "\n"
help += "Accepted arguments:\n"
help += "-a            # Test all revisions in history\n"
help += "-b [rev]      # Don't test revisions earlier than this (default "+baserev+")\n"
help += "-r [rev]      # Test listed revision[s] only\n"
help += "-f [rev]      # Test listed file[s] only (default is all in regression.txt)\n"
help += "--root [path] # Set the project root\n"
help += "-i [path]     # Set in stat file  (defaults to "+instats[0]+")\n"
help += "-o [path]     # Set out stat file (defaults to "+outstats  +")\n"
help += "-t [path]     # Set tmp repo      (defaults to "+targetrepo+")\n"
help += "-v            # Print all output"

parser = optparse.OptionParser(usage=help)
for a in ["a", "v"]: # Single letter args, flags
    parser.add_option("-"+a, action="store_true")
for a in ["r", "i", "o", "t", "b", "f", "-root"]: # Long args with arguments
    parser.add_option("-"+a, action="append")

(options, cmds) = parser.parse_args()
def flag(a):
    x = getattr(options, a)
    if x:
        return x
    return []

if cmds:
    parser.error("Stray commands: %s" % cmds)

if bool( flag("a") ) == bool( flag("r") ):
    parser.error("Must specify exactly one of '-a' or '-f'")

# Extract
if flag("b"):
    baserev = flag("b")
overriderev = flag("r")
overridefiles = flag("f")
if flag("root"):
    srcrepo = flag("root")[0]
if flag("i"):
    instats = flag("i")
if flag("o"):
    outstats = flag("o")[0]
if flag("t"):
    targetrepo = flag("t")[0]

# Normalize -- this should be the only stretch where starting cwd matters
srcrepo = os.path.abspath(srcrepo)
outstats = os.path.abspath(outstats)
targetrepo = os.path.abspath(targetrepo)

line()
for inpath in instats:
    try:
        with open(inpath) as infile:
            print "Loading stats from "+inpath
            db += json.load(infile)
    except IOError:
        print "Couldn't find/open "+inpath+", skipping load"

overriderevargs = []    # "override rev args" ... maybe i should use camelcase...
for rev in overriderev:
    overriderevargs += ["-r", rev]

# Check out repo
line()
print "Loading repo to "+targetrepo
if subprocess.call(["mkdir", "-p", targetrepo]):
    print "Failed on mkdir-- something is very wrong"
    sys.exit(1)

if subprocess.call(["hg","init"], cwd=targetrepo):
    print "Repo seems to already exist"

print ["hg","pull"] + (overriderevargs if overriderevargs else ["-r", "tip"]) + [srcrepo]
if subprocess.call(["hg","pull"] + (overriderevargs if overriderevargs else ["-r", "tip"]) + [srcrepo], cwd=targetrepo):
    print "Could not pull requested revisions"
    sys.exit(1)

if overriderevargs:
    testrevs = overriderevargs
else:
    result,outstr,errstr = fullExec(["hg","log","-r",".","--template","{node}"], cwd=srcrepo)
    if result:
        print "Need hg id for "+srcrepo+", but couldn't get it for some reason."
        sys.exit(1)
    print outstr
    testrevs = ["-r", baserev+":"+outstr.strip()]

# THESE NEXT FOUR LINES ARE JUST FOR DEBUGGING
# When this next line executes, it prints:
# ['hg', 'log', '--template', '{rev};{node};{date};{parents}\\n', '-r', '12c627cd1ede:2439431f842ec7fb7a85817a27377cbe2c9fd479']
print ["hg","log","--template",
    "{rev};{node};{date};{parents}\\n"] + testrevs
print "TEST"
# When this next line executes, HGPLAIN=1 is clearly seen
result,outstr,errstr=fullExec(["env"])
print outstr

result,outstr,errstr = fullExec(["hg","log","--template",
    "{rev};{node};{date};{parents}\\n"] + testrevs, cwd=targetrepo)
if result:
    print "Need mercurial log for "+targetrepo+", but couldn't get it for some reason."
    sys.exit(1)
print "Returned successfully from hg log" # FOR DEBUGGING, REMOVE LATER
searchspace = [ line.split(";") for line in outstr.split("\n") if line ]
searchspace.sort(key=lambda x:x[0])
print "OK:"
print searchspace

line()
print "OK"

# NOTHING AFTER THIS LINE IS KNOWN TO WORK
sys.exit(1)

indexcommentp = re.compile(r'#.+$', re.S) # Allow comments in .txt file
for filename in indices:
    dirname = os.path.dirname(filename)
    with open(filename) as f:
        for line in f.readlines():
            line = indexcommentp.sub("", line)
            line = line.rstrip()
            if line:
                files += [projectRelative(os.path.join(dirname, line))]

stdcall = [projectRelative("install/emily")]
if flag("s"):
    stdcall = ["emily"]

expectp = re.compile(r'# Expect(\s*failure)?(\:?)', re.I)
linep = re.compile(r'# ?(.+)$', re.S)
inline_expectp = re.compile(r'# Expect:\s*(.+)$', re.S|re.I)
startp = re.compile(r'^', re.MULTILINE)
argp = re.compile(r'# Arg:\s*(.+)$', re.I)
envp = re.compile(r'# Env:\s*(.+)$', re.I)
kvp = re.compile(r'(\w+)=(.+)$')
omitp = re.compile(r'# Omit\s*file', re.I)

def pretag(tag, str):
    tag = "\t%s: " % (tag)
    return startp.sub(tag, str)

failures = 0

for filename in files:
    expectfail = False
    scanning = False
    outlines = ''
    env = None
    args = []
    omit = False
    earlyfail = False

    # Pre-scan the file for magic comments with test instructions
    with open(filename) as f:
        for line in f.readlines():
            # First determine if this is an expect directive
            expect = expectp.match(line) # Expect:
            inline = inline_expectp.match(line)
            # If the inline pattern matches and the inline body isn't empty,
            # then we're looking at an inline expect directive
            if inline and not inline.group(1).isspace():
                outlines += inline.group(1)
            # Otherwise, if it's an expect we're beginning a multiline expect
            elif expect:
                expectfail = bool(expect.group(1))
                scanning = bool(expect.group(2))
            else:
                if scanning: # If currently inside an expect block
                    outline = linep.match(line)
                    if outline:
                        outlines += outline.group(1)
                    else:
                        scanning = False

                # Only an expect directive can end an expect block
                if not scanning: # Other directives:
                    argline = argp.match(line) # Arg:
                    if argline:
                        args += [argline.group(1)]

                    envline = envp.match(line) # Env:
                    if envline:
                        if not env:
                            env = copy.deepcopy( globalEnv )
                        kvline = kvp.match( envline.group(1) )
                        if not kvline:
                            print "\tMALFORMED TEST: \"Env:\" line not of form KEY=VALUE"
                            earlyfail = True
                            break
                        env[kvline.group(1)] = kvline.group(2)

                    if omitp.match(line):
                        omit = True

    if earlyfail:
        failures += 1
        continue

    print "Running %s..." % (filename)
    try:
        result,outstr,errstr = fullExec(stdcall+args+([] if omit else [filename]),env=env)
    except OSError as e:
        print "\nCATASTROPHIC FAILURE: Couldn't find emily?:"
        print e
        print "Make sure you ran a plain `make` first."
        sys.exit(1)

    result = bool(result)
    expectfail = bool(expectfail)
    outlines = outlines.rstrip()
    outstr = outstr.rstrip()
    errstr = errstr.rstrip()

    if result ^ expectfail:
        print "\tFAIL:   Process failure " + ("expected" if expectfail else "not expected") + " but " + ("seen" if result else "not seen")
        if errstr:
            print "\n"+pretag("STDERR",errstr)
        failures += 1
    elif outstr != outlines:
        print "\tFAIL:   Output differs"
        print "\n%s\n\n%s" % ( pretag("EXPECT", outlines), pretag("STDOUT", outstr) )
        failures += 1
    elif flag("v"):
        if outstr:
            print pretag("STDOUT", outstr)
        if outstr and errstr:
            print
        if errstr:
            print pretag("STDERR",errstr)

print "\n%d tests failed of %d" % (failures, len(files))

sys.exit(0 if failures == 0 else 1)