Subprocess profiling, maxsize

README.md

Alternative to Subprocess profiling.

Using stdout.read() instead of .readline()

root@3ac1680da5be:/# python subprocess_profile_maxsize.py --iterations=10000 --size=100
Read: 1,980.560 MB/s
Write: 1,596.243 MB/s
Iterations: 10,000
Avarage roundtrip time: 0.125 ms/request
Min roundtrip time: 0.049 ms/request
Max roundtrip time: 1.538 ms/request
Delta roundtrip time: 1.489 ms

strict.svg

subprocess_profile_maxsize.py

import os
import sys
import time
import argparse
import subprocess
from collections import defaultdict

KB = 1 << 10
MB = 1 << 20

BUFSIZE = 64 * KB

parser = argparse.ArgumentParser()
parser.add_argument("--iterations", default=10000, type=int)
parser.add_argument("--size", default=64, type=int)
parser.add_argument("--plot", action="store_true")
opt = parser.parse_args()

iterations = opt.iterations
size = opt.size * KB
data = b"0" * size
totalsize = size * iterations
timings = defaultdict(list)


if not os.getenv("CHILD"):
    popen = subprocess.Popen(
        [sys.executable, "-u", __file__] + sys.argv[1:],
        env=dict(os.environ, **{"CHILD": "1"}),
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        bufsize=BUFSIZE,
    )

    writetime = 0

    for i in range(iterations):
        t0 = time.clock()
        popen.stdin.write(data)
        popen.stdin.flush()
        t1 = time.clock()

        duration = t1 - t0
        writetime += duration

        response = popen.stdout.read(size)
        t2 = time.clock()
        timings["write"].append(t2 - t0)
        timings["roundtrip"].append(t2 - t0)

    popen.stdin.close()
    popen.stdout.close()
    popen.wait()

    mbps = float(totalsize) / MB / writetime
    avgtime = (sum(timings["roundtrip"][1:]) / iterations) * 1000
    mintime = min(timings["roundtrip"]) * 1000
    maxtime = max(timings["roundtrip"]) * 1000
    deltime = maxtime - mintime

    print("""\
Write: {mbps:,.3f} MB/s
Iterations: {iterations:,}
Avarage roundtrip time: {avgtime:.3f} ms/request
Min roundtrip time: {mintime:.3f} ms/request
Max roundtrip time: {maxtime:.3f} ms/request
Delta roundtrip time: {deltime:.3f} ms\
""".format(**locals()))

    if opt.plot:
        import pygal
        fname = os.path.join(os.getcwd(), "strict.svg")
        print("Plotting to %s.." % fname)

        plot = pygal.Line(width=2000, height=500)
        plot.title = "Time per iteration"
        for name, values in timings.items():
            plot.add(name, values[1:], show_dots=False)
        plot.render_to_file(fname)


else:
    readtime = 0

    while True:
        response = sys.stdin.read(size)

        try:
            t0 = time.clock()
            sys.stdout.write(data)
            sys.stdout.flush()
            t1 = time.clock()
        except IOError:
            break

        readtime += t1 - t0

    assert readtime > 0, "Too fast"
    rate = float(totalsize) / MB / readtime
    sys.stderr.write("Read: {mbps:,.3f} MB/s\n".format(
        mbps=rate
    ))