Micro benchmark for PEP 649 and PEP 563

ann_test.py

import random
import timeit
import sys
import marshal
import statistics
import tracemalloc


def generate_source(s):
    if s not in (0,1,2,3):
        raise ValueError("semantics must be 1 (py39), 2 (PEP 563), and 3 (PEP 649)")
    lines = []
    if s == 2:
        lines.append("from __future__ import annotations")
    elif s == 3:
        lines.append("from __future__ import co_annotations")

    ts = ["int", "str", "foo.bar.baz"]

    def quote(t):
        if s != 1:  # quote is needed only for py39 semantics
            return t
        if "." not in t:
            return t
        return "'" + t + "'"

    random.seed("hello, world")
    for i in range(1000):
        t1, t2, t3, t4 = map(quote, random.choices(ts, k=4))
        if s:
            lines.append(f"""\
def func{i}(a: {t1}, b: {t2}, c: {t3}) -> {t4}:
     pass
""")
        else:
            lines.append(f"""\
def func{i}(a, b, c):
     pass
""")
    return "\n".join(lines)


def main():
    s = int(sys.argv[1])
    src = generate_source(s)
    code = compile(src, "test.py", "exec")
    pyc = marshal.dumps(code)
    print(f"code size: {len(pyc)} bytes")

    tracemalloc.start()
    mod = exec(code)
    print("memory:", tracemalloc.get_traced_memory()[1], "bytes")
    del mod
    tracemalloc.stop()

    timing = timeit.repeat(lambda: marshal.loads(pyc), repeat=10, number=1000)
    #print(timing)
    print(f"unmarshal: avg: {statistics.fmean(timing)*1000:.3f}ms +/-{statistics.stdev(timing)*1000:.3f}ms")

    timing = timeit.repeat(lambda: exec(code), repeat=10, number=1000)
    #print(timing)
    print(f"exec: avg: {statistics.fmean(timing)*1000:.3f}ms +/-{statistics.stdev(timing)*1000:.3f}ms")

main()

ann_test_method.py

import random
import timeit
import sys
import marshal
import statistics
import tracemalloc
import dis


def generate_source(s):
    if s not in (0,1,2,3):
        raise ValueError("semantics must be 1 (py39), 2 (PEP 563), and 3 (PEP 649)")
    lines = []
    if s == 2:
        lines.append("from __future__ import annotations")
    elif s == 3:
        lines.append("from __future__ import co_annotations")

    ts = ["int", "str", "foo.bar.baz"]

    def quote(t):
        if s != 1:  # quote is needed only for py39 semantics
            return t
        if "." not in t:
            return t
        return "'" + t + "'"

    random.seed("hello, world")
    lines.append("class Klass:")
    for i in range(1000):
        t1, t2, t3, t4 = map(quote, random.choices(ts, k=4))
        if s:
            lines.append(f"""\
    def func{i}(a: {t1}, b: {t2}, c: {t3}) -> {t4}:
         pass
""")
        else:
            lines.append(f"""\
    def func{i}(a, b, c):
        pass
""")
    return "\n".join(lines)


def main():
    s = int(sys.argv[1])
    src = generate_source(s)
    code = compile(src, "test.py", "exec")
    pyc = marshal.dumps(code)
    print(f"code size: {len(pyc)} bytes")

    #with open("test.code.bin", "wb") as f:
    #    f.write(pyc)
    #with open("test.dis", "w", encoding="utf-8") as f:
    #    dis.dis(code, file=f)

    tracemalloc.start()
    mod = exec(code)
    print("memory:", tracemalloc.get_traced_memory()[1], "bytes")
    del mod
    tracemalloc.stop()

    timing = timeit.repeat(lambda: marshal.loads(pyc), repeat=10, number=1000)
    #print(timing)
    print(f"unmarshal: avg: {statistics.fmean(timing)*1000:.3f}ms +/-{statistics.stdev(timing)*1000:.3f}ms")

    timing = timeit.repeat(lambda: exec(code), repeat=10, number=1000)
    #print(timing)
    print(f"exec: avg: {statistics.fmean(timing)*1000:.3f}ms +/-{statistics.stdev(timing)*1000:.3f}ms")

main()