plotting.py

import pandas as pd
import seaborn as sns

data = pd.read_json("results.jsonl", lines=True)   

sns.heatmap(data.pivot("astroid", "pylint", "testpd.py"))

pylint_astroid_timings.py

import just
import sys
import time
import subprocess

pylints = ["1.7.0", "1.8.0", "1.9.0", "1.9.1", "1.9.2", "1.9.3", "2.0.0.dev0", "2.0.0", "2.1.0"]
astroids = ["1.5.1", "1.6.1", "1.6.2", "1.6.5", "2.0.0.dev0", "2.0", "2.0.1", "2.0.4"]


def install(package, version):
    subprocess.call([sys.executable, "-m", "pip", "install", "{}=={}".format(package, version)])


for pylint in pylints:
    install("pylint", pylint)
    for astroid in astroids:
        install("astroid", astroid)
        dc = {"pylint": pylint, "astroid": astroid}
        # expected_code is the return_code we expect from pylint
        for fname, expected_code in zip(["test.py", "testpd.py"], [0, 20]):
            t1 = time.time()
            res = subprocess.run(["pylint", fname])
            t2 = time.time()
            t = t2 - t1
            # if not expected error code, then there was an error
            # unfortunately it is otherwise not possible to tell whether there was an error
            if res.returncode != expected_code:
                t = None
            dc[fname] = t

        just.append(dc, "results.jsonl")