performance tests

builds.csv

perf_tests.py

import os
import re
import subprocess
import sys
from pathlib import Path

import numpy as np

test_pattern = re.compile(r"(\.{1,})   Passed")


def process_times(save_path: Path) -> float:
    with open(save_path, "r") as f:
        lines = f.readlines()
        lines = [x.strip() for x in lines]

    d = {}

    for line in lines:
        if "Test #" in line:
            line = re.sub(pattern=test_pattern, repl="", string=line)
            tokens = line.split(" ")
            tokens = [x for x in tokens if x != ""]
            test_name = tokens[3].split(".")[1]
            test_time = float(tokens[4])
            if test_name in d:
                d[test_name].append(test_time)
            else:
                d[test_name] = [test_time]
        elif "Total Test time (real) =" in line:
            line = line.replace("Total Test time (real) =", "")
            tokens = line.split(" ")
            tokens = [x.strip() for x in tokens if x != ""]
            time = float(tokens[0])
            if "total_time" in d:
                d["total_time"].append(time)
            else:
                d["total_time"] = [time]

    keys = []
    vals = []
    for k, v in d.items():
        d[k] = np.mean(v)
        keys.append(k)
        vals.append(d[k])

    # sort lists together
    keys, vals = [list(t) for t in zip(*sorted(zip(keys, vals)))]

    out_file = save_path.parent / f"{save_path.stem}.csv"

    with open(out_file, "w") as f:
        for idx, k in enumerate(keys):
            f.write(f"{k},{vals[idx]}\n")

    return d["total_time"]


def build_and_run(build_path: Path, f_name: str):
    num_threads = 10
    num_tests = 2

    make_cmds = ["make", "-j", f"{num_threads}", "energyplus"]
    subprocess.check_call(make_cmds, cwd=build_path)

    log_file = Path(build_path / f"{f_name}")

    if log_file.exists():
        os.remove(log_file)

    perf_cmds = ["ctest", "-j", f"{num_threads}", "-R", "performance*"]
    for i in range(num_tests):
        print(f"Run: {i + 1}")
        with open(log_file, "a+") as f:
            subprocess.check_call(perf_cmds, cwd=build_path, stdout=f, stderr=f)


if __name__ == "__main__":

    build_dir = Path("/Users/mmitchel/Projects/EnergyPlus/dev/EnergyPlus/build")
    save_name = "perf_tests.txt"

    try:
        print("*** Attempting to build and run tests ***")
        build_and_run(build_dir, save_name)
    except subprocess.CalledProcessError:
        print("*** An error has occurred trying to build and run tests -- returning 125 to SKIP this commit ***")
        sys.exit(125)

    try:
        tot_time = process_times(build_dir / save_name)
        time_limit = 200
        if tot_time > time_limit:
            print(f"*** Performance time ({tot_time:0.2f} s) exceeds limit ({time_limit} s) ***")
            print("*** returning 1 to mark commit as BAD ***")
            sys.exit(1)
        else:
            print(f"*** Performance time ({tot_time:0.2f} s) is below limit ({time_limit} s) ***")
            print("*** returning 0 to mark commit as GOOD ***")
            sys.exit(0)
    except subprocess.CalledProcessError:
        print("*** An error has occurred trying to compute run times -- returning 125 to SKIP this commit ***")
        sys.exit(125)

perf_tests2.py

import json
import os
import re
import subprocess
import shutil
from pathlib import Path
from typing import Union

test_pattern = re.compile(r"(\.{1,})   Passed")


def format_json(input_dict: dict, indent: int = 2) -> str:
    return json.dumps(input_dict, sort_keys=True, indent=indent, separators=(',', ': '))


def write_json(write_path: Union[str, Path], input_dict: dict, indent: int = 2) -> None:
    with open(write_path, 'w') as f:
        f.write(format_json(input_dict, indent))


def process_times(save_path: Path) -> dict:
    with open(save_path, "r") as f_log:
        lines = f_log.readlines()
        lines = [x.strip() for x in lines]

    d = {}

    for line in lines:
        if "Test #" in line:
            line = re.sub(pattern=test_pattern, repl="", string=line)
            tokens = line.split(" ")
            tokens = [x for x in tokens if x != ""]
            test_name = tokens[3].split(".")[1]
            test_time = float(tokens[4])
            if test_name in d:
                d[test_name].append(test_time)
            else:
                d[test_name] = [test_time]
        elif "Total Test time (real) =" in line:
            line = line.replace("Total Test time (real) =", "")
            tokens = line.split(" ")
            tokens = [x.strip() for x in tokens if x != ""]
            time = float(tokens[0])
            if "total_time" in d:
                d["total_time"].append(time)
            else:
                d["total_time"] = [time]

    # average results
    for k, v in d.items():
        d[k] = sum(v) / len(v)

    return d


def build_and_run(build_path: Path, tag: str, message: str, threads: int, iterations: int):
    # checkout
    try:
        subprocess.check_call(["git", "checkout", "develop"], cwd=build_path.parent)
        subprocess.check_call(["git", "checkout", f"{tag}"], cwd=build_path.parent)
    except subprocess.CalledProcessError:
        return {"status": "failed to checkout", "tag": tag, "commit_message": message}

    # delete, remake build folder
    if build_path.exists():
        shutil.rmtree(build_path)
    os.mkdir(build_path)

    # run cmake, build
    try:
        subprocess.check_call(
            ["cmake", "..", "-DBUILD_TESTING=ON", "-DBUILD_PERFORMANCE_TESTS=ON", "-DBUILD_FORTRAN=ON"], cwd=build_path)
        subprocess.check_call(["cmake", "--build", ".", "--", "-j", f"{threads}"], cwd=build_path)
    except subprocess.CalledProcessError:
        return {"status": "failed to build", "tag": tag, "commit_message": message}

    # run tests
    try:
        f_name = f"{tag}.txt"
        log_file = build_path.parent / f_name

        if log_file.exists():
            os.remove(log_file)

        for i in range(iterations):
            print(f"Run: {i + 1}")
            with open(log_file, "a+") as f_log:
                subprocess.check_call(["ctest", "-R", "performance"], cwd=build_path, stdout=f_log, stderr=f_log)

        d_message = {"status": "success", "tag": tag, "commit_message": message}
        return {**d_message, **process_times(log_file)}

    except subprocess.CalledProcessError:
        return {"status": "failed to run tests", "tag": tag, "commit_message": message}


if __name__ == "__main__":

    num_threads = 10
    num_iter = 3

    build_dir = Path("/Users/mmitchel/Projects/EnergyPlus/dev/EnergyPlus2/build")
    builds_file = Path("/Users/mmitchel/Projects/EnergyPlus/dev/IPYNotebooks/2021_12-performance/builds.csv")

    with open(builds_file, "r") as f:
        lines = f.readlines()

    lines = [x.strip() for x in lines]

    d = {}

    for idx, line in enumerate(lines[1:]):
        tokens = line.split(",")
        this_tag = tokens[0]
        this_commit_message = tokens[1]
        d[idx] = build_and_run(build_dir, this_tag, this_commit_message, num_threads, num_iter)

    summary_file = build_dir.parent / "perf_summary.json"

    write_json(summary_file, d)