Benchmark Script for Combinator Saves

benchmark_combinators.py

# Requires Python 3.6+

import subprocess
import re
from io import StringIO
import pandas as pd

__all__ = ['run_benchmark']


def run_benchmark(save_name, num_ticks=600, runs=10, verbose="all",
                  disable_audio=True, setup_ticks=10):
    args = ['"Factorio/bin/x64/factorio.exe"', ]
    args.append(f'--benchmark {save_name}')
    args.append(f'--benchmark-ticks {num_ticks + setup_ticks}')
    args.append(f'--benchmark-runs {runs}')
    args.append(f'--benchmark-verbose {verbose}')

    if disable_audio:
        args.append('--disable-audio')

    a = subprocess.check_output(" ".join(args), shell=True).decode('utf-8')
    a = "\n".join(a.splitlines()[:-1])
    s = re.split(r'run \d+:\n', a)[1:]
    d = [pd.read_csv(StringIO(x.strip())) for x in s]
    df = pd.concat(d).groupby(level=0).min()
    df = df.loc[:, ~df.columns.str.contains('^Unnamed')]
    df = df.loc[:, ~df.columns.str.contains('timestamp')]
    df = df.loc[:, ~df.columns.str.contains('tick')]
    return df[setup_ticks:].reset_index(drop=True) / 1000


if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser(description='Benchmark Factorio saves.')
    parser.add_argument('-t', '--ticks', type=int, default=1200,
                        help='No. of benchmark ticks (Default: 600)')
    parser.add_argument('-r', '--runs', type=int, default=20,
                        help='No. of benchmark runs (Default: 10)')
    parser.add_argument('-s', '--setup_ticks', type=int, default=10,
                        help='Setup ticks (Default: 10)')
    args = parser.parse_args()

    print("Factorio Benchmark:")
    print(f"  Runs per save: {args.runs}")
    print(f"  Ticks per run: {args.ticks} (+ {args.setup_ticks} setup ticks)")
    print("  (Times shown in microseconds)\n")

    saves = {'+ 0': "[CombTest]-Plus",
             '- 0': "[CombTest]-Minus",
             '* 1': "[CombTest]-Mul",
             '/ 1': "[CombTest]-Div",
             '^ 1': "[CombTest]-Power",
             '<< 0': "[CombTest]-LShift",
             '>> 0': "[CombTest]-RShift",
             'OR 0': "[CombTest]-OR"}

    results = dict()
    for s in saves:
        df = run_benchmark(saves[s], num_ticks=args.ticks, runs=args.runs,
                           setup_ticks=args.setup_ticks, verbose='all')
        results[s] = df.mean()

    print(pd.DataFrame(results).T[["circuitNetworkUpdate"]])