fio_bruteforce.py

import itertools
import subprocess as sp
import json
import pprint
from collections import namedtuple


JQ_PATH = "/usr/local/bin/jq"
FIO_PATH = "/usr/local/bin/fio"
JQ_QUERY = namedtuple("JQ_QUERY", ["query", "name", "parse_fn"])

PARAMS = {
    "CONST": {
        "name": "randrw",
        "rw":   "write",  # maybe variable
        "direct": "1",
        # "ioengin": "libaio",
        "ioengin": "sync",
        "numjobs": "1",
        # "size": "3450G",
        "size": "32M",
        "group_reporting": None,
        "runtime": "1",
        "time_based": None,
        "loops": 2,
        # "filename": "/var/lib/elasticsearch/FIO",
        "filename": "TEST_FIO_FILE",
        "output-format": "json",
    },
    "CARTESIAN": {
        "bs": list(map(lambda x: "%sk" % x, [4, 8, 16, 32, 64, 128, 256, 512, 1024])),
        "iodepth": [1, 2, 4, 8, 16, 32, 64, 128],
    },
    "CAPTURE_OUTPUT": [
        JQ_QUERY(".jobs[].write.bw_bytes", "Write bytes/s", int),
        JQ_QUERY(".jobs[].write.iops_mean", "write IOPS mean", int),
    ]
}


def unwind_dict(d):
    variable_options = d["CARTESIAN"]
    r = list(itertools.product(*variable_options.values()))
    k = list(variable_options.keys())
    out = []
    for x in r:
        obj = d["CONST"].copy()
        obj.update(dict(zip(k, x)))
        out.append(obj)

    return out


def convert_dict_to_args(d):
    out = []
    for k, v in d.items():
        if v is None:
            out += [f"--{k}"]
        else:
            out += [f"--{k}={v}"]

    return out


def jq_it(s, query):
    out = sp.check_output([JQ_PATH, "-rc", query], input=s)
    return out.strip()


def main():
    r = unwind_dict(PARAMS)
    OUTPUT = {}

    for obj in r:
        args_str = convert_dict_to_args(obj)
        res = sp.check_output([FIO_PATH] + args_str)
        out_key = tuple([f"{x}={obj[x]}" for x in PARAMS["CARTESIAN"].keys()])
        OUTPUT[out_key] = {}

        for jq_query in PARAMS["CAPTURE_OUTPUT"]:
            value = jq_it(res, jq_query.query)
            OUTPUT[out_key][jq_query.query] = {
                "raw_value": value,
                "parsed_value": jq_query.parse_fn(value),
                "name": jq_query.name,
                "query": jq_query.query
            }

    pprint.pprint(OUTPUT)


if __name__ == "__main__":
    main()