Alternative benchmark suit for Mir

bench.ts

// TODO: Support every "c2m" mode
// TODO: Add support for startupt times
import { exit } from "node:process"
import { unlinkSync, existsSync } from "node:fs"

class BenchConfig { /* The benchmark options */
  comptime:    boolean         /* Show the compile times */
  runtime:     boolean         /* Show the run times */
  file_size:   boolean         /* Show the file sizes */
  c_compilers: Array<string[]> /* Compiles to use ("c2m" is ALWAYS run!) */

  constructor(comptime, runtime, file_size, c_compilers) {
    this.comptime = comptime
    this.runtime = runtime
    this.file_size = file_size
    this.c_compilers = c_compilers
  }
}

let config = new BenchConfig(true, true, true, [ [ "all" ] ]) // Hold the options
let i = 2 // Used inside the loop to parse the arguments
let stats: number[][][] = [] // Keeps the stats of each compiler so they can be compared

/* Compilers (including their options) to search for in case there weren't specified */
let default_compilers = [ [ "clang", "-O3", "-march=native" ], ["gcc", "-O3", "-march=native" ],
  ["tcc"], ["pcc"], ["cproc", "-O3"], ["cparser", "-O3"], ["lacc", "-O3"], ["chibicc"]
]

let files_to_be_removed: string[] = [] // Output files that will be removed in the end

// let benchmarks = [ "array", "binary-trees", "except", "funnkuch-reduce",
  // "hash", "hash2", "heapsort", "lists", "matrix", "method-call",
  // "mandelbrot", "nbody", "sieve", "spectral-norm", "strcat"
// ]

// let benchmarks = [ "array", "binary-trees", "except" ]
//
let benchmarks = [ "array", "binary-trees" ]

function start() {
  if (Bun.argv >= 2) { parse_options() }
  run_benchmarks()
}

function parse_options() {
  for (; i < Bun.argv.length; i++) { // Skip the program name an the file name
    switch (Bun.argv[i]) {
      case "--help":
      case "-h":
        show_help()
      case "-no_comptime":
        config.comptime = false
        break;
      case "-no_runtime":
        config.runtime = false
        break;
      case "-no_size":
        config.file_size = false
        break;
      case "-cc":
        handle_cc()
        break;
      default:
        wrong_opt()
    }
  }
}

function show_help() {
  console.log("Available options:"
    + "  -no_comptime: Do not show the compile time benchmarks\n"
    + "  -no_runtime: Do not show the run time benchmarks\n"
    + "  -no_size: Do not show the output file sizes\n"
    + "  -cc: A C compiler to use (including its args)\n"
    + "\nIf there is not a single `-cc` option passed, then there is a\n"
    + " default set of compilers that is searched and used if found!"
  )

  exit(0)
}

function wrong_opt() {
  console.error("Error: The following option is invalid:", Bun.argv[i])
  exit(1)
}

function handle_cc() {
  i += 1 // Go to the next argument

  if (i < Bun.argv.length) {
    /* If even one "-cc" option is given, we do not check the default compilers */
    if (config.c_compilers[0][0] == "all") { config.c_compilers = [] }

    config.c_compilers[0].push([Bun.argv[i]])
  }

  else {
    console.error("Option `-cc` expects a value but was not given one")
    exit(2)
  }
}

async function run_benchmarks() {
  check_compilers()
  fill_stats()

  await compile()
  // remove_files()
}

function check_compilers() {
  let check_fn = (warn: boolean) => {
    for (let c = 0; c < config.c_compilers.length; c++) {
      const cc_exists = compile_test(config.c_compilers[c], 0, false);

      if (!cc_exists) { /* The compiler does not exist! */
        if (warn) console.log(`warning: could not execute: ${c_compilers[c]}`)

        config.c_compilers.splice(c, 1)
        c -= 1 // Don't forget to reduce the index so we don't skip the next compiler
      }
    }

    if (config.c_compilers.length == 0) {
      console.error("No available C compilers were found to compare against `c2m`!")
      exit(3)
    }


    console.log("Compiler test over!\n")
  }

  if (config.c_compilers[0][0] == "all") { // Default compilers are used
    config.c_compilers = default_compilers
    check_fn(false)
  }

  else { // User specified compilers are used
    /* Transform the given compiles to the proper format for "Bun.spawn" */
    for (let c = 0; c < config.c_compilers.length; c++) {
      config.c_compilers[c] = config.c_compilers[c][0].split(' ')
    }

    check_fn(true)
  }
}


function fill_stats() { /* Fill the stats array to default values */
  if (!(config.comptime | config.runtime | config.file_size)) {
    console.error("At least one check needs to be enabled!")
    exit(4)
  }

  /* Number of tests */
  let test_array: number[] = []
  if (config.comptime)  { test_array.push(0) }
  if (config.runtime)   { test_array.push(0) }
  if (config.file_size) { test_array.push(0) }

  /* Number of compilers for every benchmark */
  let compilers_array: number[][] = []
  for (let c = 0; c < config.c_compilers.length; c++) {
    compilers_array.push(test_array)
  }

  /* Number of benchmarks */
  for (let b = 0; b < benchmarks.length; b++) {
    stats.push(compilers_array)
  }
}

async function compile() {
  console.log("Actual compilation:")
  for (let b = 0; b < benchmarks.length; b++) {
    const benchmark = benchmarks[b]
    const arg = Bun.file(`c-benchmarks/${benchmark}.arg`) // ".arg" file
    let text = await arg.text()
    text = text.slice(0, text.length -1)

    if (config.comptime) timed_compilation(b)
    else                 untimed_compilation(b)

    if (config.runtime) {
      for (let c = 0; c < config.c_compilers.length; c++) {
        const compiler = config.c_compilers[c]
        const start = Bun.nanoseconds()
        run_test(compiler[0], arg, b)
        const end = Bun.nanoseconds()
        stats[b][c][config.comptime] = end - start
      }
    }

    if (config.file_size) {
      try {
        const proc = Bun.spawnSync(["du", "-b", `c-benchmarks/${benchmark}.${compiler[0]}`])
        const size = Number(proc.stdout.toString())
        stats[b][c][config.comptime + config.runtime] = size
      }

      catch {}
    }
  }

  // await print_stats()
}

function timed_compilation(b: number) {
  for (let c = 0; c < config.c_compilers.length; c++) {
    const start = Bun.nanoseconds()
    compile_test(config.c_compilers[c], b, true)
    const end = Bun.nanoseconds()
    stats[b][c][0] = end - start
  }
}

function untimed_compilation(b: number) {
  for (let c = 0; c < config.c_compilers; c++) {
    compile_test(config.c_compilers[c], b, true)
  }
}

async function print_stats() {
  // console.log("Number of compilers: ", config.c_compilers.length)
  // console.log(`comptime: ${config.comptime}, runtime: ${config.runtime}, file_size: ${config.file_size}`)

  for (let b = 0; b < benchmarks.length; b++) {
    const bench = benchmarks[b]
    const arg = Bun.file(`c-benchmarks/${bench}.arg`) // ".arg" file
    let text = await arg.text()

    if (b != 0) { console.log("") }
    console.log(`Benchmark #${b + 1}: ${bench} (${text.slice(0, text.length - 1)})`)

    let best_comptime = find_best(b, 0)
    let best_runtime = find_best(b, config.comptime)
    let best_size = find_best(b, config.comptime + config.runtime)
    console.log("best_comptime:", best_comptime)
    console.log("best_runtime:", best_runtime)
    console.log("best_size:", best_size)

    for (let c = 0; c < config.c_compilers.length; c++) {
      // console.log("WTF?????")
      const cc_joined = config.c_compilers[c].join(" ")
      console.log("\n", cc_joined)

      if (config.comptime) { printc("  compile time", b, 0, 0, best_comptime) }
      if (config.runtime)  { printc("  runtime", b, c, config.comptime, best_runtime) }

      if (config.file_size) {
        printc("  file size", b, c, config.comptime + config.runtime, best_size)
      }
    }
  }
}

function printc(msg: message, b: number, c: number, index: number, best: number) {
  let diff = stats[b][c][index] / best
  console.log(`${msg}:\t\t${stats[b][c][index]} nanoseconds (${diff.toFixed(2)})`)
}

function find_best(b: number, index: number) {
  let best = Number.MAX_SAFE_INTEGER
  const stat = stats[b]

  for (let c = 0; c < config.c_compilers.length; c++) {
    if (stat[c][index] < best) { best = stat[c][index] }
  }

  return best
}

function remove_files() {
  for (let f = 0; f < files_to_be_removed.length; f++) {
    unlinkSync(files_to_be_removed[f])
  }
}

function compile_test(compiler: string[], b: number, add_file: boolean = true) {
  try {
    const output = `./c-benchmarks/${benchmarks[b]}.${compiler[0]}`
    // console.log(`Source: c-benchmarks/${benchmarks[c]}.c`)
    console.log(`Output: ${output}`)
    const proc = Bun.spawnSync([...compiler, `./c-benchmarks/${benchmarks[b]}.c`,
      "-o", output
    ])

    if (!existsSync(output)) {
      console.error("command run sucessfully but file was not created!!!")
      exit(10)
    }

    if (add_file) files_to_be_removed.push(output)
    return proc
  }

  catch (error) {
    // console.error("Compilation failed!", error)
  }
}

function run_test(name: string, arg: string, c: number) {
  try {
    if (name == "c2m") {
      // console.log(`Trying to run: ./c-benchmarks/${benchmarks[c]}.c2m`)
      Bun.spawnSync(["./c2m", `./c-benchmarks/${benchmarks[c]}.c2m`,
        "-eg", arg
      ])
    }

    else  {
      console.log(`Trying to run: ./c-benchmarks/${benchmarks[c]}.${name}`)
      Bun.spawnSync([`./c-benchmarks/${benchmarks[c]}.${name}`, arg])
    }
  }

  catch (error) {
    console.error(error)
  }
}

start()