Test compiler speeds!

CompileSpeed.md

This script generates long repetitive programs in various languages (statically typed ones that target native code) and measures the time taken to compile them.

Every program is roughly of the form:

main() {
    var x = 0
    {% n*2 times: %}
    x = 2
    print(i)
    x = 3
    print(i)
    {% end %}
}

Compilers tested:

• C (Apple clang 12.0.0)
• D (dmd 2.095.0)
• LLVM IR (llc 12.0.0, release build with assertions)
• OCaml (ocamlopt 4.09.0)
• Nim (nim 1.4.2)
• Rust (rustc 1.48.0)
• V (v 0.2 f713597)
• Zig (zig 0.7.1)

Included are timings on my machine for selected values of n. My tentative ranking of compilation speed on these benchmarks is something like C > V ≈ LLVM IR > Nim > OCaml > D > Rust > Zig. Note that this is not representative of what most code looks like in practice, however.

compilespeed.py

#!/usr/bin/env python3

import sys

langs = {"c", "d", "ll", "ml", "nim", "rs", "v", "zig"}

snippets = {
    "head": {
        "c":
            '#include <stdio.h>\n'
            'int main() {'
            'int x = 0;',
        "d":
            'import std.stdio;'
            'void main() {'
            'int x = 0;',
        "ll":
            '@fmt = private unnamed_addr constant [4 x i8] c"%d\\0a\\00"\n'
            'declare i32 @printf(i8* nocapture, ...) nounwind\n'
            'define i32 @main() {\n'
            '%fmt = getelementptr [4 x i8], [4 x i8]* @fmt, i32 0, i32 0\n'
            '%x = alloca i32\n',
        "ml":
            'let x = ref 0\n'
            'let () = ',
        "nim":
            'var x = 0\n',
        "rs":
            'fn main() {'
            'let mut x: u8;',
        "v":
            'fn main() {\n'
            'mut x := 0\n',
        "zig":
            'const print = @import("std").debug.print;'
            'pub fn main() void {'
            'var x: u8 = 0;',
    },

    "content": {
        "c":   'x = $VAL; printf("%d\\n", x);',
        "d":   'x = $VAL; writefln("%d", x);',
        "ll":  'store i32 $VAL, i32* %x\n'
               '%tmp$IDX = load i32, i32* %x\n'
               'call i32 (i8*, ...) @printf(i8* %fmt, i32 %tmp$IDX)\n',
        "ml":  'x := $VAL; print_endline (string_of_int !x);',
        "nim": 'x = $VAL; echo $x;',
        "rs":  'x = $VAL; println!("{}", x);',
        "v":   "x = $VAL\nprintln('$x')\n",
        "zig": 'x = $VAL; print("{}\\n", .{x});',
    },

    "end": {
        "c":   '}',
        "d":   '}',
        "ll":  'ret i32 0\n}',
        "ml":  '',
        "nim": '',
        "rs":  '}',
        "v":   '}',
        "zig": '}',
    },

    "compile": {
        "c":   'cc main.c',
        "d":   'dmd main.d',
        "ll":  'llc --filetype=obj -O=0 main.ll && cc main.o',
        "ml":  'ocamlopt main.ml',
        "nim": 'nim compile main.nim',
        "rs":  'rustc main.rs',
        "v":   'v main.v',
        "zig": 'zig build-exe main.zig',
    },

    "clean": {
        "c":   'rm main.c a.out',
        "d":   'rm main.d main.o main',
        "ll":  'rm main.ll main.o a.out',
        "ml":  'rm main.ml main.cmi main.cmx main.o a.out',
        "nim": 'rm main.nim main',
        "rs":  'rm main.rs main',
        "v":   'rm main.v main',
        "zig": 'rm -r zig-cache main.zig main',
    },
}

if len(sys.argv) != 3:
    print("usage: {} <language> <num. of statements>".format(sys.argv[0]))
    sys.exit(1)

lang = sys.argv[1]
iters = int(sys.argv[2])

if lang not in langs:
    print("no such language: {}".format(lang))
    sys.exit(1)

print("generating...", end=" ", flush=True)

def snippet(name):
    return snippets[name].get(lang, "")

filename = "main." + lang

with open(filename, "w") as f:
    f.write(snippet("head"))

    for i in range(iters):
        s = snippet("content") \
            .replace("$VAL", str(i % 2 + 2)) \
            .replace("$IDX", str(i + 1))
        f.write(s)

    f.write(snippet("end"))

print("wrote to {}".format(filename))

import os, time

start = time.time()
print("compiling...")
os.system(snippet("compile"))
end = time.time()
print("total time: {}s".format(end - start))

print("cleaning up...")
os.system(snippet("clean"))

my_machine.txt

lang  lines     time

c     10        0.11
c     100       0.11
c     1000      0.16
c     10000     0.49
c     100000    4.29
c     1000000   58.7

d     10        0.60
d     100       0.55
d     1000      0.68
d     2000      0.86
d     5000      2.22
d     10000     13.3
d     20000     92.5

ll    10        0.10
ll    100       0.09
ll    1000      0.11
ll    10000     0.43
ll    100000    13.5
ll    200000    72.8

ml    10        0.21
ml    100       0.29
ml    1000      0.46
ml    2000      0.79
ml    5000      1.70
ml    >10000    <err>

nim   10        0.39
nim   100       0.46
nim   1000      0.58
nim   2000      0.83
nim   5000      1.25
nim   10000     2.19
nim   20000     4.31
nim   50000     13.6
nim   100000    31.3

rs    10        0.43
rs    100       0.48
rs    200       0.71
rs    500       1.21
rs    1000      2.80
rs    2000      7.78
rs    5000      50.8

v     10        0.65
v     100       0.54
v     1000      0.67
v     10000     1.55
v     20000     2.47
v     50000     5.42
v     100000    10.2
v     200000    22.7

zig   10        1.73
zig   100       2.20
zig   200       2.79
zig   500       4.46
zig   1000      7.53
zig   2000      17.6
zig   5000      81.8