gistfile1.txt

import subprocess
import time
from statistics import mean

# Function to measure the execution time of a command
def measure_time(command):
    start_time = time.time()
    subprocess.run(command, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
    end_time = time.time()
    return end_time - start_time

# Compare the execution time of two commands
def compare_commands(command1, command2, iterations):
    # prewarm
    measure_time(command1)
    measure_time(command2)

    print("llama.cpp\tcrabml")
    for _ in range(iterations):
        time1 = measure_time(command1)
        time2 = measure_time(command2)
        print("%0.3f \t| %0.3f" % (time1, time2))

# The two commands to compare
command1 = '~/code/llama.cpp/main -p "captain" --model ~/code/crabml/testdata/open-llama-3b-q8_0.gguf -n 100 -t 2 --temp 0.0'
command2 = '~/code/crabml/target/release/crabml-cli -m ~/code/crabml/testdata/open-llama-3b-q8_0.gguf "captain" --steps 100 -t 0 -T 2'

# Number of iterations
iterations = 8

# Run the comparison
compare_commands(command1, command2, iterations)