ollama-benchmark.sh

#!/bin/sh
# ollamark.aaartist.dev

# Usage
# curl -O https://ollamark.aaartist.dev/shell/ollama-benchmark.sh
# chmod +x ./ollama-benchmark.sh
# ./ollama-benchmark.sh

# Let's go

# Prompts
# prompts=("why is the sky blue" "Please roll a virtual six-sided dice for me. Do that two times and tell me the random outcome." "Alice has 2 brothers and she also has 3 sisters. How many sisters does Alice's brother have?")
prompts=("please count from 1 to 10 and after this recite the alphabet from a-z. be as precise as possible.")
prompts_length=${#prompts[@]}

# Models
models=("llama3.1:latest" "phi3:latest")

# Function to check if a command exists
command_exists() {
    type "$1" &> /dev/null ;
}

check_program_installed() {
    local program=$1

    if ! command -v $program > /dev/null 2>&1; then
        echo "$program is not installed. Exiting the script."
        exit 1
    fi
}

check_model_installed() {
    for model in "${models[@]}"; do
        if ! ollama show $model > /dev/null 2>&1; then
            echo "$model is not installed. Please download model and try again. \`ollama pull $model\`"
            exit 1
        fi
    done
}

# Get the current user
user="Unknown" # don't do $(whoami) to avoid leaking personal information

# Detect the operating system
OS="Unknown"
cpu_info="Unknown"
gpu_info="Unknown"
ram="0"
rawram=0
vram="0"
rawvram=0
num_cores=0
case "$(uname -s)" in
    Linux*)     OS="Linux"
    check_program_installed "lscpu"
    check_program_installed "lspci"
    check_program_installed "xargs"
    check_program_installed "free"
    check_program_installed "nproc"
    check_program_installed "nvidia-smi"
    check_program_installed "bc"

    if [ -f "/proc/sys/fs/binfmt_misc/WSLInterop" ]; then
        OS="Windows Subsystem for Linux"
        OS="WSL"
    fi

    cpu_info=$(lscpu | grep 'Model name:' | cut -d':' -f2 | xargs)
    gpu_info=$(lspci | grep -E 'VGA|3D' | cut -d":" -f3 | xargs)
    ram=$(free -m | awk '/^Mem:/ {print $2 / 1024}')
    rawram=$(printf "%.0f" "$ram")
    ram=$rawram" GB"
    num_cores=$(nproc)

    if command_exists nvidia-smi; then
        vram=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits | xargs echo -n)
        vram=$(echo $vram | awk '{sum=0; for(i=1; i<=NF; i++) if ($i+0==$i) sum+=$i; print sum}')
        echo $total_vram
        rawvram=$(echo "$vram / 1024" | bc -l)
        vram=$(printf "%.0f GB" "$rawvram")
        totalram=$(echo "$rawvram + $rawram" | bc -l)
        totalram=$(printf "%.0f" "$totalram")
    else
        vram="0"
    fi
    ;;
    Darwin*)    OS="MacOS"
    check_program_installed "sysctl"
    check_program_installed "system_profiler"
    ram=$(($(sysctl -n hw.memsize) / 1024 / 1024 / 1024))" GB"
    rawram=${ram%% *}
    rawram=$(printf "%.0f" "$rawram")
    # vram=$(system_profiler SPDisplaysDataType | awk -F': ' '/VRAM \(Total\)/ {print $2}')
    vram=$ram
    rawvram=$rawram
    totalram=$rawvram
    totalram=$(printf "%.0f" "$totalram")
    num_cores=$(sysctl -n hw.ncpu)
    cpu_info=$(sysctl -n machdep.cpu.brand_string)
    gpu_info=$(system_profiler SPDisplaysDataType | awk -F': ' '/Chipset Model:/ {print $2}')
    ;;
    msys*)      OS="Windows"
    check_program_installed "systeminfo"
    check_program_installed "bc"
    ram=$(systeminfo | awk -F': ' '/Total Physical Memory:/ {print $2}')
    rawram=${ram%% *}
    rawram=$(printf "%.0f" "$rawram")
    vram=$(systeminfo | awk -F': ' '/Total Virtual Memory:/ {print $2}')
    rawvram=${vram%% *}
    totalram=$(echo "$rawvram + $rawram" | bc -l)
    totalram=$(printf "%.0f" "$totalram")
    num_cores=$(systeminfo | awk -F': ' '/Number of Processors:/ {print $2}')
    cpu_info=$(systeminfo | awk -F': ' '/System Manufacturer:/ {print $2}')
    gpu_info=$(systeminfo | awk -F': ' '/Video Controller:/ {print $2}')
    ;;
    *)          OS="Other"
    ;;
esac

check_program_installed "ollama"
check_model_installed ""

echo ""
echo "Running on $OS"
echo "CPU: $cpu_info"
echo "Cores: $num_cores"
echo "GPU: $gpu_info"
echo "RAM: $ram"
echo "VRAM: $vram"
echo "Total Memory: $totalram GB"
echo ""

read -p "This will run the benchmark and send the results, including your system information from above, to ollamark.aaartist.dev. Do you want to continue? (y/n) " -n 1 -r
echo ""
if [[ $REPLY =~ ^[Yy]$ ]]; then
    # Run ollama commands
    for model in "${models[@]}"; do
        echo ""
        echo "Benchmark ollama $model"
        for prompt in "${prompts[@]}"; do
            #
            if [ "$prompts_length" -gt 1 ];
                then echo -n "⌞ Running prompt \"$prompt\" ...";
            fi
            
            output=$(ollama run --verbose $model "$prompt" 2>&1 >/dev/null | grep -v -E "prompt eval count|prompt eval duration|eval count|eval duration")
            evalrate=$(echo $output | awk -F'eval rate: ' '{print $3}' | cut -d' ' -f1)

            if [ "$prompts_length" -gt 1 ];
                then echo " $evalrate";
            else
                echo "Result: $evalrate tokens/s"
            fi

            # Send benchmark results to Ollamark (ollamark.aaartist.dev)
            # echo "Sending benchmark results to Ollamark (https://ollamark.aaartist.dev)"
            json="{ \"sql\": \"INSERT INTO benchmarks (evalrate, model, cpu, cores, gpu, ram, vram, os, user) VALUES (?, \\\"$model\\\", \\\"$cpu_info\\\", $num_cores, \\\"$gpu_info\\\", $rawram, $rawvram, \\\"$OS\\\", \\\"$user\\\");\", \"params\": [$evalrate] }"
            # echo "$json" # for debugging
            post=$(curl --data "$json" --silent --request POST --url 'https://api.cloudflare.com/client/v4/accounts/f7941adf3d567f44149ccf5e0f713756/d1/database/543c0072-cdb3-48b6-b6e1-8bc8a947614c/query' -H "Content-Type: application/json" -H "Authorization: Bearer rZRGyB-T5eiaibr8YUp6V8wkIPJ-MB-65g-FMkGv")
        done
    done

    echo "Done. Compare your results on https://ollamark.aaartist.dev."

    #if [ "$totalram" -gt 31 ]; then
    #    echo "This is a lot of RAM"
    #fi
fi