ollama perf

operf.sh

#!/bin/bash

# Function to check if a command exists
command_exists() {
    type "$1" &> /dev/null ;
}

check_program_installed() {
    local program=$1

    if ! command -v $program > /dev/null 2>&1; then
        echo "$program is not installed. Exiting the script."
        exit 1
    fi
}

# Detect the operating system
OS="Unknown"
cpu_info="Unknown"
gpu_info="Unknown"
ram="0"
rawram=0
vram="0"
rawvram=0
num_cores=0
case "$(uname -s)" in
    Linux*)     OS="Linux"
    check_program_installed "lscpu"
    check_program_installed "lspci"
    check_program_installed "xargs"
    check_program_installed "free"
    check_program_installed "nproc"
    check_program_installed "nvidia-smi"
    check_program_installed "bc"

    echo "."
    cpu_info=$(lscpu | grep 'Model name:' | cut -d':' -f2 | xargs)
    gpu_info=$(lspci | grep -E 'VGA|3D' | cut -d":" -f3 | xargs)
    ram=$(free -m | awk '/^Mem:/ {print $2 / 1024}')
    rawram=$(printf "%.0f" "$ram")
    ram=$rawram" GB"
    num_cores=$(nproc)

    if command_exists nvidia-smi; then
        vram=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits | xargs echo -n)
	vram=$(echo $vram | awk '{sum=0; for(i=1; i<=NF; i++) if ($i+0==$i) sum+=$i; print sum}')
	echo $total_vram
	rawvram=$(echo "$vram / 1024" | bc -l)
	vram=$(printf "%.0f GB" "$rawvram")
        totalram=$(echo "$rawvram + $rawram" | bc -l)
        totalram=$(printf "%.0f" "$totalram")
    else
        vram="0"
    fi
    ;;
    Darwin*)    OS="MacOS"
    echo "_"
    check_program_installed "sysctl"
    check_program_installed "system_profiler"
    check_program_installed "bc"
    echo "."
    ram=$(($(sysctl -n hw.memsize) / 1024 / 1024 / 1024))" GB"
    rawram=${ram%% *}
    rawram=$(printf "%.0f" "$rawram")
    # vram=$(system_profiler SPDisplaysDataType | awk -F': ' '/VRAM \(Total\)/ {print $2}')
    vram=$ram
    rawvram=$rawram
    # rawvram=${vram%% *}
    # totalram=$(echo "$rawvram + $rawram" | bc -l)
    totalram=$rawvram
echo ".."
    totalram=$(printf "%.0f" "$totalram")
    num_cores=$(sysctl -n hw.ncpu)
    cpu_info=$(sysctl -n machdep.cpu.brand_string)
    gpu_info=$(system_profiler SPDisplaysDataType | awk -F': ' '/Chipset Model:/ {print $2}')
    ;;
    *)          OS="Other"
    ;;
esac

# Check if 'ollama' is installed
# echo "Checking if 'ollama' is installed..."
# if command_exists ollama; then
#     echo "ollama found."
# else
#     echo "'ollama' is not installed. Exiting."
#     exit 1
# fi
check_program_installed "ollama"
# Run ollama commands
echo "Pulling orca-mini (to help reset model load time)..."
ollama pull orca-mini &> /dev/null

echo "Running orca-mini..."
ollama run orca-mini "why is the sky blue" >/dev/null

echo "Pulling llama2:7b..."
ollama pull llama2:7b &>/dev/null

echo "Running llama2:7b the first time..."
first_output=$(ollama run --verbose llama2:7b "why is the sky blue" 2>&1 >/dev/null | grep -v -E "prompt eval count|prompt eval duration|eval count|eval duration")
echo "Running llama2:7b the second time..."
second_output=$(ollama run --verbose llama2:7b "why is the sky blue" 2>&1 >/dev/null | grep -v -E "prompt eval count|prompt eval duration|eval count|eval duration")

# Print the outputs
echo ""
echo "Running on $OS"
echo "CPU: $cpu_info"
echo "Cores: $num_cores"
echo "GPU: $gpu_info"
echo "RAM: $ram"
echo "VRAM: $vram"
echo "Total Memory: $totalram GB"
echo ""
echo "First Run of llama2 7b:"
echo "$first_output"
echo "Second Run of llama2 7b:"
echo "$second_output"

if [ "$totalram" -gt 15 ]; then
echo "You can probably run llama2 13b"
echo "Pulling llama2:13b"
ollama pull llama2:13b &>/dev/null
echo "Running llama2:13b the first time..."
third_output=$(ollama run --verbose llama2:13b "why is the sky blue" 2>&1 >/dev/null | grep -v -E "prompt eval count|prompt eval duration|eval count|eval duration")
echo "Running llama2:13b the second time..."
fourth_output=$(ollama run --verbose llama2:13b "why is the sky blue" 2>&1 >/dev/null | grep -v -E "prompt eval count|prompt eval duration|eval count|eval duration")
echo ""
echo "First Run of 13b:"
echo "$third_output"
echo "Second Run of 13b:"
echo "$fourth_output"
fi

if [ "$totalram" -gt 60 ]; then
    echo "You can probably run llama2 70b"
    echo "Pulling llama2:70b"
    ollama pull llama2:70b &>/dev/null
    echo "Running llama2:70b the first time..."
    fifth_output=$(ollama run --verbose llama2:70b "why is the sky blue" 2>&1 >/dev/null | grep -v -E "prompt eval count|prompt eval duration|eval count|eval duration")
    echo "Running llama2:70b the second time..."
    sixth_output=$(ollama run --verbose llama2:70b "why is the sky blue" 2>&1 >/dev/null | grep -v -E "prompt eval count|prompt eval duration|eval count|eval duration")
    echo ""
    echo "First Run of 70b:"
    echo "$fifth_output"
    echo "Second Run of 70b:"
    echo "$sixth_output"
fi