NATS Memory Benchmark

bench.sh

#!/bin/bash

msgs_per_publisher="1 10 100 1000"
publishers="1 10 100 1000"
mem_limits="33554432 67108864 134217728 268435456"
seconds=10
timeout_seconds=13
passed=""

# set to 1 to enable jetstream
js_enabled=0
js_helm=(
    "--set-json" "nats.jetstream.enabled=true"
    "--set-json" "nats.jetstream.memStorage.enabled=false"
    "--set-json" "nats.jetstream.fileStorage.enabled=true"
    "--set" "nats.jetstream.fileStorage.size=10Gi"
)
js_bench=(
    "--stream=benchstream"
    "--js"
    "--purge"
    "--storage=file"
)
if [ "$js_enabled" = "0" ]; then
    js_helm=()
    js_bench=()
fi

# set to 1 to enable cluster of 3
cluster_enabled=0
cluster_helm=(
    "--set-json" "cluster.enabled=true"
    "--set-json" "cluster.replicas=3"
)
cluster_bench=("--replicas" "3")
if [ "$cluster_enabled" = "0" ]; then
    cluster_helm=()
    cluster_bench=()
fi

function stat_ln {
    printf "%-15s %10s %15s %15s %15s \n" "$@"
}

stat_ln Status Memory Publishers Msg/Publisher Msg/Second

for mem_limit in $mem_limits; do
    go_mem_lemit="$(($mem_limit / 10 * 9 / 1048576))MiB"
    k8s_mem_limit="$((mem_limit/1048576))Mi"
    helm upgrade --install --wait \
        --set "nats.gomemlimit=$go_mem_lemit" \
        --set nats.resources.limits.cpu=1000m \
        --set "nats.resources.limits.memory=$k8s_mem_limit" \
        "${js_helm[@]}" "${cluster_helm[@]}" \
        nats \
        nats/nats >/dev/null 2>&1
    for publisher in $publishers; do
        for msg_per_publisher in $msgs_per_publisher; do
            msgs_total_per_sec="$(($msg_per_publisher * $publisher))"
            msgs_total="$(($msgs_total_per_sec * $seconds))"
            pubsleep="$(bc -l <<< "1/$msg_per_publisher")s"
            if echo "$passed" | grep -qF "|$publisher|$msg_per_publisher|$msgs_total_per_sec|"; then
                continue
            fi
            if [ "$js_enabled" = "0" ]; then
                kubectl exec deploy/nats-box -- nats stream delete -f benchstream >/dev/null 2>&1
            fi
            for try in $(seq 1 2); do
                # try twice and take any successful run that doesn't OOM
                if kubectl exec deploy/nats-box -- \
                    timeout "$timeout_seconds" nats bench benchtest --no-progress \
                        --pub "$publisher" --pubsleep "$pubsleep" --sub "1" --msgs "$msgs_total" \
                        "${js_bench[@]}" "${cluster_bench[@]}" \
                    >/dev/null 2>&1; then
                    failed=0
                    passed="$passed"$'\n'"|$publisher|$msg_per_publisher|$msgs_total_per_sec|"
                    stat_ln "PASSED" "$k8s_mem_limit" "$publisher" "$msg_per_publisher" "$msgs_total_per_sec"
                    break
                else
                    failed=1
                    if kubectl get pods -o json | jq -r '.items[].status.containerStatuses[] | select(.name == "nats") | .lastState.terminated.reason' | grep -qF "OOMKilled"; then
                        stat_ln "FAILED - OOM" "$k8s_mem_limit" "$publisher" "$msg_per_publisher" "$msgs_total_per_sec"
                        kubectl rollout restart sts/nats >/dev/null 2>&1
                        kubectl rollout status sts/nats >/dev/null 2>&1
                        break
                    elif [ "$try" = "2" ]; then
                        stat_ln "FAILED - slow" "$k8s_mem_limit" "$publisher" "$msg_per_publisher" "$msgs_total_per_sec"
                    fi
                fi
            done
            if [ "$failed" = "1" ]; then
                break
            fi
        done
    done
done