Nginx-lua-prometheus benchmark #3

csv.sh

#!/bin/bash

set -e -u -o pipefail
commits="1e13e22c 0.20200420 avoid-get_keys"
modes="parallel serial none"
output=results.csv

echo -n 'num_metrics' > $output
for mode in $modes; do
  for commit in $commits; do
    for measure in qps p50 p99 p9999 max; do
      echo -n ",$mode $commit $measure" >> $output
    done
  done
done
echo >> $output

for num_metrics in $(ls wrk_* | sed -e 's/wrk_//' | sed -e 's/_.*//' | sort -n -u); do
  echo -n "$num_metrics" >> $output
  for mode in $modes; do
    for commit in $commits; do
      suffix="${num_metrics}_${commit}_${mode}.log"
      qps=$(grep ^Requests/sec wrk_$suffix | awk '{ print $NF }')
      p50=$(grep ^50%, wrk_$suffix | cut -f 2 -d ,)
      p99=$(grep ^99%, wrk_$suffix | cut -f 2 -d ,)
      p9999=$(grep ^99.99%, wrk_$suffix | cut -f 2 -d ,)
      max=$(grep ^max, wrk_$suffix | cut -f 2 -d ,)
      echo -n ",$qps,$p50,$p99,$p9999,$max" >> $output
    done
  done
  echo >> $output
done

nginx.conf

daemon off;
user www-data;
pid /var/run/nginx.pid;
worker_processes 4;
include /etc/nginx/modules-enabled/*.conf;
events {}

http {
    sendfile on;
    tcp_nodelay on;

    lua_shared_dict prometheus_metrics 100M;
    lua_package_path "/nginx-lua-prometheus/?.lua;;";

    log_format short_log "$time_local [$pid] $request_uri ($status) $request_time";
    access_log /dev/stdout short_log buffer=1k flush=1s ;
    error_log stderr info;

    init_by_lua_block {
        prometheus = require("prometheus").init("prometheus_metrics")
        my_counter = prometheus:counter("counter1", "Benchmark counter", {"n"})
        my_hist = prometheus:histogram("hist1", "Benchmark histogram", {"latency"},
          {0.005, 0.01, 0.02, 0.03, 0.05, 0.075, 0.1, 0.2, 0.3, 0.4, 0.5, 0.75,
          1, 1.5, 2, 3, 4, 5, 10, 15, 30, 45, 60, 90, 120, 180, 300})

    }
    init_worker_by_lua_block {
        if prometheus.init_worker then
            prometheus:init_worker(1)
        end

        local metric_count = 200000

        function prepare()
            ngx.log(ngx.INFO, "Preparing data...")
            for i = 0, metric_count do
                my_counter:inc(1, {i})
                if i % 10000 == 0 then
                   ngx.update_time()
                   ngx.log(ngx.INFO, "  ", (100.0*i/metric_count),"%")
                end
            end
            ngx.log(ngx.INFO, "Done!")
            ngx.sleep(1)
            ngx.log(ngx.INFO, metric_count, " metrics were generated.")
        end

        -- only run this on one of the workers
        if ngx.worker.id() == 0 then
            -- launch from timer, so other workers can initialize first
            ngx.timer.at(1, prepare)
        end

    }

    log_by_lua_block {
        my_hist:observe(tonumber(ngx.var.request_time), {ngx.var.uri})
    }

    server {
        listen 18003;
        server_name benchmark;
        location / {
            content_by_lua_block {
                ngx.print("Hello world!\n")
            }
        }
        location /metrics {
            content_by_lua_block {
                prometheus:collect()
            }
        }
    }
}

run.sh

#!/bin/bash

set -x -e -u -o pipefail

base_dir="$(cd "$(dirname "$0")"; pwd -P)"
bench_dir="$(basename ${base_dir})"
container_name="nginx_lua_prometheus_benchmark"
image_name="${container_name}_image"
log_dir="$HOME/benchmark.logs"

mkdir -p $log_dir

cat > ${base_dir}/Dockerfile <<EOF
FROM debian:stable-slim
RUN apt-get update
RUN apt-get install --no-install-recommends -y nginx-light libnginx-mod-http-lua
EOF

docker build -t ${image_name} ${base_dir}

function cleanup {
  docker rm -f ${container_name} || true
}
trap cleanup EXIT

run_metrics() {
    set +x
    case "$1" in
    parallel)
        CMD="curl -sS -o /dev/null --max-time 5 -w 'metrics %{time_starttransfer}\n' http://localhost:18003/metrics & sleep 1" ;;
    serial)
        CMD="curl -sS -o /dev/null --max-time 5 -w 'metrics %{time_starttransfer}\n' http://localhost:18003/metrics" ;;
    none)
        CMD="sleep 10" ;;
    esac
    while true; do
        eval "${CMD}"
    done
    set -x
}

run_benchmark() {
    num_metrics=$1
    commit=$2
    mode=$3

    log_suffix="${num_metrics}_${commit}_${mode}.log"

    sed -i -e "s/local metric_count = .*/local metric_count = $num_metrics/g" ${base_dir}/nginx.conf

    cleanup
    docker run -d --name ${container_name} -p 18003:18003 \
      -v ${base_dir}/../:/nginx-lua-prometheus ${image_name} \
      nginx -c /nginx-lua-prometheus/${bench_dir}/nginx.conf

    # wait for metrics to be prepared
    docker logs -f ${container_name} 2>&1 | sed '/Done!/q;' || true

    run_metrics "$mode" &> $log_dir/metrics_${log_suffix} &
    METRICS_PID=$!
    wrk -t 10 -c 100 -d 60s --latency --timeout 30s --script ~/wrk/scripts/report.lua "http://localhost:18003/test" | tee ${log_dir}/wrk_${log_suffix} || true
    kill $METRICS_PID || true

    curl -sS -o ${log_dir}/metrics_final_${log_suffix} --max-time 5 http://localhost:18003/metrics || true

    docker logs ${container_name} &> ${log_dir}/docker_${log_suffix}
    docker kill ${container_name}
}


for NUM_METRICS in 1000 2000 3000 4000 5000 7000 10000 15000 20000 30000 40000 50000 60000 70000 80000 90000 100000 120000 140000 160000 180000 200000 250000 300000 350000 400000; do
  for COMMIT in "1e13e22c" "0.20200420" "avoid-get_keys"; do
    git checkout "$COMMIT"
    for MODE in "parallel" "serial" "none"; do
      run_benchmark "$NUM_METRICS" "$COMMIT" "$MODE"
    done
  done
done