scrape_etcd_metrics.sh

#!/usr/bin/env bash

set -x

mkdir -p /tmp/etcd-metrics

jobName="e2e-azure-serial"
jobVersion="4.5"

# NOTE: Change this to 'stat' if you run on Linux
statCommand="gstat"

for jobID in {500..717}; do
  if [[ -f /tmp/etcd-metrics/${jobName}-${jobVersion}-${jobID}.json ]]; then
    continue
  fi

  workdir=$(mktemp -d "${TMPDIR:-/tmp}/e2e-serial-${jobID}.XXXXXXXXX")
  pushd ${workdir} >/dev/null

  url="https://storage.googleapis.com/origin-ci-test/logs/release-openshift-ocp-installer-${jobName}-${jobVersion}/${jobID}/artifacts/${jobName}/metrics/prometheus.tar"
  curl -s $url | tar xf -
  if [[ "$?" != "0" ]]; then
    echo "Failed to fetch $url" && continue
  fi

  if [[ ! -f "wal/00000000" ]]; then
    echo "Corrupted prometheus data $url" && continue
  fi

  prometheus --storage.tsdb.path=. --config.file=/Users/mfojtik//bin/prometheus.yml &
  prometheus_pid=$!

  # wait for prometheus to be ready
  while true; do
    status=$(curl -s http://localhost:9090/api/v1/status/runtimeinfo | jq -cr .status || true)
    [[ "${status}" == "success" ]] && break
    sleep 1
  done
  

  timestamp=$($statCommand -c%Y wal/00000000)
  start_time="$(perl -e "print $timestamp - 18000").00"
  end_time="$(perl -e "print $timestamp + 18000").00"

  # run query
  curl -s "http://localhost:9090/api/v1/query_range" --data-urlencode "query=histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m]))" \
    --data-urlencode "start=${start_time}" --data-urlencode "end=${end_time}" --data-urlencode "step=5" > /tmp/etcd-metrics/${jobName}-${jobVersion}-${jobID}.json

  # kill prometheus
  kill -9 ${prometheus_pid}
  popd >/dev/null
  
  rm -rf ${workdir}
done