Sysbench: AWS EC2 Userdata script

README.md

This script will run sysbench CPU and MEM tests on an EC2 instance as it launches, commit the results in CSV format into an S3 bucket of your choice, and automatically terminate the instance (provided it has appropriate IAM permissions).

Important:

Change the S3 bucket setting above to a real S3 bucket name.

export RESULTS_BUCKET=<bucket-name>

Also, make sure your EC2 instance has sufficient IAM permissions to:

• write to the S3 bucket of your choice
• modify its own instance attributes (to enable self-destruct)

Usage:

Embed the script below as part of the EC2 Userdata script on launch.

---

home · @techlifemusic

userdata.sh

#!/usr/bin/env bash

# :: init vars
export RESULTS_BUCKET=<bucket-name>

export INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id)
export INSTANCE_TYPE=$(curl http://169.254.169.254/latest/meta-data/instance-type)
export INSTANCE_REGION=$(curl http://169.254.169.254/latest/meta-data/placement/availability-zone | sed 's/.$//')

BASE=0
MAX_THREADS=$(nproc --all)

# :: ---
# :: IMPORTANT: ensure the EC2 instance is assigned a sufficient IAM role

apt update -y
apt install -y sysbench unzip awscli

# :: set this instance up to self-destruct
aws ec2 modify-instance-attribute \
  --region $INSTANCE_REGION \
  --instance-id $INSTANCE_ID \
  --instance-initiated-shutdown-behavior terminate

#for THREADS in 1 2; do
while : ; do
  THREADS=$((2**BASE))
  if (( THREADS > MAX_THREADS )); then
    break
  fi
  
  # :: run sysbench tests
  sysbench cpu --cpu-max-prime=20000 --threads=$THREADS run > sysbench-cpu.log
  sysbench memory --threads=$THREADS run > sysbench-mem.log

  # :: parse result in CSV
  # :: adapted from https://openlife.cc/blogs/2011/august/one-liner-condensing-sysbench-output-csv-file
  
  # columns:
  # threads | total time | total events | min latency | avg latency | max latency | 95p latency | thread events avg | thread events stddev | thread exec time avg | thread exec time stddev
  cat sysbench-cpu.log \
    | egrep " cat|threads:|time:|events:|min:|avg:|max:|percentile:|stddev):" \
    | tr -d "\n" \
    | sed 's/Number of threads: //g' \
    | sed 's/\[/\n/g' \
    | sed 's/[A-Za-z\/]\{1,\}://g' \
    | sed 's/ \.//g' \
    | sed \
      -e 's/read\/write//g' \
      -e 's/approx\.  95//g' \
      -e 's/per sec.)//g' \
      -e 's/ms//g' \
      -e 's/(//g' \
      -e 's/^.*cat //g' \
      -e 's/95th//g' \
      -e 's/events//g' \
      -e 's/avg\/stddev)\://g' \
      -e 's/execution//g' \
      -e 's/time//g' \
      -e 's/total//g' \
      -e 's/number//g' \
      -e 's/of//g' \
      -e 's/s//g' \
    | sed 's/ \{1,\}/,/g' \
    | sed 's/\//,/g' \
  >> sysbench-cpu.csv
  echo "" >> sysbench-cpu.csv
  
  # columns:
  # threads | ops | ops/sec | data transferred (MiB) | data transferred/src (MiB) | time | total events | min latency | avg latency | max latency | 95p latency | thread events avg | thread events stddev | thread exec time avg | thread exec time stddev
  cat sysbench-mem.log \
    | egrep " cat|threads:|operations:|transferred|time:|events:|min:|avg:|max:|percentile:|stddev):" \
    | tr -d "\n" \
    | sed 's/Number of threads: //g' \
    | sed 's/\[/\n/g' \
    | sed 's/[A-Za-z\/]\{1,\}://g' \
    | sed 's/ \.//g' \
    | sed \
      -e 's/read\/write//g' \
      -e 's/approx\.  95//g' \
      -e 's/per sec.)//g' \
      -e 's/ms//g' \
      -e 's/(//g' \
      -e 's/^.*cat //g' \
      -e 's/95th//g' \
      -e 's/events//g' \
      -e 's/avg\/stddev)\://g' \
      -e 's/execution//g' \
      -e 's/time//g' \
      -e 's/[Tt]otal//g' \
      -e 's/number//g' \
      -e 's/of//g' \
      -e 's/per//g' \
      -e 's/second)//g' \
      -e 's/transferred//g' \
      -e 's/MiB\/sec)//g' \
      -e 's/MiB//g' \
      -e 's/s//g' \
    | sed 's/ \{1,\}/,/g' \
    | sed 's/\//,/g' \
  >> sysbench-mem.csv
  echo "" >> sysbench-mem.csv
  
  # :: increment
  BASE=$((BASE+1))
done
  
# :: send the result to S3
aws s3 cp ./sysbench-cpu.csv s3://$RESULTS_BUCKET/result_type=cpu/instance_type=$INSTANCE_TYPE/instance_id=$INSTANCE_ID/results.csv
aws s3 cp ./sysbench-mem.csv s3://$RESULTS_BUCKET/result_type=mem/instance_type=$INSTANCE_TYPE/instance_id=$INSTANCE_ID/results.csv
  
# :: kill the instance
# :: because of the prep above, this should also terminate the EC2 
shutdown -h now