docker-registry-benchmark.sh

#!/bin/bash

regions=(local ca-central-1 eu-central-1 ap-southeast-1 ap-northeast-1 us-east-1)

image_name_base="cache-bench"
image_sizes_in_mb=(64 512 1024)

dockerfile="Dockerfile.sample"
log_file=docker-bench.log

local_registry_dir=${SEMAPHORE_CACHE_DIR:-'registry_data'}

function init-dockerfile() {
  if ! [ -e $dockerfile ]; then
    echo "| Creating ${dockerfile}"

    cat <<DOCKERFILE > $dockerfile
FROM alpine
ARG SIZE
RUN dd if=/dev/urandom of=outputfile.out bs=1024k count=\$SIZE
RUN ls -lah outputfile.out
DOCKERFILE
  fi
}

function stopwatch() {
  /usr/bin/time -f '|> runtime: %E\n' $@
}

function docker() {
  stopwatch docker $@
}

function setup-local-registry() {
  echo '{"insecure-registries" : ["localhost:5000"]}' | sudo tee /etc/docker/daemon.json
  sudo service docker restart
  docker run -d -p 5000:5000 -v $local_registry_dir/registry:/var/lib/registry --restart=always --name registry registry:2
  sleep 5
  curl http://localhost:5000/v2/_catalog
}

function benchmark() {
  # a variable passed as a first argument will be set to
  # contain the runtime
  local runtime="$1"; shift

  # hook up new file descriptors to capture time
  exec 3>&1 4>&2

  eval $runtime=$(TIMEFORMAT="%E"; { time $@ 1>&3 2>&4; } 2>&1)

  # clean up the custom file descriptors
  exec 3>&- 4>&-
}

function benchmark-push-pull() {
  image=$1
  registry_url=$2
  report_string=""

  echo "| Tagging '$image' for region '${region}'"
  docker tag $image $registry_url

  report_string+="$region|"
  report_string+="$image|"

  echo "| Pushing image to ${registry_url}"
  benchmark push_runtime docker push $registry_url
  report_string+="$push_runtime|"

  echo "| Removing local images and tags..."
  docker rmi -f $image
  docker rmi -f ${registry_url}

  echo "| Pulling image from ${registry_url}"
  benchmark pull_runtime docker pull $registry_url
  report_string+="$pull_runtime|"

  echo "| Removing local image..."
  docker rmi -f ${registry_url}

  echo $report_string >> $log_file
}

function build-images() {
  init-dockerfile
  
  echo "| Building local images..."

  echo '.semaphore-cache' > .dockerignore

  if ! [ -e $dockerfile ]; then
    echo "$dockerfile missing!"

    exit 1
  fi

  for size in ${image_sizes_in_mb[@]}; do
    benchmark build_runtime docker build --build-arg SIZE=$size -t "${image_name_base}-${size}" -f $dockerfile .
  done
}

function run-benchmark() {
  for region in ${regions[@]}; do

    build-images

    if ! [ $region == "local" ]; then
      aws ecr get-login --region $region --no-include-email | bash
    fi

    for size in ${image_sizes_in_mb[@]}; do
      image="${image_name_base}-${size}"
      ecr_regional_url="501965974906.dkr.ecr.${region}.amazonaws.com/cache-bench:latest"
      local_url="localhost:5000/cache-bench:latest"

      echo -e "\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
      echo -e "Image: ${image}\nRegion: ${region}\nSize: ${size}MB\n" | column -t -s $' '
      echo "--------------------------------"

      if [ $region == "local" ]; then
        benchmark-push-pull $image $local_url
      else
        benchmark-push-pull $image $ecr_regional_url

        echo "| Removing image from ECR"
        aws ecr batch-delete-image --region $region --repository-name cache-bench --image-ids imageTag=latest
      fi

      echo -e "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n"
    done
  done
}

function print-report() {
  report=docker-benchmark-report.log
  unique_images=$(cat $log_file | cut -d '|' -f2 | tail -n+2 | sort -k2,2 | uniq)

  echo "Region|Image|Push time|Pull time" > $report

  for image in $unique_images; do
    grep "$image" $log_file | sort -t '|' -k3,3n >> $report
  done
  
  echo -e "\nPrinting raw table for easier importing to Numbers:\n"
  cat $report
  
  echo -e "\nFormatted table:\n"
  cat $report | column -t -s $'|'
}

setup-local-registry

run-benchmark

print-report

Measuring Docker 'pull' and 'push' times on different ECR regions

This utility will create images of different sizes (64mb, 512mb, 1024mb) and measure the upload and download times to the defined
AWS regions. The images are deleted after the measurements are completed.

As a bonus, a local Docker registry is also spawned and tested.

Docker image

The Docker image which is used for testing is generated by the script. The content of the image is generated with /dev/urandom to avoid any possible compression on the line; the runs can be considered as a worst case scenario for pulls and pushes.

If you'd like to use your own image, create Dockerfile.sample in the directory where the script is downloaded.

Usage

0. Create the image repository on ECR named cache-bench in all the regions you'd like to test
1. Configure your AWS credentials by exporting the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables, or by using a configuration file
2. Download the script
   https://gist.githubusercontent.com/ervinb/224359235ee046c68fbc9456d6c4662b/raw/docker-registry-benchmark.sh
3. Run

$ bash docker-registry-benchmark.sh

The output is a sorted table, comparing AWS region performance:

Region          Image             Push time  Pull time
local           cache-bench-1024  7.783      4.938
eu-central-1    cache-bench-1024  9.867      5.669
us-east-1       cache-bench-1024  13.745     8.319
ca-central-1    cache-bench-1024  13.869     8.569
ap-southeast-1  cache-bench-1024  16.944     12.557
ap-northeast-1  cache-bench-1024  23.505     24.433
ap-southeast-2  cache-bench-1024  26.260     152.607
local           cache-bench-512   4.022      2.534
eu-central-1    cache-bench-512   6.304      3.164
us-east-1       cache-bench-512   9.393      5.609
ca-central-1    cache-bench-512   10.124     5.365
ap-southeast-1  cache-bench-512   14.082     8.151
ap-northeast-1  cache-bench-512   19.101     17.412
ap-southeast-2  cache-bench-512   26.161     21.561
local           cache-bench-64    0.587      0.386
eu-central-1    cache-bench-64    3.069      0.925
ca-central-1    cache-bench-64    7.857      2.686
us-east-1       cache-bench-64    8.328      3.014
ap-southeast-1  cache-bench-64    15.328     4.676
ap-northeast-1  cache-bench-64    19.095     7.713
ap-southeast-2  cache-bench-64    22.778     8.814

Can we make this script little more generic and can do benchmark any registry? by just passing registry url and image name?

This is really good work, i want to use to benchmark our quay based internal registry