Are We Compressed Yet? Deployment Script (Docker)

awcy-deploy.sh

#!/bin/bash

# Parse flags
get_deps="false"
get_media="false"
do_build="false"
worker_count=4
worker_cores=4
password="vimeo"

print_usage() {
  printf "Usage: awcy-deploy.sh [-d] [-m] [-b] [-w <WORKER_COUNT>] [-c <WORKER_CORES>] [-p <PASSWORD>]\n \
  -d: Fetch dependencies (default: false)\n \
  -m: Fetch objective-1-fast and subset1 media (default: false)\n \
  -b: Build a new AWCY image (default: false)\n \
  -w: Number of workers to deploy (default: 4)\n \
  -c: Number of CPU cores per worker (default: 4)\n \
  -p: Password needed to create new jobs (default: vimeo)\n"
}

while getopts "dmbw:c:p:" flag; do
  case "${flag}" in
    d) get_deps="true" ;;
    m) get_media="true" ;;
    b) do_build="true" ;;
    w) worker_count=${OPTARG} ;;
    c) worker_cores=${OPTARG} ;;
    p) password=${OPTARG} ;;
    *) print_usage
       exit 1 ;;
  esac
done

if [ $get_deps = "true" ]; then
  # Install dependencies
  sudo apt -y install jq git
  curl -fsSL https://get.docker.com -o get-docker.sh
  sudo sh get-docker.sh
fi

if [ $get_media = "true" ]; then
  # Fetch media
  media_path=${HOME}/xiph-media-files
  mkdir -p $media_path
  if ! [ -d $media_path/objective-1-fast ]; then
    curl -o $media_path/objective-1-fast.tar.gz \
      https://media.xiph.org/video/derf/objective-1-fast.tar.gz
    tar -C $media_path -xzf $media_path/objective-1-fast.tar.gz
    # Copy a particular file that has a misspelled name in the AWCY set
    cp $media_path/objective-1-fast/dark720p.y4m $media_path/objective-1-fast/dark70p.y4m
  fi
  if ! [ -d $media_path/subset1 ]; then
    curl -o $media_path/subset1.tar.gz \
      https://media.xiph.org/video/derf/subset1-y4m.tar.gz
    tar -C $media_path -xzf $media_path/subset1.tar.gz
    # Correct the directory name to match the AWCY data set
    mv $media_path/subset1-y4m $media_path/subset1
  fi
fi

sudo chmod -R 777 /tmp/awcy

# Clean up any files from previous deploys
rm -r /tmp/awcy
mkdir /tmp/awcy

# Clean up containers
sudo docker stop xiph-awcy
sudo docker rm xiph-awcy
for i in $(seq 1 $worker_count); do
  sudo docker stop xiph-awcy-worker-$i
  #sudo docker rm xiph-awcy-worker-$i
done

if [ $do_build = "true" ]; then
  # Initialize the repository
  git init .
  git remote add -t \* -f origin https://github.com/xiph/awcy.git
  git checkout master

  # Build container images
  sudo docker build --no-cache --tag xiph/awcy:latest .
  if [ $worker_count -ge 1 ]; then
    sudo docker build --no-cache --file Dockerfile.worker --tag xiph/awcy-worker:latest .
  fi
fi

# Run containers
if [ $worker_count -ge 1 ]; then
  sudo docker run -itd --name xiph-awcy --publish 80:3000 --publish 2222:22 --volume /tmp/awcy:/data \
    --volume ${HOME}/xiph-media-files:/media --env MEDIAS_SRC_DIR=/media --env AWCY_API_KEY=$password \
    --env LOCAL_WORKER_ENABLED=false xiph/awcy:latest > /tmp/awcy/host-id
else
  sudo docker run -itd --name xiph-awcy --publish 80:3000 --publish 2222:22 --volume /tmp/awcy:/data \
    --volume ${HOME}/xiph-media-files:/media --env MEDIAS_SRC_DIR=/media --env AWCY_API_KEY=$password \
    --env LOCAL_WORKER_ENABLED=true --env LOCAL_WORKER_SLOTS=$worker_cores xiph/awcy:latest > /tmp/awcy/host-id
fi

config_path="/tmp/awcy/conf/machines.json"
pubkey_path="/tmp/awcy/conf/awcy.pub"

# Wait until the host container is fully initialized
while [ ! -f $pubkey_path ]
do
  sleep 10s
done

# Update host configuration directory permissions
sudo chmod -R 777 /tmp/awcy/conf

for i in $(seq 1 $worker_count)
do
sudo docker run -itd --rm --name xiph-awcy-worker-$i --volume ${HOME}/xiph-media-files:/media \
        --env SSH_PUBKEY="$(cat $pubkey_path)" xiph/awcy-worker:latest
done

if [ $worker_count -ge 1 ]; then
  sudo docker stop $(cat /tmp/awcy/host-id)
  
  echo "[" > $config_path

  for i in $(seq 1 $worker_count); do
    sudo docker inspect xiph-awcy-worker-$i | jq -r '.[].NetworkSettings.IPAddress' > /tmp/awcy/worker-id-$i
    cat >>$config_path <<EOF
{
  "host": "$(cat /tmp/awcy/worker-id-$i)",
  "user": "xiph",
  "cores": $worker_cores,
  "port": 22,
  "work_root": "/data/work",
  "media_path": "/media"
},
EOF
  done

  sed -i "$ s/.$//" $config_path
  echo "]" >> $config_path
  sleep 10s
  sudo docker start $(cat /tmp/awcy/host-id)
fi