Are We Compressed Yet? Web Server Deployment Script (for Debian Buster)

awcy-deploy-web.sh

#!/bin/bash

# Parse flags
worker_cores=16
worker_address="127.0.0.1"
user="vimeo"
password="vimeo"

print_usage() {
  printf "Usage: awcy-deploy-web.sh [-w <WORKER_ADDRESS>] [-c <WORKER_CORES>] [-u <USER>] [-p <PASSWORD>]\n \
  -w: Specify the worker server address (default: 127.0.0.1)\n \
  -c: Number of CPU cores assigned to the worker (default: 16)\n \
  -u: User as whom AWCY should run (default: vimeo)\n \
  -p: Password needed to create new jobs (default: vimeo)\n"
}

while getopts "w:c:u:p:" flag; do
  case "${flag}" in
    w) worker_address=${OPTARG} ;;
    c) worker_cores=${OPTARG} ;;
    u) user=${OPTARG} ;;
    p) password=${OPTARG} ;;
    *) print_usage
       exit 1 ;;
  esac
done

pkill node && pkill python
cd ${HOME}

# Install dependencies
sudo apt update
sudo apt install -y nodejs npm cmake build-essential alien \
  python3 python3-pip nginx git rsync libicu-dev yasm libtool
pip3 install --user cython tornado boto3 numpy scipy matplotlib \
  pandas scikit-learn scikit-image h5py sureal
npm upgrade
curl -L -O https://www.nasm.us/pub/nasm/releasebuilds/2.15.04/linux/nasm-2.15.04-0.fc31.x86_64.rpm
sudo alien -d nasm*.rpm
sudo dpkg -i nasm*.deb
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -yq
source ~/.cargo/env

# Initialize AWCY
mkdir -p awcy && cd awcy
git init .
git remote add -t \* -f origin https://github.com/xiph/awcy.git
git pull origin master

# Initialize rd_tool/rd_server
mkdir -p rd_tool && cd rd_tool
git init .
git remote add -t \* -f origin https://github.com/xiph/rd_tool.git
git pull origin master
cd ..

# Set up runtime dependencies
./setup.sh
mkdir -p rav1e && cd rav1e
git init .
git remote add -t \* -f origin https://github.com/xiph/rav1e.git
git pull origin master
cd ..

# Configure SSH host and client
if [ ! -f "/home/$user/.ssh/config" ]; then
  mkdir -p /home/$user/.ssh
  cat >/home/$user/.ssh/config <<-EOF
    Host *
    UserKnownHostsFile /dev/null
    StrictHostKeyChecking no
    LogLevel quiet
    Port 22
EOF
  sudo chown -R $user:$user /home/$user/.ssh
fi

if [ ! -f /etc/ssh/ssh_host_rsa_key ]; then
  sudo sshd-keygen
fi

CONFIG_DIR="/home/$user/.awcy"
sudo mkdir -p ${CONFIG_DIR}
if [ ! -f "${CONFIG_DIR}/awcy.pem" ]; then
  sudo ssh-keygen -t rsa -f ${CONFIG_DIR}/awcy.pem -P ''
  sudo mv ${CONFIG_DIR}/awcy.pem.pub ${CONFIG_DIR}/awcy.pub
  sudo chmod 0600 ${CONFIG_DIR}/awcy.pem
  sudo ln -s ${CONFIG_DIR}/awcy.pem rd_tool/daala.pem
fi
sudo chown -R $user:$user ${CONFIG_DIR}

# Create configuration files
cat >config.json <<EOF
{
  "channel": "#daalatest",
  "have_aws": false,
  "port": 3000,
  "rd_server_url": "http://localhost:4000"
}
EOF
echo $password > secret_key

# Set up and start the npm backend
cd www
npm install && npm run build
cd ..
npm install

# Generate the worker configuration
echo "[" > machines.json

cat >>machines.json <<EOF
  {
    "host": "$worker_address",
    "user": "$user",
    "cores": $worker_cores,
    "port": 22,
    "work_root": "/home/$user/awcy/work",
    "media_path": "/home/$user/awcy/media"
  },
EOF

sed -i "$ s/.$//" machines.json
echo "]" >> machines.json

# Configure and start nginx
sudo sed -i '/^\s*proxy_pass/d' /etc/nginx/sites-enabled/default
sudo sed -i '/^\s*try_files/d' /etc/nginx/sites-enabled/default
sudo sed -i '/^\s*location \/ {/a proxy_pass http:\/\/127.0.0.1:3000;' /etc/nginx/sites-enabled/default
sudo service nginx restart

# Launch rd_server and AWCY
cat >start.sh <<EOF # Create a convenience server (re)start script
  #!/bin/bash
  cd "\$(dirname "\$0")"
  pkill node && pkill python
  cd rd_tool
  nohup ./rd_server.py -machineconf ../machines.json > rd_server.log 2>&1 &
  cd ..
  nohup npm start > awcy.log 2>&1 &
EOF
chmod +x start.sh
./start.sh
node generate_list.js

# Display additional instructions for work that cannot be automated
echo "-------------------"
echo "Deployment complete. Make sure to configure workers as follows:"
echo "- Include the key at ${CONFIG_DIR}/awcy.pub in /home/$user/.ssh/authorized_keys"
echo "- Create work and media directories in /home/$user/awcy, with data sets in the latter"
echo "Sets are available at https://media.xiph.org/sets/ and must be extracted into folders of the same names."