Are We Compressed Yet? Deployment Script (Standalone)

awcy-deploy.sh

#!/bin/bash

# Parse flags
get_deps="false"
get_media="false"
worker_cores=16
user="vimeo"
password="vimeo"

print_usage() {
  printf "Usage: awcy-deploy.sh [-d] [-m] [-c <WORKER_CORES>] [-u <USER>] [-p <PASSWORD>]\n \
  -d: Fetch dependencies (default: false)\n \
  -m: Fetch objective-1-fast and subset1 media (default: false)\n \
  -c: Number of CPU cores assigned to the worker (default: 16)\n \
  -u: User as whom AWCY should run (default: vimeo)\n \
  -p: Password needed to create new jobs (default: vimeo)\n"
}

while getopts "dmc:u:p:" flag; do
  case "${flag}" in
    d) get_deps="true" ;;
    m) get_media="true" ;;
    c) worker_cores=${OPTARG} ;;
    u) user=${OPTARG} ;;
    p) password=${OPTARG} ;;
    *) print_usage
       exit 1 ;;
  esac
done

# Kill running processes for AWCY and rd_server
pkill node
pkill python3

cd ${HOME}

if [ $get_deps = "true" ]; then
  # Install dependencies
  curl -sL https://rpm.nodesource.com/setup_10.x | sudo sh -
  curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -yq
  source ~/.cargo/env
  sudo yum install -y nodejs libicu-devel yasm libtool git python3 \
    gcc gcc-c++ cmake3 python-devel lbzip2 libogg-devel libjpeg-devel \
    libpng-devel ninja-build nginx bc time dav1d aom
  sudo ln -s /bin/cmake3 /bin/cmake
  pip3 install --user cython tornado boto3 numpy scipy matplotlib pandas scikit-learn scikit-image h5py sureal
  curl -O https://www.nasm.us/pub/nasm/releasebuilds/2.14.02/linux/nasm-2.14.02-0.fc27.x86_64.rpm
  sudo yum localinstall -y nasm-*.rpm
  curl -LO https://github.com/mesonbuild/meson/releases/download/0.54.0/meson-0.54.0.tar.gz
  tar -xzf meson-*.tar.gz
  cd meson-*
  sudo python3 setup.py install
fi

PATH=$PATH:/usr/local/bin

if [ $get_media = "true" ]; then
  # Fetch media
  media_path=${HOME}/awcy/media
  mkdir -p $media_path
  if ! [ -d $media_path/objective-1-fast ]; then
    curl -o $media_path/objective-1-fast.tar.gz \
      https://media.xiph.org/video/derf/objective-1-fast.tar.gz
    tar -C $media_path -xzf $media_path/objective-1-fast.tar.gz
    # Copy a particular file that has a misspelled name in the AWCY set
    cp $media_path/objective-1-fast/dark720p.y4m $media_path/objective-1-fast/dark70p.y4m
  fi
  if ! [ -d $media_path/subset1 ]; then
    curl -o $media_path/subset1.tar.gz \
      https://media.xiph.org/video/derf/subset1-y4m.tar.gz
    tar -C $media_path -xzf $media_path/subset1.tar.gz
    # Correct the directory name to match the AWCY data set
    mv $media_path/subset1-y4m $media_path/subset1
  fi
fi

# Initialize the repository
mkdir -p awcy && cd awcy
git init .
git remote add -t \* -f origin https://github.com/xiph/awcy.git
git pull origin master

# Set up runtime dependencies
./setup.sh
mkdir -p rav1e && cd rav1e
git init .
git remote add -t \* -f origin https://github.com/xiph/rav1e.git
git pull origin master
cd ..
mkdir -p data/work/daalatool && cd data/work/daalatool
git init .
git remote add -t \* -f origin https://gitlab.xiph.org/xiph/daala.git
git pull origin master
./autogen.sh
./configure --disable-unit-tests --disable-player
make tools
cd ..
mkdir -p dump_ciede2000 && cd dump_ciede2000
git init .
git remote add -t \* -f origin https://github.com/KyleSiefring/dump_ciede2000.git
git pull origin master
cargo build --release
cd ..
mkdir -p vmaf && cd vmaf
git init .
git remote add -t \* -f origin https://github.com/Netflix/vmaf.git
git pull origin master
cd libvmaf
meson build --buildtype release && cd build
ninja-build
sudo ninja-build install
VMAF_DIR=/opt/vmaf
cd ../../python
pip3 install --user .
cd ../../../..

# Configure SSH host and client
if [ ! -f "/home/$user/.ssh/config" ]; then
  mkdir -p /home/$user/.ssh
  cat >/home/$user/.ssh/config <<-EOF
    Host *
    UserKnownHostsFile /dev/null
    StrictHostKeyChecking no
    LogLevel quiet
    Port 22
EOF
  sudo chown -R $user:$user /home/$user/.ssh
fi

if [ ! -f /etc/ssh/ssh_host_rsa_key ]; then
  sudo sshd-keygen
fi

CONFIG_DIR="/home/$user/.awcy"
sudo mkdir -p ${CONFIG_DIR}
if [ ! -f "${CONFIG_DIR}/awcy.pem" ]; then
  sudo ssh-keygen -t rsa -f ${CONFIG_DIR}/awcy.pem -P ''
  sudo mv ${CONFIG_DIR}/awcy.pem.pub ${CONFIG_DIR}/awcy.pub
  sudo chmod 0600 ${CONFIG_DIR}/awcy.pem
  sudo ln -s ${CONFIG_DIR}/awcy.pem rd_tool/daala.pem
fi
sudo chown -R $user:$user ${CONFIG_DIR}

if [ ! -f "/home/$user/.ssh/authorized_keys" ]; then
  sudo cp ${CONFIG_DIR}/awcy.pub /home/$user/.ssh/authorized_keys
fi
sudo chown -R $user:$user /home/$user/.ssh
sudo chmod 0600 /home/$user/.ssh/*
sudo chmod 0700 /home/$user/.ssh

# Create configuration files
cat >config.json <<EOF
{
  "channel": "#daalatest",
  "have_aws": false,
  "port": 3000,
  "rd_server_url": "http://localhost:4000"
}
EOF
echo $password > secret_key

# Set up and start the npm backend
cd www
npm install && npm run build
cd ..
npm install

# Generate the worker configuration
echo "[" > machines.json

cat >>machines.json <<EOF
  {
    "host": "localhost",
    "user": "$user",
    "cores": $worker_cores,
    "port": 22,
    "work_root": "/home/$user/awcy/data/work",
    "media_path": "/home/$user/awcy/media"
  },
EOF

sed -i "$ s/.$//" machines.json
echo "]" >> machines.json

# Set up rd_server
mkdir -p rd_tool
cd rd_tool
git init .
git remote add -t \* -f origin https://github.com/xiph/rd_tool.git
git pull origin master

# Launch rd_server and AWCY
nohup ./rd_server.py -machineconf ../machines.json > rd_server.log 2>&1 &
cp sets.json ..
cd ..

sudo sed -i '/^\s*proxy_pass/d' /etc/nginx/nginx.conf
sudo sed -i '/^\s*location \/ {/a proxy_pass http:\/\/127.0.0.1:3000;' /etc/nginx/nginx.conf
sudo service nginx restart
sudo setsebool httpd_can_network_connect 1 -P
nohup npm start > awcy.log 2>&1 &
node generate_list.js