AZURE NV* BOOTSTRAP SCRIPT

bootstrap.sh

#!/bin/sh -e
# /etc/rc.local

### BEGIN INIT INFO
# Provides:          vmbootstrap
# Required-Start:    $remote_fs $syslog
# Required-Stop:     $remote_fs $syslog
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: vmbootstrap
# Description:       vmbootstrap
### END INIT INFO

# VARS
HOMEFOLDER={{HOMEFOLDER}}
DOMAIN={{DOMAIN}}
INFRAMAIL={{INFRAMAIL}}
SLACKHOOK={{SLACKHOOK}}

before_reboot(){

    # START
    curl -X POST --data-urlencode "payload={'channel': '#infrastructure', 'username': 'azure:deploy', 'text': 'DEPLOYMENT PHASE 1 STARTED ${DOMAIN}', 'icon_emoji': ':monkey:'}" $SLACKHOOK
    logger –t vmbootstrap "START PHASE 1"

    # PACKAGES
    sudo apt update
    sudo apt install build-essential cmake git unzip pkg-config libopenblas-dev liblapack-dev linux-image-generic linux-image-extra-virtual linux-source linux-headers-generic libssl-dev openssl libkrb5-dev libsasl2-dev -y
    logger –t vmbootstrap "PACKAGES"

    # BLACKLIST NOUVEAU
    sudo echo "blacklist nouveau"         >> /etc/modprobe.d/blacklist-nouveau.conf
    sudo echo "blacklist lbm-nouveau"     >> /etc/modprobe.d/blacklist-nouveau.conf
    sudo echo "options nouveau modeset=0" >> /etc/modprobe.d/blacklist-nouveau.conf
    sudo echo "alias nouveau off"         >> /etc/modprobe.d/blacklist-nouveau.conf
    sudo echo "alias lbm-nouveau off"     >> /etc/modprobe.d/blacklist-nouveau.conf
    sudo echo options nouveau modeset=0 | sudo tee -a /etc/modprobe.d/nouveau-kms.conf
    sudo update-initramfs -u
    logger –t vmbootstrap "BLACKLIST NOUVEAU"

    # FIREWALL
    sudo ufw app list
    sudo ufw allow ssh
    sudo ufw allow http
    sudo ufw allow 443/tcp
    sudo ufw allow 25/tcp
    sudo ufw show added
    sudo ufw --force enable
    logger –t vmbootstrap "FIREWALL"

    # POSTFIX
    export DEBIAN_FRONTEND="noninteractive"
    echo "postfix postfix/mailname string ${DOMAIN}" | sudo debconf-set-selections 
    echo "postfix postfix/main_mailer_type string 'Internet Site'" | sudo debconf-set-selections 
    sudo apt install postfix mailutils -y
    logger –t vmbootstrap "POSTFIX"

    # FAIL2BAN
    echo iptables-persistent iptables-persistent/autosave_v4 boolean true | sudo debconf-set-selections
    echo iptables-persistent iptables-persistent/autosave_v6 boolean true | sudo debconf-set-selections
    sudo apt install fail2ban iptables-persistent -y
    sudo cp /etc/fail2ban/jail.conf /etc/fail2ban/jail.local
    sudo sed -i "/destemail/s/= .*/= ${INFRAMAIL}/" /etc/fail2ban/jail.local
    sudo sed -i "s/^action = %(action_)s/action = %(action_mwl)s/" /etc/fail2ban/jail.local
    sudo service fail2ban stop
    sudo service fail2ban start
    sudo /etc/init.d/fail2ban status
    logger –t vmbootstrap "FAIL2BAN"

    # TIMEZONE
    sudo timedatectl set-timezone Europe/Amsterdam
    sudo apt install ntp -y
    logger –t vmbootstrap "TIMEZONE"

    # SWAP
    sudo fallocate -l 8G /swapfile
    sudo chmod 600 /swapfile
    sudo mkswap /swapfile
    sudo swapon /swapfile
    sudo sh -c 'echo "/swapfile none swap sw 0 0" >> /etc/fstab'
    logger –t vmbootstrap "SWAP"

    # PYTHON
    sudo apt install python-pip -y
    sudo pip install --upgrade pip
    logger –t vmbootstrap "PYTHON"

    # AWS
    sudo pip install awscli
    logger –t vmbootstrap "AWSCLI"

    # NGINX
    sudo apt install nginx -y
    ip addr show eth0 | grep inet | awk '{ print $2; }' | sed 's/\/.*$//'
    logger –t vmbootstrap "NGINX"

}

after_reboot(){

    # START
    curl -X POST --data-urlencode "payload={'channel': '#infrastructure', 'username': 'azure:deploy', 'text': 'DEPLOYMENT PHASE 2 STARTED ${DOMAIN}', 'icon_emoji': ':monkey:'}" $SLACKHOOK
    logger –t vmbootstrap "START PHASE 2"
    
    # ACTION > save following to $DOMAIN in folder of this script
    sudo echo "server {"                                                  >> $HOMEFOLDER/$DOMAIN
    sudo echo "    #listen 80;"                                           >> $HOMEFOLDER/$DOMAIN
    sudo echo "    server_name ${DOMAIN}"                                 >> $HOMEFOLDER/$DOMAIN
    sudo echo "    client_max_body_size 100M;"                            >> $HOMEFOLDER/$DOMAIN
    sudo echo "    location / {"                                          >> $HOMEFOLDER/$DOMAIN
    sudo echo "        proxy_set_header   X-Forwarded-For \$remote_addr;" >> $HOMEFOLDER/$DOMAIN
    sudo echo "        proxy_set_header   Host \$http_host;"              >> $HOMEFOLDER/$DOMAIN
    sudo echo "        proxy_pass         \"http://127.0.0.1:3000\";"     >> $HOMEFOLDER/$DOMAIN
    sudo echo "    }"                                                     >> $HOMEFOLDER/$DOMAIN
    sudo echo "}"                                                         >> $HOMEFOLDER/$DOMAIN

    # NGINX:CONFIG
    sudo cp $HOMEFOLDER/$DOMAIN /etc/nginx/sites-available/$DOMAIN
    sudo ln -s /etc/nginx/sites-available/$DOMAIN /etc/nginx/sites-enabled/$DOMAIN
    sudo nginx -t
    sudo service nginx restart
    sudo rm $HOMEFOLDER/$DOMAIN
    logger –t vmbootstrap "NGINX:CONFIG"

    # TOOLKIG & DRIVER
    cd $HOMEFOLDER
    sudo wget https://developer.nvidia.com/compute/cuda/8.0/prod/local_installers/cuda_8.0.44_linux-run
    sudo mv cuda_8.0.44_linux-run cuda_8.0.44_linux.run
    sudo chmod +x cuda_8.0.44_linux.run
    sudo mkdir installers
    sudo ./cuda_8.0.44_linux.run --extract=`pwd`/installers
    cd installers
    sudo ./NVIDIA-Linux-x86_64-367.48.run -a -silent
    sudo modprobe nvidia
    sudo ./cuda-linux64-rel-8.0.44-21122537.run -noprompt
    sudo ./cuda-samples-linux-8.0.44-21122537.run -prefix=$HOMEFOLDER/installers/samples -cudaprefix=/usr/local/cuda -noprompt
    sudo mv ./../cuda_8.0.44_linux.run .
    cd ..
    logger –t vmbootstrap "NVIDIA TOOLKIT & DRIVER"
    
    # CUDnn
    curl -fsSL http://developer.download.nvidia.com/compute/redist/cudnn/v5.1/cudnn-8.0-linux-x64-v5.1.tgz -O
    sudo tar xvf cudnn-8.0-linux-x64-v5.1.tgz
    sudo cp -P cuda/include/* /usr/local/cuda/include/
    sudo cp -P cuda/lib64/* /usr/local/cuda/lib64/
    sudo chmod a+r /usr/local/cuda/lib64/libcudnn*
    sudo ldconfig
    sudo rm -rf cuda/
    sudo rm cudnn*
    logger –t vmbootstrap "CUDNN"

    # UPDATE ~/.bashrc
    sudo echo "# CUDA Toolkit"                                                >> $HOMEFOLDER/.bashrc
    sudo echo "export CUDA_HOME=/usr/local/cuda"                              >> $HOMEFOLDER/.bashrc
    sudo echo "export LD_LIBRARY_PATH=\${CUDA_HOME}/lib64:\$LD_LIBRARY_PATH"  >> $HOMEFOLDER/.bashrc
    sudo echo "export PATH=\${CUDA_HOME}/bin:\${PATH}"                        >> $HOMEFOLDER/.bashrc
    logger –t vmbootstrap "BASHRC"

    # FINISHED
    curl -X POST --data-urlencode "payload={'channel': '#infrastructure', 'username': 'azure:deploy', 'text': 'DEPLOYMENT FINISHED ${DOMAIN}', 'icon_emoji': ':monkey:'}" $SLACKHOOK
    logger –t vmbootstrap "FINISHED"

}

if [ -f /var/opt/vmbootstrap-rebootflag ]; then
    logger –t vmbootstrap "AFTER REBOOT"
    after_reboot
    sudo rm /var/opt/vmbootstrap-rebootflag
    sudo mv $HOMEFOLDER/rc.backup /etc/rc.local
    exit 0
else
    logger –t vmbootstrap "BEFORE REBOOT"
    before_reboot
    sudo touch /var/opt/vmbootstrap-rebootflag
    sudo reboot
fi

run.sh

# TITLE
echo "GPU VM BOOTSTRAP"

# ENV VARS CHECK
if [ -z "$1" ] ;  then
  echo "DOMAIN arg1 is not set" >&2
  exit 1
fi
if [ -z "$2" ] ;  then
  echo "INFRAMAIL arg2 is not set" >&2
  exit 1
fi
if [ -z "$3" ] ;  then
  echo "HOMEFOLDER arg3 is not set" >&2
  exit 1
fi
if [ -z "$4" ] ;  then
  echo "SLACKHOOK arg4 is not set" >&2
  exit 1
fi

# ENV VARS OUTPUT
echo "domain: ${1}"
echo "inframail: ${2}"
echo "home: ${3}"
echo "slack: ${4}"

# BOOTSTAP SERVICE
wget https://gist.githubusercontent.com/timkock/213e2cc00ebe40edeabddd7a5bea9467/raw/8ea8d61761c4f5475217dcd95b65e286afe118cb/bootstrap.sh
sed -ie "s#{{DOMAIN}}#${1}#g" bootstrap.sh
sed -ie "s#{{INFRAMAIL}}#${2}#g" bootstrap.sh
sed -ie "s#{{HOMEFOLDER}}#${3}#g" bootstrap.sh
sed -ie "s#{{SLACKHOOK}}#${4}#g" bootstrap.sh
rm *.she
sudo mv /etc/rc.local $3/rc.backup
sudo mv bootstrap.sh /etc/rc.local
sudo chmod +x /etc/rc.local
sudo sh /etc/rc.local

You might notice initializing the GPU(s) takes a lot of time (or seems slow). This is because without the x window system for every call the set of resources required for executing GPU calls needs to be reloaded.

sudo nvidia-smi -pm 1 fixes this