g2-ml keras spot instance init: cpu and gpu

README.md

These are a few scripts that help me install requirements on fresh AWS EC2 instances for deep learning

init-ami-e2dc379a.sh

#!/bin/sh
# ami-e2dc379a  is an AMI on aws with keras, tensorflow, ... pre-installed
# I started using it recently instead of my own custom AMI
# After launching it, just run the below to set it up as per my needs
# ssh file.pem user@ip "wget https://gist.github.com/shadiakiki1986/0c9ea999113691fb9a7ae64e3541fe29/raw/init-ami-e2dc379a.sh -O - | /bin/sh"
##########################

# run my init-cpu.sh script
sudo dpkg --configure -a # it seems this AMI had dpkg interrupted at some point
wget https://gist.github.com/shadiakiki1986/0c9ea999113691fb9a7ae64e3541fe29/raw/init-cpu.sh -O - | /bin/sh

# for gpu
pew in G2ML pip install tensorflow-gpu

# other pip
pew in G2ML pip install -U Keras
pew in G2ML pip install hyperas networkx==1.11 statsmodels

# reload nvidia drivers for GPU
# https://stackoverflow.com/a/45319156/4126114
nvidia-smi > /dev/null
if [ $? -ne 0 ]; do
  sudo rmmod nvidia_drm
  sudo rmmod nvidia_modeset
  sudo rmmod nvidia_uvm
  sudo rmmod nvidia
  nvidia-smi # should output GPU
  if [ $? -ne 0 ]; do
    echo "failed to load gpu"
    exit 1
  done
done

# check GPU ok .. should display GPU alongside CPU
pew in G2ML python -c "from tensorflow.python.client import device_lib; print(device_lib.list_local_devices())"

init-cpu.sh

#!/bin/sh
# Use this when launching a new server, e.g. AWS spot instance
# ssh file.pem user@ip "wget https://gist.github.com/shadiakiki1986/0c9ea999113691fb9a7ae64e3541fe29/raw/init-cpu.sh -O - | /bin/sh"
#
# If launching a GPU instance, check also init-gpu.sh
# Note: permalink to gist from
# https://gist.github.com/atenni/5604615

set +x
set -e

# commenting out this section since pip3 already available usually
export DEBIAN_FRONTEND=noninteractive
# sudo apt-get -qq -y update
# sudo apt-get -qq -y install python3 python3-pip

sudo pip3 install -U pip
sudo pip3 install pew

# If using tensorboard, will probably want to wait
# until https://github.com/fchollet/keras/pull/7566/files/f2b66a02067cd5a0bc7291231c5fe59f355ff2ad#r134733445
# and use keras 2.0.6 and not 2.0.7
pew new \
  --python=python3 \
  -d \
  -i sklearn \
  -i pandas \
  -i numpy \
  -i scipy \
  -i matplotlib \
  -i Keras \
  -i tensorflow \
  -i h5py \
  -i slackclient \
  -i scikit-image \
  -i jupyter \
  -i jupyterlab \
  -i tensorboard \
  -i hyperas \
  -i networkx==1.11 \
  -i statsmodels \
  -i tables \
  G2ML

echo "PS1='# '">>~/.bashrc

mkdir -p ~/.jupyter/
echo "c.NotebookApp.password = u'sha1:8e63b3cd2b5e:5faf64fbd48ae73d6b14488f181d727326e573b4'" > ~/.jupyter/jupyter_notebook_config.py

init-gpu.sh

#!/bin/sh
# Run this AFTER init-cpu.sh
# Notes
# - apt-get ... nvidia-*
#    - https://stackoverflow.com/a/45725925/4126114
#    - slowest command, takes 2-3 mins
#    - requires `export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/`
#      - https://stackoverflow.com/a/43568508/4126114
#    - removed `nvidia-375 nvidia-375-dev` because tensorflow 1.3.0 asks for libcusolver.8.0.so
#      - https://stackoverflow.com/a/44993396/4126114
# - tensorflow-gpu
# - wget ...nvidia.com...
#
# Installing CUDA8 required 2GB free space, and the default EC2 /dev/xvda has only 8GB
# Pay attention to increasing this beforehand
# Ref: https://stackoverflow.com/a/44993396/4126114
#--------------------------------------------------

set -e

# install nvidia driver
wget http://us.download.nvidia.com/XFree86/Linux-x86_64/384.66/NVIDIA-Linux-x86_64-384.66.run
sudo /bin/sh NVIDIA-Linux-x86_64-384.66.run

# verify nvidia driver installed ok
nvidia-smi -q|head

# install cuda 8
# https://developer.nvidia.com/cuda-downloads
# Requires downloading 2 GB
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/cuda-repo-ubuntu1604_8.0.61-1_amd64.deb
sudo dpkg -i cuda-repo-ubuntu1604_8.0.61-1_amd64.deb
export DEBIAN_FRONTEND=noninteractive
sudo apt-get -qq -y update
sudo apt-get -qq -y install cuda

# install nvidia cudnn 6 (works with tensorflow-gpu 1.3.0)
# https://developer.nvidia.com/cudnn
# https://developer.nvidia.com/rdp/cudnn-download
# Log in / Download / Download cuDNN v6.0 (April 27, 2017), for CUDA 8.0 / cuDNN v6.0 Runtime Library for Ubuntu16.04 (Deb)
#
# WILL NEED TO DOWNLOAD THIS MANUALLY EVERY TIME BECAUSE OF THE TOKEN IN THE URL
# Example cudnn url below: http://developer2.download.nvidia.com/compute/machine-learning/cudnn/secure/v6/prod/8.0_20170427/Ubuntu16_04-x64/libcudnn6_6.0.21-1%2Bcuda8.0_amd64.deb?FJ-9yn3c0yI6peaW7lzlH4amGhYcJoDq1yvNaw8FW5M0n9s1uZwxvlkfQ7roCvVQJZ0TbTZPRFJOuv6cXTWG5QmrGON5SVGtBRlU08suwHCmcvQSmpLo9TFcy9ALmSwe9Y-UGbhPWaUMywCiEebE3IXvoPKsmtaofUaqZ9pRIwdmb3FXD-BIS4Xm1_CGU6oOQSZMPJS8Ng3B1bgi-KCeWJKJ-f2mn_DxLMPyCzbrMOo
read -p "cudnn url: " CUDNN_URL
wget \
  $CUDNN_URL \
  -O libcudnn.deb
sudo dpkg -i libcudnn.deb
  
# install tensorflow-gpu
pew in G2ML pip install tensorflow-gpu

# check no trouble with finding libraries
pew in G2ML python -c "import tensorflow"

# check tensorflow can see gpu
# https://stackoverflow.com/a/44547144/4126114
pew in G2ML python -c "from tensorflow.python.client import device_lib; print(device_lib.list_local_devices())"