Spark GPU Demo

# Prepare Steps:
# We are assuming we’re using CUDA 7.5
# Clone and Build latest Mesos with GPUs (skipping instructions here…)
# CUDA prepare script for ubuntu 14.04 with AWS G2 instance. 
# http://tleyden.github.io/blog/2015/11/22/cuda-7-dot-5-on-aws-gpu-instance-running-ubuntu-14-dot-04/
# Start mesos master and slave
# ./mesos-master --work_dir=/tmp/mesos
# sudo GLOG_v=1 ./mesos-slave --isolation=gpu/nvidia --master=`hostname -i`:5050 --nvidia_gpu_devices=0

# Add cuda bin to PATH
export PATH=$PATH:/usr/local/cuda/bin

## Install cudnn
wget http://developer.download.nvidia.com/compute/redist/cudnn/v4/cudnn-7.0-linux-x64-v4.0-prod.tgz
tar xvf cudnn-7.0-linux-x64-v4.0-prod.tgz
cd cuda; sudo cp lib64/* /usr/local/cuda/lib64/; sudo cp include/* /usr/local/cuda/include; sudo ldconfig; cd ..

# Install Tensorflow
## Install pip
sudo apt-get install python-pip python-dev
## Install tensorflow pip package
sudo pip install --upgrade https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.8.0-cp27-none-linux_x86_64.whl
## Test tensorflow installation
python -c "import tensorflow as tf; hello = tf.constant('Hello, TensorFlow'); sess = tf.Session(); print(sess.run(hello))"

# Build Spark GPU code
git clone https://github.com/tnachen/spark
cd spark; git checkout gpu_mesos; ./dev/make-distribution.sh --name custom_spark -Phadoop-2.4 
## Setup configuration
cp conf/spark-default.conf.template conf/spark-default.conf
cp conf/spark-env.sh.template conf/spark-env.sh
echo "MESOS_NATIVE_JAVA_LIBRARY=/usr/local/lib/libmesos.so" >> conf/spark-env.sh
echo "LD_LIBRARY_PATH=/usr/local/cuda/lib64/" >> conf/spark-env.sh
echo "spark.executorEnv.LD_LIBRARY_PATH=/usr/local/cuda/lib64/" >> conf/spark-defaults.conf
# Turn this off if don't want to use GPU
echo "spark.mesos.gpu.enabled=true" >> conf/spark-defaults.conf

# Download demo script
wget -o demo.py https://gist.githubusercontent.com/tnachen/e004539cdd38e5941d0cb712d0ad3b91/raw/4f1df9f7674c6d9e29214ddaa2eefe2e9a094776/Tensorflow%2520spark%2520demo

# Run tensorflow spark demo
./bin/pyspark --master mesos://`hostname -i`:5050
# Run execfile("demo.py")
# Run local_labelled_images