EC2 g2.xlarge with NVIDIA GRID Environment Setup

environment_setup.sh

#!/bin/bash

# --------------------------------------------------------------------------------------
# Preparation
# To execute this script, please follow:

# wget https://gist.github.com/sinhrks/ba9876e7aa2a18a601f8/raw/environment_setup.sh
# chmod +x environment_setup.sh
# ./environment_setup.sh
# --------------------------------------------------------------------------------------

function usage() {
  echo "Usage: $0"
  echo ""
  echo "This is pylearn2 environment setup script for:"
  echo "- Amazon Linux AMI"
  echo "        or" 
  echo "- Amazon Linux AMI with NVIDIA GRID GPU Driver"
  echo "http://goo.gl/3KeXXW"
  echo ""
  exit 1
}

if [ "$OPTIND" = 1 ]; then
  while getopts abf:h OPT
  do
    case $OPT in
      h)
        usage
        ;;
      \?)
        usage
        ;;
    esac
  done
else
  echo "No installed getopts-command." 1>&2
  exit 1
fi

mkdir scripts
cd scripts
wget https://gist.github.com/sinhrks/ba9876e7aa2a18a601f8/raw/install_common.sh
chmod +x install_common.sh
./install_common.sh 

# additionals
sudo python -m pip install pystan
sudo python -m pip install sympy
sudo python -m pip install milk

# Theano configurations derived from:
# http://www.kurtsp.com/deep-learning-in-python-with-pylearn2-and-amazon-ec2.html
# http://r9y9.github.io/blog/2014/07/20/pylearn2-on-ec2-g2-2xlarge/

# theano
python -c 'import theano' 2>/dev/null || {
  # sudo pip install theano
  git clone git://github.com/Theano/Theano.git
  cd Theano
  sudo python setup.py develop
  cd ..
  
echo '[global]
floatX = float32
device = gpu0

[nvcc]
fastmath = True' > .theanorc
}

# DeepLearningTutorials
# http://deeplearning.net/tutorial/contents.html
if [ ! -e DeepLearningTutorials ]; then 
  git clone git://github.com/lisa-lab/DeepLearningTutorials.git
  cd DeepLearningTutorials/data
  ./download.sh &
  cd
fi

# Reload
source .bashrc

# chainer
cd 
sudo yum-config-manager --enable epel
git clone https://github.com/pfnet/chainer.git
cd chainer
sudo CUDA_PATH=/opt/nvidia/cuda python setup.py install chainer
cd

# Check Environment
wget https://gist.github.com/sinhrks/ba9876e7aa2a18a601f8/raw/test_environment.py
chmod +x test_environment.py
nosetests test_environment.py

install_common.sh

#!/bin/bash

sudo yum update -y
sudo yum -y install make automake gcc gcc-c++ git
sudo yum -y install "kernel-devel-uname-r == $(uname -r)" 

sudo yum -y install gcc48-gfortran.x86_64
sudo yum -y install cmake boost-devel.x86_64 gcc48-gfortran.x86_64

sudo yum -y install java-1.7.0-openjdk-devel.x86_64
sudo yum -y install gcc-c++ curl make cmake zlib zlib-devel 

# link to math/image modules
sudo yum -y install blas-devel lapack-devel atlas-devel
sudo yum -y install libpng-devel libjpeg-devel freetype-devel

sudo yum install -y hdf5-devel

sudo python -m pip install --upgrade awscli

# install conda
wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
bash -b -p Miniconda3-latest-Linux-x86_64.sh
export PATH="~/miniconda/bin:$PATH"
# PATH を bashrc に追加するかどうかで yes にする 

source .bashrc

# configure conda
conda config --set always_yes yes --set changeps1 no
conda update -q conda

conda install cython numpy pillow scipy matplotlib pandas nose
conda install numexpr bottleneck pytables
conda install blaze dask bokeh
sudo yum -y install graphviz
python -m pip install graphviz
# statistics / machine learning
conda install scikit-learn scikit-image statsmodels
# NLP
# conda install nltk gensim

# Jupyter
conda install jupyter 
jupyter notebook --generate-config
echo "c.IPKernelApp.pylab = 'inline'" >> ~/.jupyter/jupyter_notebook_config.py
echo "c.NotebookApp.ip = '*'" >> ~/.jupyter/jupyter_notebook_config.py 
echo "c.NotebookApp.open_browser = False" >> ~/.jupyter/jupyter_notebook_config.py
echo "c.NotebookApp.port = 9999" >> ~/.jupyter/jupyter_notebook_config.py
jupyter notebook --profile=nbserver &
# access to http://<address>:9999/

conda install -c r r
conda install -c https://conda.anaconda.org/r rpy2

# matplotlib フォント
# vi /home/ec2-user/miniconda/lib/python2.7/site-packages/matplotlib/mpl-data/matplotlibrc
# font.family に IPAexGothic
wget http://dl.ipafont.ipa.go.jp/IPAexfont/IPAexfont00301.zip
unzip IPAexfont00301.zip
rm IPAexfont00301.zip
mv IPAexfont00301/ipaexg.ttf ~/miniconda/lib/python2.7/site-packages/matplotlib/mpl-data/fonts/ttf
mv IPAexfont00301/ipaexm.ttf ~/miniconda/lib/python2.7/site-packages/matplotlib/mpl-data/fonts/ttf
rm -rf IPAexfont00301

rm -rf ~/.cache

install_cudnn.sh

tar zxvf cudnn-6.5-linux-x64-v2.tgz
sudo cp lib* /opt/nvidia/cuda/lib64
sudo cp cudnn.h /opt/nvidia/cuda/include
ls /opt/nvidia/cuda/include

install_rdevel.sh

sudo yum update -y
sudo yum -y install "kernel-devel-uname-r == $(uname -r)" 
sudo yum -y install make automake gcc gcc-c++ cmake gcc48-gfortran.x86_64 git
sudo yum -y install blas-devel lapack-devel atlas-devel

sudo yum -y install curl readline-devel libcurl-devel libxml2-devel openssl-devel bzip2-devel pcre-devel
sudo yum -y install libXt-devel libX11-devel mesa-libGL-devel mesa-libGLU-devel
sudo yum -y install cairo cairo-devel
sudo yum -y install texlive texinfo

# PCRE
wget ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-8.38.tar.gz
tar zxf pcre-8.38.tar.gz
cd pcre-8.38
./configure
make
sudo make install
cd ../
rm pcre-8.38.tar.gz
rm -rf pcre-8.38

# https://stat.ethz.ch/R/daily/
wget https://stat.ethz.ch/R/daily/R-devel.tar.gz
tar zxf R-devel.tar.gz
cd R-devel
./configure 
make
sudo make install
cd

Rscript -e "install.packages('devtools', repos='http://cran.ism.ac.jp/')"
Rscript -e "install.packages(c('RcppEigen', 'rgl'), repos='http://cran.ism.ac.jp/')"
Rscript -e "install.packages(c('lfda', 'forecast'), repos='http://cran.ism.ac.jp/')"
Rscript -e "install.packages('ggfortify', repos='http://cran.ism.ac.jp/', dependencies=c('Depends', 'Imports', 'Suggests'))"

# R CMD check --as-cran 

# .Renviron
# _R_CHECK_FORCE_SUGGESTS_=false

install_spark.sh

mkdir scripts
cd scripts
wget https://gist.github.com/sinhrks/ba9876e7aa2a18a601f8/raw/install_common.sh
chmod +x install_common.sh
./install_common.sh 

sudo python -m pip install pandas
cd 

##########################
# Hadoop
##########################

cd
cd scripts

wget http://ftp.riken.jp/net/apache/hadoop/common/hadoop-2.6.0/hadoop-2.6.0.tar.gz
tar -xzvf hadoop-2.6.0.tar.gz
sudo mv hadoop-2.6.0 /usr/local/

echo "export HADOOP_HOME=/usr/local/hadoop-2.6.0" >> ~/.bashrc
# export を都度反映するために更新
source ~/.bashrc 
echo "export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native" >> ~/.bashrc
echo "export HADOOP_OPTS='-Djava.library.path=$HADOOP_HOME/lib/native'" >> ~/.bashrc
source ~/.bashrc
echo "export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$JAVA_HOME/bin:$PATH" >> ~/.bashrc
source ~/.bashrc

# sshでlocalhostにパスなしでログインができるよう設定
cd
ssh-keygen -t dsa # require input
cat .ssh/id_dsa.pub >> .ssh/authorized_keys
chmod 600 .ssh/authorized_keys
ssh localhost

hadoop version

##########################
# etcファイルのプロパティ追加
##########################

cd
cd $HADOOP_HOME/etc/hadoop
vi core-site.xml
    /* 以下を追記
    <property>
      <name>fs.defaultFS</name>
      <value>hdfs://localhost:9000</value>
    </property>
    */
vi hdfs-site.xml
    /* 以下を追記
    <property>
      <name>dfs.replication</name>
      <value>1</value>
    </property>
    */
$ vi yarn-site.xml
    /* 以下を追記
    <property>
      <name>yarn.nodemanager.aux-services</name>
      <value>mapreduce_shuffle</value>
    </property>
    <property>
       <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
       <value>org.apache.hadoop.mapred.ShuffleHandler</value>
    </property>
    */

hadoop namenode -format
start-dfs.sh
start-yarn.sh
jps

##########################
# Hive
##########################

cd
cd scripts
wget ftp://ftp.riken.jp/net/apache/hive/hive-1.2.0/apache-hive-1.2.0-bin.tar.gz
tar -xzvf apache-hive-1.2.0-bin.tar.gz
sudo mv apache-hive-1.2.0-bin /usr/local/hive

echo "export HIVE_HOME=/usr/local/hive" >> ~/.bashrc
source ~/.bashrc
echo "export PATH=$HIVE_HOME/bin:$PATH" >> ~/.bashrc
source ~/.bashrc

##########################
# Spark
##########################

cd
cd scripts
# Note: Scala 2.11 users should download the Spark source package and build with Scala 2.11 support.
wget http://www.scala-lang.org/files/archive/scala-2.10.4.tgz
tar -xzvf scala-2.10.4.tgz
sudo mv scala-2.10.4 /usr/local/scala

# from git
sudo wget http://repos.fedorapeople.org/repos/dchen/apache-maven/epel-apache-maven.repo -O /etc/yum.repos.d/epel-apache-maven.repo
sudo sed -i s/\$releasever/6/g /etc/yum.repos.d/epel-apache-maven.repo
sudo yum install -y apache-maven
mvn --version
cd
git clone git://github.com/apache/spark.git
cd spark
mvn -DskipTests clean package

# or official
cd 
cd scripts
wget http://archive.apache.org/dist/spark/spark-1.4.0/spark-1.4.0-bin-hadoop2.6.tgz
tar -xzvf spark-1.4.0-bin-hadoop2.6.tgz
sudo mv spark-1.4.0-bin-hadoop2.6 /usr/local/spark

echo "export SCALA_HOME=/usr/local/scala" >> ~/.bashrc
echo "export SPARK_HOME=/usr/local/spark" >> ~/.bashrc
source ~/.bashrc
echo "export PATH=$SCALA_HOME/bin:$PATH" >> ~/.bashrc
source ~/.bashrc

cd $SPARK_HOME

pandas_asv.sh

conda create -n asvtest pandas
source activate asvtest

# clone and install asv
git clone git://github.com/spacetelescope/asv.git
cd asv
sudo python setup.py install

cd

# clone pandas
git clone git://github.com/sinhrks/pandas.git
cd pandas/asv_bench

nohup asv continuous --bench sparse.* HEAD^ HEAD
asv compare commit1 commit2