Instalation routine for Apache Spark in an Ubuntu Machine

spark_install.sh

#!/bin/sh

#################################################################
### Instalation routine for Apache Spark in an Ubuntu Machine ###
### Spark 1.5.1 / Scala 2.11.1 / Java 8 / Maven 3.3.3         ###
### Modified Routine for ATLAS Install                        ###
### Ricardo B. Lourenco - RDCEP.org                           ###
################################################################# 

#Setup for folder destination (in this case, /mnt/atlas)
sudo mkdir /mnt/atlas
sudo chmod -r 777 /mnt/atlas
cd /mnt/atlas
 
# installation of Oracle Java JDK.
sudo apt-get -y update
sudo apt-get -y install python-software-properties
sudo add-apt-repository -y ppa:webupd8team/java
sudo apt-get -y update
sudo apt-get -y install oracle-java8-installer

# Installation of commonly used python scipy tools // added gdal; pip
sudo apt-get -y install python-numpy python-scipy python-matplotlib ipython ipython-notebook python-pandas python-sympy python-nose python-gdal python-pip python-dev build-essential maven
sudo apt-get -y remove maven # Maven installed and removed just to fulfill dependencies for maven 3.3.3 install

# Installation of scala
cd /mnt/atlas
wget http://www.scala-lang.org/files/archive/scala-2.11.1.deb
sudo dpkg -i scala-2.11.1.deb
sudo apt-get -y update
sudo apt-get -y install scala

# Sbt Installation
cd /mnt/atlas
echo "deb https://dl.bintray.com/sbt/debian /" | sudo tee -a /etc/apt/sources.list.d/sbt.list
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 642AC823
sudo apt-get update
sudo apt-get install sbt

# Installation of Maven 3.3.3 (Not supported on Ubuntu repo)
cd /mnt/atlas
wget http://mirrors.sonic.net/apache/maven/maven-3/3.3.3/binaries/apache-maven-3.3.3-bin.tar.gz
tar -zxf apache-maven-3.3.3-bin.tar.gz
sudo cp -R apache-maven-3.3.3 /usr/local
sudo ln -s /usr/local/apache-maven-3.3.3/bin/mvn /usr/bin/mvn

# Downloading spark
cd /mnt/atlas
wget http://d3kbcqa49mib13.cloudfront.net/spark-1.5.1.tgz
tar -xf spark-1.5.1.tgz

# Building spark
cd /mnt/atlas/spark-1.5.1
build/mvn -Pyarn -Phadoop-2.4 -Dhadoop.version=2.4.0 -DskipTests clean package

# Clean-up
cd /mnt/atlas
sudo chmod -R 777 /mnt/atlas/.
rm scala-2.11.1.deb
rm apache-maven-3.3.3-bin.tar.gz
sudo rm -R apache-maven-3.3.3
rm spark-1.5.1.tgz
sudo apt-get -y autoremove