config

Host hadoop_name
    HostName 34.207.74.173
    User ubuntu
    IdentityFile ~/.ssh/hadoopkey.pem

Host hadoop_data1
    HostName 54.89.92.59
    User ubuntu
    IdentityFile ~/.ssh/hadoopkey.pem

Host hadoop_data2
    HostName 54.84.228.100
    User ubuntu
    IdentityFile ~/.ssh/hadoopkey.pem

Host hadoop_data3
    HostName 54.210.73.141
    User ubuntu
    IdentityFile ~/.ssh/hadoopkey.pem

hadoop_env

export HADOOP_CONF=$HOME/hadoop/etc/hadoop
export HADOOP_PREFIX=$HOME/hadoop

export JAVA_HOME=/usr/lib/jvm/java-8-openjdk/amd64

export PATH=$PATH:$HADOOP_PREFIX/bin:$HOME/bin

provision-1.sh

#!/usr/bin/env bash

set -e

sudo apt update
sudo rm /boot/grub/menu.lst
sudo update-grub-legacy-ec2 -y
sudo apt -yqq dist-upgrade
sudo reboot

provision-2.sh

#!/usr/bin/env bash

set -e

wget -qO- https://gist.githubusercontent.com/mammothbane/ef6b3e63715dcb0e59afc9922a4eca04/raw/9c4ae0c11bfe95ef35d855ab62f6a1cd0e8e5179/hadoop_env > $HOME/.hadoop_env
echo 'source $HOME/.hadoop_env' >> $HOME/.bashrc
source $HOME/.bashrc

sudo apt install -yqq emacs-nox openjdk-8-jdk-headless wget curl python3 python3-pip
wget -qO- https://github.com/sbt/sbt/releases/download/v1.0.1/sbt-1.0.1.tgz | tar xvz -C $HOME
mkdir -p $HOME/bin
ln -s $HOME/sbt/bin/sbt $HOME/bin/sbt

mkdir tmp
pushd tmp
sbt update
popd
rm -rf tmp

mkdir -p $HOME/hadoop
wget -qO- http://apache.cs.utah.edu/hadoop/common/hadoop-2.7.4/hadoop-2.7.4.tar.gz | tar xvz -C $HOME/hadoop --strip-components=1