Last active
March 29, 2016 20:16
-
-
Save camilosampedro/b15bce5c3ef019b75a0b to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Verifies if Java is installed, and if it's not installed, install it. | |
function isJavaInstalled { | |
echo -ne "=> Verifying Java installation" | |
# Trying to execute java | |
java -version | |
# Getting the exit code of the execution. | |
EXITCODE="$?" | |
echo -ne "... " | |
# If the code is different from zero, it is not installed | |
if [ "$EXITCODE" != "0" ]; | |
then | |
echo -ne "Java is not installed :(.\n" | |
# Proceed to install java | |
installJava | |
else | |
echo -ne "Java is installed :).\n" | |
fi | |
} | |
# Java Ubuntu install | |
function installJava { | |
# Clone my another gist with Java installation for Ubuntu | |
git clone https://gist.github.com/7128552e33c6c4c6ab51.git | |
cd 7128552e33c6c4c6ab51 || exit | |
# Give it permissions and execute it | |
chmod +x Install\ Java.sh | |
./Install\ Java.sh | |
} | |
# Installs Hadoop | |
function installHadoop { | |
# Stand in /opt/ directory | |
cd /opt/ || exit | |
echo -ne "(1/4) => Downloading Hadoop" | |
sudo wget http://www.eu.apache.org/dist/hadoop/common/current/hadoop-2.7.1.tar.gz | |
echo -ne "... :) \n" | |
echo -ne "=> Decompressing Hadoop..." | |
sudo tar -xvzf hadoop-2.7.1.tar.gz | |
echo -ne "... :) \n" | |
echo -ne "=> Creating a symbolic link to hadoop" | |
sudo ln -s hadoop-2.7.1 hadoop | |
echo -ne "... :) \n" | |
echo -ne "=> Adding permissions..." | |
sudo chmod -R +rx hadoop | |
echo -ne "... :) \n" | |
} | |
# Create Hadoop user | |
function createUser { | |
echo -ne "=> Creating Hadoop user..." | |
sudo useradd -d /home/hadoop -m hadoop | |
echo -ne "... :) \n" | |
echo "Enter the new password for the Hadoop user. (It will have sudo's permissions)" | |
sudo passwd hadoop | |
usermod -a -G sudo hadoop | |
usermod -s /bin/bash hadoop | |
echo "Log with the new Hadoop password here. It's now adding the key and testing the hadoop user." | |
sudo -u hadoop exit | |
echo -ne "=> Adding environment variables to ~/.bashrc" | |
{ | |
echo "export HADOOP_HOME=/usr/local/hadoop" | |
echo "export PATH=\$PATH:\$HADOOP_HOME/bin" | |
echo "export PATH=\$PATH:\$HADOOP_HOME/sbin" | |
echo "export HADOOP_MAPRED_HOME=\${HADOOP_HOME}" | |
echo "export HADOOP_COMMON_HOME=\${HADOOP_HOME}" | |
echo "export HADOOP_HDFS_HOME=\${HADOOP_HOME}" | |
echo "export YARN_HOME=\${HADOOP_HOME}" | |
} >> /home/hadoop/.bashrc | |
echo -ne "... :) \n" | |
} | |
# Adds network settings for Hadoop | |
function networkSettings { | |
echo "=> Installing ssh..." | |
sudo apt-get install -y ssh | |
echo "=> Generating public key for hadoop user" | |
sudo -u hadoop ssh-keygen -t rsa -f ~/.ssh/id_rsa | |
echo "=> Adding localhost public key to authorized keys" | |
cat ~/.ssh/id_rsa.pub | sudo -u hadoop tee -a ~/.ssh/authorized_keys > /dev/null | |
echo "=> Updating ssh files permissions..." | |
sudo -u hadoop sudo chmod go-w "$HOME" "$HOME/.ssh" | |
sudo -u hadoop chmod 600 "$HOME"/.ssh/authorized_keys | |
sudo -u hadoop chown "$(whoami)" "$HOME/.ssh/authorized_keys" | |
echo "=> Disabling IPv6" | |
{ | |
echo "net.ipv6.conf.all.disable_ipv6 = 1" | |
echo "net.ipv6.conf.default.disable_ipv6 = 1" | |
echo "net.ipv6.conf.lo.disable_ipv6 = 1" | |
} >> /etc/sysctl.conf | |
} | |
# Edit Hadoop single host configuration | |
function editMainConfig { | |
echo "Editing the config files under /opt/hadoop/etc/hadoop/" | |
cd /opt/hadoop/etc/hadoop/ || exit | |
echo -ne "[1/] core-site.xml... " | |
{ | |
echo "<?xml version=\"1.0\"?>" | |
echo "<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>" | |
echo "<configuration>" | |
echo " <property>" | |
echo " <name>fs.default.name</name>" | |
echo " <value>hdfs://localhost:8020</value>" | |
echo " <description>Nombre del filesystem por defecto.</description>" | |
echo " </property>" | |
echo "</configuration>" | |
} > core-site.xml | |
echo -ne " OK\n" | |
echo -ne "[2/] hdfs-site.xml... " | |
{ | |
echo "<?xml version=\"1.0\"?>" | |
echo "<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>" | |
echo "<configuration>" | |
echo " <property>" | |
echo " <name>dfs.namenode.name.dir</name>" | |
echo " <value>file:/home/hadoop/workspace/dfs/name</value>" | |
echo " <description>Path del filesystem donde el namenode almacenará los metadatos.</description>" | |
echo " </property>" | |
echo " <property>" | |
echo " <name>dfs.datanode.data.dir</name>" | |
echo " <value>file:/home/hadoop/workspace/dfs/data</value>" | |
echo " <description>Path del filesystem donde el datanode almacenará los bloques.</description>" | |
echo " </property>" | |
echo " <property>" | |
echo " <name>dfs.replication</name>" | |
echo " <value>1</value>" | |
echo " <description>Factor de replicación. Lo ponemos a 1 porque sólo tenemos 1 máquina.</description>" | |
echo " </property>" | |
echo "</configuration>" | |
} > hdfs-site.xml | |
echo -ne "OK\n" | |
echo -ne "[3/] Creating hdfs directories /home/hadoop/workspace/dfs/name... " | |
sudo -u hadoop mkdir -p /home/hadoop/workspace/dfs/name | |
sudo -u hadoop mkdir -p /home/hadoop/workspace/dfs/data | |
echo -ne "OK \n" | |
echo -ne "[4/] Copying the mapred-site.xml template file... " | |
sudo cp mapred-site.xml.template mapred-site.xml | |
echo -ne "OK \n" | |
{ | |
echo "<configuration>" | |
echo "<property>" | |
echo " <name>yarn.nodemanager.aux-services</name>" | |
echo " <value>mapreduce_shuffle</value>" | |
echo "</property>" | |
echo "<property>" | |
echo " <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>" | |
echo " <value>org.apache.hadoop.mapred.ShuffleHandler</value>" | |
echo "</property>" | |
echo "</configuration>" | |
} >> yarn-site.xml | |
} | |
# Verify if Java is installed | |
isJavaInstalled | |
# Install Hadoop | |
installHadoop | |
sudo su | |
# User creation | |
createUser | |
# Network settings | |
networkSettings | |
# Config files | |
editMainConfig | |
# Format hdfs | |
sudo hdfs namenode -format | |
echo "Installation complete, after you log in with hadoop user, run Hadoop using:" | |
echo "start-dfs.sh & star-yarn.sh" | |
echo "" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment