Automatically set up a Hadoop cluster provisioned through Axemblr Provisionr and configured through Rundeck.

provisionr-rundeck-config.sh

#!/bin/bash
if [ $# -ne 2 ]; then
	echo "The script must take two arguments: namenode internal hostname and current machine internal hostname."
	exit 0
fi

# JAVA_HOME is not set by default
sudo sh -c "echo 'export JAVA_HOME=/usr/lib/jvm/java-6-openjdk' >> /etc/profile"
sudo ln -s /usr/lib/jvm/java-6-openjdk /usr/lib/jvm/java-openjdk # needed because bigtop-utils do not recognize the prev path
source /etc/profile

# set up the configuration folder for our cluster on each node
sudo cp -r /etc/hadoop/conf.empty /etc/hadoop/conf.axemblr
sudo update-alternatives --install /etc/hadoop/conf hadoop-conf /etc/hadoop/conf.axemblr 50
sudo update-alternatives --set hadoop-conf /etc/hadoop/conf.axemblr

# setup the core-site.xml config with the hostname of the namenode
cat /etc/hadoop/conf.axemblr/core-site.xml | \
xmlstarlet ed -s /configuration -t elem -n property -v "" | \
xmlstarlet ed -s /configuration/property -t elem -n name -v "fs.defaultFS" | \
xmlstarlet ed -s /configuration/property -t elem -n value -v "hdfs://$1/" > tmp.xml
sudo mv ./tmp.xml /etc/hadoop/conf.axemblr/core-site.xml

if [ "$1" == "$2" ]; then
	# only for the namenode
	cat /etc/hadoop/conf.axemblr/hdfs-site.xml | \
	xmlstarlet ed -d /configuration/property | \
	xmlstarlet ed -s /configuration -t elem -n property -v "" | \
	xmlstarlet ed -s /configuration/property -t elem -n name -v "dfs.namenode.name.dir" | \
	xmlstarlet ed -s /configuration/property -t elem -n value -v "/mnt/dfs/nn" > tmp.xml
	sudo mv ./tmp.xml /etc/hadoop/conf.axemblr/hdfs-site.xml
	sudo mkdir -p /mnt/dfs/nn
	sudo chown -R hdfs /mnt/dfs/nn
	sudo chmod go-rx /mnt/dfs/nn
	yes Y | sudo -i -u hdfs hadoop namenode -format
	sudo -i service hadoop-hdfs-namenode start
else
	# for the datanodes
	cat /etc/hadoop/conf.axemblr/hdfs-site.xml | \
	xmlstarlet ed -d /configuration/property | \
	xmlstarlet ed -s /configuration -t elem -n property -v "" | \
	xmlstarlet ed -s /configuration/property -t elem -n name -v "dfs.datanode.data.dir" | \
	xmlstarlet ed -s /configuration/property -t elem -n value -v "/mnt/dn" > tmp.xml
	sudo mv ./tmp.xml /etc/hadoop/conf.axemblr/hdfs-site.xml
	# allow access to localhost via ssh
	echo -e "\n\n\n" | ssh-keygen -t rsa
	cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
	sudo mkdir -p /mnt/dn
	sudo chown -R hdfs /mnt/dn
	sudo chmod go-rx /mnt/dn
	sudo -i service hadoop-hdfs-datanode start
fi