Skip to content

Instantly share code, notes, and snippets.

@llk23r
Last active April 21, 2022 11:18
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 2 You must be signed in to fork a gist
  • Save llk23r/2d8c97d10410eff363c9828fb6e9c566 to your computer and use it in GitHub Desktop.
Save llk23r/2d8c97d10410eff363c9828fb6e9c566 to your computer and use it in GitHub Desktop.
#!/usr/bin/env bash
sudo apt-get update
clear
echo "--------------UPDATED--------"
sudo apt-get install default-jdk
clear
java -version
echo "-------------JAVA INSTALLED----"
clear
sudo addgroup hadoop
echo "============HADOOP GROUP ADDED========"
clear
sudo adduser --ingroup hadoop hduser
echo "===========HDUSER USER ADDED=========="
clear
sudo apt-get install ssh
which ssh
which sshd
echo "=============SSH INSTALLED=========="
clear
su hduser
echo "==============INSIDE HDUSER=========="
ssh-keygen -t rsa -P ""
echo "===============KEY GENERATED SUCCESSFULLY=============="
cat $HOME/.ssh/id_rsa.pub >> $HOME/.ssh/authorized_keys
ssh localhost
echo "===================DOWNLOADING HADOOP================"
wget http://www-eu.apache.org/dist/hadoop/common/hadoop-2.8.1/hadoop-2.8.1.tar.gz
clear
echo "==================DOWNLOADED SUCCESSFULLY=============="
clear
echo "====================UNtar================"
tar -xvzf hadoop-2.8.1.tar.gz
clear
echo "=====untar successful=============="
cd /
exit
exit
sudo adduser hduser sudo
su hduser
sudo mkdir /usr/local/hadoop
sudo cp -R $HOME/Downloads/hadoop-2.8.1/* /usr/local/hadoop
sudo chown -R hduser:hadoop /usr/local/hadoop
echo "===================writing to bashrc========"
sudo cat<<EOF >>~/.bashrc
#HADOOP VARIABLES START
export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
export HADOOP_INSTALL=/usr/local/hadoop
export PATH=$PATH:$HADOOP_INSTALL/bin
export PATH=$PATH:$HADOOP_INSTALL/sbin
export HADOOP_MAPRED_HOME=$HADOOP_INSTALL
export HADOOP_COMMON_HOME=$HADOOP_INSTALL
export HADOOP_HDFS_HOME=$HADOOP_INSTALL
export YARN_HOME=$HADOOP_INSTALL
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_INSTALL/lib/native
export HADOOP_OPTS="-Djava.library.path=$HADOOP_INSTALL/lib"
#HADOOP VARIABLES END
EOF
sudo cat<<EOF >>/usr/local/hadoop/etc/hadoop/hadoop-env.sh
export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
EOF
sudo mkdir -p /app/hadoop/tmp
sudo chown hduser:hadoop /app/hadoop/tmp
sudo cat<<EOF >>/usr/local/hadoop/etc/hadoop/core-site.xml
<configuration>
<property>
<name>hadoop.tmp.dir</name>
<value>/app/hadoop/tmp</value>
<description>A base for other temporary directories.</description>
</property>
<property>
<name>fs.default.name</name>
<value>hdfs://localhost:54310</value>
<description>The name of the default file system. A URI whose scheme and authority determine the FileSystem implementation. The
uri's scheme determines the config property (fs.SCHEME.impl) naming
the FileSystem implementation class. The uri's authority is used to
determine the host, port, etc. for a filesystem.</description>
</property>
</configuration>
EOF
sudo cp /usr/local/hadoop/etc/hadoop/mapred-site.xml.template /usr/local/hadoop/etc/hadoop/mapred-site.xml
cat<<EOF >>/usr/local/hadoop/etc/hadoop/mapred-site.xml
<property>
<name>mapred.job.tracker</name>
<value>localhost:54311</value>
<description>The host and port that the MapReduce job tracker runs
at. If "local", then jobs are run in-process as a single map
and reduce task.
</description>
</property>
EOF
sudo mkdir -p /usr/local/hadoop_store/hdfs/namenode
sudo mkdir -p /usr/local/hadoop_store/hdfs/datanode
sudo chown -R hduser:hadoop /usr/local/hadoop_store
cat<<EOF >>/usr/local/hadoop/etc/hadoop/hdfs-site.xml
<property>
<name>dfs.replication</name>
<value>1</value>
<description>Default block replication.
The actual number of replications can be specified when the file is created.
The default is used if replication is not specified in create time.
</description>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/usr/local/hadoop_store/hdfs/namenode</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/usr/local/hadoop_store/hdfs/datanode</value>
</property>
EOF
hadoop namenode -format
start-dfs.sh
jps
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment