Skip to content

Instantly share code, notes, and snippets.

@hex108
Created November 27, 2014 03:40
Show Gist options
  • Save hex108/efac2433495a1b19098f to your computer and use it in GitHub Desktop.
Save hex108/efac2433495a1b19098f to your computer and use it in GitHub Desktop.
Configure Hadoop so that we could start Hadoop in a local machine. Then it will start Hadoop which includes HDFS and Yarn.
#!/bin/bash
set -u
set -e
if [ $# -ne 1 ]; then
echo "Usage: $0 hadoop_home_directory"
echo " e.g. $0 /home/test/hadoop-2.4.1"
exit -1
fi
HADOOP_HOME=$1
echo "Installing to $HADOOP_HOME"
mkdir -p $HADOOP_HOME/data/hadoop/hdfs/nn
mkdir -p $HADOOP_HOME/data/hadoop/hdfs/snn
mkdir -p $HADOOP_HOME/data/hadoop/hdfs/dn
# configure core-site.xml
echo "
<configuration>
<property>
<name>fs.default.name</name>
<value>hdfs://localhost:9000</value>
</property>
</configuration>" > $HADOOP_HOME/etc/hadoop/core-site.xml
# configure hdfs-site.xml
echo "
<configuration>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:$HADOOP_HOME/data/hadoop/hdfs/nn</value>
</property>
<property>
<name>fs.checkpoint.dir</name>
<value>file:$HADOOP_HOME/data/hadoop/hdfs/snn</value>
</property>
<property>
<name>fs.checkpoint.edits.dir</name>
<value>file:$HADOOP_HOME/data/hadoop/hdfs/snn</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:$HADOOP_HOME/data/hadoop/hdfs/dn</value>
</property>
</configuration>" > $HADOOP_HOME/etc/hadoop/hdfs-site.xml
# configure mapred-site.xml
echo "
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
</configuration>" > $HADOOP_HOME/etc/hadoop/mapred-site.xml
# configure yarn-site.sml
echo "
<configuration>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
</configuration>" > $HADOOP_HOME/etc/hadoop/yarn-site.xml
BIN=$HADOOP_HOME/bin
SBIN=$HADOOP_HOME/sbin
echo "Start service : "
echo "\$ hdfs namenode -format"
$BIN/hdfs namenode -format >/dev/null
echo "\$ hadoop-daemon.sh start namenode"
$SBIN/hadoop-daemon.sh start namenode
echo "\$ hadoop-daemon.sh start secondarynamenode"
$SBIN/hadoop-daemon.sh start secondarynamenode >/dev/null
echo "\$ hadoop-daemon.sh start datanode"
$SBIN/hadoop-daemon.sh start datanode >/dev/null
echo "\$ yarn-daemon.sh start resourcemanager"
$SBIN/yarn-daemon.sh start resourcemanager >/dev/null
echo "\$ yarn-daemon.sh start nodemanager"
$SBIN/yarn-daemon.sh start nodemanager >/dev/null
echo "\$ jps"
jps
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment