Skip to content

Instantly share code, notes, and snippets.

@hex108
Last active January 15, 2016 05:01
Show Gist options
  • Save hex108/112acdc5b606f3788975 to your computer and use it in GitHub Desktop.
Save hex108/112acdc5b606f3788975 to your computer and use it in GitHub Desktop.
#!/bin/bash
set -u
set -e
run_cmd() {
echo "$ $@"
eval "$@"
if [ $? -ne 0 ] ; then
echo "Failed to run $@, exiting..."
exit -1
fi
}
if [ $# -ne 1 ]; then
echo "Usage: $0 hadoop_home_directory"
echo " e.g. $0 /home/test/hadoop-2.4.1"
exit -1
fi
export HADOOP_HOME=$1
if [ "${HADOOP_HOME:0:1}" != "/" ]; then
HADOOP_HOME="$(pwd)/$HADOOP_HOME"
fi
export RECOVERY_DIR="$HADOOP_HOME/recovery"
echo "Installing to $HADOOP_HOME"
mkdir -p $HADOOP_HOME/local1
mkdir -p $HADOOP_HOME/local2
mkdir -p $HADOOP_HOME/local3
mkdir -p $HADOOP_HOME/log1
mkdir -p $HADOOP_HOME/log2
mkdir -p $HADOOP_HOME/log3
# configure core-site.xml
echo "
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://localhost:9000</value>
</property>
</configuration>" > $HADOOP_HOME/etc/hadoop/core-site.xml
# configure hdfs-site.xml
echo "
<configuration>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
</configuration>" > $HADOOP_HOME/etc/hadoop/hdfs-site.xml
# configure mapred-site.xml
echo "
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
</configuration>" > $HADOOP_HOME/etc/hadoop/mapred-site.xml
# configure yarn-site.sml
# Reference: http://www.cloudera.com/content/www/en-us/documentation/enterprise/5-3-x/topics/cdh_hag_rm_ha_config.html
echo "
<configuration>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
</property>
<property>
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</property>
<property>
<name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<property>
<name>yarn.resourcemanager.ha.automatic-failover.embedded</name>
<value>true</value>
</property>
<property>
<name>yarn.resourcemanager.cluster-id</name>
<value>test-jun</value>
</property>
<property>
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2</value>
</property>
<property>
<name>yarn.resourcemanager.ha.id</name>
<value>rm1</value>
</property>
<property>
<name>yarn.resourcemanager.recovery.enabled</name>
<value>true</value>
</property>
<property>
<name>yarn.resourcemanager.store.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
</property>
<property>
<name>yarn.resourcemanager.zk-address</name>
<value>localhost:2181</value>
</property>
<property>
<name>yarn.app.mapreduce.am.scheduler.connection.wait.interval-ms</name>
<value>5000</value>
</property>
<property>
<name>yarn.resourcemanager.work-preserving-recovery.enabled</name>
<value>true</value>
</property>
<property>
<name>yarn.resourcemanager.am.max-attempts</name>
<value>2</value>
</property>
<!-- NM restart: https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerRestart.html -->
<property>
<name>yarn.nodemanager.recovery.enabled</name>
<value>true</value>
</property>
<property>
<name>yarn.nodemanager.recovery.dir</name>
<value>$RECOVERY_DIR</value>
</property>
<property>
<name>yarn.nodemanager.address</name>
<value>localhost:8888</value>
</property>
<property>
<name>yarn.nodemanager.recovery.supervised</name>
<value>true</value>
</property>
<property>
<name>yarn.nodemanager.local-dirs</name>
<value>$HADOOP_HOME/local1,$HADOOP_HOME/local2,$HADOOP_HOME/local3</value>
</property>
<property>
<name>yarn.nodemanager.log-dirs</name>
<value>$HADOOP_HOME/log1,$HADOOP_HOME/log2,$HADOOP_HOME/log3</value>
</property>
<property>
<name>yarn.nodemanager.delete.debug-delay-sec</name>
<value>3000000</value>
</property>
<!-- RM1 configs -->
<property>
<name>yarn.resourcemanager.address.rm1</name>
<value>localhost:23140</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address.rm1</name>
<value>localhost:23130</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.https.address.rm1</name>
<value>localhost:23189</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address.rm1</name>
<value>localhost:23188</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address.rm1</name>
<value>localhost:23125</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address.rm1</name>
<value>localhost:23141</value>
</property>
<!-- RM2 configs -->
<property>
<name>yarn.resourcemanager.address.rm2</name>
<value>localhost:23140</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address.rm2</name>
<value>localhost:23130</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.https.address.rm2</name>
<value>localhost:23189</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address.rm2</name>
<value>localhost:23188</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address.rm2</name>
<value>localhost:23125</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address.rm2</name>
<value>localhost:23141</value>
</property>
</configuration>" > $HADOOP_HOME/etc/hadoop/yarn-site.xml
BIN=$HADOOP_HOME/bin
SBIN=$HADOOP_HOME/sbin
echo "Start service : "
run_cmd "$BIN/hdfs namenode -format >/dev/null"
run_cmd "$SBIN/start-dfs.sh >/dev/null"
run_cmd "$SBIN/start-yarn.sh >/dev/null"
run_cmd "$SBIN/yarn-daemon.sh start resourcemanager >/dev/null"
jps
# Generate stop-all.sh: stop-dfs.sh + stop-yarn.sh
echo "$SBIN/stop-dfs.sh
$SBIN/stop-yarn.sh
$SBIN/yarn-daemon.sh stop resourcemanager
rm -rf $HADOOP_HOME/logs/*
rm -rf $RECOVERY_DIR
rm -rf /private/tmp/hadoop-jungong" > stop-all.sh
chmod +x stop-all.sh
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment