Last active
January 15, 2016 05:01
-
-
Save hex108/112acdc5b606f3788975 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
set -u | |
set -e | |
run_cmd() { | |
echo "$ $@" | |
eval "$@" | |
if [ $? -ne 0 ] ; then | |
echo "Failed to run $@, exiting..." | |
exit -1 | |
fi | |
} | |
if [ $# -ne 1 ]; then | |
echo "Usage: $0 hadoop_home_directory" | |
echo " e.g. $0 /home/test/hadoop-2.4.1" | |
exit -1 | |
fi | |
export HADOOP_HOME=$1 | |
if [ "${HADOOP_HOME:0:1}" != "/" ]; then | |
HADOOP_HOME="$(pwd)/$HADOOP_HOME" | |
fi | |
export RECOVERY_DIR="$HADOOP_HOME/recovery" | |
echo "Installing to $HADOOP_HOME" | |
mkdir -p $HADOOP_HOME/local1 | |
mkdir -p $HADOOP_HOME/local2 | |
mkdir -p $HADOOP_HOME/local3 | |
mkdir -p $HADOOP_HOME/log1 | |
mkdir -p $HADOOP_HOME/log2 | |
mkdir -p $HADOOP_HOME/log3 | |
# configure core-site.xml | |
echo " | |
<configuration> | |
<property> | |
<name>fs.defaultFS</name> | |
<value>hdfs://localhost:9000</value> | |
</property> | |
</configuration>" > $HADOOP_HOME/etc/hadoop/core-site.xml | |
# configure hdfs-site.xml | |
echo " | |
<configuration> | |
<property> | |
<name>dfs.replication</name> | |
<value>1</value> | |
</property> | |
</configuration>" > $HADOOP_HOME/etc/hadoop/hdfs-site.xml | |
# configure mapred-site.xml | |
echo " | |
<configuration> | |
<property> | |
<name>mapreduce.framework.name</name> | |
<value>yarn</value> | |
</property> | |
</configuration>" > $HADOOP_HOME/etc/hadoop/mapred-site.xml | |
# configure yarn-site.sml | |
# Reference: http://www.cloudera.com/content/www/en-us/documentation/enterprise/5-3-x/topics/cdh_hag_rm_ha_config.html | |
echo " | |
<configuration> | |
<property> | |
<name>yarn.nodemanager.aux-services</name> | |
<value>mapreduce_shuffle</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.scheduler.class</name> | |
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.ha.enabled</name> | |
<value>true</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.ha.automatic-failover.enabled</name> | |
<value>true</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.ha.automatic-failover.embedded</name> | |
<value>true</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.cluster-id</name> | |
<value>test-jun</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.ha.rm-ids</name> | |
<value>rm1,rm2</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.ha.id</name> | |
<value>rm1</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.recovery.enabled</name> | |
<value>true</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.store.class</name> | |
<value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.zk-address</name> | |
<value>localhost:2181</value> | |
</property> | |
<property> | |
<name>yarn.app.mapreduce.am.scheduler.connection.wait.interval-ms</name> | |
<value>5000</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.work-preserving-recovery.enabled</name> | |
<value>true</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.am.max-attempts</name> | |
<value>2</value> | |
</property> | |
<!-- NM restart: https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerRestart.html --> | |
<property> | |
<name>yarn.nodemanager.recovery.enabled</name> | |
<value>true</value> | |
</property> | |
<property> | |
<name>yarn.nodemanager.recovery.dir</name> | |
<value>$RECOVERY_DIR</value> | |
</property> | |
<property> | |
<name>yarn.nodemanager.address</name> | |
<value>localhost:8888</value> | |
</property> | |
<property> | |
<name>yarn.nodemanager.recovery.supervised</name> | |
<value>true</value> | |
</property> | |
<property> | |
<name>yarn.nodemanager.local-dirs</name> | |
<value>$HADOOP_HOME/local1,$HADOOP_HOME/local2,$HADOOP_HOME/local3</value> | |
</property> | |
<property> | |
<name>yarn.nodemanager.log-dirs</name> | |
<value>$HADOOP_HOME/log1,$HADOOP_HOME/log2,$HADOOP_HOME/log3</value> | |
</property> | |
<property> | |
<name>yarn.nodemanager.delete.debug-delay-sec</name> | |
<value>3000000</value> | |
</property> | |
<!-- RM1 configs --> | |
<property> | |
<name>yarn.resourcemanager.address.rm1</name> | |
<value>localhost:23140</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.scheduler.address.rm1</name> | |
<value>localhost:23130</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.webapp.https.address.rm1</name> | |
<value>localhost:23189</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.webapp.address.rm1</name> | |
<value>localhost:23188</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.resource-tracker.address.rm1</name> | |
<value>localhost:23125</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.admin.address.rm1</name> | |
<value>localhost:23141</value> | |
</property> | |
<!-- RM2 configs --> | |
<property> | |
<name>yarn.resourcemanager.address.rm2</name> | |
<value>localhost:23140</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.scheduler.address.rm2</name> | |
<value>localhost:23130</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.webapp.https.address.rm2</name> | |
<value>localhost:23189</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.webapp.address.rm2</name> | |
<value>localhost:23188</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.resource-tracker.address.rm2</name> | |
<value>localhost:23125</value> | |
</property> | |
<property> | |
<name>yarn.resourcemanager.admin.address.rm2</name> | |
<value>localhost:23141</value> | |
</property> | |
</configuration>" > $HADOOP_HOME/etc/hadoop/yarn-site.xml | |
BIN=$HADOOP_HOME/bin | |
SBIN=$HADOOP_HOME/sbin | |
echo "Start service : " | |
run_cmd "$BIN/hdfs namenode -format >/dev/null" | |
run_cmd "$SBIN/start-dfs.sh >/dev/null" | |
run_cmd "$SBIN/start-yarn.sh >/dev/null" | |
run_cmd "$SBIN/yarn-daemon.sh start resourcemanager >/dev/null" | |
jps | |
# Generate stop-all.sh: stop-dfs.sh + stop-yarn.sh | |
echo "$SBIN/stop-dfs.sh | |
$SBIN/stop-yarn.sh | |
$SBIN/yarn-daemon.sh stop resourcemanager | |
rm -rf $HADOOP_HOME/logs/* | |
rm -rf $RECOVERY_DIR | |
rm -rf /private/tmp/hadoop-jungong" > stop-all.sh | |
chmod +x stop-all.sh |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment