Skip to content

Instantly share code, notes, and snippets.

@jcrobak

jcrobak/start-hdfs.sh

Created July 6, 2012 15:31
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save jcrobak/3060908 to your computer and use it in GitHub Desktop.
Save jcrobak/3060908 to your computer and use it in GitHub Desktop.
Download ZIP
simple start/stop scripts for hadoop-2.0.0 / CDH4
Raw
start-hdfs.sh
#!/bin/bash
set -ex
HADOOP_HOME=`dirname $0`/..
HADOOP_CONF_DIR="$HADOOP_HOME/etc/hadoop"
HADOOP_LOG_DIR="$HADOOP_HOME/log"
HADOOP_PID_DIR="$HADOOP_HOME"
HADOOP_OPTS="-Xmx512m -Djava.net.preferIPv4Stack=true"
HADOOP_NN_OPTS="-Dproc_namenode $HADOOP_OPTS"
HADOOP_DN_OPTS="-Dproc_datanode $HADOOP_OPTS"
HADOOP_SNN_OPTS="-Dproc_secondarynamenode $HADOOP_OPTS"
HDFS_CLASSPATH="$HADOOP_CONF_DIR:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/hdfs:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*"
nohup java -cp $HDFS_CLASSPATH $HADOOP_NN_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR -Dhadoop.log.file=hadoop-namenode.log -Dhadoop.home.dir=$HADOOP_HOME -Dhadoop.id.str=$USER -Dhadoop.root.logger=INFO,RFA -Djava.library.path=$HADOOP_HOME/lib/native -Dhadoop.policy.file=hadoop-policy.xml -Djava.net.preferIPv4Stack=true -Dhadoop.security.logger=INFO,RFAS -Dhdfs.audit.logger=INFO,NullAppender -Dhadoop.security.logger=INFO,NullAppender org.apache.hadoop.hdfs.server.namenode.NameNode &
echo $! > "$HADOOP_PID_DIR/nn.pid"
nohup java -cp $HDFS_CLASSPATH $HADOOP_DN_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR -Dhadoop.log.file=hadoop-datatanode.log -Dhadoop.home.dir=$HADOOP_HOME -Dhadoop.id.str=$USER -Dhadoop.root.logger=INFO,RFA -Djava.library.path=$HADOOP_HOME/lib/native -Dhadoop.policy.file=hadoop-policy.xml -Djava.net.preferIPv4Stack=true -Dhadoop.security.logger=INFO,RFAS -Dhdfs.audit.logger=INFO,NullAppender -Dhadoop.security.logger=INFO,NullAppender org.apache.hadoop.hdfs.server.datanode.DataNode &
echo $! > "$HADOOP_PID_DIR/dn.pid"
nohup java -cp $HDFS_CLASSPATH $HADOOP_SNN_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR -Dhadoop.log.file=hadoop-secondarynamenode.log -Dhadoop.home.dir=$HADOOP_HOME -Dhadoop.id.str=$USER -Dhadoop.root.logger=INFO,RFA -Djava.library.path=$HADOOP_HOME/lib/native -Dhadoop.policy.file=hadoop-policy.xml -Djava.net.preferIPv4Stack=true -Dhadoop.security.logger=INFO,RFAS -Dhdfs.audit.logger=INFO,NullAppender -Dhadoop.security.logger=INFO,NullAppender org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode &
echo $! > "$HADOOP_PID_DIR/snn.pid"
Raw
start-mapred.sh
#!/bin/bash
# starts MRv1 for cloudera's cdh4
set -ex
HADOOP_HOME=`dirname $0`/..
HADOOP_CONF_DIR="$HADOOP_HOME/conf"
HADOOP_LOG_DIR="$HADOOP_HOME/log"
HADOOP_PID_DIR="$HADOOP_HOME"
HADOOP_OPTS="-Xmx512m -Djava.net.preferIPv4Stack=true"
HADOOP_JT_OPTS="-Dproc_jobtracker $HADOOP_OPTS"
HADOOP_TT_OPTS="-Dproc_tasktracker $HADOOP_OPTS"
MAPRED_CLASSPATH="$HADOOP_CONF_DIR:$HADOOP_HOME:$HADOOP_HOME/*:$HADOOP_HOME/lib/*"
nohup java -cp $MAPRED_CLASSPATH $HADOOP_JT_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR -Dhadoop.log.file=hadoop-jobtracker.log -Dhadoop.home.dir=$HADOOP_HOME -Dhadoop.id.str=$USER -Dhadoop.root.logger=INFO,RFA -Djava.library.path=$HADOOP_HOME/lib/native -Dhadoop.policy.file=hadoop-policy.xml -Djava.net.preferIPv4Stack=true -Dhadoop.security.logger=INFO,RFAS -Dhdfs.audit.logger=INFO,NullAppender -Dhadoop.security.logger=INFO,NullAppender org.apache.hadoop.mapred.JobTracker &
echo $! > $HADOOP_PID_DIR/jt.pid
nohup java -cp $MAPRED_CLASSPATH $HADOOP_TT_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR -Dhadoop.log.file=hadoop-tasktracker.log -Dhadoop.home.dir=$HADOOP_HOME -Dhadoop.id.str=$USER -Dhadoop.root.logger=INFO,RFA -Djava.library.path=$HADOOP_HOME/lib/native -Dhadoop.policy.file=hadoop-policy.xml -Djava.net.preferIPv4Stack=true -Dhadoop.security.logger=INFO,RFAS -Dhdfs.audit.logger=INFO,NullAppender -Dhadoop.security.logger=INFO,NullAppender org.apache.hadoop.mapred.TaskTracker &
echo $! > $HADOOP_PID_DIR/tt.pid
Raw
stop-hdfs.sh
#!/bin/bash
set -ex
HADOOP_HOME=`dirname $0`/..
HADOOP_PID_DIR="$HADOOP_HOME"
[ -f "$HADOOP_PID_DIR/dn.pid" ] && kill `cat "$HADOOP_PID_DIR/dn.pid"`
[ -f "$HADOOP_PID_DIR/nn.pid" ] && kill `cat "$HADOOP_PID_DIR/nn.pid"`
[ -f "$HADOOP_PID_DIR/snn.pid" ] && kill `cat "$HADOOP_PID_DIR/snn.pid"`
Raw
stop-mapred.sh
#!/bin/bash
set -ex
HADOOP_HOME=`dirname $0`/..
HADOOP_PID_DIR="$HADOOP_HOME"
[ -f "$HADOOP_PID_DIR/jt.pid" ] && kill `cat "$HADOOP_PID_DIR/jt.pid"`
[ -f "$HADOOP_PID_DIR/tt.pid" ] && kill `cat "$HADOOP_PID_DIR/tt.pid"`
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment