Build and Start A Single Node Hadoop Cluster From Source

myhadoop

#!/bin/bash

export HADOOP_HOME=$HOME/Applications/hadoop

main() {
    if [ "$1" == "rebuild" ]; then
        myhadoop_build
        myhadoop_config
        rm -f $HADOOP_HOME/logs/*.* # Delete old logs
    fi
    myhadoop_start
}

myhadoop_build() {
    echo "Building...."
	mvn install package -DskipTests -Dtest=none -Dtar -Pdist -q || exit 1
    # deploy
    rm -rf $HADOOP_HOME
    #cp -r $(pwd)/hadoop-dist/target/hadoop-*-SNAPSHOT $HADOOP_HOME
    cp -r $(pwd)/hadoop-dist/target/hadoop-2.8.0 $HADOOP_HOME
}

myhadoop_config() {
    echo "Configuring..."
    cd $HADOOP_HOME
    if [ -d $HADOOP_HOME/etc/hadoop/conf ]; then
        HADOOP_CONF_HOME=$HADOOP_HOME/etc/hadoop/conf
    else
        HADOOP_CONF_HOME=$HADOOP_HOME/etc/hadoop
    fi

    sed -i '' '/HADOOP_NAMENODE_OPTS/i \
    HDFS_AUDIT_LOGGER=INFO,RFAAUDIT\
    ' $HADOOP_CONF_HOME/hadoop-env.sh
    echo 'export HADOOP_OPTIONAL_TOOLS="hadoop-aws"' >> $HADOOP_CONF_HOME/hadoop-env.sh

cat > $HADOOP_CONF_HOME/core-site.xml << EOF
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
    <include xmlns="http://www.w3.org/2001/XInclude" href="/Users/mliu/.ssh/auth-keys.xml"/>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://localhost:9000</value>
    </property>
    <property>
        <name>hadoop.caller.context.enabled</name>
        <value>true</value>
    </property>
    <property>
          <name>fs.s3a.metadatastore.impl</name>
          <value>org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore</value>
    </property>
</configuration>
EOF

cat > $HADOOP_CONF_HOME/hdfs-site.xml << EOF
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
</configuration>
EOF

cat > $HADOOP_CONF_HOME/mapred-site.xml << EOF
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property>
        <name>yarn.app.mapreduce.am.env</name>
        <value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
    </property>
    <property>
        <name>mapreduce.map.env</name>
        <value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
    </property>
    <property>
        <name>mapreduce.reduce.env</name>
        <value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
    </property>
    <property>
        <name>mapreduce.map.java.opts</name>
        <value>-Xmx512m</value>
    </property>
    <property>
        <name>mapreduce.reduce.java.opts</name>
        <value>-Xmx512m</value>
    </property>
    <property>
        <name>mapreduce.application.classpath</name>
        <value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_MAPRED_HOME/share/hadoop/tools/lib/*</value>
    </property>
</configuration>
EOF

cat > $HADOOP_CONF_HOME/yarn-site.xml << EOF
<configuration>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
</configuration>
EOF
}

myhadoop_start() {
    echo "Stopping old services..."
    $HADOOP_HOME/sbin/stop-dfs.sh
    $HADOOP_HOME/sbin/stop-yarn.sh
    echo "Starting..."
    rm -rf /tmp/hadoop-$USER/
    $HADOOP_HOME/bin/hdfs namenode -format -force
    $HADOOP_HOME/sbin/start-dfs.sh
    $HADOOP_HOME/sbin/start-yarn.sh
}

main "$@"