Hadoop 3.1 on Debian 9.5

Hadoop-3.1.md

INSTALL Hadoop 3.1 on Debian 9.5

1. Install some dependencies:

   su -
   apt -y install build-essential autoconf automake libtool cmake zlib1g-dev pkg-config libssl-dev libsasl2-dev \
   libsnappy-dev bzip2 libbz2-dev libjansson-dev fuse libfuse-dev zstd \
   clang libc6-dev libtool libzip-dev libncurses-dev liblzo2-dev  \
   software-properties-common dirmngr python python-dev python-setuptools \
   libffi6 libffi-dev doxygen ant unzip subversion git curl
   
   curl -sL https://deb.nodesource.com/setup_8.x | bash -
   apt install -y nodejs
   
   npm install -g bower
   npm install -g ember-cli

Install Java (JDK) 8:

   su -
   add-apt-repository ppa:webupd8team/java
   apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C2518248EEA14886
   apt update
   apt install oracle-java8-installer
   update-alternatives --config java
   update-alternatives --config javac
   nano /etc/environment
       JAVA_HOME="/usr/lib/jvm/java-8-oracle/jre"
   source /etc/environment

Install Apache Maven:

   su -
   wget http://espejito.fder.edu.uy/apache/maven/maven-3/3.5.4/binaries/apache-maven-3.5.4-bin.tar.gz
   tar xzf apache-maven-3.5.4-bin.tar.gz
   mkdir /usr/local/apache-maven
   mv apache-maven-3.5.4/* /usr/local/apache-maven
   nano /etc/profile.d/apache-maven.sh
       export M2_HOME=/usr/local/apache-maven
       export MAVEN_HOME=/usr/local/apache-maven
       export PATH=${M2_HOME}/bin:${PATH}
   chmod +x /etc/profile.d/apache-maven.sh
   source /etc/profile.d/apache-maven.sh
   
   # it is recommended to restart
   reboot

2. Compile protobuf:

   su -
   wget https://github.com/protocolbuffers/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz
   tar -zxf protobuf-2.5.0.tar.gz
   cd protobuf-2.5.0
   ./configure
   make
   make check
   make install
   ldconfig

3. Compile Hadoop:

   su -
   mkdir -p /usr/local/hadoop
   chown hadoop:hadoop /usr/local/hadoop
   nano /etc/profile.d/apache-hadoop.sh
       export HADOOP_HOME=/usr/local/hadoop
       export HADOOP_INSTALL=$HADOOP_HOME
       export HADOOP_MAPRED_HOME=$HADOOP_HOME
       export HADOOP_COMMON_HOME=$HADOOP_HOME
       export HADOOP_HDFS_HOME=$HADOOP_HOME
       export YARN_HOME=$HADOOP_HOME
       export HADOOP_COMMON_LIB_NATIVE_DIR=${HADOOP_HOME}/lib/native
       export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop
       export PATH=${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin:${PATH}
   chmod +x /etc/profile.d/apache-hadoop.sh
   source /etc/profile.d/apache-hadoop.sh
   su - hadoop
   cd
   wget http://apache.spinellicreations.com/hadoop/common/hadoop-3.1.1/hadoop-3.1.1-src.tar.gz
   tar -zxf hadoop-3.1.1-src.tar.gz
   cd hadoop-3.1.1-src
   mvn package -Dmaven.javadoc.skip=true -Pdist,native -DskipTests -Dtar

4. The compiled hadoop tarball locates at hadoop-3.1.1-src/hadoop-dist/target/hadoop-3.1.1.tar.gz

   su - hadoop
   cd
   cp hadoop-3.1.1-src/hadoop-dist/target/hadoop-3.1.1.tar.gz /usr/local/hadoop/
   cd /usr/local/hadoop/
   tar -zxf hadoop-3.1.1.tar.gz
   mv hadoop-3.1.1/* .
   rm -rf hadoop-3.1.1

SET UP Hadoop 3.1 on Debian 9.5

• How To Change Your Hostname on Debian

   nano /etc/hostname
   nano /etc/hosts
   hostnamectl set-hostname nodeX
   hostname

• How To Change Your Network on Debian

   ip addr
   nano /etc/network/interfaces
      allow-hotplug enp0s8
      iface enp0s8 inet static
      address 192.168.56.10X
      netmask 255.255.255.0
      network 192.168.56.0