Build Hadoop 2.9.1 with SNAPPY support in CentOS 6.5

process.txt

######## execute as root ######
sudo su
service iptables stop
chkconfig iptables off
# centos 7:
# systemctl disable firewalld
# systemctl stop firewalld
sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
reboot
sudo su
cd /root
wget --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/8u181-b13/96a7b8442fe848ef90c96a2fad6ed6d1/jdk-8u181-linux-x64.tar.gz
tar -xzf jdk-8u181-linux-x64.tar.gz -C /opt/
alternatives --install /usr/bin/java java /opt/jdk1.8.0_181/bin/java 2
alternatives --config java
java -version
groupadd hadgroup
useradd haduser -G hadgroup -m
passwd haduser

######## execute as haduser
su - haduser
ssh-keygen -t rsa -P ""
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 0600 ~/.ssh/authorized_keys
ssh localhost date
exit # back to root

###### execute as root ######
cd /root
yum groupinstall "Development Tools" "Development Libraries" -y
yum install openssl-devel cmake snappy snappy-devel -y
wget https://archive.apache.org/dist/maven/maven-3/3.3.3/binaries/apache-maven-3.3.3-bin.tar.gz
tar -zxf apache-maven-3.3.3-bin.tar.gz -C /opt/
cat << EOF > /etc/profile.d/maven.sh
export JAVA_HOME=/opt/jdk1.8.0_181
export M3_HOME=/opt/apache-maven-3.3.3
export PATH=/opt/apache-maven-3.3.3/bin:\$PATH
EOF
exit
sudo su # run as root
cd /root
echo $M3_HOME
mvn -version
wget https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz
tar -xzf protobuf-2.5.0.tar.gz -C /root
cd protobuf-2.5.0
./configure
make
make install

######## execute as haduser
su - haduser
cat >> ~/.bash_profile << EOF
export PATH=\$PATH:/usr/local/bin
EOF
source ~/.bash_profile
protoc --version

wget https://archive.apache.org/dist/hadoop/common/hadoop-2.9.1/hadoop-2.9.1-src.tar.gz
tar xf hadoop-2.9.1-src.tar.gz
cd hadoop-2.9.1-src
mvn package -Pdist,native -DskipTests -Dtar -Drequire.snappy

#### execute as root
mv /home/haduser/hadoop-2.9.1-src/hadoop-dist/target/hadoop-2.9.1 /usr/local/
ln -s /usr/local/hadoop-2.9.1 /usr/local/hadoop

### execute as haduser
cat >> ~/.bash_profile << EOF
# Hadoop
export HADOOP_HOME=/usr/local/hadoop
export PATH=\$PATH:\$HADOOP_HOME/bin:\$HADOOP_HOME/sbin
EOF
source ~/.bash_profile
ln -s /usr/lib64/libsnappy.so $HADOOP_HOME/lib/native/libsnappy.so
vi $HADOOP_HOME/etc/hadoop/core-site.xml
-------------
<property>
  <name>fs.default.name</name>
    <value>hdfs://localhost:9000</value>
</property>
<property>
  <name>hadoop.tmp.dir</name>
  <value>/usr/local/hadoop/tmp</value>
</property>
<property>
<name>io.compression.codecs</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
</property>
---------------
cp $HADOOP_HOME/etc/hadoop/mapred-site.xml.template $HADOOP_HOME/etc/hadoop/mapred-site.xml
vi $HADOOP_HOME/etc/hadoop/mapred-site.xml
-----------
<property>
   <name>mapreduce.framework.name</name>
   <value>yarn</value>
</property>
-----------
vi $HADOOP_HOME/etc/hadoop/yarn-site.xml
-----------
<property>
  <name>mapreduce.map.output.compress</name>
  <value>true</value>
</property>
<property>
  <name>mapred.map.output.compress.codec</name>
  <value>org.apache.hadoop.io.compress.SnappyCodec</value>
</property>
-----------
cat >> $HADOOP_HOME/etc/hadoop/hadoop-env.sh << EOF
export JAVA_LIBRARY_PATH="/usr/local/hadoop/lib/native"
EOF

hdfs namenode -format
start-dfs.sh
start-yarn.sh
hadoop fs -mkdir hdfs://localhost:9000/test
hadoop fs -ls hdfs://localhost:9000/

#visit http://localhost:50070