HDFS_KNOX_PERF_TEST.sh

#!/bin/sh
SIZES="1 10 20 50"
MBGB="MB"   # Options are:  GB OR MB
SAMPLE_SIZE=5  # Number of time to run.
USER="admin"
PASSWORD="admin-password"
ENDPOINT="namenode2-1.utsaics.edu"

# Please also note that the port changed from 50070 to 9870 in Hadoop 3.0.
echo "Which version of Hadoop are you using?"
options=("Version 2" "Version 3")
select HADOOP_VERSION in "${options[@]}"
do
    case $HADOOP_VERSION in
        "Version 2")
            PORT=50070
            break;;
        "Version 3")
            PORT=9870
            break;;
    esac
done

GENERATE_FILE()
{
  echo "Checking if $i$MBGB file exists in HDFS..."
  i=$1
      if hdfs dfs -test -e /tmp/"$i$MBGB".txt; then
          echo "\t</tmp/$i$MBGB.txt> file exists in HDFS, continuing."
      else
          echo "GENERATING FILE $i $MBGB FILE FOR HDFS"
              if [ $MBGB = "GB" ]; then
                  REAL_SIZE=$(($i*1024))
              else
                  REAL_SIZE=$(($i))
              fi
              FILE=/tmp/"$i$MBGB".txt
              if [ ! -f "$FILE" ]; then
                  echo "CREATING FILE IN /tmp/$i$MBGB.txt"
                  dd if=/dev/zero of=/tmp/"$i$MBGB".txt count="$REAL_SIZE" bs=1048576 >/dev/null 2>/dev/null
                  ls -lrht /tmp/*.txt | grep "$i$MBGB"
              fi
          echo "\tUploading $i $MBGB file to HDFS"
          echo "\tThis might take some time, please wait."
          time hdfs dfs -copyFromLocal -f /tmp/$i$MBGB.txt hdfs:///tmp/
          echo "\t$i $MBGB File upload complete, continuing.\n\n"
      fi
}

TEST_DIRECT_HDFS()
{
  i=$1
  echo "\n\tTest direct HDFS communications ($i$MBGB FILE): Test $j of $SAMPLE_SIZE"
  now=$(TZ=CST6CDT date +"%T")
  echo "\tCurrent time : $now"
  command time hdfs dfs -text hdfs:///tmp/"$i$MBGB".txt >/dev/null
}

TEST_webHDFS()
{
  i=$1
  echo "\n\tTest webHDFS communications ($i$MBGB FILE): Test $j of $SAMPLE_SIZE"
  now=$(TZ=CST6CDT date +"%T")
  echo "\tCurrent time : $now"
  command time curl -s -k -u $USER:$USER -L --location-trusted "http://$ENDPOINT:$PORT/webhdfs/v1/tmp/$i$MBGB.txt?op=OPEN"
}

TEST_knox()
{
  i=$1
  echo "\n\tTest KNOX HDFS communications ($i$MBGB FILE): Test $j of $SAMPLE_SIZE"
  now=$(TZ=CST6CDT date +"%T")
  echo "\tCurrent time : $now"
  command time curl -k -u $USER:$PASSWORD -L --location-trusted "https://$ENDPOINT:$PORT/gateway/default/webhdfs/v1/tmp/$i$MBGB.txt?op=OPEN"
}

for i in $SIZES; do
    GENERATE_FILE $i
    for j in $(seq $SAMPLE_SIZE); do
        TEST_knox $i
    done
    for j in $(seq $SAMPLE_SIZE); do
        TEST_webHDFS $i
    done
    echo "Removing file locally"
    rm /tmp/"$i$MBGB".txt
    echo "Removing file from HDFS"
    sudo -H -u hdfs bash -c "hdfs dfs -rm -skipTrash /tmp/$i$MBGB.txt"
    echo "\n\n\n"
done

# DEBUGGING KNOX:
# curl -s -k -u admin:admin -L --location-trusted "http://namenode2-1.utsaics.edu:50070/webhdfs/v1/tmp/?op=GETHOMEDIRECTORY"
# > {"Path":"/user/dr.who"}
#curl -k -u admin:admin-password https://namenode2-1.utsaics.edu:8443/gateway/default/webhdfs/v1/?op=GETHOMEDIRECTORY