spark-hdp-install for spark-1.5.1-bin-hadoop2.6

spark-hdp-install

#!/bin/bash

: ${SPARK_VERSION:=1.5.1}
: ${SPARK_DIST_PREFIX_VERSION:=2.6}
: ${SPARK_HADOOP_VERSION:=2.6.0}

SPARK_DIST_VERSION=$SPARK_VERSION-bin-hadoop$SPARK_DIST_PREFIX_VERSION
SPARK_ASSEMBLY_VERSION=$SPARK_VERSION-hadoop$SPARK_HADOOP_VERSION

set-spark-envs() {
  export SPARK_HOME=/usr/local/spark
  export YARN_CONF_DIR=/usr/hdp/$HDP_VERSION/hadoop/conf
  export PATH=$PATH:$SPARK_HOME/bin
  export HADOOP_USER_NAME=hdfs
  export SPARK_JAR=hdfs:///spark/spark-assembly-$SPARK_DIST_VERSION.jar
}

set-default-spark-conf() {
(cat <<- EOF
spark.driver.extraJavaOptions -Dhdp.version=$HDP_VERSION
spark.yarn.am.extraJavaOptions -Dhdp.version=$HDP_VERSION
EOF
) > $SPARK_HOME/conf/spark-defaults.conf
}

install-local-spark() {
  curl -s http://d3kbcqa49mib13.cloudfront.net/spark-$SPARK_DIST_VERSION.tgz | tar -xz -C /usr/local/

  cd /usr/local && ln -s spark-$SPARK_DIST_VERSION spark
  set-spark-envs
  set-default-spark-conf
}

install-spark() {
  install-local-spark
  hadoop fs -rm -r -f /spark
  echo "upload assembly jar to hdfs"
  hadoop fs -mkdir /spark
  hadoop fs -put $SPARK_HOME/lib/spark-assembly-$SPARK_ASSEMBLY_VERSION.jar /spark/
}

main() {
  if [[ "$1" == "install" ]]; then
    echo "install spark..."
    install-spark
  elif [[ "$1" == "install-local" ]]; then
    echo "install spark without uploading jar to the hdfs..."
    install-local-spark
  else 
    echo  "command not supported"
  fi
}

[[ "$0" == "$BASH_SOURCE" ]] && main "$@"