spark2_submit_hdp.sh

#!/usr/bin/env bash
export SPARK_MAJOR_VERSION=2

/usr/hdp/current/spark2-client/bin/spark-submit --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
--files /path/to/log4j.properties \
--conf spark.yarn.executor.memoryOverhead=1024  \
--conf spark.port.maxRetries=64  \
--conf spark.driver.extraJavaOptions='-Dlog4j.debug -Dlog4j.configuration=file:/path/to/log4j.properties -Da=a1' \
--conf spark.executor.extraJavaOptions='-Dlog4j.debug -Dlog4j.configuration=log4j.properties' \
--master yarn \
--deploy-mode client \
--queue myqueue \
--num-executors 4 \
--driver-memory 2g \
--executor-cores 4 \
--executor-memory 15g \
--proxy-user myuser \
--class com.example.MainClass \
/path/to/project/project-assembly-1.0.jar \