Run mxnet training for MNIST on spark standalone

run-mxnet-on-spark.sh

#!/bin/bash                                                                                                                                              
CURR_DIR=$(cd `dirname $0`; pwd)
ROOT_DIR=$(cd $CURR_DIR/../../; pwd)
                               
MODULE_DIR=/opt/mxnet          
LIB_DIR=${MODULE_DIR}/lib      
JAR=${MODULE_DIR}/mxnet-spark_2.10-0.1.2-SNAPSHOT.jar
LIBS=${MODULE_DIR}/mxnet-full_2.10-linux-x86_64-cpu-0.1.2-SNAPSHOT.jar
for jar in `ls ${LIB_DIR}/*.jar`; do                
  [ ! -z ${LIBS} ] && LIBS="${LIBS},"               
  LIBS="${LIBS}$jar";          
done                           
                               
SPARK_OPTS+=" --name mxnet"    
SPARK_OPTS+=" --driver-memory 1g"  
SPARK_OPTS+=" --executor-memory 1g"
SPARK_OPTS+=" --num-executors 2"   
SPARK_OPTS+=" --executor-cores 1"  
#SPARK_OPTS+=" --spark.network.timeout 120"         
SPARK_OPTS+=" --jars ${LIBS}"  
                               
# running opts                 
RUN_JARS="${MODULE_DIR}/*:${MODULE_DIR}/lib/*"      
RUN_OPTS+=" --input s3a://mxnet.liuyz/data/mnist/val.txt"            
RUN_OPTS+=" --jars ${RUN_JARS}"
RUN_OPTS+=" --num-server 1"    
RUN_OPTS+=" --num-worker 2"    
RUN_OPTS+=" --java /opt/jdk1.8.0_74/bin/java"       
                               
SPARK_HOME=/opt/spark          
${SPARK_HOME}/bin/spark-submit --master spark://localhost:7077 \     
  --conf spark.driver.extraClassPath=/opt/spark/extra-libs/* \       
  --conf spark.executor.extraClassPath=/opt/spark/extra-libs/* \     
  --conf spark.hadoop.fs.s3a.access.key=XXX \       
  --conf spark.hadoop.fs.s3a.secret.key=XXXXX \
  --conf spark.dynamicAllocation.enabled=false \    
  --conf spark.speculation=false \ 
  --class ml.dmlc.mxnet.spark.MXNet \               
  ${SPARK_OPTS} \              
  ${JAR} \                     
  ${RUN_OPTS}