spark-hpc

echo '>> Start of Script'
nodes=($( cat $PBS_NODEFILE | sort | uniq ))
nnodes=${#nodes[@]}
last=$(( $nnodes - 1 ))

export SPARK_HOME=/work/{user}/sparktest/spark/
ssh ${nodes[0]} "cd ${SPARK_HOME}; ./sbin/start-master.sh"
sparkmaster="spark://${nodes[0]}:7077"
echo 'master created'

for i in $( seq 0 $last )
do
    ssh ${nodes[$i]} "cd ${SPARK_HOME}; nohup ./sbin/start-slave ${sparkmaster} &> /work/{user}/sparktest/testSucess/nohup-${nodes[$i]}.out" &
done
echo 'Worker Started'

${SPARK_HOME}/bin/spark-submit /work/{user}/sparktest/testSucess/sparkscript.py
echo 'application submitted'
echo ${nnodes[0]}

ssh ${nodes[0]} " cd ${SPARK_HOME}; ./sbin/stop-master.sh"
for i in $( seq 0 $last )
do
    ssh ${nodes[$i]} "cd ${SPARK_HOME}; ./sbin/stop-slaves.sh"
done