Using spark master REST API to submit a job as a replacement to spark-submit command for python and scala.

spark-rest-job.sh

#!/usr/bin/env bash

#python
curl -X POST http://localhost:6066/v1/submissions/create --header "Content-Type:application/json;charset=UTF-8" --data '{
   "action":"CreateSubmissionRequest",
   "appArgs":[
      "/Users/tilak/jobs/test_job.py"
   ],
   "appResource":"file:/Users/tilak/jobs/test_job.py",
   "clientSparkVersion":"2.3.3",
   "environmentVariables":{
      "SPARK_ENV_LOADED":"1"
   },
   "mainClass":"org.apache.spark.deploy.SparkSubmit",
   "sparkProperties":{
      "spark.driver.supervise":"false",
      "spark.app.name":"Simple App",
      "spark.eventLog.enabled":"true",
      "spark.submit.deployMode":"cluster",
      "spark.master":"spark://localhost:6066"
   }
}'
# Get status of the job
curl http://localhost:6066/v1/submissions/status/driver-20190711080507-0000
# Kill the job
curl -XPOST http://localhost:6066/v1/submissions/kill/driver-20190711081255-0005


# java
curl -X POST http://localhost:6066/v1/submissions/create --header "Content-Type:application/json" --data '{
  "action": "CreateSubmissionRequest",
  "appResource": "/Users/tilak/spark-2.3.3/examples/jars/spark-examples_2.11-2.3.3.jar",
  "clientSparkVersion": "2.3.3",
  "appArgs": [ "10" ],
  "environmentVariables" : {
    "SPARK_ENV_LOADED" : "1"
  },
  "mainClass": "org.apache.spark.examples.SparkPi",
  "sparkProperties": {
    "spark.jars": "/Users/tilak/spark-2.3.3/examples/jars/spark-examples_2.11-2.3.3.jar",
    "spark.driver.supervise":"false",
    "spark.executor.memory": "512m",
    "spark.driver.memory": "512m",
    "spark.submit.deployMode":"cluster",
    "spark.app.name": "SparkPi",
    "spark.master": "spark://localhost:6066"
  }
}'

# Get status of the job
curl http://localhost:6066/v1/submissions/status/driver-20190711080507-0000

# Kill the job
curl -XPOST http://localhost:6066/v1/submissions/kill/driver-20190711081255-0005

@tilakpatidar, is it possible to implement the same for YARN based remote spark cluster?