Skip to content

Instantly share code, notes, and snippets.

@ebongzzang
Created July 25, 2018 07:05
Show Gist options
  • Save ebongzzang/e208eeb108b6e593730687457b19cda6 to your computer and use it in GitHub Desktop.
Save ebongzzang/e208eeb108b6e593730687457b19cda6 to your computer and use it in GitHub Desktop.
submit spark-job to livy as gradle task
task submitLivy(type: Exec, dependsOn: shadowJar ) {
def targetUrl = 'http://localhost:8998/batches'
def sparkOption = file("livy.json").getText("utf-8")
def curlComm = ['curl', '-X', 'POST', '-H', 'Content-Type: application/json', targetUrl, '-d', sparkOption]
doLast {
ProcessBuilder builder = new ProcessBuilder(curlComm)
println builder.command()
Process process = builder.start()
process.waitFor()
println process.err.text
println process.text
process.destroy()
}
}
{
"conf": {
"spark.master": "yarn-cluster",
"spark.driver.extraJavaOptions": "-DENV=dev",
"spark.driver.extraClassPath": "/home/ebong2/Downloads/spark-1.6.2-bin-hadoop2.6/conf/"
},
"file": "file:///home/ebong2/workspace/spark-modules/jobs/awesome-job/build/libs/awesome-job-1.0.0-all.jar",
"className": "org.ebong2.awesome.Application",
"name": "awesome-",
"executorCores": 1,
"executorMemory": "1g",
"driverMemory": "1g",
"queue": "default",
"jars": [
"file:///home/ebong2/Downloads/spark-1.6.2-bin-hadoop2.6/lib/datanucleus-api-jdo-3.2.6.jar",
"file:///home/ebong2/Downloads/spark-1.6.2-bin-hadoop2.6/lib/datanucleus-core-3.2.10",
"file:///home/ebong2/Downloads/spark-1.6.2-bin-hadoop2.6/lib/datanucleus-rdbms-3.2.9"
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment