Created
July 25, 2018 07:05
-
-
Save ebongzzang/e208eeb108b6e593730687457b19cda6 to your computer and use it in GitHub Desktop.
submit spark-job to livy as gradle task
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
task submitLivy(type: Exec, dependsOn: shadowJar ) { | |
def targetUrl = 'http://localhost:8998/batches' | |
def sparkOption = file("livy.json").getText("utf-8") | |
def curlComm = ['curl', '-X', 'POST', '-H', 'Content-Type: application/json', targetUrl, '-d', sparkOption] | |
doLast { | |
ProcessBuilder builder = new ProcessBuilder(curlComm) | |
println builder.command() | |
Process process = builder.start() | |
process.waitFor() | |
println process.err.text | |
println process.text | |
process.destroy() | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"conf": { | |
"spark.master": "yarn-cluster", | |
"spark.driver.extraJavaOptions": "-DENV=dev", | |
"spark.driver.extraClassPath": "/home/ebong2/Downloads/spark-1.6.2-bin-hadoop2.6/conf/" | |
}, | |
"file": "file:///home/ebong2/workspace/spark-modules/jobs/awesome-job/build/libs/awesome-job-1.0.0-all.jar", | |
"className": "org.ebong2.awesome.Application", | |
"name": "awesome-", | |
"executorCores": 1, | |
"executorMemory": "1g", | |
"driverMemory": "1g", | |
"queue": "default", | |
"jars": [ | |
"file:///home/ebong2/Downloads/spark-1.6.2-bin-hadoop2.6/lib/datanucleus-api-jdo-3.2.6.jar", | |
"file:///home/ebong2/Downloads/spark-1.6.2-bin-hadoop2.6/lib/datanucleus-core-3.2.10", | |
"file:///home/ebong2/Downloads/spark-1.6.2-bin-hadoop2.6/lib/datanucleus-rdbms-3.2.9" | |
] | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment