-
-
Save sathiyarajanm/a55be18534ad27fb05b1f2360e760ddc to your computer and use it in GitHub Desktop.
An example Ansible playbook that validates a Spark cluster (including source for test Spark App)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package com.newsweaver.spark.test | |
import org.apache.spark.{SparkConf, SparkContext} | |
/** | |
* simple Spark app to calculate Pi | |
*/ | |
object EstimatePiSparkApp { | |
def main(args: Array[String]) { | |
val sparkConf = new SparkConf() | |
.setAppName("EstimatePiSparkApp") | |
val sc = new SparkContext(sparkConf) | |
runApp(sc) | |
sc.stop() | |
} | |
def runApp(sc: SparkContext) = { | |
val count = sc.parallelize(1 to 1000).map { i => | |
val x = Math.random() | |
val y = Math.random() | |
if (x * x + y * y < 1) 1 else 0 | |
}.reduce(_ + _) | |
println("Pi is roughly " + 4.0 * count / 1000) | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
- hosts: spark_master | |
tasks: | |
- name: check connectivity to spark master ui | |
# local_action runs task on the machine running ansible | |
local_action: "uri url=http://{{ HOST_IP }}:{{ SPARK_MASTER_UI_PORT }} return_content=yes" | |
register: spark_master_ui | |
- fail: msg='spark master ui is not available' | |
when: "'Spark Master' not in spark_master_ui.content" | |
- fail: msg='spark workers are not all registered with spark master' | |
when: "'Workers:</strong> {{ groups.spark_worker|length }}</li>' not in spark_master_ui.content" | |
- name: check connectivity from spark master to spark workers | |
wait_for: host="{{ HOST_IP }}" port="{{ SPARK_MASTER_PORT }}" | |
with_items: | |
- "{{ groups.spark_worker }}" | |
#delegate_to runs the task on each of the spark workers | |
delegate_to: "{{ item }}" | |
with_items: | |
- "{{ groups.spark_worker }}" | |
- name: download spark test app to localhost | |
local_action: "get_url url=https://maven-repo.lan/repositories/releases/content/com/newsweaver/spark/test/spk-test-pi/{{ SPARK_TEST_JAR_VERSION }}/spk-test-pi-{{ SPARK_TEST_JAR_VERSION }}.jar dest=/tmp/spk-test-pi-{{ SPARK_TEST_JAR_VERSION }}.jar" | |
- name: upload spark test app to spark master | |
copy: "src=/tmp/spk-test-pi-{{ SPARK_TEST_JAR_VERSION }}.jar dest=/tmp/spk-test-pi-{{ SPARK_TEST_JAR_VERSION }}.jar" | |
- name: start spark test app | |
shell: "{{ SPARK_HOME }}/bin/spark-submit --class com.newsweaver.spark.test.EstimatePiSparkApp /tmp/spk-test-pi-{{ SPARK_TEST_JAR_VERSION }}.jar" | |
register: spark_test_app | |
- fail: msg='spark test job failed' | |
when: "'Pi is roughly' not in spark_test_app.stdout" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment