Chef Recipe for remote spark-submit setup to YARN running on Amazon EMR

spark-yarn-emr-client.rb

# setup for the hadoop + spark env for airflow

remote_file "/home/airflow/spark.tgz" do
  source "/remote/spark/download/url"
  owner "airflow"
  group "airflow"
  mode  '0755'
  not_if { File.exists?("/home/airflow/spark.tgz") }
end

directory "/home/airflow/spark" do
  owner 'airflow'
  group 'airflow'
  mode '0755'
end

execute 'extract spark' do
  command 'tar xvf spark.tgz -C spark/ --strip-components=1'
  cwd "/home/airflow"
  not_if { File.exists?("/home/airflow/spark/RELEASE") }
end

["emrfs-hadoop-assembly-2.12.0.jar", "s3-dist-cp-2.4.0.jar", "jersey-bundle-1.19.4.jar"].each do |package|
  remote_file "/home/airflow/spark/jars/#{package}" do
    source "https://remote/path/to/server/with/#{package}"
    owner "airflow"
    group "airflow"
    mode  '0755'
    not_if { File.exists?("/home/airflow/#{package}") }
  end
end


directory "/home/airflow/hadoop-config" do
  owner 'airflow'
  group 'airflow'
  mode '0755'
end

["core-site.xml", "yarn-site.xml"].each do |config_file|
  template "/home/airflow/hadoop-config/#{config_file}" do
    source "#{config_file}.erb"
    owner 'airflow'
    group 'airflow'
    mode '0755'
    variables(
      :emr_master_node_ip => <emr_master_node_ip>,
      :emr_master_node_dns => <emr_master_node_dns>
    )
  end
end


# hadoop.sh.erb
#export HADOOP_CONF_DIR=<%= @hadoop_config_dir %>
#export SPARK_HOME=<%= @spark_home %>
#export AWS_ACCESS_KEY_ID=<%= @aws_access_key %>
#export AWS_SECRET_ACCESS_KEY=<%= @aws_secret_key %>
#export HADOOP_USER_NAME=hdfs


template "/home/airflow/conf/spark-env.sh" do
  source 'hadoop.sh.erb'
  owner 'airflow'
  group 'airflow'
  mode '0775'
  variables(
    :hadoop_config_dir => "/home/airflow/hadoop-config/",
    :spark_home => "/home/airflow/spark",
    :aws_access_key => <aws_access_key>,
    :aws_secret_key => <aws_secret_key>
  )
end