run_spark_app.py

def run_spark_app(config_name, dag):
    eks_namespace = config.dim_value(config_name, 'eks_namespace')
    eks_in_cluster = bool(config.dim_value(config_name, 'eks_in_cluster'))
    eks_service_account_name = config.dim_value(config_name, 'eks_service_account_name')
    eks_context = None if eks_in_cluster else config.dim_value(config_name, 'eks_context')
    eks_commands = format_spark_submit(config_name, config.dim_value(config_name, 'eks_commands'))

    return KubernetesPodOperator(
      dag=dag,
      cmds=eks_commands,
      namespace=eks_namespace,
      in_cluster=eks_in_cluster,
      cluster_context=eks_context,
      task_id='livy_sensor_log_ingest',
      service_account_name=eks_service_account_name,
      image=config.dim_value(config_name, 'eks_image_spark'),
      name='spark_job_{}_{}'.format('vpa_ingest', uuid.uuid4())
   )