Skip to content

Instantly share code, notes, and snippets.

@jamiekt
Last active April 14, 2024 20:29
Show Gist options
  • Save jamiekt/983e05878c89dc97705d2f2c67060a03 to your computer and use it in GitHub Desktop.
Save jamiekt/983e05878c89dc97705d2f2c67060a03 to your computer and use it in GitHub Desktop.
Airflow XCom problem
import datetime
from airflow.decorators import task, dag
from airflow.providers.amazon.aws.operators.emr import EmrServerlessCreateApplicationOperator
@dag(
dag_id="demo-xcom-problem",
start_date=datetime.datetime(2021, 1, 1),
catchup=False
)
def taskflow():
@task(multiple_outputs=True)
def config() -> dict:
return {
"name": "my-spark-app",
}
create_app = EmrServerlessCreateApplicationOperator(
task_id="create_spark_app",
job_type="SPARK",
release_label="emr-6.9.0",
config=config(),
aws_conn_id="",
)
create_app
taskflow()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment