Created
May 8, 2018 17:00
-
-
Save dkhabarov/5b1c3b231e325c6cf2ac4170da8a1e03 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
Code that goes along with the Airflow located at: | |
http://airflow.readthedocs.org/en/latest/tutorial.html | |
""" | |
import airflow | |
from airflow import DAG | |
from airflow.operators.bash_operator import BashOperator | |
from airflow.operators.latest_only_operator import LatestOnlyOperator | |
from datetime import datetime, timedelta | |
default_args = { | |
'owner': 'airflow', | |
'depends_on_past': False, | |
'start_date': airflow.utils.dates.days_ago(0), | |
'email': ['airflow@airflow.com'], | |
'email_on_failure': False, | |
'email_on_retry': False, | |
'retries': 1, | |
'retry_delay': timedelta(minutes=5), | |
# 'queue': 'bash_queue', | |
# 'pool': 'backfill', | |
# 'priority_weight': 10, | |
# 'end_date': datetime(2016, 1, 1), | |
} | |
dag = DAG( | |
'tutorial', default_args=default_args, schedule_interval="*/10 * * * *") | |
latest_only = LatestOnlyOperator(task_id='latest_only', dag=dag) | |
# t1, t2 and t3 are examples of tasks created by instantiating operators | |
t1 = BashOperator( | |
task_id='print_date', | |
bash_command='date', | |
dag=dag) | |
t2 = BashOperator( | |
task_id='sleep', | |
bash_command='sleep 5', | |
retries=3, | |
dag=dag) | |
templated_command = """ | |
{% for i in range(5) %} | |
echo "{{ ds }}" | |
echo "{{ macros.ds_add(ds, 7)}}" | |
echo "{{ params.my_param }}" | |
{% endfor %} | |
""" | |
t3 = BashOperator( | |
task_id='templated', | |
bash_command=templated_command, | |
params={'my_param': 'Parameter I passed in'}, | |
dag=dag) | |
latest_only >> t1 >> t2 >> t3 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment