Skip to content

Instantly share code, notes, and snippets.

@boxysean
Created December 17, 2015 16:33
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save boxysean/5b5d73f4fc3541d63c4b to your computer and use it in GitHub Desktop.
Save boxysean/5b5d73f4fc3541d63c4b to your computer and use it in GitHub Desktop.
DAG that crashes Airflow scheduler quickly
# airflow.cfg
[core]
airflow_home = /opt/titan/airflow
dags_folder = /opt/titan/airflow/dags
s3_log_folder = None
executor = CeleryExecutor
sql_alchemy_conn = mysql://airflow:airflow@mysql/airflow
parallelism = 32
dag_concurrency = 16
max_active_runs_per_dag = 16
load_examples = False
plugins_folder = /opt/titan/airflow/plugins
fernet_key = M4_FERNET_KEY
donot_pickle = False
[webserver]
base_url = M4_BASE_URL
web_server_host = 0.0.0.0
web_server_port = 8080
secret_key = MY_KEY
workers = 4
worker_class = sync
expose_config = true
authenticate = False
filter_by_owner = False
[celery]
celery_app_name = airflow.executors.celery_executor
celeryd_concurrency = 16
worker_log_server_port = 8793
broker_url = redis://redis/0
celery_result_backend = redis://redis/0
flower_port = 5555
default_queue = default
[scheduler]
job_heartbeat_sec = 5
scheduler_heartbeat_sec = 5
# dags/dag_that_crashes_scheduler.py
from __future__ import print_function
from datetime import datetime, timedelta
from airflow.models import DAG
from airflow.operators.bash_operator import BashOperator
args = {
'owner': 'boxysean',
'start_date': datetime.utcnow(),
'depends_on_past': False,
}
dag = DAG(
dag_id='crash_dag',
default_args=args,
schedule_interval=timedelta(minutes=1))
bash_crash = BashOperator(
task_id='bash_crash',
bash_command="echo we will we will crash you && sleep 5",
dag=dag)
airflow==1.6.1
alembic==0.8.3
amqp==1.4.7
anyjson==0.3.3
Babel==2.1.1
backports-abc==0.4
backports.ssl-match-hostname==3.4.0.2
beautifulsoup4==4.4.1
billiard==3.3.0.21
celery==3.1.19
certifi==2015.11.20.1
cffi==1.3.1
chartkick==0.4.2
colander==1.0
croniter==0.3.10
cryptography==1.1.1
cx-Oracle==5.2
dill==0.2.4
Django==1.9
docker-compose==1.5.1
docker-py==1.6.0
dockerpty==0.3.4
docopt==0.6.2
enum34==1.1.1
Flask==0.10.1
Flask-Admin==1.2.0
Flask-Cache==0.13.1
Flask-Login==0.2.11
Flask-WTF==0.12
flower==0.8.3
functools32==3.2.3-2
future==0.15.2
futures==3.0.3
geopy==1.11.0
gunicorn==19.4.1
idna==2.0
ipaddress==1.0.15
iso8601==0.1.11
itsdangerous==0.24
JayDeBeApi==0.2.0
Jinja2==2.8
JPype1==0.6.1
jsonschema==2.5.1
kombu==3.0.29
lxml==3.5.0
Mako==1.0.3
Markdown==2.6.5
MarkupSafe==0.23
mysqlclient==1.3.7
nltk==3.1
numpy==1.10.1
pandas==0.17.1
petl==1.0.11
pipdeptree==0.5.0
psycopg2==2.6.1
pyasn1==0.1.9
pycparser==2.14
Pygments==2.0.2
pymongo==3.2
python-dateutil==2.4.2
python-editor==0.4
pytz==2015.7
PyYAML==3.11
redis==2.10.5
requests==2.7.0
setproctitle==1.1.9
singledispatch==3.4.0.3
six==1.10.0
slackclient==0.16-dp
SQLAlchemy==1.0.9
texttable==0.8.4
thrift==0.9.3
titlecase==0.8.0
tornado==4.3
translationstring==1.3
websocket-client==0.34.0
Werkzeug==0.11.2
wheel==0.24.0
WTForms==2.0.2
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment