Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
How to profile the import of an Airflow DAG definition file
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"%load_ext snakeviz"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[2016-11-18 15:00:48,485] {__init__.py:36} INFO - Using executor SequentialExecutor\n"
]
}
],
"source": [
"from airflow.bin import cli"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"parser = cli.CLIFactory.get_parser()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"args = parser.parse_args(['test', 'atl', 'atl_is_done', '2016-11-16T00:00:00', '--dry_run'])"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"Namespace(dag_id='atl', dry_run=True, execution_date=datetime.datetime(2016, 11, 16, 0, 0), func=<function test at 0x11093bea0>, subcommand='test', subdir='/Users/mattdavis/airflow/dags', task_id='atl_is_done', task_params=None)"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"args"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[2016-11-18 15:00:51,489] {models.py:154} INFO - Filling up the DagBag from /Users/mattdavis/airflow/dags\n",
"[INFO ] 2016-11-18 15:00:51.815000: Using connection to: localhost\n",
"[INFO ] 2016-11-18 15:00:51.817000: Using connection to: localhost\n",
"[INFO ] 2016-11-18 15:00:53.691000: timing Set up dependencies for all tasks\n",
"[INFO ] 2016-11-18 15:00:54.848000: elapsed time Set up dependencies for all tasks: 1.156\n",
"[INFO ] 2016-11-18 15:00:54.855000: timing make_last_task\n",
"[INFO ] 2016-11-18 15:00:54.865000: elapsed time make_last_task: 0.009\n",
"[INFO ] 2016-11-18 15:00:54.868000: timing make_first_task\n",
"[INFO ] 2016-11-18 15:00:55.748000: elapsed time make_first_task: 0.88\n",
"[INFO ] 2016-11-18 15:00:58.431000: Using connection to: localhost\n",
"[INFO ] 2016-11-18 15:00:58.434000: Using connection to: localhost\n",
"[INFO ] 2016-11-18 15:00:58.437000: Using connection to: localhost\n",
"[INFO ] 2016-11-18 15:00:58.440000: Using connection to: localhost\n",
"[INFO ] 2016-11-18 15:00:59.358000: Using connection to: localhost\n",
"[INFO ] 2016-11-18 15:00:59.360000: Using connection to: localhost\n",
"[INFO ] 2016-11-18 15:00:59.362000: Using connection to: localhost\n",
"[INFO ] 2016-11-18 15:00:59.365000: Using connection to: localhost\n",
"[INFO ] 2016-11-18 15:01:00.280000: timing make_first_task\n",
"[INFO ] 2016-11-18 15:01:03.887000: elapsed time make_first_task: 3.607\n",
"[INFO ] 2016-11-18 15:01:04.075000: Dry run\n",
" \n",
"*** Profile stats marshalled to file '/var/folders/x6/p7mnf01514l7nkl52k9mlbbh0000gp/T/tmp0eb4jfsy'. \n"
]
}
],
"source": [
"%snakeviz cli.test(args)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"anaconda-cloud": {},
"kernelspec": {
"display_name": "Python [conda env:provider_pipe]",
"language": "python",
"name": "conda-env-provider_pipe-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
}
},
"nbformat": 4,
"nbformat_minor": 1
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.