Created
May 24, 2021 17:33
-
-
Save aktech/4ec2b8cd275a0e3205e0bb76365a7d3c to your computer and use it in GitHub Desktop.
qhub-dask-worker-environment-variable-example
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"cells": [ | |
{ | |
"cell_type": "code", | |
"execution_count": 1, | |
"id": "2943c876-298d-4411-8e4b-6b245df0e29e", | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"from dask.distributed import Client\n", | |
"import dask_gateway\n", | |
"\n", | |
"gateway = dask_gateway.Gateway()\n", | |
"options = gateway.cluster_options()" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 2, | |
"id": "4852872a-bcc2-4b5a-8621-b120cb3ecc4f", | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"data": { | |
"application/vnd.jupyter.widget-view+json": { | |
"model_id": "c5612325bed948ce9e8568b5736c5710", | |
"version_major": 2, | |
"version_minor": 0 | |
}, | |
"text/plain": [ | |
"VBox(children=(HTML(value='<h2>Cluster Options</h2>'), GridBox(children=(HTML(value=\"<p style='font-weight: bo…" | |
] | |
}, | |
"metadata": {}, | |
"output_type": "display_data" | |
} | |
], | |
"source": [ | |
"options" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 3, | |
"id": "c6dc4e4f-980f-4c52-8154-5fff145346f3", | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"env_vars = {\n", | |
" \"AMIT\": \"KUMAR\",\n", | |
" \"ENV_1\": \"VALUE_1\"\n", | |
"}\n", | |
"options.environment_vars = env_vars" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 4, | |
"id": "d5da2b34-1c7b-4e59-ac8c-791fa8b46912", | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/html": [ | |
"<table style=\"border: 2px solid white;\">\n", | |
"<tr>\n", | |
"<td style=\"vertical-align: top; border: 0px solid white\">\n", | |
"<h3 style=\"text-align: left;\">Client</h3>\n", | |
"<ul style=\"text-align: left; list-style: none; margin: 0; padding: 0;\">\n", | |
" <li><b>Scheduler: </b>gateway://qhub-amit-aws.qhub.dev:8786/dev.6f13f27a129246c88e0bd9a868c31bf5</li>\n", | |
" <li><b>Dashboard: </b><a href='https://qhub-amit-aws.qhub.dev/gateway/clusters/dev.6f13f27a129246c88e0bd9a868c31bf5/status' target='_blank'>https://qhub-amit-aws.qhub.dev/gateway/clusters/dev.6f13f27a129246c88e0bd9a868c31bf5/status</a></li>\n", | |
"</ul>\n", | |
"</td>\n", | |
"<td style=\"vertical-align: top; border: 0px solid white\">\n", | |
"<h3 style=\"text-align: left;\">Cluster</h3>\n", | |
"<ul style=\"text-align: left; list-style:none; margin: 0; padding: 0;\">\n", | |
" <li><b>Workers: </b>0</li>\n", | |
" <li><b>Cores: </b>0</li>\n", | |
" <li><b>Memory: </b>0 B</li>\n", | |
"</ul>\n", | |
"</td>\n", | |
"</tr>\n", | |
"</table>" | |
], | |
"text/plain": [ | |
"<Client: 'tls://10.10.1.28:8786' processes=0 threads=0, memory=0 B>" | |
] | |
}, | |
"execution_count": 4, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"cluster = gateway.new_cluster(options)\n", | |
"cluster.scale(1)\n", | |
"client = Client(cluster)\n", | |
"client" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 5, | |
"id": "ea7501dd-4c34-4b23-b049-80767ad53f84", | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"def func():\n", | |
" import os\n", | |
" return os.environ" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 6, | |
"id": "c76f6262-55e4-4b12-a319-b6504e031368", | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/html": [ | |
"<b>Future: func</b> <font color=\"gray\">status: </font><font color=\"black\">pending</font>, <font color=\"gray\">key: </font>func-12e19fd149fc0661fbb8ea46bb18b4b2" | |
], | |
"text/plain": [ | |
"<Future: pending, key: func-12e19fd149fc0661fbb8ea46bb18b4b2>" | |
] | |
}, | |
"execution_count": 6, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"f = client.submit(func)\n", | |
"f" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 7, | |
"id": "9d404f79-5665-41e6-a9ea-b268acddf223", | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/plain": [ | |
"'KUMAR'" | |
] | |
}, | |
"execution_count": 7, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"f.result()['AMIT']" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 8, | |
"id": "a513b89d-453e-4ca9-a9d3-0d818ffdcf73", | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/plain": [ | |
"'VALUE_1'" | |
] | |
}, | |
"execution_count": 8, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"f.result()['ENV_1']" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "66296e22-ccba-4cc2-89ec-9d89e10262e1", | |
"metadata": {}, | |
"outputs": [], | |
"source": [] | |
} | |
], | |
"metadata": { | |
"kernelspec": { | |
"display_name": "Python [conda env:dask]", | |
"language": "python", | |
"name": "conda-env-dask-py" | |
}, | |
"language_info": { | |
"codemirror_mode": { | |
"name": "ipython", | |
"version": 3 | |
}, | |
"file_extension": ".py", | |
"mimetype": "text/x-python", | |
"name": "python", | |
"nbconvert_exporter": "python", | |
"pygments_lexer": "ipython3", | |
"version": "3.8.10" | |
} | |
}, | |
"nbformat": 4, | |
"nbformat_minor": 5 | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment