Skip to content

Instantly share code, notes, and snippets.

@alimanfoo
Created July 1, 2021 13:36
Show Gist options
  • Save alimanfoo/1b3d3a776d5561cac153a72672e63619 to your computer and use it in GitHub Desktop.
Save alimanfoo/1b3d3a776d5561cac153a72672e63619 to your computer and use it in GitHub Desktop.
Display the source blob
Display the rendered blob
Raw
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "2dcd01db-fc9e-42b6-8c13-df5ff7876595",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"hello\n"
]
}
],
"source": [
"print('hello')"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "9e9ef947-c9b1-4706-8d9e-f3226e9a583f",
"metadata": {},
"outputs": [],
"source": [
"from dask_gateway import Gateway\n",
"gateway = Gateway()"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "a4f6cdad-1926-4bfc-9920-a18bf23a702e",
"metadata": {},
"outputs": [],
"source": [
"# options = gateway.cluster_options()\n",
"# options"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5ef8e5b2-c073-4eed-859b-44a363cb9403",
"metadata": {},
"outputs": [],
"source": [
"# launch a cluster with the options selected via the widgets above\n",
"# cluster = gateway.new_cluster(options)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "f991632c-560d-49a5-a6d5-396159f61809",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "3def748988bc444cbbdbadbd2a767054",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"VBox(children=(HTML(value='<h2>GatewayCluster</h2>'), HBox(children=(HTML(value='\\n<div>\\n<style scoped>\\n …"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# launch a cluster specifying environment and profile as arguments\n",
"# may take a few minutes to provision VM for the scheduler\n",
"cluster = gateway.new_cluster(environment=\"abafar-20210701a\", profile=\"High Memory Worker\")\n",
"cluster"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "697a5cd3-34ec-41bc-828c-00aad0894f37",
"metadata": {},
"outputs": [],
"source": [
"cluster.scale(8)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "a4e63eaf-ffbd-4c69-b32e-b179b7fdf95d",
"metadata": {},
"outputs": [],
"source": [
"client = cluster.get_client()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "b513edaa-3fef-48e0-8333-74732e51f1b4",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([1.00083673, 1.00276507, 0.99661714, ..., 1.00159093, 1.00030677,\n",
" 0.99948974])"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import dask.array as da\n",
"x = da.random.random((50000, 50000), chunks=(1000, 1000))\n",
"y = x + x.T\n",
"z = y[::2, 5000:].mean(axis=1)\n",
"z.compute()"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "87ef4049-2c96-4256-bbb2-494dc84fa3d0",
"metadata": {},
"outputs": [],
"source": [
"cluster.scale(0)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "de4c410b-dcc8-47ad-8b6b-2c05b58f22a7",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:abafar-20210701a]",
"language": "python",
"name": "conda-env-abafar-20210701a-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment