Skip to content

Instantly share code, notes, and snippets.

@rsignell-usgs

rsignell-usgs/create_coiled_env.ipynb

Created June 15, 2022 14:57
Show Gist options
  • Save rsignell-usgs/1821962b2a2f3ddbb642e8b37e35e2f6 to your computer and use it in GitHub Desktop.
Save rsignell-usgs/1821962b2a2f3ddbb642e8b37e35e2f6 to your computer and use it in GitHub Desktop.
Download ZIP
create_coiled_env.ipynb
Display the source blob
Display the rendered blob
Raw
create_coiled_env.ipynb
{
"cells": [
{
"cell_type": "markdown",
"id": "c9c67231-e69f-4a15-9b6a-b29f7553f209",
"metadata": {},
"source": [
"# Create a Coiled software environment that matches the local environment"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "6c76c093-5cc4-4b8c-b495-3baefe74512c",
"metadata": {},
"outputs": [],
"source": [
"import coiled"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2b3682da-0a02-4a49-9a3d-a5b98c559f8a",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Traceback (most recent call last):\n",
" File \"/usr/src/app/cloud/consumers/common.py\", line 424, in websocket_error_context\n",
" yield\n",
" File \"/usr/src/app/cloud/consumers/common.py\", line 184, in receive\n",
" return await self.receive_json(data, **kwargs)\n",
" File \"/usr/src/app/cloud/consumers/common.py\", line 339, in receive_json\n",
" return await handler(content)\n",
" File \"/usr/src/app/cloud/consumers/common.py\", line 362, in wrapper\n",
" return await f(self, *args, **kwargs)\n",
" File \"/usr/src/app/cloud/consumers/software_environments.py\", line 129, in handle_build\n",
" await create_software_environment(\n",
" File \"<string>\", line 21, in func_wrapper\n",
" File \"/usr/src/app/software_environments/core.py\", line 150, in create_software_environment\n",
" build_location, read_creds, write_creds = await registry.prepare_build(\n",
" File \"/usr/src/app/software_environments/registry.py\", line 259, in prepare_build\n",
" repo_description = await aws.create_repository(\n",
" File \"/usr/src/app/software_environments/aws.py\", line 176, in create_repository\n",
" raise error\n",
" File \"/usr/src/app/software_environments/aws.py\", line 169, in create_repository\n",
" res = await ecr.create_repository(**kwargs)\n",
" File \"/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/aiobotocore/patch.py\", line 110, in _wrapped_api_call\n",
" result = await original_func(*args, **kwargs)\n",
" File \"/opt/conda/lib/python3.8/site-packages/backoff/_async.py\", line 133, in retry\n",
" ret = await target(*args, **kwargs)\n",
" File \"/usr/src/app/backends/__init__.py\", line 15, in _wrapper\n",
" return await original_func(*args, **kwargs)\n",
" File \"/opt/conda/lib/python3.8/site-packages/aiobotocore/client.py\", line 154, in _make_api_call\n",
" raise error_class(parsed_response, operation_name)\n",
"botocore.exceptions.ClientError: An error occurred (AccessDeniedException) when calling the CreateRepository operation: User: arn:aws:iam::097532040392:user/rsignell is not authorized to perform: ecr:CreateRepository on resource: arn:aws:ecr:us-west-2:097532040392:repository/prod/rsignell-odc with an explicit deny in a service control policy\n",
"\n"
]
},
{
"ename": "ServerError",
"evalue": "Traceback (most recent call last):\n File \"/usr/src/app/cloud/consumers/common.py\", line 424, in websocket_error_context\n yield\n File \"/usr/src/app/cloud/consumers/common.py\", line 184, in receive\n return await self.receive_json(data, **kwargs)\n File \"/usr/src/app/cloud/consumers/common.py\", line 339, in receive_json\n return await handler(content)\n File \"/usr/src/app/cloud/consumers/common.py\", line 362, in wrapper\n return await f(self, *args, **kwargs)\n File \"/usr/src/app/cloud/consumers/software_environments.py\", line 129, in handle_build\n await create_software_environment(\n File \"<string>\", line 21, in func_wrapper\n File \"/usr/src/app/software_environments/core.py\", line 150, in create_software_environment\n build_location, read_creds, write_creds = await registry.prepare_build(\n File \"/usr/src/app/software_environments/registry.py\", line 259, in prepare_build\n repo_description = await aws.create_repository(\n File \"/usr/src/app/software_environments/aws.py\", line 176, in create_repository\n raise error\n File \"/usr/src/app/software_environments/aws.py\", line 169, in create_repository\n res = await ecr.create_repository(**kwargs)\n File \"/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/aiobotocore/patch.py\", line 110, in _wrapped_api_call\n result = await original_func(*args, **kwargs)\n File \"/opt/conda/lib/python3.8/site-packages/backoff/_async.py\", line 133, in retry\n ret = await target(*args, **kwargs)\n File \"/usr/src/app/backends/__init__.py\", line 15, in _wrapper\n return await original_func(*args, **kwargs)\n File \"/opt/conda/lib/python3.8/site-packages/aiobotocore/client.py\", line 154, in _make_api_call\n raise error_class(parsed_response, operation_name)\nbotocore.exceptions.ClientError: An error occurred (AccessDeniedException) when calling the CreateRepository operation: User: arn:aws:iam::097532040392:user/rsignell is not authorized to perform: ecr:CreateRepository on resource: arn:aws:ecr:us-west-2:097532040392:repository/prod/rsignell-odc with an explicit deny in a service control policy\n",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mServerError\u001b[0m Traceback (most recent call last)",
"Input \u001b[0;32mIn [2]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mcoiled\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate_software_environment\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2\u001b[0m \u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43modc\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[43m \u001b[49m\u001b[43mconda\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43menvironment.yml\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/.conda/envs/odc/lib/python3.9/site-packages/coiled/core.py:2919\u001b[0m, in \u001b[0;36mcreate_software_environment\u001b[0;34m(name, account, conda, pip, container, log_output, post_build, conda_env_name, backend_options, private, force_rebuild, environ)\u001b[0m\n\u001b[1;32m 2878\u001b[0m \u001b[38;5;124;03m\"\"\"Create a software environment\u001b[39;00m\n\u001b[1;32m 2879\u001b[0m \n\u001b[1;32m 2880\u001b[0m \u001b[38;5;124;03mParameters\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 2916\u001b[0m \u001b[38;5;124;03m Dictionary of environment variables.\u001b[39;00m\n\u001b[1;32m 2917\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 2918\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m Cloud() \u001b[38;5;28;01mas\u001b[39;00m cloud:\n\u001b[0;32m-> 2919\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mcloud\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate_software_environment\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2920\u001b[0m \u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2921\u001b[0m \u001b[43m \u001b[49m\u001b[43maccount\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43maccount\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2922\u001b[0m \u001b[43m \u001b[49m\u001b[43mconda\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconda\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2923\u001b[0m \u001b[43m \u001b[49m\u001b[43mpip\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpip\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2924\u001b[0m \u001b[43m \u001b[49m\u001b[43mcontainer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontainer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2925\u001b[0m \u001b[43m \u001b[49m\u001b[43mpost_build\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpost_build\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2926\u001b[0m \u001b[43m \u001b[49m\u001b[43mlog_output\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlog_output\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2927\u001b[0m \u001b[43m \u001b[49m\u001b[43mconda_env_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconda_env_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2928\u001b[0m \u001b[43m \u001b[49m\u001b[43mbackend_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbackend_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2929\u001b[0m \u001b[43m \u001b[49m\u001b[43mprivate\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprivate\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2930\u001b[0m \u001b[43m \u001b[49m\u001b[43mforce_rebuild\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mforce_rebuild\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2931\u001b[0m \u001b[43m \u001b[49m\u001b[43menviron\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43menviron\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2932\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/.conda/envs/odc/lib/python3.9/site-packages/coiled/core.py:1227\u001b[0m, in \u001b[0;36mCloud.create_software_environment\u001b[0;34m(self, name, account, conda, pip, container, post_build, conda_env_name, backend_options, log_output, private, force_rebuild, environ)\u001b[0m\n\u001b[1;32m 1211\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcreate_software_environment\u001b[39m(\n\u001b[1;32m 1212\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 1213\u001b[0m name: \u001b[38;5;28mstr\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1225\u001b[0m environ: Optional[Dict] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 1226\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Optional[Awaitable[\u001b[38;5;28;01mNone\u001b[39;00m]]:\n\u001b[0;32m-> 1227\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sync\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1228\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_create_software_environment\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1229\u001b[0m \u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1230\u001b[0m \u001b[43m \u001b[49m\u001b[43maccount\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43maccount\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1231\u001b[0m \u001b[43m \u001b[49m\u001b[43mconda\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconda\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1232\u001b[0m \u001b[43m \u001b[49m\u001b[43mpip\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpip\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1233\u001b[0m \u001b[43m \u001b[49m\u001b[43mcontainer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontainer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1234\u001b[0m \u001b[43m \u001b[49m\u001b[43mpost_build\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpost_build\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1235\u001b[0m \u001b[43m \u001b[49m\u001b[43mlog_output\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlog_output\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1236\u001b[0m \u001b[43m \u001b[49m\u001b[43mconda_env_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconda_env_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1237\u001b[0m \u001b[43m \u001b[49m\u001b[43mbackend_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbackend_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1238\u001b[0m \u001b[43m \u001b[49m\u001b[43mprivate\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprivate\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1239\u001b[0m \u001b[43m \u001b[49m\u001b[43mforce_rebuild\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mforce_rebuild\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1240\u001b[0m \u001b[43m \u001b[49m\u001b[43menviron\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43menviron\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1241\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/.conda/envs/odc/lib/python3.9/site-packages/coiled/core.py:529\u001b[0m, in \u001b[0;36mCloud._sync\u001b[0;34m(self, func, asynchronous, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 527\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m future\n\u001b[1;32m 528\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 529\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msync\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 530\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mloop\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfunc\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallback_timeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcallback_timeout\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\n\u001b[1;32m 531\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/.conda/envs/odc/lib/python3.9/site-packages/distributed/utils.py:364\u001b[0m, in \u001b[0;36msync\u001b[0;34m(loop, func, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 362\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m error[\u001b[38;5;241m0\u001b[39m]:\n\u001b[1;32m 363\u001b[0m typ, exc, tb \u001b[38;5;241m=\u001b[39m error[\u001b[38;5;241m0\u001b[39m]\n\u001b[0;32m--> 364\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m exc\u001b[38;5;241m.\u001b[39mwith_traceback(tb)\n\u001b[1;32m 365\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 366\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m result[\u001b[38;5;241m0\u001b[39m]\n",
"File \u001b[0;32m~/.conda/envs/odc/lib/python3.9/site-packages/distributed/utils.py:349\u001b[0m, in \u001b[0;36msync.<locals>.f\u001b[0;34m()\u001b[0m\n\u001b[1;32m 347\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m callback_timeout \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 348\u001b[0m future \u001b[38;5;241m=\u001b[39m asyncio\u001b[38;5;241m.\u001b[39mwait_for(future, callback_timeout)\n\u001b[0;32m--> 349\u001b[0m result[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01myield\u001b[39;00m future\n\u001b[1;32m 350\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m:\n\u001b[1;32m 351\u001b[0m error[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m=\u001b[39m sys\u001b[38;5;241m.\u001b[39mexc_info()\n",
"File \u001b[0;32m~/.conda/envs/odc/lib/python3.9/site-packages/tornado/gen.py:762\u001b[0m, in \u001b[0;36mRunner.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 759\u001b[0m exc_info \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 761\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 762\u001b[0m value \u001b[38;5;241m=\u001b[39m \u001b[43mfuture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 763\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m:\n\u001b[1;32m 764\u001b[0m exc_info \u001b[38;5;241m=\u001b[39m sys\u001b[38;5;241m.\u001b[39mexc_info()\n",
"File \u001b[0;32m~/.conda/envs/odc/lib/python3.9/site-packages/coiled/context.py:77\u001b[0m, in \u001b[0;36mtrack_context.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 75\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 76\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m operation_context(name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfunc\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__module__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfunc\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__qualname__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m---> 77\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
"File \u001b[0;32m~/.conda/envs/odc/lib/python3.9/site-packages/coiled/core.py:1376\u001b[0m, in \u001b[0;36mCloud._create_software_environment\u001b[0;34m(self, name, account, conda, pip, container, post_build, conda_env_name, log_output, backend_options, private, force_rebuild, environ)\u001b[0m\n\u001b[1;32m 1374\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m ws\u001b[38;5;241m.\u001b[39mconnect()\n\u001b[1;32m 1375\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m ws\u001b[38;5;241m.\u001b[39msend_json(data)\n\u001b[0;32m-> 1376\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m ws\u001b[38;5;241m.\u001b[39mstream_messages()\n",
"File \u001b[0;32m~/.conda/envs/odc/lib/python3.9/site-packages/coiled/websockets.py:205\u001b[0m, in \u001b[0;36mWebsocketConnector.stream_messages\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 204\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mstream_messages\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 205\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mgather(\n\u001b[1;32m 206\u001b[0m _exhaust_async_iterator(\u001b[38;5;28mself\u001b[39m),\n\u001b[1;32m 207\u001b[0m _exhaust_async_iterator(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnotification_connector),\n\u001b[1;32m 208\u001b[0m )\n",
"File \u001b[0;32m~/.conda/envs/odc/lib/python3.9/site-packages/coiled/websockets.py:21\u001b[0m, in \u001b[0;36m_exhaust_async_iterator\u001b[0;34m(stream)\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_exhaust_async_iterator\u001b[39m(stream):\n\u001b[0;32m---> 21\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mfor\u001b[39;00m _ \u001b[38;5;129;01min\u001b[39;00m stream:\n\u001b[1;32m 22\u001b[0m \u001b[38;5;28;01mpass\u001b[39;00m\n",
"File \u001b[0;32m~/.conda/envs/odc/lib/python3.9/site-packages/coiled/websockets.py:91\u001b[0m, in \u001b[0;36mWebsocketConnectorBase.__anext__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mon_error(message)\n\u001b[1;32m 85\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m message\u001b[38;5;241m.\u001b[39mtype \u001b[38;5;129;01min\u001b[39;00m (WSMsgType\u001b[38;5;241m.\u001b[39mCLOSE, WSMsgType\u001b[38;5;241m.\u001b[39mCLOSED, WSMsgType\u001b[38;5;241m.\u001b[39mCLOSING):\n\u001b[1;32m 86\u001b[0m \u001b[38;5;66;03m# From what I could tell, whenever we get any one of\u001b[39;00m\n\u001b[1;32m 87\u001b[0m \u001b[38;5;66;03m# these, the ClientWebsocketResponse already has a close_code\u001b[39;00m\n\u001b[1;32m 88\u001b[0m \u001b[38;5;66;03m# and an exception (when there is one), so we should be able to\u001b[39;00m\n\u001b[1;32m 89\u001b[0m \u001b[38;5;66;03m# handle the close on any one of these message types, but I may\u001b[39;00m\n\u001b[1;32m 90\u001b[0m \u001b[38;5;66;03m# end up being wrong\u001b[39;00m\n\u001b[0;32m---> 91\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mon_close(message)\n\u001b[1;32m 92\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mStopAsyncIteration\u001b[39;00m\n",
"File \u001b[0;32m~/.conda/envs/odc/lib/python3.9/site-packages/coiled/websockets.py:196\u001b[0m, in \u001b[0;36mWebsocketConnector.on_close\u001b[0;34m(self, message)\u001b[0m\n\u001b[1;32m 194\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m ws_exception:\n\u001b[1;32m 195\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mWebsocket exception: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mws_exception\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, file\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlog_output)\n\u001b[0;32m--> 196\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m ServerError(error_messages[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m])\n\u001b[1;32m 197\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m ws_exception \u001b[38;5;129;01mor\u001b[39;00m is_bad_close_code:\n\u001b[1;32m 198\u001b[0m ws_error_message \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mError with connection to server: code \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mclose_code\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n",
"\u001b[0;31mServerError\u001b[0m: Traceback (most recent call last):\n File \"/usr/src/app/cloud/consumers/common.py\", line 424, in websocket_error_context\n yield\n File \"/usr/src/app/cloud/consumers/common.py\", line 184, in receive\n return await self.receive_json(data, **kwargs)\n File \"/usr/src/app/cloud/consumers/common.py\", line 339, in receive_json\n return await handler(content)\n File \"/usr/src/app/cloud/consumers/common.py\", line 362, in wrapper\n return await f(self, *args, **kwargs)\n File \"/usr/src/app/cloud/consumers/software_environments.py\", line 129, in handle_build\n await create_software_environment(\n File \"<string>\", line 21, in func_wrapper\n File \"/usr/src/app/software_environments/core.py\", line 150, in create_software_environment\n build_location, read_creds, write_creds = await registry.prepare_build(\n File \"/usr/src/app/software_environments/registry.py\", line 259, in prepare_build\n repo_description = await aws.create_repository(\n File \"/usr/src/app/software_environments/aws.py\", line 176, in create_repository\n raise error\n File \"/usr/src/app/software_environments/aws.py\", line 169, in create_repository\n res = await ecr.create_repository(**kwargs)\n File \"/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/aiobotocore/patch.py\", line 110, in _wrapped_api_call\n result = await original_func(*args, **kwargs)\n File \"/opt/conda/lib/python3.8/site-packages/backoff/_async.py\", line 133, in retry\n ret = await target(*args, **kwargs)\n File \"/usr/src/app/backends/__init__.py\", line 15, in _wrapper\n return await original_func(*args, **kwargs)\n File \"/opt/conda/lib/python3.8/site-packages/aiobotocore/client.py\", line 154, in _make_api_call\n raise error_class(parsed_response, operation_name)\nbotocore.exceptions.ClientError: An error occurred (AccessDeniedException) when calling the CreateRepository operation: User: arn:aws:iam::097532040392:user/rsignell is not authorized to perform: ecr:CreateRepository on resource: arn:aws:ecr:us-west-2:097532040392:repository/prod/rsignell-odc with an explicit deny in a service control policy\n"
]
}
],
"source": [
"coiled.create_software_environment(\n",
" name=\"odc\",\n",
" conda=\"environment.yml\",\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6904d898-08e0-4a07-9d0e-1abf56c207ed",
"metadata": {},
"outputs": [],
"source": [
"coiled.list_software_environments()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "df01aed2-a08c-49a8-9625-39659b84e770",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "odc:Python",
"language": "python",
"name": "conda-env-odc-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.12"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment