Created
September 6, 2023 23:32
-
-
Save jbusecke/6497ccba119eb6a8e0a0cd79e7e50acb to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"cells": [ | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "bb89df8d-3a0b-478c-a7fc-f846b673d819", | |
"metadata": { | |
"tags": [] | |
}, | |
"outputs": [], | |
"source": [ | |
"# !pip install pip install git+https://github.com/jbusecke/pangeo-forge-esgf.git@8303d22" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "48b89aa4-c348-4904-a67f-04a396ba0e18", | |
"metadata": { | |
"tags": [] | |
}, | |
"outputs": [], | |
"source": [ | |
"iids = [\n", | |
" 'CMIP6.CMIP.CCCma.CanESM5.historical.r33i1p2f1.Omon.mlotst.gn.v20190429',\n", | |
" 'CMIP6.ScenarioMIP.EC-Earth-Consortium.EC-Earth3.ssp245.r18i1p1f2.Omon.mlotst.gn.v20210407',\n", | |
" 'CMIP6.ScenarioMIP.EC-Earth-Consortium.EC-Earth3.ssp245.r24i1p1f2.Omon.sos.gn.v20210407',\n", | |
"]" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "2d33f725-2931-4466-8661-9dc576660e33", | |
"metadata": { | |
"tags": [] | |
}, | |
"outputs": [], | |
"source": [ | |
"# query pangeo-forge-esgf for the urls\n", | |
"# TODO: I need to write the order of urls into the attrs\n", | |
"import logging\n", | |
"import pangeo_forge_esgf\n", | |
"from pangeo_forge_esgf import get_urls_from_esgf, setup_logging\n", | |
"\n", | |
"# setup_logging('DEBUG')\n", | |
"# setup_logging('WARNING')\n", | |
"setup_logging('INFO')" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "76be991d-a987-4174-a693-70b97068f64b", | |
"metadata": { | |
"tags": [] | |
}, | |
"outputs": [], | |
"source": [ | |
"for i in range(10):\n", | |
" url_dict = await get_urls_from_esgf(iids, limit_per_host=5, max_concurrency=5, max_concurrency_response=5)\n", | |
" if 'CMIP6.CMIP.CCCma.CanESM5.historical.r33i1p2f1.Omon.mlotst.gn.v20190429' in url_dict.keys():\n", | |
" print('SUCCESS')\n", | |
" else:\n", | |
" print('SCHEISSE')" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"id": "1690e029-83c7-43d8-8438-84f4badc68f7", | |
"metadata": { | |
"tags": [] | |
}, | |
"source": [ | |
"This is suspicious...if I restart this fresh this works, but when I loop over it again and again, it fails..." | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"id": "886c9eb7-af53-4dc7-a0a5-f7a86ebb4d3e", | |
"metadata": {}, | |
"source": [ | |
"Now it does not fail anymore...ughhh this is very hard. Shooting at a moving target..." | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "ae573d6a-65ea-4761-a6da-2679b71997ce", | |
"metadata": {}, | |
"outputs": [], | |
"source": [] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "7227e453-9fbf-451a-9c50-1e6b4bc59a0f", | |
"metadata": {}, | |
"outputs": [], | |
"source": [] | |
} | |
], | |
"metadata": { | |
"kernelspec": { | |
"display_name": "Python 3 (ipykernel)", | |
"language": "python", | |
"name": "python3" | |
}, | |
"language_info": { | |
"codemirror_mode": { | |
"name": "ipython", | |
"version": 3 | |
}, | |
"file_extension": ".py", | |
"mimetype": "text/x-python", | |
"name": "python", | |
"nbconvert_exporter": "python", | |
"pygments_lexer": "ipython3", | |
"version": "3.10.12" | |
} | |
}, | |
"nbformat": 4, | |
"nbformat_minor": 5 | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment