Skip to content

Instantly share code, notes, and snippets.

@Sharan123

Sharan123/OpenMeteo_Matcher.ipynb

Last active November 16, 2023 21:11
Show Gist options
  • Save Sharan123/87597a1002fc5bb65c4280b37c91972f to your computer and use it in GitHub Desktop.
Save Sharan123/87597a1002fc5bb65c4280b37c91972f to your computer and use it in GitHub Desktop.
Download ZIP
Open Meteo Weather Feature Crawler
Display the source blob
Display the rendered blob
Raw
OpenMeteo_Matcher.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import geopandas as gpd\n",
"import requests\n",
"from pathlib import Path\n",
"import time"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
"#Load the geo data\n",
"\n",
"path_to_geojson = \"../Datasets/TransmissionLineData/BPA_TransmissionLines.geojson\"\n",
"lines = gpd.read_file(path_to_geojson)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Extracting Lines of Interest, i.e. XRefCd's from the outages for matching**"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [],
"source": [
"# Each XRefCd is a list of lists, given as a string in the form of [Xref1], [XRef2, Xref3], [Xref4] etc. for example\n",
"def extract_xrefcds(xrefcds):\n",
" xrefcds = xrefcds.replace(\"[\", \"\").replace(\"]\", \"\").split(\",\")\n",
" xrefcds = [xrefcd.strip() for xrefcd in xrefcds]\n",
" #print('Found the following XRefCds: ', xrefcds)\n",
" return xrefcds"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Cells crossed: 1020\n",
"Points considered: 44498\n"
]
}
],
"source": [
"cells = set()\n",
"count = 0\n",
"\n",
"#Note that the 'lines' geojson has multiple entries for some XRefCd; this successfully handles that\n",
"for idx, item in lines.iterrows():\n",
" if not(item['XRefCd'] in interest_xrefs): #If the current xrefcd/line not used, skip it\n",
" continue\n",
" \n",
" g = item['geometry']\n",
" for line in g.geoms: #This seems to be just 1; some sort of wrapper\n",
" for c in line.coords: #all of the pairs\n",
" lon = round(c[0],1) #Cells are centered at 0.1 degree longitude/latitudes resolution in OpenMeteo dataset\n",
" lat = round(c[1],1)\n",
" cells.add( (lon,lat) )\n",
" count += 1\n",
"\n",
"print(\"Cells crossed:\", len(cells))\n",
"print(\"Points considered: \",count)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Downloading the OpenMeteo Dataset"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [],
"source": [
"save_path = Path(\"../Datasets/open-metro-cells/\")\n",
"start_date= \"2016-01-01\" # YYYY-MM-DD, Inclusive\n",
"end_date= \"2022-04-06\" #Also inclusive\n",
"\n",
"cells_list = [c for c in cells] #The cells of interest as a list\n",
"cells_list.sort() #To maintain consistant order if something goes wrong and the requests have to be resumed\n"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"cell_-124.5_42.7 (1/1020)\n",
"File already exists for cell (-124.5, 42.7). Not Saving.\n",
"cell_-124.5_42.8 (2/1020)\n",
"File already exists for cell (-124.5, 42.8). Not Saving.\n",
"cell_-124.5_42.9 (3/1020)\n",
"File already exists for cell (-124.5, 42.9). Not Saving.\n",
"cell_-124.4_42.4 (4/1020)\n",
"File already exists for cell (-124.4, 42.4). Not Saving.\n",
"cell_-124.4_42.5 (5/1020)\n",
"File already exists for cell (-124.4, 42.5). Not Saving.\n",
"cell_-124.4_42.6 (6/1020)\n",
"File already exists for cell (-124.4, 42.6). Not Saving.\n",
"cell_-124.4_42.7 (7/1020)\n",
"File already exists for cell (-124.4, 42.7). Not Saving.\n",
"cell_-124.4_42.9 (8/1020)\n",
"File already exists for cell (-124.4, 42.9). Not Saving.\n",
"cell_-124.4_43.0 (9/1020)\n",
"File already exists for cell (-124.4, 43.0). Not Saving.\n",
"cell_-124.4_43.1 (10/1020)\n",
"File already exists for cell (-124.4, 43.1). Not Saving.\n",
"cell_-124.3_43.1 (11/1020)\n",
"cell_-124.3_48.1 (12/1020)\n",
"cell_-124.2_43.1 (13/1020)\n",
"Too many requests, waiting 1 second...\n",
"Error on cell (-124.2, 43.1). Code 429. Not Saving...\n"
]
}
],
"source": [
"count = 0\n",
"total = len(cells_list)\n",
"\n",
"if not save_path.is_dir():\n",
" save_path.mkdir()\n",
"\n",
"for cell in cells_list:\n",
" #Progress tracker\n",
" count += 1\n",
" \n",
" lon = cell[0]\n",
" lat = cell[1]\n",
" print(f\"cell_{lon}_{lat} ({count}/{total})\")\n",
"\n",
" # Check if the file for this cell already exists, if so, skip it. Use pathlib Path for this\n",
" # check if file cell_{lon}_{lat}.csv exists in save_path\n",
" if Path(save_path/f\"cell_{lon}_{lat}.csv\").is_file():\n",
" print(f\"File already exists for cell {cell}. Not Saving.\")\n",
" continue\n",
" #Note the cell selection method is 'Nearest'; there are various ways of selecting cells, see https://open-meteo.com/en/docs/overview\n",
" response = requests.get(f\"https://archive-api.open-meteo.com/v1/archive?latitude={lat}&longitude={lon}&start_date={start_date}&end_date={end_date}&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\", headers = {'User-agent': 'Weather Historical Crawler 0.2'})\n",
" if (response.status_code == 429): #Too many requests\n",
" print(\"Too many requests, waiting 1 second...\")\n",
" time.sleep(1)\n",
" response = requests.get(f\"https://archive-api.open-meteo.com/v1/archive?latitude={lat}&longitude={lon}&start_date={start_date}&end_date={end_date}&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto\")\n",
" if(response.status_code != 200): \n",
" print(f\"Error on cell {cell}. Code {response.status_code}. Not Saving...\")\n",
" break #don't continue if request failed\n",
" \n",
" #Get the json representation\n",
" j = response.json()\n",
"\n",
" #Continue if mismatched\n",
" if abs(j['latitude'] - lat) > 0.1005 or abs(j[\"longitude\"] - lon) > 0.1005: #Check to make sure it is correct cell, with tolerance for values like (-124.5, 42.699997)\n",
" print(f\"Cell did not match expected coords! actual lon/lat = {j['longitude'],j['latitude']}, expected {cell}. Not Saving...\")\n",
" print(f'Difference lon/lat = {abs(j[\"longitude\"] - lon), abs(j[\"latitude\"] - lat)}')\n",
" continue\n",
" \n",
" #create the df from the hourly data (ignoring the metadata such as units, etc.)\n",
" df = pd.DataFrame(data=j[\"hourly\"])\n",
"\n",
" # Save the data\n",
" df.to_csv(path_or_buf=save_path / f\"cell_{lon}_{lat}.csv\",index=False)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Note! We do have a couple of lines which have 2 entries for the same XRefCd in the BPA_TransmissionLines** \n",
"We are handling it by including both of the geojson shapefiles for the two XRefCds"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"-----------------------"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Trying something else since we are getting 429 Error quickly (after ~3 API calls).\n",
" **So trying to merge all cell lat,lon and see if it works**\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(-124.5, 42.7)"
]
},
"execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cells_list[0]"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"https://archive-api.open-meteo.com/v1/archive?latitude=42.7,42.8,42.9,42.4,42.5,42.6,42.7,42.9,43.0,43.1,43.1,48.1,43.1,43.2,43.3,43.4,43.5,43.6,43.7,48.1,43.1,43.2,43.4,43.7,48.1,43.2,44.0,48.1,44.0,45.2,47.0,48.1,44.0,45.1,45.2,45.3,45.4,45.5,46.2,46.4,46.9,47.0,48.1,44.0,45.1,45.4,45.5,46.2,46.4,46.7,46.9,47.0,48.1,44.0,44.6,45.4,45.5,45.6,46.2,46.3,46.4,46.5,46.6,46.7,46.9,47.0,48.1,44.0,44.6,45.4,45.6,46.2,46.3,46.5,46.6,47.0,48.1,44.0,44.6,45.3,45.4,45.6,46.1,46.2,46.3,46.5,46.6,47.0,48.1,44.0,44.6,45.3,45.6,45.7,45.8,46.1,46.6,47.0,48.1,44.0,44.1,44.2,44.3,44.4,44.5,44.6,44.7,44.8,44.9,45.3,45.6,46.1,46.6,47.0,47.3,47.4,48.0,48.1,43.7,43.8,44.0,44.1,44.6,44.7,44.8,44.9,45.5,45.6,46.1,46.6,47.0,47.1,47.2,47.3,47.4,47.5,47.6,48.0,43.8,44.0,44.6,45.0,45.1,46.1,46.2,46.3,46.4,46.5,46.6,46.9,47.0,47.3,47.6,47.7,48.0,44.6,45.1,45.2,46.1,46.2,46.6,46.8,46.9,47.0,47.3,47.4,47.7,47.8,47.9,48.0,48.5,43.9,44.6,44.7,45.2,45.3,45.4,46.1,46.6,47.0,47.4,47.5,48.5,43.9,44.7,44.8,45.3,45.7,45.9,46.0,46.1,46.6,47.0,47.5,48.5,43.8,43.9,44.8,45.3,45.7,45.8,45.9,46.5,46.6,47.0,48.9,43.7,43.8,44.7,44.8,45.3,45.6,45.7,47.0,48.8,48.9,43.7,44.7,44.8,44.9,45.3,45.4,45.5,45.6,45.7,47.0,47.1,48.8,44.7,44.8,44.9,45.0,45.4,45.5,45.6,45.7,46.9,47.0,47.1,47.2,48.7,48.8,44.7,45.0,45.4,45.5,45.6,45.7,47.0,47.1,47.2,47.5,47.8,48.3,48.4,48.5,48.6,48.7,44.7,44.8,45.0,45.4,45.5,45.6,45.7,47.1,47.5,47.9,48.1,48.2,48.3,44.8,45.0,45.4,45.5,45.6,45.7,47.1,47.4,47.5,47.6,47.7,47.9,48.0,48.1,44.8,45.0,45.4,45.6,45.7,47.1,47.2,47.3,47.4,47.5,47.7,47.8,47.9,44.8,45.0,45.4,45.5,45.7,45.8,47.2,47.3,47.4,47.5,47.9,44.8,44.9,45.0,45.1,45.5,45.7,45.8,47.2,47.3,47.4,47.5,47.9,44.9,45.1,45.5,45.7,45.8,47.2,47.4,47.8,47.9,43.6,43.7,44.9,45.1,45.5,45.7,45.8,47.2,47.4,47.8,42.0,42.1,43.5,43.6,44.9,45.0,45.1,45.5,45.6,45.7,45.8,47.2,47.3,47.4,47.7,42.0,43.4,43.5,45.0,45.1,45.2,45.5,45.6,45.7,45.8,47.3,47.7,43.4,45.1,45.2,45.6,45.7,45.8,47.2,47.3,47.7,43.3,43.4,45.1,45.2,45.3,45.6,45.7,45.8,47.2,47.7,43.0,43.1,43.3,44.3,44.4,44.5,44.6,44.7,45.2,45.3,45.4,45.6,45.7,45.8,45.9,47.2,47.7,47.8,43.1,43.2,43.3,43.4,43.5,43.6,43.7,43.8,43.9,44.0,44.1,44.2,44.3,44.8,44.9,45.0,45.1,45.2,45.3,45.4,45.5,45.6,45.7,45.8,45.9,47.2,47.8,43.4,43.8,45.3,45.4,45.5,45.6,45.7,45.8,47.1,47.2,47.8,45.3,45.4,45.5,45.7,47.1,47.2,47.7,47.8,45.4,45.5,45.7,45.8,46.8,46.9,47.0,47.1,47.2,47.7,45.4,45.5,45.7,45.8,45.9,46.7,46.8,47.1,47.7,41.7,45.4,45.5,45.6,45.7,45.9,46.0,46.6,46.7,47.1,47.2,47.7,45.2,45.3,45.4,45.6,45.7,46.0,46.1,46.5,46.6,47.0,47.1,47.2,47.3,47.5,47.7,45.0,45.1,45.2,45.7,45.8,46.1,46.5,46.9,47.0,47.2,47.3,47.4,47.5,47.7,45.7,45.8,46.1,46.5,46.9,47.3,47.7,45.7,45.8,46.1,46.2,46.3,46.5,46.9,47.3,47.7,47.8,45.8,45.9,46.3,46.4,46.5,46.7,46.8,46.9,47.3,47.8,45.8,45.9,46.2,46.3,46.5,46.6,46.7,46.8,47.4,47.8,47.9,45.5,45.6,45.8,45.9,46.0,46.3,46.6,46.8,46.9,47.0,47.4,47.5,47.9,48.0,45.6,45.7,45.8,45.9,46.0,46.1,46.2,46.3,46.6,46.7,46.8,46.9,47.0,47.1,47.5,48.0,45.9,46.3,46.4,46.6,46.7,47.1,47.2,47.3,47.5,47.6,48.0,45.9,46.3,46.4,46.5,46.6,46.7,47.3,47.4,47.6,47.7,48.0,45.8,45.9,46.0,46.2,46.4,46.5,46.7,47.4,47.5,47.7,47.8,48.0,45.7,45.8,45.9,46.0,46.1,46.2,46.4,46.5,46.7,47.5,47.6,47.7,47.8,47.9,48.0,45.7,45.9,46.1,46.2,46.3,46.4,46.5,46.6,46.7,47.7,47.8,47.9,48.0,45.6,45.7,45.9,46.0,46.2,46.3,46.5,46.6,46.7,47.8,47.9,48.0,45.6,46.0,46.1,46.2,46.3,46.6,47.9,45.6,45.7,46.1,46.2,46.3,46.6,47.9,45.6,45.7,45.8,46.1,46.3,46.4,46.6,47.6,47.7,47.8,47.9,45.6,45.8,45.9,46.1,46.4,46.5,46.6,47.8,45.5,45.6,45.9,46.0,46.1,46.5,46.6,47.8,45.4,45.5,46.0,46.1,47.7,47.8,45.3,45.4,47.7,45.3,47.7,45.3,47.7,47.7,47.7,48.4,48.5,48.6,47.7,48.3,48.4,48.6,48.7,46.4,47.7,48.1,48.2,48.3,48.7,48.8,46.4,47.7,48.0,48.1,48.7,48.8,46.4,47.7,47.8,47.9,48.0,48.7,48.8,46.4,46.6,46.7,47.7,47.8,47.9,48.3,48.4,48.5,48.6,48.7,48.8,48.9,49.0,46.4,46.6,47.7,47.8,48.2,48.3,48.4,48.5,46.4,46.5,46.6,47.7,47.8,48.2,46.4,47.7,48.2,46.4,46.5,47.7,47.8,48.2,46.5,47.8,48.2,47.8,47.9,48.2,47.9,48.2,48.3,47.8,47.9,48.3,46.7,46.8,47.8,47.9,48.0,48.3,48.4,48.5,46.5,46.6,46.7,46.8,46.9,47.7,47.8,48.0,48.1,48.5,48.6,46.9,47.0,47.7,48.1,48.6,48.7,47.0,47.1,47.7,48.1,48.7,47.1,47.2,47.6,47.7,48.1,48.6,48.7,47.2,47.3,47.6,48.0,48.1,48.5,48.6,47.3,47.4,47.5,48.0,48.4,48.5,47.4,47.5,48.0,48.4,48.5,43.2,47.4,47.5,47.9,48.0,48.4,48.5,43.2,47.4,47.5,47.9,48.0,48.4,43.3,43.4,47.4,47.5,47.9,48.0,48.4,47.4,48.0,48.4,47.4,48.0,48.1,48.4,47.4,48.1,48.4,47.3,47.4,48.1,48.2,48.3,48.4,47.3,48.2,48.3,47.2,47.3,48.7,47.2,48.6,48.7,47.1,47.2,48.6,47.1,47.5,47.6,48.5,48.6,47.0,47.1,47.5,47.6,47.7,48.4,48.5,47.0,47.4,47.5,47.7,47.8,47.9,48.2,48.4,46.9,47.0,47.3,47.4,47.7,47.8,47.9,48.0,48.1,48.2,48.4,46.8,46.9,47.2,47.3,47.7,48.0,48.1,48.2,48.3,48.4,44.4,46.8,47.0,47.1,47.2,48.3,48.4,44.3,44.4,46.7,46.8,46.9,47.0,46.7,46.7,42.5,46.7,42.5,46.6,46.7,42.5,42.6,42.7,46.6,46.5,46.6,46.4,46.5,46.4,46.4,46.5,46.5,46.5,46.5,46.5,46.3,46.4,46.5,46.3,46.3,46.3,46.3,43.3,46.3,43.3,43.4,46.2,46.3,43.4,43.5,46.2,43.5,43.6,43.7,46.2,43.5,43.6,43.7,46.2,43.5,43.6,43.7,43.8,43.9,46.2,43.5,43.8,43.9,46.2,46.3,43.5,43.9,44.0,43.5,43.6,43.8,43.9,44.0,44.5,43.6,43.7,43.8,44.5,44.6,43.5,43.6,43.7,44.6,44.7,43.5,43.5,43.5&longitude=-124.5,-124.5,-124.5,-124.4,-124.4,-124.4,-124.4,-124.4,-124.4,-124.4,-124.3,-124.3,-124.2,-124.2,-124.2,-124.2,-124.2,-124.2,-124.2,-124.2,-124.1,-124.1,-124.1,-124.1,-124.1,-124.0,-124.0,-124.0,-123.9,-123.9,-123.9,-123.9,-123.8,-123.8,-123.8,-123.8,-123.8,-123.8,-123.8,-123.8,-123.8,-123.8,-123.8,-123.7,-123.7,-123.7,-123.7,-123.7,-123.7,-123.7,-123.7,-123.7,-123.7,-123.6,-123.6,-123.6,-123.6,-123.6,-123.6,-123.6,-123.6,-123.6,-123.6,-123.6,-123.6,-123.6,-123.6,-123.5,-123.5,-123.5,-123.5,-123.5,-123.5,-123.5,-123.5,-123.5,-123.5,-123.4,-123.4,-123.4,-123.4,-123.4,-123.4,-123.4,-123.4,-123.4,-123.4,-123.4,-123.4,-123.3,-123.3,-123.3,-123.3,-123.3,-123.3,-123.3,-123.3,-123.3,-123.3,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.8,-122.8,-122.8,-122.8,-122.8,-122.8,-122.8,-122.8,-122.8,-122.8,-122.8,-122.8,-122.7,-122.7,-122.7,-122.7,-122.7,-122.7,-122.7,-122.7,-122.7,-122.7,-122.7,-122.7,-122.6,-122.6,-122.6,-122.6,-122.6,-122.6,-122.6,-122.6,-122.6,-122.6,-122.6,-122.5,-122.5,-122.5,-122.5,-122.5,-122.5,-122.5,-122.5,-122.5,-122.5,-122.4,-122.4,-122.4,-122.4,-122.4,-122.4,-122.4,-122.4,-122.4,-122.4,-122.4,-122.4,-122.3,-122.3,-122.3,-122.3,-122.3,-122.3,-122.3,-122.3,-122.3,-122.3,-122.3,-122.3,-122.3,-122.3,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.1,-122.1,-122.1,-122.1,-122.1,-122.1,-122.1,-122.1,-122.1,-122.1,-122.1,-122.1,-122.1,-122.0,-122.0,-122.0,-122.0,-122.0,-122.0,-122.0,-122.0,-122.0,-122.0,-122.0,-122.0,-122.0,-122.0,-121.9,-121.9,-121.9,-121.9,-121.9,-121.9,-121.9,-121.9,-121.9,-121.9,-121.9,-121.9,-121.9,-121.8,-121.8,-121.8,-121.8,-121.8,-121.8,-121.8,-121.8,-121.8,-121.8,-121.8,-121.7,-121.7,-121.7,-121.7,-121.7,-121.7,-121.7,-121.7,-121.7,-121.7,-121.7,-121.7,-121.6,-121.6,-121.6,-121.6,-121.6,-121.6,-121.6,-121.6,-121.6,-121.5,-121.5,-121.5,-121.5,-121.5,-121.5,-121.5,-121.5,-121.5,-121.5,-121.4,-121.4,-121.4,-121.4,-121.4,-121.4,-121.4,-121.4,-121.4,-121.4,-121.4,-121.4,-121.4,-121.4,-121.4,-121.3,-121.3,-121.3,-121.3,-121.3,-121.3,-121.3,-121.3,-121.3,-121.3,-121.3,-121.3,-121.2,-121.2,-121.2,-121.2,-121.2,-121.2,-121.2,-121.2,-121.2,-121.1,-121.1,-121.1,-121.1,-121.1,-121.1,-121.1,-121.1,-121.1,-121.1,-121.0,-121.0,-121.0,-121.0,-121.0,-121.0,-121.0,-121.0,-121.0,-121.0,-121.0,-121.0,-121.0,-121.0,-121.0,-121.0,-121.0,-121.0,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.8,-120.8,-120.8,-120.8,-120.8,-120.8,-120.8,-120.8,-120.8,-120.8,-120.8,-120.7,-120.7,-120.7,-120.7,-120.7,-120.7,-120.7,-120.7,-120.6,-120.6,-120.6,-120.6,-120.6,-120.6,-120.6,-120.6,-120.6,-120.6,-120.5,-120.5,-120.5,-120.5,-120.5,-120.5,-120.5,-120.5,-120.5,-120.4,-120.4,-120.4,-120.4,-120.4,-120.4,-120.4,-120.4,-120.4,-120.4,-120.4,-120.4,-120.3,-120.3,-120.3,-120.3,-120.3,-120.3,-120.3,-120.3,-120.3,-120.3,-120.3,-120.3,-120.3,-120.3,-120.3,-120.2,-120.2,-120.2,-120.2,-120.2,-120.2,-120.2,-120.2,-120.2,-120.2,-120.2,-120.2,-120.2,-120.2,-120.1,-120.1,-120.1,-120.1,-120.1,-120.1,-120.1,-120.0,-120.0,-120.0,-120.0,-120.0,-120.0,-120.0,-120.0,-120.0,-120.0,-119.9,-119.9,-119.9,-119.9,-119.9,-119.9,-119.9,-119.9,-119.9,-119.9,-119.8,-119.8,-119.8,-119.8,-119.8,-119.8,-119.8,-119.8,-119.8,-119.8,-119.8,-119.7,-119.7,-119.7,-119.7,-119.7,-119.7,-119.7,-119.7,-119.7,-119.7,-119.7,-119.7,-119.7,-119.7,-119.6,-119.6,-119.6,-119.6,-119.6,-119.6,-119.6,-119.6,-119.6,-119.6,-119.6,-119.6,-119.6,-119.6,-119.6,-119.6,-119.5,-119.5,-119.5,-119.5,-119.5,-119.5,-119.5,-119.5,-119.5,-119.5,-119.5,-119.4,-119.4,-119.4,-119.4,-119.4,-119.4,-119.4,-119.4,-119.4,-119.4,-119.4,-119.3,-119.3,-119.3,-119.3,-119.3,-119.3,-119.3,-119.3,-119.3,-119.3,-119.3,-119.3,-119.2,-119.2,-119.2,-119.2,-119.2,-119.2,-119.2,-119.2,-119.2,-119.2,-119.2,-119.2,-119.2,-119.2,-119.2,-119.1,-119.1,-119.1,-119.1,-119.1,-119.1,-119.1,-119.1,-119.1,-119.1,-119.1,-119.1,-119.1,-119.0,-119.0,-119.0,-119.0,-119.0,-119.0,-119.0,-119.0,-119.0,-119.0,-119.0,-119.0,-118.9,-118.9,-118.9,-118.9,-118.9,-118.9,-118.9,-118.8,-118.8,-118.8,-118.8,-118.8,-118.8,-118.8,-118.7,-118.7,-118.7,-118.7,-118.7,-118.7,-118.7,-118.7,-118.7,-118.7,-118.7,-118.6,-118.6,-118.6,-118.6,-118.6,-118.6,-118.6,-118.6,-118.5,-118.5,-118.5,-118.5,-118.5,-118.5,-118.5,-118.5,-118.4,-118.4,-118.4,-118.4,-118.4,-118.4,-118.3,-118.3,-118.3,-118.2,-118.2,-118.1,-118.1,-118.0,-117.9,-117.9,-117.9,-117.9,-117.8,-117.8,-117.8,-117.8,-117.8,-117.7,-117.7,-117.7,-117.7,-117.7,-117.7,-117.7,-117.6,-117.6,-117.6,-117.6,-117.6,-117.6,-117.5,-117.5,-117.5,-117.5,-117.5,-117.5,-117.5,-117.4,-117.4,-117.4,-117.4,-117.4,-117.4,-117.4,-117.4,-117.4,-117.4,-117.4,-117.4,-117.4,-117.4,-117.3,-117.3,-117.3,-117.3,-117.3,-117.3,-117.3,-117.3,-117.2,-117.2,-117.2,-117.2,-117.2,-117.2,-117.1,-117.1,-117.1,-117.0,-117.0,-117.0,-117.0,-117.0,-116.9,-116.9,-116.9,-116.8,-116.8,-116.8,-116.7,-116.7,-116.7,-116.6,-116.6,-116.6,-116.5,-116.5,-116.5,-116.5,-116.5,-116.5,-116.5,-116.5,-116.4,-116.4,-116.4,-116.4,-116.4,-116.4,-116.4,-116.4,-116.4,-116.4,-116.4,-116.3,-116.3,-116.3,-116.3,-116.3,-116.3,-116.2,-116.2,-116.2,-116.2,-116.2,-116.1,-116.1,-116.1,-116.1,-116.1,-116.1,-116.1,-116.0,-116.0,-116.0,-116.0,-116.0,-116.0,-116.0,-115.9,-115.9,-115.9,-115.9,-115.9,-115.9,-115.8,-115.8,-115.8,-115.8,-115.8,-115.7,-115.7,-115.7,-115.7,-115.7,-115.7,-115.7,-115.6,-115.6,-115.6,-115.6,-115.6,-115.6,-115.5,-115.5,-115.5,-115.5,-115.5,-115.5,-115.5,-115.4,-115.4,-115.4,-115.3,-115.3,-115.3,-115.3,-115.2,-115.2,-115.2,-115.1,-115.1,-115.1,-115.1,-115.1,-115.1,-115.0,-115.0,-115.0,-114.9,-114.9,-114.9,-114.8,-114.8,-114.8,-114.7,-114.7,-114.7,-114.6,-114.6,-114.6,-114.6,-114.6,-114.5,-114.5,-114.5,-114.5,-114.5,-114.5,-114.5,-114.4,-114.4,-114.4,-114.4,-114.4,-114.4,-114.4,-114.4,-114.3,-114.3,-114.3,-114.3,-114.3,-114.3,-114.3,-114.3,-114.3,-114.3,-114.3,-114.2,-114.2,-114.2,-114.2,-114.2,-114.2,-114.2,-114.2,-114.2,-114.2,-114.1,-114.1,-114.1,-114.1,-114.1,-114.1,-114.1,-114.0,-114.0,-114.0,-114.0,-114.0,-114.0,-113.9,-113.8,-113.7,-113.7,-113.6,-113.6,-113.6,-113.5,-113.5,-113.5,-113.5,-113.4,-113.4,-113.3,-113.3,-113.2,-113.1,-113.1,-113.0,-112.9,-112.8,-112.7,-112.6,-112.6,-112.6,-112.5,-112.4,-112.3,-112.2,-112.1,-112.1,-112.0,-112.0,-112.0,-112.0,-111.9,-111.9,-111.9,-111.8,-111.8,-111.8,-111.8,-111.7,-111.7,-111.7,-111.7,-111.6,-111.6,-111.6,-111.6,-111.6,-111.6,-111.5,-111.5,-111.5,-111.5,-111.5,-111.4,-111.4,-111.4,-111.3,-111.3,-111.3,-111.3,-111.3,-111.3,-111.2,-111.2,-111.2,-111.2,-111.2,-111.1,-111.1,-111.1,-111.1,-111.1,-111.0,-110.9,-110.8&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n"
]
}
],
"source": [
"# Format the api string which is in format https://archive-api.open-meteo.com/v1/archive?latitude=lat1,lat2,lat,...&longitude=lon1,lon2,lon3,...\n",
"# Where lat1 = cells_list[0][1], lon1 = cells_list[0][0] etc.\n",
"latitudes = [str(cell[1]) for cell in cells_list]\n",
"longitudes = [str(cell[0]) for cell in cells_list]\n",
"latitudes_str = \",\".join(latitudes)\n",
"longitudes_str = \",\".join(longitudes)\n",
"api_string = f\"https://archive-api.open-meteo.com/v1/archive?latitude={latitudes_str}&longitude={longitudes_str}&start_date={start_date}&end_date={end_date}&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\"\n",
"print(api_string)"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Idx: 0, URL: https://archive-api.open-meteo.com/v1/archive?latitude=42.7,42.8,42.9,42.4,42.5,42.6,42.7,42.9,43.0,43.1,43.1,48.1,43.1,43.2,43.3,43.4,43.5,43.6,43.7,48.1,43.1,43.2,43.4,43.7,48.1,43.2,44.0,48.1,44.0,45.2,47.0,48.1,44.0,45.1,45.2,45.3,45.4,45.5,46.2,46.4,46.9,47.0,48.1,44.0,45.1&longitude=-124.5,-124.5,-124.5,-124.4,-124.4,-124.4,-124.4,-124.4,-124.4,-124.4,-124.3,-124.3,-124.2,-124.2,-124.2,-124.2,-124.2,-124.2,-124.2,-124.2,-124.1,-124.1,-124.1,-124.1,-124.1,-124.0,-124.0,-124.0,-123.9,-123.9,-123.9,-123.9,-123.8,-123.8,-123.8,-123.8,-123.8,-123.8,-123.8,-123.8,-123.8,-123.8,-123.8,-123.7,-123.7&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 1, URL: https://archive-api.open-meteo.com/v1/archive?latitude=45.1,45.4,45.5,46.2,46.4,46.7,46.9,47.0,48.1,44.0,44.6,45.4,45.5,45.6,46.2,46.3,46.4,46.5,46.6,46.7,46.9,47.0,48.1,44.0,44.6,45.4,45.6,46.2,46.3,46.5,46.6,47.0,48.1,44.0,44.6,45.3,45.4,45.6,46.1,46.2,46.3,46.5,46.6,47.0,48.1&longitude=-123.7,-123.7,-123.7,-123.7,-123.7,-123.7,-123.7,-123.7,-123.7,-123.6,-123.6,-123.6,-123.6,-123.6,-123.6,-123.6,-123.6,-123.6,-123.6,-123.6,-123.6,-123.6,-123.6,-123.5,-123.5,-123.5,-123.5,-123.5,-123.5,-123.5,-123.5,-123.5,-123.5,-123.4,-123.4,-123.4,-123.4,-123.4,-123.4,-123.4,-123.4,-123.4,-123.4,-123.4,-123.4&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 2, URL: https://archive-api.open-meteo.com/v1/archive?latitude=48.1,44.0,44.6,45.3,45.6,45.7,45.8,46.1,46.6,47.0,48.1,44.0,44.1,44.2,44.3,44.4,44.5,44.6,44.7,44.8,44.9,45.3,45.6,46.1,46.6,47.0,47.3,47.4,48.0,48.1,43.7,43.8,44.0,44.1,44.6,44.7,44.8,44.9,45.5,45.6,46.1,46.6,47.0,47.1,47.2&longitude=-123.4,-123.3,-123.3,-123.3,-123.3,-123.3,-123.3,-123.3,-123.3,-123.3,-123.3,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.2,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.1&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 3, URL: https://archive-api.open-meteo.com/v1/archive?latitude=47.2,47.3,47.4,47.5,47.6,48.0,43.8,44.0,44.6,45.0,45.1,46.1,46.2,46.3,46.4,46.5,46.6,46.9,47.0,47.3,47.6,47.7,48.0,44.6,45.1,45.2,46.1,46.2,46.6,46.8,46.9,47.0,47.3,47.4,47.7,47.8,47.9,48.0,48.5,43.9,44.6,44.7,45.2,45.3,45.4&longitude=-123.1,-123.1,-123.1,-123.1,-123.1,-123.1,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-123.0,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.9,-122.8,-122.8,-122.8,-122.8,-122.8,-122.8&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 4, URL: https://archive-api.open-meteo.com/v1/archive?latitude=45.4,46.1,46.6,47.0,47.4,47.5,48.5,43.9,44.7,44.8,45.3,45.7,45.9,46.0,46.1,46.6,47.0,47.5,48.5,43.8,43.9,44.8,45.3,45.7,45.8,45.9,46.5,46.6,47.0,48.9,43.7,43.8,44.7,44.8,45.3,45.6,45.7,47.0,48.8,48.9,43.7,44.7,44.8,44.9,45.3&longitude=-122.8,-122.8,-122.8,-122.8,-122.8,-122.8,-122.8,-122.7,-122.7,-122.7,-122.7,-122.7,-122.7,-122.7,-122.7,-122.7,-122.7,-122.7,-122.7,-122.6,-122.6,-122.6,-122.6,-122.6,-122.6,-122.6,-122.6,-122.6,-122.6,-122.6,-122.5,-122.5,-122.5,-122.5,-122.5,-122.5,-122.5,-122.5,-122.5,-122.5,-122.4,-122.4,-122.4,-122.4,-122.4&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 5, URL: https://archive-api.open-meteo.com/v1/archive?latitude=45.3,45.4,45.5,45.6,45.7,47.0,47.1,48.8,44.7,44.8,44.9,45.0,45.4,45.5,45.6,45.7,46.9,47.0,47.1,47.2,48.7,48.8,44.7,45.0,45.4,45.5,45.6,45.7,47.0,47.1,47.2,47.5,47.8,48.3,48.4,48.5,48.6,48.7,44.7,44.8,45.0,45.4,45.5,45.6,45.7&longitude=-122.4,-122.4,-122.4,-122.4,-122.4,-122.4,-122.4,-122.4,-122.3,-122.3,-122.3,-122.3,-122.3,-122.3,-122.3,-122.3,-122.3,-122.3,-122.3,-122.3,-122.3,-122.3,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.2,-122.1,-122.1,-122.1,-122.1,-122.1,-122.1,-122.1&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 6, URL: https://archive-api.open-meteo.com/v1/archive?latitude=45.7,47.1,47.5,47.9,48.1,48.2,48.3,44.8,45.0,45.4,45.5,45.6,45.7,47.1,47.4,47.5,47.6,47.7,47.9,48.0,48.1,44.8,45.0,45.4,45.6,45.7,47.1,47.2,47.3,47.4,47.5,47.7,47.8,47.9,44.8,45.0,45.4,45.5,45.7,45.8,47.2,47.3,47.4,47.5,47.9&longitude=-122.1,-122.1,-122.1,-122.1,-122.1,-122.1,-122.1,-122.0,-122.0,-122.0,-122.0,-122.0,-122.0,-122.0,-122.0,-122.0,-122.0,-122.0,-122.0,-122.0,-122.0,-121.9,-121.9,-121.9,-121.9,-121.9,-121.9,-121.9,-121.9,-121.9,-121.9,-121.9,-121.9,-121.9,-121.8,-121.8,-121.8,-121.8,-121.8,-121.8,-121.8,-121.8,-121.8,-121.8,-121.8&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 7, URL: https://archive-api.open-meteo.com/v1/archive?latitude=47.9,44.8,44.9,45.0,45.1,45.5,45.7,45.8,47.2,47.3,47.4,47.5,47.9,44.9,45.1,45.5,45.7,45.8,47.2,47.4,47.8,47.9,43.6,43.7,44.9,45.1,45.5,45.7,45.8,47.2,47.4,47.8,42.0,42.1,43.5,43.6,44.9,45.0,45.1,45.5,45.6,45.7,45.8,47.2,47.3&longitude=-121.8,-121.7,-121.7,-121.7,-121.7,-121.7,-121.7,-121.7,-121.7,-121.7,-121.7,-121.7,-121.7,-121.6,-121.6,-121.6,-121.6,-121.6,-121.6,-121.6,-121.6,-121.6,-121.5,-121.5,-121.5,-121.5,-121.5,-121.5,-121.5,-121.5,-121.5,-121.5,-121.4,-121.4,-121.4,-121.4,-121.4,-121.4,-121.4,-121.4,-121.4,-121.4,-121.4,-121.4,-121.4&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 8, URL: https://archive-api.open-meteo.com/v1/archive?latitude=47.3,47.4,47.7,42.0,43.4,43.5,45.0,45.1,45.2,45.5,45.6,45.7,45.8,47.3,47.7,43.4,45.1,45.2,45.6,45.7,45.8,47.2,47.3,47.7,43.3,43.4,45.1,45.2,45.3,45.6,45.7,45.8,47.2,47.7,43.0,43.1,43.3,44.3,44.4,44.5,44.6,44.7,45.2,45.3,45.4&longitude=-121.4,-121.4,-121.4,-121.3,-121.3,-121.3,-121.3,-121.3,-121.3,-121.3,-121.3,-121.3,-121.3,-121.3,-121.3,-121.2,-121.2,-121.2,-121.2,-121.2,-121.2,-121.2,-121.2,-121.2,-121.1,-121.1,-121.1,-121.1,-121.1,-121.1,-121.1,-121.1,-121.1,-121.1,-121.0,-121.0,-121.0,-121.0,-121.0,-121.0,-121.0,-121.0,-121.0,-121.0,-121.0&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 9, URL: https://archive-api.open-meteo.com/v1/archive?latitude=45.4,45.6,45.7,45.8,45.9,47.2,47.7,47.8,43.1,43.2,43.3,43.4,43.5,43.6,43.7,43.8,43.9,44.0,44.1,44.2,44.3,44.8,44.9,45.0,45.1,45.2,45.3,45.4,45.5,45.6,45.7,45.8,45.9,47.2,47.8,43.4,43.8,45.3,45.4,45.5,45.6,45.7,45.8,47.1,47.2&longitude=-121.0,-121.0,-121.0,-121.0,-121.0,-121.0,-121.0,-121.0,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.9,-120.8,-120.8,-120.8,-120.8,-120.8,-120.8,-120.8,-120.8,-120.8,-120.8&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 10, URL: https://archive-api.open-meteo.com/v1/archive?latitude=47.2,47.8,45.3,45.4,45.5,45.7,47.1,47.2,47.7,47.8,45.4,45.5,45.7,45.8,46.8,46.9,47.0,47.1,47.2,47.7,45.4,45.5,45.7,45.8,45.9,46.7,46.8,47.1,47.7,41.7,45.4,45.5,45.6,45.7,45.9,46.0,46.6,46.7,47.1,47.2,47.7,45.2,45.3,45.4,45.6&longitude=-120.8,-120.8,-120.7,-120.7,-120.7,-120.7,-120.7,-120.7,-120.7,-120.7,-120.6,-120.6,-120.6,-120.6,-120.6,-120.6,-120.6,-120.6,-120.6,-120.6,-120.5,-120.5,-120.5,-120.5,-120.5,-120.5,-120.5,-120.5,-120.5,-120.4,-120.4,-120.4,-120.4,-120.4,-120.4,-120.4,-120.4,-120.4,-120.4,-120.4,-120.4,-120.3,-120.3,-120.3,-120.3&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 11, URL: https://archive-api.open-meteo.com/v1/archive?latitude=45.6,45.7,46.0,46.1,46.5,46.6,47.0,47.1,47.2,47.3,47.5,47.7,45.0,45.1,45.2,45.7,45.8,46.1,46.5,46.9,47.0,47.2,47.3,47.4,47.5,47.7,45.7,45.8,46.1,46.5,46.9,47.3,47.7,45.7,45.8,46.1,46.2,46.3,46.5,46.9,47.3,47.7,47.8,45.8,45.9&longitude=-120.3,-120.3,-120.3,-120.3,-120.3,-120.3,-120.3,-120.3,-120.3,-120.3,-120.3,-120.3,-120.2,-120.2,-120.2,-120.2,-120.2,-120.2,-120.2,-120.2,-120.2,-120.2,-120.2,-120.2,-120.2,-120.2,-120.1,-120.1,-120.1,-120.1,-120.1,-120.1,-120.1,-120.0,-120.0,-120.0,-120.0,-120.0,-120.0,-120.0,-120.0,-120.0,-120.0,-119.9,-119.9&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 12, URL: https://archive-api.open-meteo.com/v1/archive?latitude=45.9,46.3,46.4,46.5,46.7,46.8,46.9,47.3,47.8,45.8,45.9,46.2,46.3,46.5,46.6,46.7,46.8,47.4,47.8,47.9,45.5,45.6,45.8,45.9,46.0,46.3,46.6,46.8,46.9,47.0,47.4,47.5,47.9,48.0,45.6,45.7,45.8,45.9,46.0,46.1,46.2,46.3,46.6,46.7,46.8&longitude=-119.9,-119.9,-119.9,-119.9,-119.9,-119.9,-119.9,-119.9,-119.9,-119.8,-119.8,-119.8,-119.8,-119.8,-119.8,-119.8,-119.8,-119.8,-119.8,-119.8,-119.7,-119.7,-119.7,-119.7,-119.7,-119.7,-119.7,-119.7,-119.7,-119.7,-119.7,-119.7,-119.7,-119.7,-119.6,-119.6,-119.6,-119.6,-119.6,-119.6,-119.6,-119.6,-119.6,-119.6,-119.6&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 13, URL: https://archive-api.open-meteo.com/v1/archive?latitude=46.8,46.9,47.0,47.1,47.5,48.0,45.9,46.3,46.4,46.6,46.7,47.1,47.2,47.3,47.5,47.6,48.0,45.9,46.3,46.4,46.5,46.6,46.7,47.3,47.4,47.6,47.7,48.0,45.8,45.9,46.0,46.2,46.4,46.5,46.7,47.4,47.5,47.7,47.8,48.0,45.7,45.8,45.9,46.0,46.1&longitude=-119.6,-119.6,-119.6,-119.6,-119.6,-119.6,-119.5,-119.5,-119.5,-119.5,-119.5,-119.5,-119.5,-119.5,-119.5,-119.5,-119.5,-119.4,-119.4,-119.4,-119.4,-119.4,-119.4,-119.4,-119.4,-119.4,-119.4,-119.4,-119.3,-119.3,-119.3,-119.3,-119.3,-119.3,-119.3,-119.3,-119.3,-119.3,-119.3,-119.3,-119.2,-119.2,-119.2,-119.2,-119.2&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 14, URL: https://archive-api.open-meteo.com/v1/archive?latitude=46.1,46.2,46.4,46.5,46.7,47.5,47.6,47.7,47.8,47.9,48.0,45.7,45.9,46.1,46.2,46.3,46.4,46.5,46.6,46.7,47.7,47.8,47.9,48.0,45.6,45.7,45.9,46.0,46.2,46.3,46.5,46.6,46.7,47.8,47.9,48.0,45.6,46.0,46.1,46.2,46.3,46.6,47.9,45.6,45.7&longitude=-119.2,-119.2,-119.2,-119.2,-119.2,-119.2,-119.2,-119.2,-119.2,-119.2,-119.2,-119.1,-119.1,-119.1,-119.1,-119.1,-119.1,-119.1,-119.1,-119.1,-119.1,-119.1,-119.1,-119.1,-119.0,-119.0,-119.0,-119.0,-119.0,-119.0,-119.0,-119.0,-119.0,-119.0,-119.0,-119.0,-118.9,-118.9,-118.9,-118.9,-118.9,-118.9,-118.9,-118.8,-118.8&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 15, URL: https://archive-api.open-meteo.com/v1/archive?latitude=45.7,46.1,46.2,46.3,46.6,47.9,45.6,45.7,45.8,46.1,46.3,46.4,46.6,47.6,47.7,47.8,47.9,45.6,45.8,45.9,46.1,46.4,46.5,46.6,47.8,45.5,45.6,45.9,46.0,46.1,46.5,46.6,47.8,45.4,45.5,46.0,46.1,47.7,47.8,45.3,45.4,47.7,45.3,47.7,45.3&longitude=-118.8,-118.8,-118.8,-118.8,-118.8,-118.8,-118.7,-118.7,-118.7,-118.7,-118.7,-118.7,-118.7,-118.7,-118.7,-118.7,-118.7,-118.6,-118.6,-118.6,-118.6,-118.6,-118.6,-118.6,-118.6,-118.5,-118.5,-118.5,-118.5,-118.5,-118.5,-118.5,-118.5,-118.4,-118.4,-118.4,-118.4,-118.4,-118.4,-118.3,-118.3,-118.3,-118.2,-118.2,-118.1&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 16, URL: https://archive-api.open-meteo.com/v1/archive?latitude=45.3,47.7,47.7,47.7,48.4,48.5,48.6,47.7,48.3,48.4,48.6,48.7,46.4,47.7,48.1,48.2,48.3,48.7,48.8,46.4,47.7,48.0,48.1,48.7,48.8,46.4,47.7,47.8,47.9,48.0,48.7,48.8,46.4,46.6,46.7,47.7,47.8,47.9,48.3,48.4,48.5,48.6,48.7,48.8,48.9&longitude=-118.1,-118.1,-118.0,-117.9,-117.9,-117.9,-117.9,-117.8,-117.8,-117.8,-117.8,-117.8,-117.7,-117.7,-117.7,-117.7,-117.7,-117.7,-117.7,-117.6,-117.6,-117.6,-117.6,-117.6,-117.6,-117.5,-117.5,-117.5,-117.5,-117.5,-117.5,-117.5,-117.4,-117.4,-117.4,-117.4,-117.4,-117.4,-117.4,-117.4,-117.4,-117.4,-117.4,-117.4,-117.4&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 17, URL: https://archive-api.open-meteo.com/v1/archive?latitude=48.9,49.0,46.4,46.6,47.7,47.8,48.2,48.3,48.4,48.5,46.4,46.5,46.6,47.7,47.8,48.2,46.4,47.7,48.2,46.4,46.5,47.7,47.8,48.2,46.5,47.8,48.2,47.8,47.9,48.2,47.9,48.2,48.3,47.8,47.9,48.3,46.7,46.8,47.8,47.9,48.0,48.3,48.4,48.5,46.5&longitude=-117.4,-117.4,-117.3,-117.3,-117.3,-117.3,-117.3,-117.3,-117.3,-117.3,-117.2,-117.2,-117.2,-117.2,-117.2,-117.2,-117.1,-117.1,-117.1,-117.0,-117.0,-117.0,-117.0,-117.0,-116.9,-116.9,-116.9,-116.8,-116.8,-116.8,-116.7,-116.7,-116.7,-116.6,-116.6,-116.6,-116.5,-116.5,-116.5,-116.5,-116.5,-116.5,-116.5,-116.5,-116.4&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 18, URL: https://archive-api.open-meteo.com/v1/archive?latitude=46.5,46.6,46.7,46.8,46.9,47.7,47.8,48.0,48.1,48.5,48.6,46.9,47.0,47.7,48.1,48.6,48.7,47.0,47.1,47.7,48.1,48.7,47.1,47.2,47.6,47.7,48.1,48.6,48.7,47.2,47.3,47.6,48.0,48.1,48.5,48.6,47.3,47.4,47.5,48.0,48.4,48.5,47.4,47.5,48.0&longitude=-116.4,-116.4,-116.4,-116.4,-116.4,-116.4,-116.4,-116.4,-116.4,-116.4,-116.4,-116.3,-116.3,-116.3,-116.3,-116.3,-116.3,-116.2,-116.2,-116.2,-116.2,-116.2,-116.1,-116.1,-116.1,-116.1,-116.1,-116.1,-116.1,-116.0,-116.0,-116.0,-116.0,-116.0,-116.0,-116.0,-115.9,-115.9,-115.9,-115.9,-115.9,-115.9,-115.8,-115.8,-115.8&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 19, URL: https://archive-api.open-meteo.com/v1/archive?latitude=48.0,48.4,48.5,43.2,47.4,47.5,47.9,48.0,48.4,48.5,43.2,47.4,47.5,47.9,48.0,48.4,43.3,43.4,47.4,47.5,47.9,48.0,48.4,47.4,48.0,48.4,47.4,48.0,48.1,48.4,47.4,48.1,48.4,47.3,47.4,48.1,48.2,48.3,48.4,47.3,48.2,48.3,47.2,47.3,48.7&longitude=-115.8,-115.8,-115.8,-115.7,-115.7,-115.7,-115.7,-115.7,-115.7,-115.7,-115.6,-115.6,-115.6,-115.6,-115.6,-115.6,-115.5,-115.5,-115.5,-115.5,-115.5,-115.5,-115.5,-115.4,-115.4,-115.4,-115.3,-115.3,-115.3,-115.3,-115.2,-115.2,-115.2,-115.1,-115.1,-115.1,-115.1,-115.1,-115.1,-115.0,-115.0,-115.0,-114.9,-114.9,-114.9&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 20, URL: https://archive-api.open-meteo.com/v1/archive?latitude=48.7,47.2,48.6,48.7,47.1,47.2,48.6,47.1,47.5,47.6,48.5,48.6,47.0,47.1,47.5,47.6,47.7,48.4,48.5,47.0,47.4,47.5,47.7,47.8,47.9,48.2,48.4,46.9,47.0,47.3,47.4,47.7,47.8,47.9,48.0,48.1,48.2,48.4,46.8,46.9,47.2,47.3,47.7,48.0,48.1&longitude=-114.9,-114.8,-114.8,-114.8,-114.7,-114.7,-114.7,-114.6,-114.6,-114.6,-114.6,-114.6,-114.5,-114.5,-114.5,-114.5,-114.5,-114.5,-114.5,-114.4,-114.4,-114.4,-114.4,-114.4,-114.4,-114.4,-114.4,-114.3,-114.3,-114.3,-114.3,-114.3,-114.3,-114.3,-114.3,-114.3,-114.3,-114.3,-114.2,-114.2,-114.2,-114.2,-114.2,-114.2,-114.2&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 21, URL: https://archive-api.open-meteo.com/v1/archive?latitude=48.1,48.2,48.3,48.4,44.4,46.8,47.0,47.1,47.2,48.3,48.4,44.3,44.4,46.7,46.8,46.9,47.0,46.7,46.7,42.5,46.7,42.5,46.6,46.7,42.5,42.6,42.7,46.6,46.5,46.6,46.4,46.5,46.4,46.4,46.5,46.5,46.5,46.5,46.5,46.3,46.4,46.5,46.3,46.3,46.3&longitude=-114.2,-114.2,-114.2,-114.2,-114.1,-114.1,-114.1,-114.1,-114.1,-114.1,-114.1,-114.0,-114.0,-114.0,-114.0,-114.0,-114.0,-113.9,-113.8,-113.7,-113.7,-113.6,-113.6,-113.6,-113.5,-113.5,-113.5,-113.5,-113.4,-113.4,-113.3,-113.3,-113.2,-113.1,-113.1,-113.0,-112.9,-112.8,-112.7,-112.6,-112.6,-112.6,-112.5,-112.4,-112.3&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 22, URL: https://archive-api.open-meteo.com/v1/archive?latitude=46.3,46.3,43.3,46.3,43.3,43.4,46.2,46.3,43.4,43.5,46.2,43.5,43.6,43.7,46.2,43.5,43.6,43.7,46.2,43.5,43.6,43.7,43.8,43.9,46.2,43.5,43.8,43.9,46.2,46.3,43.5,43.9,44.0,43.5,43.6,43.8,43.9,44.0,44.5,43.6,43.7,43.8,44.5,44.6,43.5&longitude=-112.3,-112.2,-112.1,-112.1,-112.0,-112.0,-112.0,-112.0,-111.9,-111.9,-111.9,-111.8,-111.8,-111.8,-111.8,-111.7,-111.7,-111.7,-111.7,-111.6,-111.6,-111.6,-111.6,-111.6,-111.6,-111.5,-111.5,-111.5,-111.5,-111.5,-111.4,-111.4,-111.4,-111.3,-111.3,-111.3,-111.3,-111.3,-111.3,-111.2,-111.2,-111.2,-111.2,-111.2,-111.1&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n",
"Idx: 23, URL: https://archive-api.open-meteo.com/v1/archive?latitude=43.5,43.6,43.7,44.6,44.7,43.5,43.5,43.5&longitude=-111.1,-111.1,-111.1,-111.1,-111.1,-111.0,-110.9,-110.8&start_date=2016-01-01&end_date=2022-04-06&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\n"
]
}
],
"source": [
"def generate_api_strings(latitudes, longitudes, max_len=1000):\n",
" left_part = 'https://archive-api.open-meteo.com/v1/archive?latitude='\n",
" right_part = f\"&start_date={start_date}&end_date={end_date}&hourly=temperature_2m,relativehumidity_2m,dewpoint_2m,precipitation,rain,snowfall,weathercode,surface_pressure,cloudcover_low,cloudcover_mid,cloudcover_high,windspeed_10m,windspeed_100m,winddirection_10m,winddirection_100m,windgusts_10m,soil_temperature_0_to_7cm,soil_temperature_7_to_28cm,soil_moisture_0_to_7cm,soil_moisture_7_to_28cm&timezone=auto&format=csv\"\n",
" idx = 0\n",
" while idx < len(latitudes):\n",
" next_idx = idx\n",
" while next_idx < len(latitudes):\n",
" api_str = left_part + ','.join(latitudes[idx:next_idx+1]) + '&longitude=' + ','.join(longitudes[idx:next_idx+1]) + right_part\n",
" if len(api_str) > max_len:\n",
" break\n",
" next_idx += 1\n",
" yield api_str\n",
" idx = next_idx\n",
"\n",
"\n",
"for i, api_str in enumerate(generate_api_strings(latitudes, longitudes,1000)):\n",
" print(f\"Idx: {i}, URL: {api_str}\")"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Too many requests, waiting 1 second...\n",
"Error on idx=0. Code 429. Not Saving...\n",
"Too many requests, waiting 1 second...\n",
"Error on idx=1. Code 429. Not Saving...\n",
"Too many requests, waiting 1 second...\n",
"Error on idx=2. Code 429. Not Saving...\n",
"Too many requests, waiting 1 second...\n",
"Error on idx=3. Code 429. Not Saving...\n",
"Too many requests, waiting 1 second...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Exception ignored in: <finalize object at 0x7f4cbdb7da00; dead>\n",
"Traceback (most recent call last):\n",
" File \"/home/sharan/anaconda3/lib/python3.10/weakref.py\", line 591, in __call__\n",
" return info.func(*info.args, **(info.kwargs or {}))\n",
" File \"/home/sharan/anaconda3/lib/python3.10/site-packages/urllib3/connectionpool.py\", line 1129, in _close_pool_connections\n",
" if conn:\n",
"KeyboardInterrupt: \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Error on idx=4. Code 429. Not Saving...\n",
"Too many requests, waiting 1 second...\n",
"Error on idx=5. Code 429. Not Saving...\n",
"Too many requests, waiting 1 second...\n",
"Error on idx=6. Code 429. Not Saving...\n",
"Too many requests, waiting 1 second...\n",
"Error on idx=7. Code 429. Not Saving...\n",
"Too many requests, waiting 1 second...\n",
"Error on idx=8. Code 429. Not Saving...\n",
"Too many requests, waiting 1 second...\n",
"Error on idx=9. Code 429. Not Saving...\n",
"Too many requests, waiting 1 second...\n",
"Error on idx=10. Code 429. Not Saving...\n",
"Too many requests, waiting 1 second...\n"
]
}
],
"source": [
"for i, api_str in enumerate(generate_api_strings(latitudes, longitudes, 1000)):\n",
" # Check that if the file for this index already exists, if so, skip it. Use pathlib Path for this\n",
" # check if file all_cells_{i}.csv exists in save_path\n",
" save_path = Path(\"../Datasets/open-metro-cells/\")\n",
" if Path(save_path/f\"all_cells_{i}.csv\").is_file():\n",
" print(f\"File already exists for idx {i}. Not Saving.\")\n",
" continue\n",
" response = requests.get(api_str, headers = {'User-agent': 'Weather Batch Crawler 0.7'})\n",
"\n",
" if (response.status_code == 429): #Too many requests\n",
" print(\"Too many requests, waiting 1 second...\")\n",
" time.sleep(1)\n",
" response = requests.get(api_str)\n",
"\n",
" if(response.status_code != 200):\n",
" print(f\"Error on idx={i}. Code {response.status_code}. Not Saving...\")\n",
"\n",
" # Use pathlib for handling file paths\n",
" if (response.status_code == 200):\n",
" data = response.content\n",
" print(f\"Writing for Idx:{i}\")\n",
" output_file = Path(save_path/f'all_cells_{i}.csv')\n",
" output_file.write_bytes(data)"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# The api string is too long. Split the lat, long string such that we can have multiple calls. The rest of the API call string should not be changed! \n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "base",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.8"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment