Skip to content

Instantly share code, notes, and snippets.

@ashiklom
Created December 13, 2022 22:54
Show Gist options
  • Save ashiklom/6d3cf6e12ea2582221e9e7446bc94f6a to your computer and use it in GitHub Desktop.
Save ashiklom/6d3cf6e12ea2582221e9e7446bc94f6a to your computer and use it in GitHub Desktop.
EDL "wget" implementation
def wget(url):
import urllib.request
import netrc
import os
import fsspec
import base64
from urllib.parse import urlparse
# Get login credentials
URS_URL="https://urs.earthdata.nasa.gov"
info = netrc.netrc()
username, account, password = info.authenticators(urlparse(URS_URL).hostname)
credentials = '{0}:{1}'.format(username, password)
credentials = base64.b64encode(credentials.encode('ascii')).decode('ascii')
target_dir = "."
target_file = os.path.join(target_dir, os.path.basename(url))
print(f"Downloading {url} to {target_file}")
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor())
request = urllib.request.Request(url)
request.add_header("Authorization", f"Bearer {credentials}")
with opener.open(request) as response, fsspec.open(target_file, "wb") as f:
f.write(response.read())
# This works
wget("https://data.nsidc.earthdatacloud.nasa.gov/nsidc-cumulus-prod-protected/ATLAS/ATL08/005/2018/10/14/ATL08_20181014001049_02350102_005_01.h5")
# This fails
wget("https://n5eil01u.ecs.nsidc.org/DP7/ATLAS/ATL06.005/2018.10.14/ATL06_20181014051246_02380105_005_01.h5")
# This works
wget("https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/ECCO_L4_BOLUS_LLC0090GRID_MONTHLY_V4R4/OCEAN_BOLUS_VELOCITY_mon_mean_1992-03_ECCO_V4r4_native_llc0090.nc")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment