# This script queries MAST for WFC3 IR data and downloads the data from | |
# the AWS public dataset rather than from MAST servers. | |
# Working with http://astroquery.readthedocs.io/en/latest/mast/mast.html | |
from astroquery.mast import Observations | |
import boto3 | |
# This downloads all the F160W DRZ images from CANDELS in the GOODS-South field | |
obsTable = Observations.query_criteria(project='HST', | |
filters='F160W', | |
instrument_name='WFC3/IR', | |
proposal_id=['12062','12061','12062']) | |
# Grab the first 100 records | |
products = Observations.get_product_list(obsTable[:100]) | |
# Select only drizzled (DRZ) files | |
filtered = Observations.filter_products(products, | |
mrp_only=False, | |
productSubGroupDescription='DRZ') | |
# Enable 'S3 mode' for module which will return S3-like URLs for FITs files | |
# e.g. s3://stpubdata/hst/public/icde/icde43l0q/icde43l0q_drz.fits | |
Observations.enable_s3_hst_dataset() | |
# Grab the S3 URLs for each of the observations | |
s3_urls = Observations.get_hst_s3_uris(filtered) | |
s3 = boto3.resource('s3') | |
# Create an authenticated S3 session. Note, download within US-East is free | |
# e.g. to a node on EC2. | |
s3_client = boto3.client('s3', | |
aws_access_key_id='YOURAWSACCESSKEY', | |
aws_secret_access_key='YOURSECRETACCESSKEY') | |
bucket = s3.Bucket('stpubdata') | |
# Just download a few of the files (remove the [0:3] to download them all) | |
for url in s3_urls[0:3]: | |
# Extract the S3 key from the S3 URL | |
fits_s3_key = url.replace("s3://stpubdata/", "") | |
root = url.split('/')[-1] | |
bucket.download_file(fits_s3_key, root, ExtraArgs={"RequestPayer": "requester"}) | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment