Create a gist now

Instantly share code, notes, and snippets.

@arfon /astroquery.mast.py Secret
Last active Jun 4, 2018

Embed
What would you like to do?
# This script queries MAST for WFC3 IR data and downloads the data from
# the AWS public dataset rather than from MAST servers.
# Working with http://astroquery.readthedocs.io/en/latest/mast/mast.html
from astroquery.mast import Observations
import boto3
# This downloads all the F160W DRZ images from CANDELS in the GOODS-South field
obsTable = Observations.query_criteria(project='HST',
filters='F160W',
instrument_name='WFC3/IR',
proposal_id=['12062','12061','12062'])
# Grab the first 100 records
products = Observations.get_product_list(obsTable[:100])
# Select only drizzled (DRZ) files
filtered = Observations.filter_products(products,
mrp_only=False,
productSubGroupDescription='DRZ')
# Enable 'S3 mode' for module which will return S3-like URLs for FITs files
# e.g. s3://stpubdata/hst/public/icde/icde43l0q/icde43l0q_drz.fits
Observations.enable_s3_hst_dataset()
# Grab the S3 URLs for each of the observations
s3_urls = Observations.get_hst_s3_uris(filtered)
s3 = boto3.resource('s3')
# Create an authenticated S3 session. Note, download within US-East is free
# e.g. to a node on EC2.
s3_client = boto3.client('s3',
aws_access_key_id='YOURAWSACCESSKEY',
aws_secret_access_key='YOURSECRETACCESSKEY')
bucket = s3.Bucket('stpubdata')
# Just download a few of the files (remove the [0:3] to download them all)
for url in s3_urls[0:3]:
# Extract the S3 key from the S3 URL
fits_s3_key = url.replace("s3://stpubdata/", "")
root = url.split('/')[-1]
bucket.download_file(fits_s3_key, root, ExtraArgs={"RequestPayer": "requester"})
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment