Instantly share code, notes, and snippets.

@arfon /astroquery-sep.mast.py Secret
Last active Jun 5, 2018

Embed
What would you like to do?
# Import astroquery.mast (http://astroquery.readthedocs.io/en/latest/mast/mast.html)
# Note, you may need to build from source to access the HST data on AWS.
from astroquery.mast import Observations
import boto3
import json
import os
# $ cat ~/.aws/credentials
# [lambda-invoker]
# aws_access_key_id = EXAMPLEACCESSKEY
# aws_secret_access_key = EXAMPLESECRETACCESSKEY
# This is required for astroquery.mast to pick up the credentials defined in ~/.aws/credentials
os.environ['AWS_PROFILE']='lambda-invoker'
# Query MAST for some WFC3 data
obsTable = Observations.query_criteria(project='HST',
filters='F160W',
instrument_name='WFC3/IR')
# Grab 100 products:
# http://astroquery.readthedocs.io/en/latest/mast/mast.html#getting-product-lists
products = Observations.get_product_list(obsTable[:100])
# Filter out just the drizzled FITS files
filtered_products = Observations.filter_products(products,
mrp_only=False,
productSubGroupDescription='DRZ')
# Use AWS S3 URLs for the MAST records (rather than the ones at http://mast.stsci.edu)
Observations.enable_s3_hst_dataset()
# We want URLs like this: s3://stpubdata/hst/public/ibg7/ibg705080/ibg705081_drz.fits
s3_urls = Observations.get_hst_s3_uris(filtered_products)
# Auth to create a Lambda function
session = boto3.Session()
client = session.client('lambda', region_name='us-east-1')
# Loop through the URLs for the data on S3
# 'your-output-bucket' is where you want to the Lambda outputs to be written
# FunctionName is the name of the Lambda function you made earlier.
for url in s3_urls:
fits_s3_key = url.replace("s3://stpubdata/", "")
event = {
'fits_s3_key': fits_s3_key,
'fits_s3_bucket': 'stpubdata',
's3_output_bucket': 'your-output-bucket' # <- change this to your output bucket
}
# Invoke Lambda function
response = client.invoke(
FunctionName='SEPFunction',
InvocationType='Event',
LogType='Tail',
Payload=json.dumps(event)
)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment