# Import astroquery.mast (http://astroquery.readthedocs.io/en/latest/mast/mast.html) | |
# Note, you may need to build from source to access the HST data on AWS. | |
from astroquery.mast import Observations | |
import boto3 | |
import json | |
import os | |
# $ cat ~/.aws/credentials | |
# [lambda-invoker] | |
# aws_access_key_id = EXAMPLEACCESSKEY | |
# aws_secret_access_key = EXAMPLESECRETACCESSKEY | |
# This is required for astroquery.mast to pick up the credentials defined in ~/.aws/credentials | |
os.environ['AWS_PROFILE']='lambda-invoker' | |
# Query MAST for some WFC3 data | |
obsTable = Observations.query_criteria(project='HST', | |
filters='F160W', | |
instrument_name='WFC3/IR') | |
# Grab 100 products: | |
# http://astroquery.readthedocs.io/en/latest/mast/mast.html#getting-product-lists | |
products = Observations.get_product_list(obsTable[:100]) | |
# Filter out just the drizzled FITS files | |
filtered_products = Observations.filter_products(products, | |
mrp_only=False, | |
productSubGroupDescription='DRZ') | |
# Use AWS S3 URLs for the MAST records (rather than the ones at http://mast.stsci.edu) | |
Observations.enable_s3_hst_dataset() | |
# We want URLs like this: s3://stpubdata/hst/public/ibg7/ibg705080/ibg705081_drz.fits | |
s3_urls = Observations.get_hst_s3_uris(filtered_products) | |
# Auth to create a Lambda function | |
session = boto3.Session() | |
client = session.client('lambda', region_name='us-east-1') | |
# Loop through the URLs for the data on S3 | |
# 'your-output-bucket' is where you want to the Lambda outputs to be written | |
# FunctionName is the name of the Lambda function you made earlier. | |
for url in s3_urls: | |
fits_s3_key = url.replace("s3://stpubdata/", "") | |
event = { | |
'fits_s3_key': fits_s3_key, | |
'fits_s3_bucket': 'stpubdata', | |
's3_output_bucket': 'your-output-bucket' # <- change this to your output bucket | |
} | |
# Invoke Lambda function | |
response = client.invoke( | |
FunctionName='SEPFunction', | |
InvocationType='Event', | |
LogType='Tail', | |
Payload=json.dumps(event) | |
) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment