Download pilot images for the DM Challenge from an S3 bucket

download_pilot_images.py

import os
from boto.s3.connection import S3Connection
from boto.s3.connection import OrdinaryCallingFormat

## retrieve AWS credentials from environment variables
aws_key = os.getenv('AWS_KEY_ID')
aws_secret = os.getenv('AWS_SECRET_KEY')

## Ordinary calling format necessary because of dots in
## bucket name. See: https://github.com/boto/boto/issues/2836
conn = S3Connection(aws_key, aws_secret, calling_format=OrdinaryCallingFormat())

bucket = conn.get_bucket('dmchallenge.synapse.org')

prefix = 'grouphealth//'

i = 0
with open('images_crosswalk_pilot.tsv') as f:
    headers = f.readline().split('\t')
    col = headers.index('filename')
    for line in f:
        filename = line.strip().split('\t')[col]
        i += 1
        print(i, filename)
        key = bucket.get_key(prefix+filename)
        key.get_contents_to_filename(filename)