td_download.py

import os
import sys
import requests
import urlparse

apikey=sys.argv[1]
host=sys.argv[2]
database=sys.argv[3]
table=sys.argv[4]
headers={
    "Authorization": "TD1 %s" % apikey,
    "Content-Type": "application/json"
}
HOSTS={
    "dev-eu" : "https://api-plazma-development.eu01.treasuredata.com",
    "stg-eu" : "https://api-plazma-staging.eu01.treasuredata.com",
    "prod-eu" : "https://api-plazma.eu01.treasuredata.com",
}
def _url(host, path):
    global HOsTS
    
    host = HOSTS.get(host, host)
    return '%s/api/%s' % (host, path)

def _send(resp):
    print ('%s %s' % (resp.request.method, resp.url))
    if resp.status_code not in [200, 201]:
        # This means something went wrong.
        raise Exception('{} {} {}'.format(resp.url, resp.status_code, resp.text))
    return resp.json()

# start transaction
params={"database": database, "table" : table}
data = _send(requests.get(_url(host, 'partitions'), headers=headers, params=params))
print data

for p in data:
    file = os.path.split(urlparse.urlparse(p['directUrl']).path)[1]
    print ("download %s to %s" % (p["directUrl"], file))
    with open(file, 'wb') as f:
        resp = requests.get(p["directUrl"], headers={"host":"trd-marriott-result-development-eu01-eu-central-1.s3-eu-central-1.amazonaws.com"}, stream=True)
        if resp.status_code not in [200, 201]:
            # This means something went wrong.
            raise Exception('{} {} {}'.format(resp.url, resp.status_code, resp.text))
        for chunk in resp.iter_content(chunk_size=8192):
            f.write(chunk)
    assert p['fileSize'] == os.path.getsize(file)