s3 demo

S3demo

import subprocess,json,sys

def runAndGet(cmd,parse=False):
    print(cmd)
    dummy=raw_input()

    try:
        data = subprocess.check_output(cmd, shell=True)
    except:
        sys.exit(0)
    print(data)
    dummy=raw_input()

    if parse:
        #get the output
        jsonStuff=data.split()
        qid=str(jsonStuff[-1])
        return qid

def runWithProcess(cmd,qid):
    assert(qid[-4:]=='.csv')
    cmd = qid.join(cmd)
    runAndGet(cmd)

#hybrid
cmd ='aws s3 ls s3://priv-demo-customer-athena-test/output/ | tail -2 | grep -v metadata'

qid = runAndGet(cmd,True)

cmd2 = ['aws s3 cp s3://priv-demo-customer-athena-test/output/',' .']
runWithProcess(cmd2,qid)