Skip to content

Instantly share code, notes, and snippets.

@nsabharwal
Created September 18, 2019 05:23
Show Gist options
  • Save nsabharwal/aa9588a41d4bd641604f87d3ebc52372 to your computer and use it in GitHub Desktop.
Save nsabharwal/aa9588a41d4bd641604f87d3ebc52372 to your computer and use it in GitHub Desktop.
s3 demo
import subprocess,json,sys
def runAndGet(cmd,parse=False):
print(cmd)
dummy=raw_input()
try:
data = subprocess.check_output(cmd, shell=True)
except:
sys.exit(0)
print(data)
dummy=raw_input()
if parse:
#get the output
jsonStuff=data.split()
qid=str(jsonStuff[-1])
return qid
def runWithProcess(cmd,qid):
assert(qid[-4:]=='.csv')
cmd = qid.join(cmd)
runAndGet(cmd)
#hybrid
cmd ='aws s3 ls s3://priv-demo-customer-athena-test/output/ | tail -2 | grep -v metadata'
qid = runAndGet(cmd,True)
cmd2 = ['aws s3 cp s3://priv-demo-customer-athena-test/output/',' .']
runWithProcess(cmd2,qid)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment