public
Last active

create a new directory and put a checkpoint in it

  • Download Gist
checkpoint.py
Python
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
#!/usr/bin/python
# crontab -e
# */20 * * * * /home/ubuntu/checkpoint.py > /home/ubuntu/checkpoint.log 2>&1
import os
 
hosts = []
with open('/usr/local/Rmpi/hostfile.plain') as f:
for line in f:
hosts.append(line.split()[0])
 
os.chdir("/home/ubuntu/large1")
 
if os.path.exists('checkpoint_counter'):
with open('checkpoint_counter') as f:
i = int(f.read())
else:
i = 1
 
try:
os.mkdir("checkpoint%d" % i)
os.chdir("checkpoint%d" % i)
for host in hosts:
print('scp -i ~/cbare-sage.pem ubuntu@%s:/home/ubuntu/dose.large.node*.tsv .' % host)
os.system('scp -i ~/cbare-sage.pem ubuntu@%s:/home/ubuntu/dose.large.node*.tsv .' % host)
 
os.chdir('..')
with open("checkpoint_counter", "w") as f:
f.write("%d\n" % (i+1))
except Exception, e:
print("Error: " + str(e))
finally:
os.chdir("/home/ubuntu/")

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.