Skip to content

Instantly share code, notes, and snippets.

@dandye
Created July 9, 2012 21:30
Show Gist options
  • Save dandye/3079103 to your computer and use it in GitHub Desktop.
Save dandye/3079103 to your computer and use it in GitHub Desktop.
divide data among multiple wetsaws
import os,sys
def get_fips():
lines = open("counties_fips_list.txt").readlines()
fips = [line.strip() for line in lines]
return fips
if __name__ == "__main__":
wetsawn = sys.argv[1] # pass the wetsaw # as an argument
fips = get_fips()
of = open("pull_county_dirs.sh","w")
sh = open("render_all.sh","w")
fips_len = len(fips)/wetsawn
fips_start = int(wetsawn) * fips_len
for fip in fips[fips_start:fips_start+fips_len]:
if not os.path.exists("/mnt/tiger/TIGER2011/wg_fips/tl_2011_{}".format(fip)):
os.mkdir("/mnt/tiger/TIGER2011/wg_fips/tl_2011_{}".format(fip))
of.write("scp -r root@ec2-107-21-179-195.compute-1.amazonaws.com:/mnt/tiger/TIGER2011/wg_fips/tl_2011_{}/ /mnt/tiger/TIGER2011/wg_fips/tl_2011_{}\n".format(fip,fip))
sh.write("cd /mnt/tiger/TIGER2011/wg_fips/tl_2011_{}\n".format(fip))
sh.write("./render.sh\n")
of.close()
sh.close()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment