Download files using python, in this case precipitation gis files

download_precip.sh

#!/home/deepwinter/env/bin/python
# using virtual env
# http://wiki.dreamhost.com/Python
import datetime
import time
import wget
import datetime
import tarfile
import os.path
import urlgrabber
from urlgrabber import urlgrab

# file name format
# nws_precip_1day_observed_shape_20140101.tar.gz

def daterange(start_date, end_date):
  for n in range(int ((end_date - start_date).days)):
    yield start_date + datetime.timedelta(n)


errors_file = open("DownloadErrors.txt", "w")

start_date = datetime.date(2005,1,1)
end_date = datetime.date(2010,12,31)
downloads_folder = "downloads_new/" # needs final slash
shapes_folder = "shapes_new/" # needs final slash
for single_date in daterange(start_date, end_date):
  download_url = time.strftime("http://water.weather.gov/precip/p_download_new/%Y/%m/%d/nws_precip_1day_observed_shape_%Y%m%d.tar.gz", single_date.timetuple())
  print download_url
  filename = time.strftime("nws_precip_1day_observed_shape_%Y%m%d.tar.gz", single_date.timetuple())
  try:
    urlgrab(download_url, downloads_folder + filename)
    path = downloads_folder + filename
    tar = tarfile.open(path)
    tar.extractall(shapes_folder)
    tar.close()
  except urlgrabber.grabber.URLGrabError, e:
    print e
    errors_file.write("Missing File " 'download_url')

errors_file.close()