gistfile1.py

#!/usr/bin/env python
#-----------------------------------------------
# Name:  fetch_cosm.py
#
# Purpose: retrieve historical hi-res data from cosm
# Usage:  ./fetch_cosm.py >>cosm_data.csv
# History:
#   Date        Author  Remarks
#   15Oct2012   RW     Created.
#-----------------------------------------------
import sys
import urllib2
from datetime import *
from time import *
from optparse import OptionParser
COSM_API_KEY="INSERT_YOUR_API_KEY_HERE"
parser = OptionParser()
parser.add_option("-f","--feedid",type="int",dest="feedid",default=8675309,help="Set Feed ID")
(options,args) = parser.parse_args()
FEED_ID = options.feedid

def main():
    api_key= { 'X-ApiKey' : COSM_API_KEY }
    # change this interval depending on how frequent your data points are- Cosm API limits you to 1000 points per fetch
    interval=timedelta(hours=1)
    # work backwards until this date
    earlist=datetime(2012,10,24)

    # start from now (or change to another date using above syntax)
    end=datetime.utcnow()
    begin=end-interval
    retry_count=0
    while begin > earlist :
        try:
            req=urllib2.Request('http://api.cosm.com/v2/feeds/%d.csv?start=%s&end=%s&interval=0&limit=1000' % ( FEED_ID, begin.strftime('%Y-%m-%dT%H:%M:%SZ'),end.strftime('%Y-%m-%dT%H:%M:%SZ')) , headers=api_key)
	    #swap comments with above line if you want just a specific datastream (e.g. 0 in this example)
            #req=urllib2.Request('http://api.cosm.com/v2/feeds/%d/datastreams/0.csv?start=%s&end=%s&interval=0&limit=1000' % ( FEED_ID, begin.strftime('%Y-%m-%dT%H:%M:%SZ'),end.strftime('%Y-%m-%dT%H:%M:%SZ')) , headers=api_key)
            resp=urllib2.urlopen(req)
            if resp.code == 200:
                csv=resp.read()
                print csv
                end=begin
                begin=end-interval
                retry_count=0

        except urllib2.HTTPError, error:
            # sometimes requests fail (usually 500-internal server error) so try again
            ++retry_count
            print >> sys.stderr,"Error %s %s (%d)" %  (begin.strftime('%Y-%m-%dT%H:M:%SZ'),end.strftime('%Y-%m-%dT%H:%M:%SZ'),retry_count)
            print error.read()


# wait 5 seconds between fetchs to avoid API rate limit
        sleep(5)


if __name__ == '__main__':
    main()