Mock IFCB

ifcb_scraper.py

import os
import re
import time
from urllib2 import urlopen
import json

FEED_URL='http://ifcb-data.whoi.edu/mvco/feed.json'
DATA_DIR='/data'
FORMATS=('hdr','adc','roi')

def copy_new():
    feed = json.loads(urlopen(FEED_URL).read())
    pid = feed[0]['pid']
    lid = re.sub(r'.*/','',pid)
    urls = dict((fmt,'%s.%s' % (pid,fmt)) for fmt in FORMATS)
    paths = dict((fmt,os.path.join(DATA_DIR,'%s.%s' % (lid,fmt))) for fmt in FORMATS)
    for fmt in FORMATS:
        url, path = urls[fmt], paths[fmt]
        if not os.path.exists(path):
            with open(path,'w') as local:
                local.write(urlopen(url).read())

def delete_old():
    now = time.time()
    interval = 3600 * 12 # 12 hours
    elapsed = now - interval
    for f in os.listdir(DATA_DIR):
        p = os.path.join(DATA_DIR,f)
        if os.stat(p).st_mtime < elapsed:
            os.remove(p)

try:
    copy_new()
except:
    pass

try:
    delete_old()
except:
    pass