checkfeeds.py

#!/usr/bin/env python3.5
import os
import sys
from urllib.parse import urlsplit
from requests import Session
from transitfeeds import TransitFeeds

listing = sys.argv[1]
feed_ids = sys.argv[2:]
urls = []

with Session() as s:
    api = TransitFeeds(os.environ['TRANSITFEEDS_API_KEY'], session=s)
    latest = [api.latest(i) for i in feed_ids]

with open(listing, 'r+') as f:
    prev_paths = [urlsplit(x).path for x in f.read().strip().split('\n')]

    f.seek(0)

    for new in latest:
        f.write(new + '\n')
        if urlsplit(new).path not in prev_paths:
            urls.append(new)

with Session() as session:
    for url in urls:
        r = session.get(url)
        filename = urlsplit(url).path.strip('/')
        os.makedirs(os.path.dirname(filename))
        with open(filename, 'wb') as f:
            for chunk in r.iter_content(chunk_size=256):
                f.write(chunk)

        print(filename)

crontab

FEED_IDS="mta/81 mta/80 mta/85 mta/82 mta/83 mta/84"

0 4 * * 0 python3.5 checkfeeds.py feeds.txt $FEED_IDS | xargs -n 1 -I {} make load DATABASE=nycbus GTFS={}

feeds.txt

https://transitfeeds-data.s3-us-west-1.amazonaws.com/public/feeds/mta/81/20170405/gtfs.zip
https://transitfeeds-data.s3-us-west-1.amazonaws.com/public/feeds/mta/80/20170404/gtfs.zip
https://transitfeeds-data.s3-us-west-1.amazonaws.com/public/feeds/mta/85/20170404/gtfs.zip
https://transitfeeds-data.s3-us-west-1.amazonaws.com/public/feeds/mta/82/20170405/gtfs.zip
https://transitfeeds-data.s3-us-west-1.amazonaws.com/public/feeds/mta/83/20170405/gtfs.zip
https://transitfeeds-data.s3-us-west-1.amazonaws.com/public/feeds/mta/84/20170404/gtfs.zip

requirements.txt

https://github.com/fitnr/transitfeeds-api/archive/alpha.tar.gz#egg=transitfeeds-0.1