quickndirtyscraper.py

#!/usr/bin/env python
"""Dumps all translink bus timetables to csv
"""
# marty mcfly imports
from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals

# stdlib imports
import os
import re
import sys
import unicodedata as ud

# third-party imports
import opentranslink


def slugify(value):
    """Normalizes string, converts to lowercase, removes non-alpha characters,
    and converts spaces to hyphens.
    """
    value = ud.normalize('NFKD', value).encode('ascii', 'ignore')
    value = unicode(re.sub('[^\w\s-]', '', value).strip().lower())
    return re.sub('[-\s]+', '-', value)


def get_absolute_path(tt_path):
    tt_path = os.path.abspath(tt_path)
    if not os.path.exists(tt_path):
        os.makedirs(tt_path)
    return tt_path


def write_timetables_to_disk(path, service_name, route):
    print('Fetching timetable data: {0} - {1}'.format(service_name, route.code))
    dir_path = os.path.join(path, service_name, route.code)
    if not os.path.exists(dir_path):
        os.makedirs(dir_path)
    for idx, tt in enumerate(route.timetable):
        file_name = str(idx).zfill(2) + '_' + slugify(tt[0]) + '.csv'
        file_path = os.path.join(dir_path, file_name)
        with open(file_path, 'w') as f:
            f.write(tt[1].csv)
        print('Wrote timetable to disk: {0}'.format(file_path))


if __name__ == '__main__':

    tt_path = get_absolute_path(sys.argv[1])

    service_names = ['metro', 'goldline', 'ulsterbus']
    for service_name in service_names:
        service = opentranslink.Service(service_name)
        print('Fetching route data: {0}'.format(service_name))
        for route in service.routes():
            write_timetables_to_disk(tt_path, service_name, route)