Two approaches to visualizing MTA NYCT A Divsion ATS data in Google Earth

subviewer1.py

from csv import DictReader
from collections import defaultdict
import xml.etree.ElementTree as etree
from datetime import datetime

import pytz

from gtfs import Schedule
from gtfs.entity import Stop

eastern = pytz.timezone('US/Eastern')

dr = DictReader(open('data.csv'))
schedule = Schedule('google_transit.db')

trains = defaultdict(list)

for row in dr:
    trains[row['train_id']].append(row)

print len(trains)

style = etree.fromstring("""<Style id="train" xmlns="http://www.opengis.net/kml/2.2">
          <IconStyle>
            <scale>0.5</scale>
            <Icon>
              <href>http://maps.google.com/mapfiles/kml/shapes/rail.png</href>
            </Icon>
          </IconStyle>
          <LabelStyle>
            <scale>0</scale>
          </LabelStyle>
        </Style>""")

kml = etree.Element("{http://www.opengis.net/kml/2.2}kml")
folder = etree.SubElement(kml, "{http://www.opengis.net/kml/2.2}Folder")
foldername = etree.SubElement(folder, "{http://www.opengis.net/kml/2.2}name")
foldername.text = "Trains"
folder.append(style)

count = 0

for train, events in trains.iteritems():
    count += 1
    print "Processing train %s" % train
    trainfolder = etree.SubElement(folder, "{http://www.opengis.net/kml/2.2}Folder")
    trainfoldername = etree.SubElement(trainfolder, "{http://www.opengis.net/kml/2.2}name")
    direction = ['Southbound', 'Northbound'][int(events[0]['direction_id'])-1]
    trainfoldername.text = "%s %s Train %s" % (direction, events[0]['route_id'], events[0]['train_id'])

    stops = defaultdict(list)
    for event in events:
        stops[event['stop_id']].append(event)
        
    for stop, stop_events in stops.iteritems():
        assert len(stop_events) <= 2

        arr = dep = None
        for event in stop_events:
            if event['event_type'] == '1':
                arr = event['timestamp']
            if event['event_type'] == '2':
                dep = event['timestamp']
            if arr is None and dep is not None:
                arr = dep
            if dep is None and arr is not None:
                dep = arr

        placemark = etree.Element("{http://www.opengis.net/kml/2.2}Placemark")
        name = etree.SubElement(placemark, "{http://www.opengis.net/kml/2.2}name")
        name.text = train

        stop = Stop.query.get(stop)
        
        description = etree.SubElement(placemark, "{http://www.opengis.net/kml/2.2}description")
        description.text = "%s on track %s" % (stop.stop_name, stop_events[0]['track_id'])
        point = etree.SubElement(placemark, "{http://www.opengis.net/kml/2.2}Point")
        coordinates = etree.SubElement(point, "{http://www.opengis.net/kml/2.2}coordinates")
        coordinates.text="%s,%s" % (stop.stop_lon, stop.stop_lat)



        span = etree.SubElement(placemark, "{http://www.opengis.net/kml/2.2}TimeSpan")
        begin =  etree.SubElement(span, "{http://www.opengis.net/kml/2.2}begin")
        end =  etree.SubElement(span, "{http://www.opengis.net/kml/2.2}end")
        
        begin.text = eastern.localize(datetime.strptime(arr,
                                                      "%Y-%m-%d %H:%M:%S")).astimezone(pytz.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
        end.text = eastern.localize(datetime.strptime(dep,
                                                      "%Y-%m-%d %H:%M:%S")).astimezone(pytz.utc).strftime("%Y-%m-%dT%H:%M:%SZ")


        style = etree.SubElement(placemark, "{http://www.opengis.net/kml/2.2}styleUrl")
        style.text = "#train"


        trainfolder.append(placemark)


with file('out.kml', 'w') as outfile:
    et = etree.ElementTree(kml)
    et.write(outfile)

subviewer2.py

from csv import DictReader
from collections import defaultdict
import xml.etree.ElementTree as etree
from datetime import datetime

import pytz

from gtfs import Schedule
from gtfs.entity import Stop

eastern = pytz.timezone('US/Eastern')

dr = DictReader(open('data.csv'))
schedule = Schedule('google_transit.db')

trains = defaultdict(list)

for row in dr:
    trains[row['train_id']].append(row)

print len(trains)

style = etree.fromstring("""<Style id="train" xmlns="http://www.opengis.net/kml/2.2">
          <IconStyle>
            <scale>0.5</scale>
            <Icon>
              <href>http://maps.google.com/mapfiles/kml/shapes/rail.png</href>
            </Icon>
          </IconStyle>
          <LabelStyle>
            <scale>0</scale>
          </LabelStyle>
        </Style>""")

kml = etree.Element("{http://www.opengis.net/kml/2.2}kml")
folder = etree.SubElement(kml, "{http://www.opengis.net/kml/2.2}Folder")
foldername = etree.SubElement(folder, "{http://www.opengis.net/kml/2.2}name")
foldername.text = "Trains"
folder.append(style)

count = 0

for train, events in trains.iteritems():
    count += 1
    print "Processing train %s" % train
    trainfolder = etree.SubElement(folder, "{http://www.opengis.net/kml/2.2}Folder")
    trainfoldername = etree.SubElement(trainfolder, "{http://www.opengis.net/kml/2.2}name")
    direction = ['Southbound', 'Northbound'][int(events[0]['direction_id'])-1]
    trainfoldername.text = "%s %s Train %s" % (direction, events[0]['route_id'], events[0]['train_id'])

    for event in events:
        placemark = etree.Element("{http://www.opengis.net/kml/2.2}Placemark")
        name = etree.SubElement(placemark, "{http://www.opengis.net/kml/2.2}name")
        name.text = train

        stop = Stop.query.get(event['stop_id'])

        event_type = ['Arrived at', 'Departed from'][int(event['event_type'])-1]
        
        description = etree.SubElement(placemark, "{http://www.opengis.net/kml/2.2}description")
        description.text = "%s station %s on track %s" % (event_type, stop.stop_name, event['track_id'])
        point = etree.SubElement(placemark, "{http://www.opengis.net/kml/2.2}Point")
        coordinates = etree.SubElement(point, "{http://www.opengis.net/kml/2.2}coordinates")
        coordinates.text="%s,%s" % (stop.stop_lon, stop.stop_lat)

        ts = etree.SubElement(placemark, "{http://www.opengis.net/kml/2.2}TimeStamp")
        when =  etree.SubElement(ts, "{http://www.opengis.net/kml/2.2}when")

        dt = eastern.localize(datetime.strptime(event['timestamp'],
                                           "%Y-%m-%d %H:%M:%S"))
        when.text = dt.astimezone(pytz.utc).strftime("%Y-%m-%dT%H:%M:%SZ")

        style = etree.SubElement(placemark, "{http://www.opengis.net/kml/2.2}styleUrl")
        style.text = "#train"


        trainfolder.append(placemark)


with file('out.kml', 'w') as outfile:
    et = etree.ElementTree(kml)
    et.write(outfile)

Before settling on creating a video with Processing, I tried visualizing the MTA NYCT ATS data by converting it to KML to view in Google Earth. The approach worked so long as I limited the number of trains processed, but when I processed the entire file, the resulting KML (around 25 MB) was so large that Google Earth would choke on it; it never rendered fully, and usually crashed.