Instantly share code, notes, and snippets.

Embed
What would you like to do?
Two approaches to visualizing MTA NYCT A Divsion ATS data in Google Earth
from csv import DictReader
from collections import defaultdict
import xml.etree.ElementTree as etree
from datetime import datetime
import pytz
from gtfs import Schedule
from gtfs.entity import Stop
eastern = pytz.timezone('US/Eastern')
dr = DictReader(open('data.csv'))
schedule = Schedule('google_transit.db')
trains = defaultdict(list)
for row in dr:
trains[row['train_id']].append(row)
print len(trains)
style = etree.fromstring("""<Style id="train" xmlns="http://www.opengis.net/kml/2.2">
<IconStyle>
<scale>0.5</scale>
<Icon>
<href>http://maps.google.com/mapfiles/kml/shapes/rail.png</href>
</Icon>
</IconStyle>
<LabelStyle>
<scale>0</scale>
</LabelStyle>
</Style>""")
kml = etree.Element("{http://www.opengis.net/kml/2.2}kml")
folder = etree.SubElement(kml, "{http://www.opengis.net/kml/2.2}Folder")
foldername = etree.SubElement(folder, "{http://www.opengis.net/kml/2.2}name")
foldername.text = "Trains"
folder.append(style)
count = 0
for train, events in trains.iteritems():
count += 1
print "Processing train %s" % train
trainfolder = etree.SubElement(folder, "{http://www.opengis.net/kml/2.2}Folder")
trainfoldername = etree.SubElement(trainfolder, "{http://www.opengis.net/kml/2.2}name")
direction = ['Southbound', 'Northbound'][int(events[0]['direction_id'])-1]
trainfoldername.text = "%s %s Train %s" % (direction, events[0]['route_id'], events[0]['train_id'])
stops = defaultdict(list)
for event in events:
stops[event['stop_id']].append(event)
for stop, stop_events in stops.iteritems():
assert len(stop_events) <= 2
arr = dep = None
for event in stop_events:
if event['event_type'] == '1':
arr = event['timestamp']
if event['event_type'] == '2':
dep = event['timestamp']
if arr is None and dep is not None:
arr = dep
if dep is None and arr is not None:
dep = arr
placemark = etree.Element("{http://www.opengis.net/kml/2.2}Placemark")
name = etree.SubElement(placemark, "{http://www.opengis.net/kml/2.2}name")
name.text = train
stop = Stop.query.get(stop)
description = etree.SubElement(placemark, "{http://www.opengis.net/kml/2.2}description")
description.text = "%s on track %s" % (stop.stop_name, stop_events[0]['track_id'])
point = etree.SubElement(placemark, "{http://www.opengis.net/kml/2.2}Point")
coordinates = etree.SubElement(point, "{http://www.opengis.net/kml/2.2}coordinates")
coordinates.text="%s,%s" % (stop.stop_lon, stop.stop_lat)
span = etree.SubElement(placemark, "{http://www.opengis.net/kml/2.2}TimeSpan")
begin = etree.SubElement(span, "{http://www.opengis.net/kml/2.2}begin")
end = etree.SubElement(span, "{http://www.opengis.net/kml/2.2}end")
begin.text = eastern.localize(datetime.strptime(arr,
"%Y-%m-%d %H:%M:%S")).astimezone(pytz.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
end.text = eastern.localize(datetime.strptime(dep,
"%Y-%m-%d %H:%M:%S")).astimezone(pytz.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
style = etree.SubElement(placemark, "{http://www.opengis.net/kml/2.2}styleUrl")
style.text = "#train"
trainfolder.append(placemark)
with file('out.kml', 'w') as outfile:
et = etree.ElementTree(kml)
et.write(outfile)
from csv import DictReader
from collections import defaultdict
import xml.etree.ElementTree as etree
from datetime import datetime
import pytz
from gtfs import Schedule
from gtfs.entity import Stop
eastern = pytz.timezone('US/Eastern')
dr = DictReader(open('data.csv'))
schedule = Schedule('google_transit.db')
trains = defaultdict(list)
for row in dr:
trains[row['train_id']].append(row)
print len(trains)
style = etree.fromstring("""<Style id="train" xmlns="http://www.opengis.net/kml/2.2">
<IconStyle>
<scale>0.5</scale>
<Icon>
<href>http://maps.google.com/mapfiles/kml/shapes/rail.png</href>
</Icon>
</IconStyle>
<LabelStyle>
<scale>0</scale>
</LabelStyle>
</Style>""")
kml = etree.Element("{http://www.opengis.net/kml/2.2}kml")
folder = etree.SubElement(kml, "{http://www.opengis.net/kml/2.2}Folder")
foldername = etree.SubElement(folder, "{http://www.opengis.net/kml/2.2}name")
foldername.text = "Trains"
folder.append(style)
count = 0
for train, events in trains.iteritems():
count += 1
print "Processing train %s" % train
trainfolder = etree.SubElement(folder, "{http://www.opengis.net/kml/2.2}Folder")
trainfoldername = etree.SubElement(trainfolder, "{http://www.opengis.net/kml/2.2}name")
direction = ['Southbound', 'Northbound'][int(events[0]['direction_id'])-1]
trainfoldername.text = "%s %s Train %s" % (direction, events[0]['route_id'], events[0]['train_id'])
for event in events:
placemark = etree.Element("{http://www.opengis.net/kml/2.2}Placemark")
name = etree.SubElement(placemark, "{http://www.opengis.net/kml/2.2}name")
name.text = train
stop = Stop.query.get(event['stop_id'])
event_type = ['Arrived at', 'Departed from'][int(event['event_type'])-1]
description = etree.SubElement(placemark, "{http://www.opengis.net/kml/2.2}description")
description.text = "%s station %s on track %s" % (event_type, stop.stop_name, event['track_id'])
point = etree.SubElement(placemark, "{http://www.opengis.net/kml/2.2}Point")
coordinates = etree.SubElement(point, "{http://www.opengis.net/kml/2.2}coordinates")
coordinates.text="%s,%s" % (stop.stop_lon, stop.stop_lat)
ts = etree.SubElement(placemark, "{http://www.opengis.net/kml/2.2}TimeStamp")
when = etree.SubElement(ts, "{http://www.opengis.net/kml/2.2}when")
dt = eastern.localize(datetime.strptime(event['timestamp'],
"%Y-%m-%d %H:%M:%S"))
when.text = dt.astimezone(pytz.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
style = etree.SubElement(placemark, "{http://www.opengis.net/kml/2.2}styleUrl")
style.text = "#train"
trainfolder.append(placemark)
with file('out.kml', 'w') as outfile:
et = etree.ElementTree(kml)
et.write(outfile)
@kurtraschke

This comment has been minimized.

Copy link
Owner

kurtraschke commented Sep 6, 2011

Before settling on creating a video with Processing, I tried visualizing the MTA NYCT ATS data by converting it to KML to view in Google Earth. The approach worked so long as I limited the number of trains processed, but when I processed the entire file, the resulting KML (around 25 MB) was so large that Google Earth would choke on it; it never rendered fully, and usually crashed.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment