kml_tester.py

from bs4 import BeautifulSoup
import json
import pandas as pd

# Process information on Caltrans projects.  Convert this file to
# GeoJSON:

# wget https://dot.ca.gov/hq/construc/cons.kml
# ogr2ogr -f GeoJSON cons.json cons.kml


def _read_data():
    with open('cons.json') as f:
        x = json.load(f)
        return x['features']


def _process_entry(entry):
    lon, lat, _ = entry['geometry']['coordinates']
    doc = entry['properties']['Description']
    l = doc.split("<br />")

    def _process_kv(kv):
        try:
            return dict([BeautifulSoup(kv).text.split(": ")])
        except ValueError:
            return None

    a = [_process_kv(x) for x in l if _process_kv(x) is not None]
    a = a + [{'lat':lat}] + [{'lon':lon}]
    return { k: v for d in a for k, v in d.items() }


xx = map(_process_entry, _read_data())
df = pd.DataFrame(xx)
df.to_csv('tester.csv')