Sample datasource for remote tiled data in Mapnik.

RemoteGeoJSON.py

''' Sample datasource for remote tiled data in Mapnik.

See also https://github.com/mapnik/mapnik/wiki/Python-Plugin

Use in Mapnik:

    <Datasource>
      <Parameter name="type">python</Parameter>
      <Parameter name="factory">RemoteGeoJSON:Datasource</Parameter>
      
      <!-- URL template for remote data, with {Z}, {X}, {Y} parameters -->
      <Parameter name="url_template">http://tile.osm.osuosl.org/tiles/osmus_vector_line/{Z}/{X}/{Y}.geojson</Parameter>
      
      <!-- number of threads to use when requesting data -->
      <Parameter name="thread_count">8</Parameter>
    </Datasource>
'''
from math import pi
from sys import stderr
from itertools import product
from thread import start_new_thread
from urllib import urlopen
from time import sleep
from json import load

import mapnik
from shapely.geometry import asShape

from ModestMaps.OpenStreetMap import Provider
from ModestMaps.Core import Coordinate
from ModestMaps.Geo import Location

radius = 6378137 * pi
resolutions = [(2 ** (8 + zoom) / (radius * 2), zoom) for zoom in range(32)]
mercator = mapnik.Projection('+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs')
osm = Provider()

def resolution_zoom(resolution):
    ''' Determine a tile zoom level for resolution expressed in pixels-per-meter.
    '''
    resolutions.sort(key=lambda (res, zoom): abs(res - resolution))
    return resolutions[0][1]

def de_unicode(thing):
    ''' Convert unicode parts of json.load to strings for mapnik's use.
    '''
    if type(thing) is unicode:
        return thing.encode('utf-8')
    
    if type(thing) is list:
        return [de_unicode(val) for val in thing]
    
    if type(thing) is dict:
        return dict([map(de_unicode, (k, v)) for (k, v) in thing.items()])
    
    return thing

def query_coords(query):
    ''' Get a list of tile coordinates for a mapnik.Query object.
    
        Make some dangerous assumptions here about mercator projection.
    '''
    zoom = resolution_zoom(query.resolution[0]/2 + query.resolution[1]/2)
    bbox = query.bbox.inverse(mercator)
    
    nw = osm.locationCoordinate(Location(bbox.maxy, bbox.minx)).zoomTo(zoom)
    se = osm.locationCoordinate(Location(bbox.miny, bbox.maxx)).zoomTo(zoom)
    
    cols = range(int(nw.column), int(se.column + 1))
    rows = range(int(nw.row), int(se.row + 1))
    
    coords = [Coordinate(row, col, zoom) for (row, col) in product(rows, cols)]
    
    return coords

def fetch_geojson_tiles(coords, url_template, geodata):
    ''' Request and parse GeoJSON data from coords list to geodata list.
    
        Intended to be run on a thread, with shared coords and geodata lists.
        Thread-safety is ensured by atomicity of coords.pop and geodata.append.
    '''
    tmpl = url_template.replace('{Y}', '{y}') \
                       .replace('{X}', '{x}') \
                       .replace('{Z}', '{z}')
    
    while len(coords):
        try:
            coord = coords.pop()
        
        except IndexError:
            return
        
        try:
            url = tmpl.replace('{y}', '%d' % coord.row) \
                      .replace('{x}', '%d' % coord.column) \
                      .replace('{z}', '%d' % coord.zoom)

            data = load(urlopen(url))
            geodata.append(data)
        
        except Exception, e:
            geodata.append(e)

def fetch_geojson_features(coords, url, thread_count):
    ''' Retrieve remote features for a list of tile coordinates.
    
        Use fetch_geojson_tiles() on multiple threads to perform requests.
    '''
    geodata, needed = [], len(coords)
    
    for job in range(thread_count):
        start_new_thread(fetch_geojson_tiles, (coords, url, geodata))
    
    while len(geodata) < needed:
        sleep(.1)
    
    features = []
    
    for collection in geodata:
        for feature in collection['features']:
            features.append(feature)
    
    return features

def merge_wkb_features(geojson_features):
    ''' Merge features into WKB-format with keys for mapnik.PythonDatasource.
    '''
    features = []
    seen_ids = set()
    keys = set()
    
    for feature in geojson_features:
        fid = feature.get('id', feature['properties'].get('id', None))
        
        # if fid in seen_ids and fid is not None:
        #     continue

        seen_ids.add(fid)

        properties = de_unicode(feature['properties'])
        geometry = asShape(feature['geometry']).wkb

        features.append((geometry, properties))
        keys = keys.union(properties.keys())
            
    return features, tuple(keys)

class Datasource(mapnik.PythonDatasource):
    ''' Tiled GeoJSON datasource for Mapnik Python Plugin.
    '''
    def __init__(self, url_template, thread_count=4):
        '''
        '''
        self.url_template = url_template
        self.thread_count = int(thread_count or 4)
        super(Datasource, self).__init__(envelope=mapnik.Envelope(-radius, -radius, radius, radius), geometry_type=mapnik.DataType.Vector)

    def features(self, query):
        '''
        '''
        coords = query_coords(query)
        fetch_args = self.url_template, self.thread_count
        geojson_features = fetch_geojson_features(coords, *fetch_args)
        features, keys = merge_wkb_features(geojson_features)
        
        return mapnik.PythonDatasource.wkb_features(keys=keys, features=features)

style.xml

<Map background-color="rgb(255,255,255)" srs="+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over">
  <Style name="lines">
    <Rule>
      <LineSymbolizer stroke-opacity="0.25" stroke-width="8" />
    </Rule>
  </Style>
  <Layer name="py" srs="+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over">
    <StyleName>lines</StyleName>
    <Datasource>
      <Parameter name="type">python</Parameter>
      <Parameter name="factory">RemoteGeoJSON:Datasource</Parameter>
      <Parameter name="url_template">http://tile.osm.osuosl.org/tiles/osmus_vector_line/{Z}/{X}/{Y}.geojson</Parameter>
      <Parameter name="thread_count">8</Parameter>
    </Datasource>
  </Layer>
</Map>