Skip to content

Instantly share code, notes, and snippets.

@migurski
Created August 3, 2012 21:15
Show Gist options
  • Save migurski/3251607 to your computer and use it in GitHub Desktop.
Save migurski/3251607 to your computer and use it in GitHub Desktop.
Two scripts for creating heatmap of OSM data size, e.g. https://tiles.mapbox.com/migurski/map/osmsize
from sys import argv, stdout, stderr
from subprocess import Popen
from os import stat, unlink
from math import log, ceil
from stat import ST_SIZE
from time import time
from ModestMaps.OpenStreetMap import Provider
from ModestMaps.Core import Coordinate
osm = Provider()
if __name__ == '__main__':
stdout.write('coord\tfile\tsize\ttime\n')
stdout.flush()
tile, original_file = argv[1:]
z, x, y = [int(part) for part in tile.split('/')]
coords = [(Coordinate(y, x, z), original_file)]
while len(coords):
coord, file = coords.pop(0)
size = stat(file)[ST_SIZE]
if size < 1024 or coord.zoom >= 9:
# nothing useful fits inside 1KB, and don't descend past z9
stdout.write('%d/%d/%d\t%s\t%d\t\n' % (coord.zoom, coord.column, coord.row, file, size))
else:
ul = coord.zoomBy(1)
ur = ul.right()
lr = ur.down()
ll = lr.left()
osmosis = '/usr/local/bin/osmosis --rb ___ --log-progress interval=60 --tee outputCount=4'.split()
osmosis[2] = file
for other_coord in (ul, ll, ur, lr):
ne = osm.coordinateLocation(other_coord.right())
sw = osm.coordinateLocation(other_coord.down())
zoom = other_coord.zoom
length = '%d' % max(1, ceil(log(2**zoom) / log(10)))
x, y = [('%0' + length + 'd') % val for val in (other_coord.column, other_coord.row)]
other_file = 'planet-%(zoom)d-%(x)s-%(y)s.osm.pbf' % locals()
osmosis += ('--bb top=%.6f right=%.6f bottom=%.6f left=%.6f --wb ___' % (ne.lat, ne.lon, sw.lat, sw.lon)).split()
osmosis[-1] = other_file
coords.insert(0, (other_coord, other_file))
start_time = time()
osmosis = Popen(osmosis)
osmosis.wait()
stdout.write('%d/%d/%d\t%s\t%d\t%.1f\n' % (coord.zoom, coord.column, coord.row, file, size, time() - start_time))
if file != original_file:
unlink(file)
stdout.flush()
from psycopg2 import connect
db = connect(user='osm', database='planet_osm')
cur = db.cursor()
cur.execute('''select SUM(ST_NumPoints(way)), COUNT(osm_id)
from planet_osm_line
where way && ST_Transform(ST_SetSRID(ST_MakeBox2D(ST_MakePoint(-123, 37), ST_MakePoint(-122, 38)), 4326), 900913)''')
vertex_count, way_count = cur.fetchone()
from math import log, floor, ceil
from psycopg2 import connect
from PIL.ImageDraw import ImageDraw
from PIL import Image
from ModestMaps.Core import Coordinate
from ModestMaps.Tiles import toMicrosoft
colors = [
(8,29,88),
(37,52,148),
(34,94,168),
(29,145,192),
(65,182,196),
(127,205,187),
(199,233,180),
(237,248,177),
(255,255,217)
]
min_size, max_size = 126, 16206851105
log_base = (max_size - min_size) ** (1./len(colors))
def size_color(size):
""" Return an interpolated color for a given byte size.
"""
try:
index = log(size - min_size) / log(log_base)
except ValueError:
index = 0
low_index = int(index)
high_index = low_index + 1
high_mix = index - low_index
low_mix = 1 - high_mix
r1, g1, b1 = colors[low_index]
try:
r2, g2, b2 = colors[high_index]
except IndexError:
return colors[-1]
return (int(r1 * low_mix + r2 * high_mix),
int(g1 * low_mix + g2 * high_mix),
int(b1 * low_mix + b2 * high_mix))
class Provider:
def __init__(self, layer):
"""
"""
pass
def renderTile(self, width, height, srs, coord):
"""
"""
db = connect(user='osm', database='planet_osm').cursor()
quad = toMicrosoft(coord.column, coord.row, coord.zoom)
db.execute("""SELECT z, x, y, size
FROM data_sizes
WHERE quad LIKE '%s%%'
AND z = %d"""
% (quad, min(9, coord.zoom + 4)))
img = Image.new('RGB', (width, height), colors[0])
draw = ImageDraw(img)
coord = coord.zoomBy(8)
for (zoom, column, row, size) in db.fetchall():
sub_coord = Coordinate(row, column, zoom)
ul = sub_coord.zoomTo(coord.zoom)
lr = sub_coord.right().down().zoomTo(coord.zoom)
x1 = int(ul.column - coord.column)
x2 = int(lr.column - coord.column - 1)
y1 = int(ul.row - coord.row)
y2 = int(lr.row - coord.row - 1)
draw.rectangle((x1, y1, x2, y2), size_color(size))
return img
{
"cache": {"name": "test"},
"layers":
{
"size":
{
"provider": {"class": "sizerender:Provider"},
"preview": {"zoom": 5},
"bounds": {"high": 9}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment