sgillies

"""Concurrent read-process-write example"""

import concurrent.futures
from itertools import islice
from time import sleep

import rasterio


CHUNK = 100

sgillies

sgillies

{"geometry": {"coordinates": [[[-111.73527526855469, 41.995094299316406], [-111.65931701660156, 41.99627685546875], [-111.6587142944336, 41.9921875], [-111.65888977050781, 41.95676803588867], [-111.67082977294922, 41.91230010986328], [-111.67332458496094, 41.905494689941406], [-111.67088317871094, 41.90049362182617], [-111.66474914550781, 41.893211364746094], [-111.6506576538086, 41.875465393066406], [-111.64759826660156, 41.87091827392578], [-111.64640808105469, 41.86273956298828], [-111.64334869384766, 41.858192443847656], [-111.63720703125, 41.85499572753906], [-111.633544921875, 41.847267150878906], [-111.63053894042969, 41.83409118652344], [-111.6330337524414, 41.82728576660156], [-111.63983154296875, 41.8227653503418], [-111.6484603881836, 41.82188034057617], [-111.66077423095703, 41.82327651977539], [-111.6712417602539, 41.82330322265625], [-111.67618560791016, 41.82013702392578], [-111.68803405761719, 41.78792953491211], [-111.69361114501953, 41.77931594848633], [-111.70162200927734, 41.77797317504883

sgillies

Latency Comparison Numbers
--------------------------
L1 cache reference                           0.5 ns
Branch mispredict                            5   ns
L2 cache reference                           7   ns                      14x L1 cache
Mutex lock/unlock                           25   ns
Main memory reference                      100   ns                      20x L2 cache, 200x L1 cache
Compress 1K bytes with Zippy             3,000   ns        3 us
Send 1K bytes over 1 Gbps network       10,000   ns       10 us
Read 4K randomly from SSD*             150,000   ns      150 us          ~1GB/sec SSD

sgillies

sgillies

sgillies

# Comparing serialization of complex GeoJSON geometries using:
# 
# - standard lib json, marshal, pickle, cPickle
# - umsgpack
# - shapely.wkb
# - geobuf (protobuf)
#
# The test case is a nearly circular polygon with 128 vertices.
#
# Python 2.7 because geobuf isn't possible on Python 3 (because

sgillies

sgillies

sgillies

{"features": [{"geometry": {"coordinates": [[[-111.73527526855469, 41.995094299316406], [-111.65931701660156, 41.99627685546875], [-111.6587142944336, 41.9921875], [-111.65888977050781, 41.95676803588867], [-111.67082977294922, 41.91230010986328], [-111.67332458496094, 41.905494689941406], [-111.67088317871094, 41.90049362182617], [-111.66474914550781, 41.893211364746094], [-111.6506576538086, 41.875465393066406], [-111.64759826660156, 41.87091827392578], [-111.64640808105469, 41.86273956298828], [-111.64334869384766, 41.858192443847656], [-111.63720703125, 41.85499572753906], [-111.633544921875, 41.847267150878906], [-111.63053894042969, 41.83409118652344], [-111.6330337524414, 41.82728576660156], [-111.63983154296875, 41.8227653503418], [-111.6484603881836, 41.82188034057617], [-111.66077423095703, 41.82327651977539], [-111.6712417602539, 41.82330322265625], [-111.67618560791016, 41.82013702392578], [-111.68803405761719, 41.78792953491211], [-111.69361114501953, 41.77931594848633], [-111.70162200927734, 41.