parallel_fiona_generators.py

import fiona
from shapely.geometry import shape, mapping
import multiprocessing
import time
import functools
import copy

def factory(f, chunksize=50):
    def wrapped(features, **kwargs):
        pool = kwargs.pop("pool", None)
        if pool:
            _f = functools.partial(
                f,
                **kwargs
            )
            for feature in pool.imap(_f, features, chunksize=chunksize):
                yield feature
        else:
            for feature in features:
                yield f(feature, **kwargs)
    return wrapped

def _buffer(feature, distance):
    feature = feature.copy()
    geometry = shape(feature["geometry"])
    geometry = geometry.buffer(distance)
    feature["geometry"] = mapping(geometry)
    return feature

buffer = factory(_buffer)

if __name__ == "__main__":
    pool = multiprocessing.Pool(4)

    with fiona.open("example.shp", "r") as src:
    
        meta = copy.deepcopy(src.meta)
        meta["schema"]["geometry"] = "Polygon"
        with fiona.open("result.shp", "w", **meta) as dst:
        
            t0 = time.time()
            features = buffer(src, distance=1, pool=pool)
            dst.writerecords(features)
            print("{:.2f} seconds".format(time.time()-t0))