Harvey contour processing

gdb2json.py

from __future__ import print_function

from subprocess import call
from multiprocessing import Process, Queue, cpu_count


def process():
    chunk_size = 1300
    max_size = 4233236 + chunk_size

    start = 1
    end = chunk_size
    chunk_num = 1

    queue = Queue()
    processes = [setup_processes(queue) for i in range(cpu_count())]

    while end < max_size:
        # Shell out ogr call to a process
        queue.put((chunk_num, start, end))

        # Increment to another chunk
        start += chunk_size
        end += chunk_size
        chunk_num += 1

    # Put enough kill messages in the queue for each process to receive
    for p in processes:
        queue.put([None] * 3)


def setup_processes(queue):
    handler = Process(target=convert_chunk, args=((queue),))

    # Allow the main process to block until the spawned processes terminate
    handler.daemon = False
    handler.start()
    return handler


def convert_chunk(queue):
    while True:
        cnt, lower, upper = queue.get()
        if cnt is not None:
            print(cnt)
            where = 'fid >= {} and fid < {}'.format(lower, upper)

            call(['ogr2ogr',
                  '-f', 'GeoJSON',
                  '-t_srs', 'epsg:4326',
                  '-select', 'CONTOUR',
                  '-where', '{}'.format(where),
                  'output/contours_{}.json'.format(cnt),
                  'elev.gdb',
                  'HCFCD_1ftContours',
                  ])

        else:
            # Kill message received, exit the read loop
            break


if __name__ == '__main__':
    process()