Skip to content

Instantly share code, notes, and snippets.

Last active Aug 18, 2017
What would you like to do?
Python multiprocessing

Python Multiprocessing

This gist exists to keep track of modifications to code from a blog post.

The first revision of this gist is the code as it exists on the blog post at the time of this writing. Follow-up revisions are proposed improvments.

from __future__ import print_function
from concurrent.futures import ThreadPoolExecutor
import logging
import time
LOG = logging.getLogger(__name__) # we'll use this later for error-reporting
class FakeDiff(object):
Fake diff implementation for demonstration only
def __init__(self, a, b):
self.diffs = []
def fetch(url):
LOG.debug('Fetching %r using fake GET', url)
from time import sleep
from random import randint
sleep(randint(0, 3)) # simulate slow network
return {
'results': [{
'types': {'postal_code'},
'address_components': {'United States'}
import requests
return requests.get(url).json()
def diff(a, b):
LOG.debug('Diffing %r with %r using fake diff', a, b)
return FakeDiff(a, b)
from datadiff import diff
return diff(a, b)
def getzip(code):
code = str(code)
url = "{}".format(code)
res = fetch(url)['results']
if len(res) < 1: # Re-try
return getzip(code)
iszip = 'postal_code' in res[0]['types'] and "United States" in str(res[0]['address_components'])
except Exception:
# Let's not silence the traceback. This contains valuable information
# for debugging. Using the logging framework makes this ease. Either
# like this:
LOG.exception('In error') # high severity message including the traceback
# ... or like this:
LOG.debug('In error', exc_info=True) # Low severity, not displayed by default
iszip = False
return (code, iszip)
ziprange = range(94400, 94420)
print("Range is: %s" % len(ziprange)) # Using %-formatting is a bit cleaner
print("Using one thread")
start = time.time()
syncres = [getzip(c) for c in ziprange]
print("took %s" % (time.time() - start))
print("Using multiple threads")
start = time.time()
with ThreadPoolExecutor(max_workers=10) as executor:
results =, ziprange)
asyncres = sorted(results)
print("took %s" % (time.time() - start))
# Make sure results are equal
d = diff(syncres, asyncres)
if len(d.diffs) > 0:
print("diff is")
for r in asyncres:
print("Zip code {} is {} US code".format(r[0], "valid" if r[1] else "invalid"))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment