requests-futures with a ProcessPoolExecutor

processpoolexecutorrequests.log

# Without callback
2016-03-04 12:43:26,342 - 6782 - DEBUG - root : Executing session.request in child process
2016-03-04 12:43:26,346 - 6782 - DEBUG - root : Executing session.request in child process
2016-03-04 12:43:26,347 - 6782 - DEBUG - root : Executing session.request in child process
2016-03-04 12:43:26,347 - 6782 - DEBUG - root : Executing session.request in child process
2016-03-04 12:43:26,375 - 6823 - INFO - requests.packages.urllib3.connectionpool : Starting new HTTP connection (1): test.dev
2016-03-04 12:43:26,375 - 6824 - INFO - requests.packages.urllib3.connectionpool : Starting new HTTP connection (1): test.dev
2016-03-04 12:43:26,427 - 6824 - DEBUG - requests.packages.urllib3.connectionpool : "GET / HTTP/1.1" 404 None
2016-03-04 12:43:26,428 - 6823 - DEBUG - requests.packages.urllib3.connectionpool : "GET / HTTP/1.1" 404 None
2016-03-04 12:43:26,432 - 6824 - INFO - requests.packages.urllib3.connectionpool : Starting new HTTP connection (1): test.dev
2016-03-04 12:43:26,432 - 6823 - INFO - requests.packages.urllib3.connectionpool : Starting new HTTP connection (1): test.dev
2016-03-04 12:43:26,449 - 6824 - DEBUG - requests.packages.urllib3.connectionpool : "GET / HTTP/1.1" 404 None
2016-03-04 12:43:26,449 - 6823 - DEBUG - requests.packages.urllib3.connectionpool : "GET / HTTP/1.1" 404 None
[<Response [404]>, <Response [404]>, <Response [404]>, <Response [404]>]

# With callback
2016-03-04 12:43:26,451 - 6782 - DEBUG - root : Executing session.request in child process
2016-03-04 12:43:26,451 - 6782 - DEBUG - root : Executing session.request in child process
2016-03-04 12:43:26,451 - 6782 - DEBUG - root : Executing session.request in child process
2016-03-04 12:43:26,452 - 6782 - DEBUG - root : Executing session.request in child process
2016-03-04 12:43:26,456 - 6823 - INFO - requests.packages.urllib3.connectionpool : Starting new HTTP connection (1): test.dev
2016-03-04 12:43:26,456 - 6824 - INFO - requests.packages.urllib3.connectionpool : Starting new HTTP connection (1): test.dev
2016-03-04 12:43:26,471 - 6823 - DEBUG - requests.packages.urllib3.connectionpool : "GET / HTTP/1.1" 404 None
2016-03-04 12:43:26,471 - 6824 - DEBUG - requests.packages.urllib3.connectionpool : "GET / HTTP/1.1" 404 None
2016-03-04 12:43:26,476 - 6824 - INFO - requests.packages.urllib3.connectionpool : Starting new HTTP connection (1): test.dev
2016-03-04 12:43:26,476 - 6823 - INFO - requests.packages.urllib3.connectionpool : Starting new HTTP connection (1): test.dev
2016-03-04 12:43:26,490 - 6823 - DEBUG - requests.packages.urllib3.connectionpool : "GET / HTTP/1.1" 404 None
2016-03-04 12:43:26,490 - 6824 - DEBUG - requests.packages.urllib3.connectionpool : "GET / HTTP/1.1" 404 None
[{'error': {'message': 'Not Found', 'code': 404}}, {'error': {'message': 'Not Found', 'code': 404}}, {'error': {'message': 'Not Found', 'code': 404}}, {'error': {'message': 'Not Found', 'code': 404}}]

processpoolexecutorrequests.py

import concurrent.futures
import functools
import logging
import requests

LOG_FORMAT = '%(asctime)s - %(process)s - %(levelname)s - %(name)s : %(message)s'
logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)

session = requests.Session()
executor = concurrent.futures.ProcessPoolExecutor(max_workers=2)


def wraps(f, bg_cb, *args, **kwargs):
    resp = f(*args, **kwargs)
    return bg_cb(resp)


def request(bg_cb=None, *args, **kwargs):
    if bg_cb:
        func = functools.partial(wraps, session.request, bg_cb)
    else:
        func = session.request
    logging.debug('Executing session.request in child process')
    return executor.submit(func, *args, **kwargs)


# I must be top level!
def test_callback(res):
    return res.json()


if __name__ == '__main__':
    futures = [request(method='get', url='http://test.dev') for _ in range(4)]
    responses = [f.result() for f in concurrent.futures.as_completed(futures)]
    print(responses)

    # same but with a callback wrapper
    futures = [request(bg_cb=test_callback, method='get',
                       url='http://test.dev') for _ in range(4)]
    responses = [f.result() for f in concurrent.futures.as_completed(futures)]
    print(responses)