Created
June 14, 2012 20:37
-
-
Save thatguystone/2932793 to your computer and use it in GitHub Desktop.
connection leaking in urllib3
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Required pacakges: gunicorn gevent urllib3 | |
1) Start gunicorn: gunicorn -k gevent -w 9 test:app --access-logfile=- | |
2) Run the test: python test.py | |
By the end of the test, there should be 5 connections in the pool (5 False 624 1007), but in reality, it's showing 0 (0 True 5 9) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from gevent import monkey | |
monkey.patch_all() | |
import gevent | |
import os | |
import sys | |
import time | |
import urllib3 | |
URL = 'http://localhost:8000/' | |
TIMEOUT = 1 | |
def app(environ, start_response): | |
data = "success" | |
start_response("404 Not Found", [ | |
("Content-Type", "text/plain"), | |
("Content-Length", str(len(data))) | |
]) | |
time.sleep(TIMEOUT+1) | |
return iter([data]) | |
def main(): | |
pool = urllib3.PoolManager(num_pools=25, maxsize=5, block=True, timeout=TIMEOUT) | |
def stats(): | |
p = pool.connection_from_host('localhost', 8000) | |
sys.stderr.write('%d %r %d %d\n' % (p.pool.qsize(), p.pool.empty(), p.num_connections, p.num_requests)) | |
def hit(): | |
class MyCrazyTimeout(Exception): | |
pass | |
with gevent.Timeout(TIMEOUT, MyCrazyTimeout) as timeout_ex: | |
try: | |
print pool.urlopen('GET', URL).data | |
except: | |
# print sys.exc_info() | |
pass | |
wait = [gevent.Greenlet.spawn(hit) for i in xrange(1000)] | |
time.sleep(4) | |
stats() | |
wait += [gevent.Greenlet.spawn(hit) for i in xrange(1000)] | |
stats() | |
gevent.joinall(wait) | |
time.sleep(TIMEOUT+1) | |
stats() | |
if __name__ == "__main__": | |
main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment