import requests | |
from requests.adapters import HTTPAdapter | |
from threading import Thread | |
import logging | |
logging.basicConfig(filename='out.txt', level=logging.DEBUG, filemode='w') | |
requests_log = logging.getLogger("requests.packages.urllib3") | |
requests_log.setLevel(logging.DEBUG) | |
requests_log.propagate = True | |
def test_pool_connections(): | |
# 测试访问 zhihu.com 和 zhihu.com/question/xxx 不会新建 connection | |
s = requests.Session() | |
s.mount('https://www.zhihu.com', HTTPAdapter(pool_connections=1)) | |
s.get('https://www.zhihu.com/question/36612174') | |
s.get('https://www.zhihu.com') | |
"""output | |
INFO:requests.packages.urllib3.connectionpool:Starting new HTTPS connection (1): www.zhihu.com | |
DEBUG:requests.packages.urllib3.connectionpool:"GET /question/36612174 HTTP/1.1" 200 21854 | |
DEBUG:requests.packages.urllib3.connectionpool:"GET / HTTP/1.1" 200 2623 | |
""" | |
logging.getLogger().info('----------------------------------------') | |
# 测试访问不同 host 会新建两个 connection | |
s = requests.Session() | |
s.mount('https://', HTTPAdapter(pool_connections=1)) | |
s.get('https://www.baidu.com') | |
s.get('https://www.zhihu.com') | |
s.get('https://www.baidu.com') | |
"""output | |
INFO:requests.packages.urllib3.connectionpool:Starting new HTTPS connection (1): www.baidu.com | |
DEBUG:requests.packages.urllib3.connectionpool:"GET / HTTP/1.1" 200 None | |
INFO:requests.packages.urllib3.connectionpool:Starting new HTTPS connection (1): www.zhihu.com | |
DEBUG:requests.packages.urllib3.connectionpool:"GET / HTTP/1.1" 200 2621 | |
INFO:requests.packages.urllib3.connectionpool:Starting new HTTPS connection (1): www.baidu.com | |
DEBUG:requests.packages.urllib3.connectionpool:"GET / HTTP/1.1" 200 None | |
""" | |
logging.getLogger().info('----------------------------------------') | |
# 测试 pool_connection 对连接的缓存 | |
s = requests.Session() | |
s.mount('https://', HTTPAdapter(pool_connections=2)) | |
s.get('https://www.baidu.com') | |
s.get('https://www.zhihu.com') | |
s.get('https://www.baidu.com') | |
"""output | |
INFO:requests.packages.urllib3.connectionpool:Starting new HTTPS connection (1): www.baidu.com | |
DEBUG:requests.packages.urllib3.connectionpool:"GET / HTTP/1.1" 200 None | |
INFO:requests.packages.urllib3.connectionpool:Starting new HTTPS connection (1): www.zhihu.com | |
DEBUG:requests.packages.urllib3.connectionpool:"GET / HTTP/1.1" 200 2623 | |
DEBUG:requests.packages.urllib3.connectionpool:"GET / HTTP/1.1" 200 None | |
""" | |
logging.getLogger().info('----------------------------------------') | |
def test_pool_max_size(): | |
# 测试 pool_maxsize 对多线程访问的影响 | |
def thread_get(url): | |
s.get(url) | |
s = requests.Session() | |
s.mount('https://', HTTPAdapter(pool_connections=1, pool_maxsize=1)) | |
t1 = Thread(target=thread_get, args=('https://www.zhihu.com',)) | |
t2 = Thread(target=thread_get, args=('https://www.zhihu.com/question/36612174',)) | |
t1.start() | |
t2.start() | |
t1.join();t2.join() | |
t3 = Thread(target=thread_get, args=('https://www.zhihu.com/question/39420364',)) | |
t4 = Thread(target=thread_get, args=('https://www.zhihu.com/question/21362402',)) | |
t3.start();t4.start() | |
t3.join();t4.join() | |
""" | |
INFO:requests.packages.urllib3.connectionpool:Starting new HTTPS connection (1): www.zhihu.com | |
INFO:requests.packages.urllib3.connectionpool:Starting new HTTPS connection (2): www.zhihu.com | |
DEBUG:requests.packages.urllib3.connectionpool:"GET /question/36612174 HTTP/1.1" 200 21906 | |
DEBUG:requests.packages.urllib3.connectionpool:"GET / HTTP/1.1" 200 2606 | |
WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: www.zhihu.com | |
INFO:requests.packages.urllib3.connectionpool:Starting new HTTPS connection (3): www.zhihu.com | |
DEBUG:requests.packages.urllib3.connectionpool:"GET /question/39420364 HTTP/1.1" 200 28739 | |
DEBUG:requests.packages.urllib3.connectionpool:"GET /question/21362402 HTTP/1.1" 200 57556 | |
WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: www.zhihu.com | |
""" | |
logging.getLogger().info('----------------------------------------') | |
s = requests.Session() | |
s.mount('https://', HTTPAdapter(pool_connections=1, pool_maxsize=2)) | |
t1 = Thread(target=thread_get, args=('https://www.zhihu.com',)) | |
t2 = Thread(target=thread_get, args=('https://www.zhihu.com/question/36612174',)) | |
t1.start() | |
t2.start() | |
t1.join();t2.join() | |
t3 = Thread(target=thread_get, args=('https://www.zhihu.com/question/39420364',)) | |
t4 = Thread(target=thread_get, args=('https://www.zhihu.com/question/21362402',)) | |
t3.start();t4.start() | |
""" | |
INFO:requests.packages.urllib3.connectionpool:Starting new HTTPS connection (1): www.zhihu.com | |
INFO:requests.packages.urllib3.connectionpool:Starting new HTTPS connection (2): www.zhihu.com | |
DEBUG:requests.packages.urllib3.connectionpool:"GET /question/36612174 HTTP/1.1" 200 21906 | |
DEBUG:requests.packages.urllib3.connectionpool:"GET / HTTP/1.1" 200 2606 | |
DEBUG:requests.packages.urllib3.connectionpool:"GET /question/21362402 HTTP/1.1" 200 57556 | |
DEBUG:requests.packages.urllib3.connectionpool:"GET /question/39420364 HTTP/1.1" 200 28739 | |
""" | |
def test_mount_prefix(): | |
# 测试 HTTPAdapter 是相互独立的 | |
def thread_get(url): | |
s.get(url) | |
s = requests.Session() | |
s.mount('https://', HTTPAdapter(pool_connections=1, pool_maxsize=2)) | |
s.mount('https://baidu.com', HTTPAdapter(pool_connections=1, pool_maxsize=1)) | |
t1 = Thread(target=thread_get, args=('https://www.zhihu.com',)) | |
t2 =Thread(target=thread_get, args=('https://www.zhihu.com/question/36612174',)) | |
t1.start() | |
t2.start() | |
t1.join();t2.join() | |
t3 = Thread(target=thread_get, args=('https://www.zhihu.com/question/39420364',)) | |
t4 = Thread(target=thread_get, args=('https://www.zhihu.com/question/21362402',)) | |
t3.start();t4.start() | |
t3.join();t4.join() | |
"""output | |
INFO:requests.packages.urllib3.connectionpool:Starting new HTTPS connection (1): www.zhihu.com | |
INFO:requests.packages.urllib3.connectionpool:Starting new HTTPS connection (2): www.zhihu.com | |
DEBUG:requests.packages.urllib3.connectionpool:"GET /question/36612174 HTTP/1.1" 200 21906 | |
DEBUG:requests.packages.urllib3.connectionpool:"GET / HTTP/1.1" 200 2623 | |
DEBUG:requests.packages.urllib3.connectionpool:"GET /question/39420364 HTTP/1.1" 200 28739 | |
DEBUG:requests.packages.urllib3.connectionpool:"GET /question/21362402 HTTP/1.1" 200 57669 | |
""" | |
if __name__ == '__main__': | |
test_pool_connections() | |
test_pool_max_size() | |
test_mount_prefix() |
This comment has been minimized.
This comment has been minimized.
@quietin True. |
This comment has been minimized.
This comment has been minimized.
great, resolve my question, thanks |
This comment has been minimized.
This comment has been minimized.
我觉得你用知乎举例,应该是个中国人,哈哈 |
This comment has been minimized.
This comment has been minimized.
@yuan6785 啥。。你一点我的 profile 不就知道了 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
This comment has been minimized.
logging.getLogger().info('----------------------------------------') can be replaced with logging.getLogger().info('-' * 40)