Skip to content

Instantly share code, notes, and snippets.

@mrluanma

mrluanma/README.md

Last active August 1, 2017 01:22
Show Gist options
  • Star 2 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save mrluanma/69ab1d7127291acd0fec to your computer and use it in GitHub Desktop.
Save mrluanma/69ab1d7127291acd0fec to your computer and use it in GitHub Desktop.
Download ZIP
使用 requests + concurrent.futures 并发的访问 HTTP 服务示例
Raw
README.md 
$ python t.py
INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (1): example.net
INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (2): example.net
INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (3): example.net
INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (4): example.net
INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (5): example.net
INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (6): example.net
INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (7): example.net
INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (8): example.net
INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (9): example.net
INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (10): example.net
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#3 200: 1128 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#8 200: 1128 HIT
#6 200: 1128 HIT
#4 200: 1128 HIT
#5 200: 1128 HIT
#2 200: 1128 HIT
#9 200: 1128 HIT
#7 200: 1128 HIT
#1 200: 1128 HIT
#0 200: 1128 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#19 200: 1129 HIT
#10 200: 1129 HIT
#15 200: 1129 HIT
#14 200: 1129 HIT
#12 200: 1129 HIT
#11 200: 1129 HIT
#18 200: 1129 HIT
#13 200: 1129 HIT
#16 200: 1129 HIT
#17 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#24 200: 1129 HIT
#26 200: 1129 HIT
#25 200: 1129 HIT
#29 200: 1129 HIT
#22 200: 1129 HIT
#20 200: 1129 HIT
#23 200: 1129 HIT
#21 200: 1129 HIT
#28 200: 1129 HIT
#27 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#37 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#32 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#39 200: 1129 HIT
#33 200: 1129 HIT
#38 200: 1129 HIT
#35 200: 1129 HIT
#36 200: 1129 HIT
#31 200: 1129 HIT
#30 200: 1129 HIT
#34 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#42 200: 1129 HIT
#40 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#41 200: 1129 HIT
#47 200: 1129 HIT
#48 200: 1129 HIT
#46 200: 1129 HIT
#45 200: 1129 HIT
#44 200: 1129 HIT
#43 200: 1129 HIT
#49 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#55 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#53 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#52 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#57 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#50 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#51 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#54 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#59 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#58 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#56 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#60 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#61 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#63 200: 1129 HIT
#62 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#66 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#64 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#65 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#67 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#68 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#69 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#70 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#71 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#73 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#72 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#76 200: 1129 HIT
#74 200: 1129 HIT
#75 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#77 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#78 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#79 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#80 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#81 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#82 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#83 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#85 200: 1129 HIT
#86 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#88 200: 1129 HIT
#84 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#87 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#89 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#90 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#91 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#92 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#93 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#97 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#94 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#95 200: 1129 HIT
#96 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#98 200: 1129 HIT
DEBUG:requests.packages.urllib3.connectionpool:"POST /tag HTTP/1.1" 200 None
#99 200: 1129 HIT
clock: 3.540749s, elapsed: 32.766114s

用 3.540749s 完成顺序执行的话需要耗时 32.766114s 的 HTTP 请求！

Raw
requirements.txt
futures==3.0.2
requests==2.7.0
requests-futures==0.9.5
Raw
t.py
from __future__ import absolute_import, division, print_function, unicode_literals
import time
import logging
import concurrent.futures
from requests_futures.sessions import FuturesSession
logging.basicConfig(level=logging.DEBUG)
session = FuturesSession(max_workers=10)
def main():
now = time.time()
elapsed = 0
contents = {seq: 'ASDF ' * 100 + str(seq) for seq in range(100)}
futures = {
session.post('http://example.net/tag', json=content): seq
for seq, content in contents.iteritems()
}
for future in concurrent.futures.as_completed(futures):
seq = futures[future]
resp = future.result()
elapsed += resp.elapsed.total_seconds()
print('#%d %d: %d %s' % (seq, resp.status_code, len(resp.content),
resp.headers['X-Cache-Status']))
print('clock: %fs, elapsed: %fs' % (time.time() - now, elapsed))
if __name__ == '__main__':
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment