Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
python-yield-http.py
#coding:utf-8
# http://stackoverflow.com/questions/2719017/how-to-set-timeout-on-pythons-socket-recv-method
# http://stackoverflow.com/questions/16745409/what-does-pythons-socket-recv-return-for-non-blocking-sockets-if-no-data-is-r
import select
import socket
def receive(s):
try:
s.settimeout(0)
data = s.recv(1024)
except socket.error, e:
# [Errno 11] Resource temporarily unavailable Linux
# [Errno 35] Resource temporarily unavailable Mac
if e.errno in (11, 35):
return
else:
print "socket error"
return "error"
else:
s.close()
return data
def download(host, port, file, worker):
print "{} start".format(worker)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host, port))
s.send("GET %s HTTP/1.0\r\n\r\n" % (file,))
while 1:
data = receive(s)
yield data, worker
threads = []
def get(host, port, file, worker):
co = download(host, port, file, worker)
threads.append(co)
def dispatch():
while 1:
i = 0
while i < len(threads):
try:
data, worker = threads[i].next()
if data:
print data
print "{} done".format(worker)
threads.pop(i)
else:
i += 1
except StopIteration:
threads.pop(i)
if len(threads) == 0:
return
if __name__ == "__main__":
import time
st = time.time()
for x in xrange(0, 10):
get("httpbin.org", 80 , "/delay/3", "worker {}".format( x ) )
dispatch()
print "spend time:{}".format(time.time() - st)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment