python-yield-http.py

python-yield-socket.py

#coding:utf-8

# http://stackoverflow.com/questions/2719017/how-to-set-timeout-on-pythons-socket-recv-method
# http://stackoverflow.com/questions/16745409/what-does-pythons-socket-recv-return-for-non-blocking-sockets-if-no-data-is-r

import select
import socket


def receive(s):
    try:
        s.settimeout(0)
        data = s.recv(1024)
    except socket.error, e:
        # [Errno 11] Resource temporarily unavailable Linux
        # [Errno 35] Resource temporarily unavailable Mac
        if e.errno in (11, 35):
            return
        else:
            print "socket error"
            return "error"
    else:
        s.close()
        return data

def download(host, port, file, worker):

    print "{} start".format(worker)
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.connect((host, port))
    s.send("GET %s HTTP/1.0\r\n\r\n" % (file,))
    while 1:
        data = receive(s)
        yield data, worker

threads = []
def get(host,  port,  file, worker):
    co  = download(host, port, file, worker)
    threads.append(co)

def dispatch():
    while 1:
        i = 0
        while i < len(threads):
            try:
                data, worker = threads[i].next()
                if  data:
                    print data
                    print "{}  done".format(worker)
                    threads.pop(i)
                else:
                    i += 1
            except StopIteration:
                threads.pop(i)

        if len(threads) == 0:
            return

if __name__ == "__main__":
    import time
    st = time.time()
    for x in xrange(0, 10):
        get("httpbin.org",  80 ,  "/delay/3", "worker {}".format( x ) )
    dispatch()
    print "spend time:{}".format(time.time() - st)