大数据下载tornado - Big download demo with Tornado 3.0.1 and pycurl.

client.py

from tornado.httpclient import HTTPRequest, AsyncHTTPClient
import tornado.ioloop
import tornado.web
from tornado import gen

GB = 1024 * 1024 * 1024
total_downloaded = 0


def streaming_callback(chunk):
    global total_downloaded
    total_downloaded += len(chunk)
    print 'downloaded', len(chunk), 'total is now', total_downloaded / float(GB), 'GB'


@gen.coroutine
def fetch():
    request = HTTPRequest(
        'http://localhost:8888', streaming_callback=streaming_callback)

    http_client = AsyncHTTPClient()
    response = yield http_client.fetch(request)
    print response
    tornado.ioloop.IOLoop.instance().stop()


if __name__ == "__main__":
    AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
    fetch()
    tornado.ioloop.IOLoop.instance().start()

server.py

import tornado.ioloop
import tornado.web
import tornado.options
from tornado import gen

GB = 1024 * 1024 * 1024
body_size = 4 * GB


class StreamingRequestHandler(tornado.web.RequestHandler):
    @tornado.web.asynchronous
    @gen.coroutine
    def get(self):
        total_sent = 0
        chunk = 'a' * 1024 * 1024

        self.set_header('Content-Length', body_size)
        while total_sent < body_size:
            self.write(chunk)
            yield gen.Task(self.flush)
            total_sent += len(chunk)
            print 'sent', total_sent / float(GB), 'GB'

        self.finish()


if __name__ == "__main__":
    tornado.options.parse_command_line()
    application = tornado.web.Application([
        (r"/", StreamingRequestHandler),
    ])
    application.listen(8888)
    tornado.ioloop.IOLoop.instance().start()