Created
September 1, 2013 21:20
-
-
Save minrk/6407393 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
"""Simple server for testing code against a slow-responding server. | |
requests to '/INTEGER' are replied to with 'INTEGER', | |
after an artificial delay. | |
""" | |
import time | |
from tornado import web, ioloop, options, httpserver | |
options.define("port", default=8888, help="run on the given port", type=int) | |
options.define("delay", default=0.2, help="delay builtin to each request", type=float) | |
class SlowHandler(web.RequestHandler): | |
@web.asynchronous | |
def get(self, page): | |
loop = ioloop.IOLoop.instance() | |
loop.add_timeout(loop.time() + self.settings['delay'], | |
lambda : self.finish(page), | |
) | |
if __name__ == '__main__': | |
options.parse_command_line() | |
app = web.Application([ | |
(r'/(\d+)', SlowHandler) | |
], delay=options.options.delay, | |
) | |
http_server = httpserver.HTTPServer(app) | |
http_server.listen(options.options.port) | |
ioloop.IOLoop.instance().start() | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"metadata": { | |
"name": "" | |
}, | |
"nbformat": 3, | |
"nbformat_minor": 0, | |
"worksheets": [ | |
{ | |
"cells": [ | |
{ | |
"cell_type": "heading", | |
"level": 1, | |
"metadata": {}, | |
"source": [ | |
"Using ThreadPool for concurrent downloads" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"This is a simple illustration of using `ThreadPool` to parallelize downloads.\n", | |
"Assumes that bandwidth is not the limiting factor, in which case concurrency doesn't help." | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"collapsed": false, | |
"input": [ | |
"import requests\n", | |
"from multiprocessing.pool import ThreadPool" | |
], | |
"language": "python", | |
"metadata": {}, | |
"outputs": [], | |
"prompt_number": 1 | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"Test a simple request to my slow server\n", | |
"It just replies to any request for `/NUMBER` with the number requested,\n", | |
"but the server is artificially slow in its handling of requests." | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"collapsed": false, | |
"input": [ | |
"%time r = requests.get(\"http://localhost:8888/10\")\n", | |
"r.content" | |
], | |
"language": "python", | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"output_type": "stream", | |
"stream": "stdout", | |
"text": [ | |
"CPU times: user 18.1 ms, sys: 4.54 ms, total: 22.6 ms\n", | |
"Wall time: 224 ms\n" | |
] | |
}, | |
{ | |
"metadata": {}, | |
"output_type": "pyout", | |
"prompt_number": 2, | |
"text": [ | |
"'10'" | |
] | |
} | |
], | |
"prompt_number": 2 | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"Our test function downloads the URL for a given ID,\n", | |
"and parses the result (casts str of int to int)." | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"collapsed": false, | |
"input": [ | |
"def get_data(ID):\n", | |
" \"\"\"function for getting data from our slow server\"\"\"\n", | |
" r = requests.get(\"http://localhost:8888/%i\" % ID)\n", | |
" return int(r.content)" | |
], | |
"language": "python", | |
"metadata": {}, | |
"outputs": [], | |
"prompt_number": 3 | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"Now test using a threadpool to get the data, using a varying number of concurrent threads" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"collapsed": false, | |
"input": [ | |
"IDs = range(128)\n", | |
"for nthreads in [1, 2, 4, 8, 16, 32]:\n", | |
" pool = ThreadPool(nthreads)\n", | |
" tic = time.time()\n", | |
" result = pool.map(get_data, IDs)\n", | |
" toc = time.time()\n", | |
" print \"%i threads: %3.1f seconds\" % (nthreads, toc-tic)" | |
], | |
"language": "python", | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"output_type": "stream", | |
"stream": "stdout", | |
"text": [ | |
"1 threads: 26.2 seconds\n", | |
"2 threads: 13.3 seconds" | |
] | |
}, | |
{ | |
"output_type": "stream", | |
"stream": "stdout", | |
"text": [ | |
"\n", | |
"4 threads: 6.7 seconds" | |
] | |
}, | |
{ | |
"output_type": "stream", | |
"stream": "stdout", | |
"text": [ | |
"\n", | |
"8 threads: 3.4 seconds" | |
] | |
}, | |
{ | |
"output_type": "stream", | |
"stream": "stdout", | |
"text": [ | |
"\n", | |
"16 threads: 1.8 seconds" | |
] | |
}, | |
{ | |
"output_type": "stream", | |
"stream": "stdout", | |
"text": [ | |
"\n", | |
"32 threads: 1.1 seconds" | |
] | |
}, | |
{ | |
"output_type": "stream", | |
"stream": "stdout", | |
"text": [ | |
"\n" | |
] | |
} | |
], | |
"prompt_number": 4 | |
} | |
], | |
"metadata": {} | |
} | |
] | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment