Skip to content

Instantly share code, notes, and snippets.

@davidjmerritt
Last active November 6, 2016 22:22
Show Gist options
  • Save davidjmerritt/3f536707efbd805ad2ee7c3bcbb4f725 to your computer and use it in GitHub Desktop.
Save davidjmerritt/3f536707efbd805ad2ee7c3bcbb4f725 to your computer and use it in GitHub Desktop.
Python multiprocessing test
#!/usr/bin/python
from multiprocessing import Pool
import time
import urllib
start = time.time()
urls = [
"https://docs.python.org/2/library/urllib.html",
"http://www.nytimes.com",
"https://www.google.com/webhp?sourceid=chrome-instant&ion=1&espv=2&ie=UTF-8#q=test",
"https://www.wrglive.com/about",
"http://wrgtexas.com/",
"https://en.wikipedia.org/wiki/Tyranids",
"http://www.morewords.com/contains/uuhjk/",
"https://www.abcmouse.com/landing/dis:goog?gclid=CLHquoeQldACFQkMaQodv5IKGw",
"http://wikiwordbook.info/uuhjk"
]
def loadData(url):
return urllib.urlopen(urls[1]).read()
def thread():
start = time.time()
pool = Pool()
return {
"data": pool.map(loadData, urls),
"duration_in_seconds": time.time() - start
}
def loop():
start = time.time()
data = []
for url in urls:
data.append(loadData(url))
return {
"data": data,
"duration_in_seconds": time.time() - start
}
if __name__ == "__main__":
t = thread()['duration_in_seconds']
l = loop()['duration_in_seconds']
print time.time() - start, t, l, t/l
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment