Skip to content

Instantly share code, notes, and snippets.

@oubiwann
Created October 13, 2012 21:01
Show Gist options
  • Save oubiwann/3886139 to your computer and use it in GitHub Desktop.
Save oubiwann/3886139 to your computer and use it in GitHub Desktop.
Async Batching with Twisted: A Walkthrough
from twisted.internet import defer, reactor
from twisted.web.client import getPage
def listCallback(results):
print results
def finish(ign):
reactor.stop()
def test():
d1 = getPage('http://www.google.com')
d2 = getPage('http://yahoo.com')
dl = defer.DeferredList([d1, d2])
dl.addCallback(listCallback)
dl.addCallback(finish)
test()
reactor.run()
from twisted.internet import defer, reactor
from twisted.web.client import getPage
def listCallback(results):
for isSuccess, content in results:
print "Successful? %s" % isSuccess
print "Content Length: %s" % len(content)
def finish(ign):
reactor.stop()
def test():
d1 = getPage('http://www.google.com')
d2 = getPage('http://yahoo.com')
dl = defer.DeferredList([d1, d2])
dl.addCallback(listCallback)
dl.addCallback(finish)
test()
reactor.run()
from twisted.internet import defer, reactor
from twisted.web.client import getPage
def pageCallback(result):
return len(result)
def listCallback(result):
print result
def finish(ign):
reactor.stop()
def test():
d1 = getPage('http://www.google.com')
d1.addCallback(pageCallback)
d2 = getPage('http://yahoo.com')
d2.addCallback(pageCallback)
dl = defer.DeferredList([d1, d2])
dl.addCallback(listCallback)
dl.addCallback(finish)
test()
reactor.run()
from twisted.internet import defer, reactor
from twisted.web.client import getPage
def pageCallback(result):
data = {
'length': len(result),
'content': result[:10],
}
return data
def listCallback(result):
for isSuccess, data in result:
if isSuccess:
print "Call to server succeeded with data %s" % str(data)
def finish(ign):
reactor.stop()
def test():
d1 = getPage('http://www.google.com')
d1.addCallback(pageCallback)
d2 = getPage('http://yahoo.com')
d2.addCallback(pageCallback)
dl = defer.DeferredList([d1, d2])
dl.addCallback(listCallback)
dl.addCallback(finish)
test()
reactor.run()
from twisted.internet import defer, reactor
from twisted.web.client import getPage
def pageCallback(result, url):
data = {
'length': len(result),
'content': result[:10],
'url': url,
}
return data
def getPageData(url):
d = getPage(url)
d.addCallback(pageCallback, url)
return d
def listCallback(result):
for isSuccess, data in result:
if isSuccess:
print "Call to %s succeeded with data %s" % (data['url'], str(data))
def finish(ign):
reactor.stop()
def test():
d1 = getPageData('http://www.google.com')
d2 = getPageData('http://yahoo.com')
dl = defer.DeferredList([d1, d2])
dl.addCallback(listCallback)
dl.addCallback(finish)
test()
reactor.run()
from twisted.internet import defer, reactor
from twisted.web.client import getPage
urls = [
'http://yahoo.com',
'http://www.google.com',
'http://www.google.com/MicrosoftRules.html',
'http://bogusdomain.com',
]
def pageCallback(result, url):
data = {
'length': len(result),
'content': result[:10],
'url': url,
}
return data
def pageErrback(error, url):
return {
'msg': error.getErrorMessage(),
'err': error,
'url': url,
}
def getPageData(url):
d = getPage(url, timeout=5)
d.addCallback(pageCallback, url)
d.addErrback(pageErrback, url)
return d
def listCallback(result):
for ignore, data in result:
if data.has_key('err'):
print "Call to %s failed with data %s" % (data['url'], str(data))
else:
print "Call to %s succeeded with data %s" % (data['url'], str(data))
def finish(ign):
reactor.stop()
def test():
deferreds = []
for url in urls:
d = getPageData(url)
deferreds.append(d)
dl = defer.DeferredList(deferreds, consumeErrors=1)
dl.addCallback(listCallback)
dl.addCallback(finish)
test()
reactor.run()
from twisted.internet import defer, reactor
from twisted.web.client import getPage
maxRun = 1
urls = [
'http://twistedmatrix.com',
'http://yahoo.com',
'http://www.google.com',
]
def listCallback(results):
for isSuccess, result in results:
print len(result)
def finish(ign):
reactor.stop()
def test():
deferreds = []
sem = defer.DeferredSemaphore(maxRun)
for url in urls:
d = sem.run(getPage, url)
deferreds.append(d)
dl = defer.DeferredList(deferreds)
dl.addCallback(listCallback)
dl.addCallback(finish)
test()
reactor.run()
from twisted.internet import defer, reactor, task
from twisted.web.client import getPage
maxRun = 2
urls = [
'http://twistedmatrix.com',
'http://yahoo.com',
'http://www.google.com',
]
def pageCallback(result):
print len(result)
return result
def doWork():
for url in urls:
d = getPage(url)
d.addCallback(pageCallback)
yield d
def finish(ign):
reactor.stop()
def test():
deferreds = []
coop = task.Cooperator()
work = doWork()
for i in xrange(maxRun):
d = coop.coiterate(work)
deferreds.append(d)
dl = defer.DeferredList(deferreds)
dl.addCallback(finish)
test()
reactor.run()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment