Skip to content

Instantly share code, notes, and snippets.

@lvh
Created July 6, 2012 08:37
Show Gist options
  • Star 4 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save lvh/3058974 to your computer and use it in GitHub Desktop.
Save lvh/3058974 to your computer and use it in GitHub Desktop.
Twisted file upload
import random
import StringIO
from zope import interface
from twisted.internet import abstract, defer, reactor
from twisted.web import client, http_headers, iweb
class _FileProducer(object): # pragma: no cover
"""
A simple producer that produces a body.
"""
interface.implements(iweb.IBodyProducer)
def __init__(self, bodyFile):
self._file = bodyFile
self._consumer = self._deferred = self._delayedProduce = None
self._paused = False
self.length = iweb.UNKNOWN_LENGTH
def startProducing(self, consumer):
"""
Starts producing to the given consumer.
"""
self._consumer = consumer
self._deferred = defer.Deferred()
reactor.callLater(0, self._produceSome)
return self._deferred
def _scheduleSomeProducing(self):
"""
Schedules some producing.
"""
self._delayedProduce = reactor.callLater(0, self._produceSome)
def _produceSome(self):
"""
Reads some data from the file synchronously, writes it to the
consumer, and cooperatively schedules the next read. If no data is
left, fires the deferred and loses the reference to it.
If paused, does nothing.
"""
if self._paused:
return
data = self._file.read(abstract.FileDescriptor.bufferSize)
if data:
self._consumer.write(data)
self._scheduleSomeProducing()
else:
if self._deferred is not None:
self._deferred.callback(None)
self._deferred = None
def pauseProducing(self):
"""
Pauses the producer.
"""
self._paused = True
if self._delayedProduce is not None:
self._delayedProduce.cancel()
def resumeProducing(self):
"""
Unpauses the producer.
"""
self._paused = False
if self._deferred is not None:
self._scheduleSomeProducing()
def stopProducing(self):
"""
Loses a reference to the deferred.
"""
if self._delayedProduce is not None:
self._delayedProduce.cancel()
self._deferred = None
def _encodeForm(a, b, f):
"""
Encodes two form values and a fake image file as multipart form encoding.
"""
randomChars = [str(random.randrange(10)) for _ in xrange(28)]
boundary = "-----------------------------" + "".join(randomChars)
lines = [boundary]
def add(name, content, isImage=False):
header = "Content-Disposition: form-data; name={}".format(name)
if isImage:
header += "; filename=img.jpeg"
header += "\r\nContent-Type: image/jpeg"
lines.extend([header, "", content, "", boundary])
add("a", a)
add("b", b)
add("f", f.read(), isImage=True)
lines.extend([boundary + "--", ""])
return boundary, "\r\n".join(lines)
def send():
f = StringIO.StringIO("xyzzy") # pretend this is a real file
boundary, body = _encodeForm("1", "2", f)
assert body.startswith(boundary) # 3 char junk not added here!
producer = _FileProducer(StringIO.StringIO(body))
headers = http_headers.Headers()
contentType = "multipart/form-data; boundary={}".format(boundary)
headers.setRawHeaders("Content-Type", [contentType])
headers.setRawHeaders("Content-Length", [len(body)])
agent = client.Agent(reactor)
d = agent.request("POST", "http://localhost:8080", headers, producer)
return d
if __name__ == "__main__":
d = send()
d.addCallback(lambda _result: reactor.stop())
reactor.run()
import sys
from twisted.internet import reactor
from twisted.python import log
from twisted.web import resource, server
class Resource(resource.Resource):
isLeaf = True
def render_GET(self, request):
return """
<html>
<body>
<form enctype="multipart/form-data" action="/?q=1" method="post">
<input name="xyzzy">
<input type="file" name="image">
<input type="submit">
</form>
</body>
</html>
"""
def render_POST(self, request):
import pdb; pdb.set_trace()
return ""
log.startLogging(sys.stdout)
reactor.listenTCP(8080, server.Site(Resource()))
reactor.run()
@prat0318
Copy link

Thanks for the example. It helped me write a file upload for twisted web (I wonder why there is not implicit support for this). The curious thing is setting the content-length as len(body) is throwing up bad request at server end. And removing it worked. But having the content-length in the request is a good thing. Did you face any issues with it?

@prat0318
Copy link

It turned out that the content-length is automatically added by twisted, so the line need not be manually added.

@lucagiovagnoli
Copy link

Fixing fido regarding the content-length problem: Yelp/fido#26

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment