tornado: chunked handler (RFC2616)

.gitmodules

[submodule "iostream_callback"]
	path = iostream_callback
	url = git://gist.github.com/753987.git

__init__.py

from chunked_handler import ChunkedHandler

README.md

Description

Implementations for receiving chunked requests (RFC2616) for Tornado. Two are available here.

The first in naive.py is based on IOLoop.add_callback(). We wrap callbacks with add_callback() to avoid busting the stack limit in Python.

A more complex implementation is available in chunked_handler.py. It's built on callback technique in this gist; for more, see this post: http://rctay.tuletech.com/2010/12/tornado-presenting-a-new-paradigm-for-IOStream-read-callbacks

While both work the same, it's recommended that you go with the latter for performance.

Usage

First, be sure to checkout submodule dependencies after cloning with

$ git submodule init
$ git submodule update

A simple standalone server is provided in main.py. It binds to port 8000, and both handler implementations can be hit with requests to /chunked and /chunked_naive.

For example, you can test the naive implementation with

$ curl -v -H "Transfer-Encoding: chunked" --data-binary @some.file http://localhost:8000/chunked_naive

To add chunked functionality to your own handler, extend the handler implementation of your choice and override _on_chunks(); it will be called when all the data is received with the chunked data as a file-like object (actually, a cStringIO instance).

chunked_handler.py

from tornado import web

from iostream_callback import (
    Callback,
    Data,
    DONE,
)

from cStringIO import StringIO

WAIT_LENGTH = (1, )
WAIT_CHUNK = (2, )


class ChunkedData(Data):

    def __init__(self):
        self.chunk = StringIO()
        self.chunk_length = 0

        super(ChunkedData, self).__init__()


class LengthCallback(Callback):
    start_state = WAIT_LENGTH

    def _handle(self, data):
        assert data[-2:] == '\r\n', "chunk size ends with CRLF"
        self.data.chunk_length = int(data[:-2], 16)

        if self.data.chunk_length:
            self.data.state = WAIT_CHUNK
        else:
            self.data.state = DONE


class DataCallback(Callback):
    start_state = WAIT_CHUNK

    def _handle(self, data):
        assert data[-2:] == '\r\n', "chunk data ends with CRLF"
        self.data.chunk.write(data[:-2])

        self.data.state = WAIT_LENGTH


class ChunkReader(object):
    def __init__(self, handler):
        self.handler = handler

        stream = handler.request.connection.stream

        data = ChunkedData()
        func = Callback.make_entry_callback(data, (
                LengthCallback(data,
                    lambda self: stream.read_until('\r\n', self)),
                DataCallback(data,
                    lambda self: stream.read_bytes(data.chunk_length + 2, self)),
            ), self._done_callback)

        data.state = WAIT_LENGTH
        func()

    def _done_callback(self, data):
        self.handler._on_chunks(data.chunk)


class ChunkedHandler(web.RequestHandler):
    def _handle_chunked(self, *args, **kwargs):
        # we assume that the wrapping server has not sent/flushed the
        # 100 (Continue) response
        if self.request.headers.get('Expect', None) == '100-continue' and \
            not 'Content-Length' in self.request.headers and \
            self.request.headers.get('Transfer-Encoding', None) == 'chunked':

            self._auto_finish = False
            ChunkReader(self)

            self.request.write("HTTP/1.1 100 (Continue)\r\n\r\n")

            return True
        return False

    def _on_chunks(self, all_chunks):
        self.finish()

main.py

from tornado import (
	httpserver,
	ioloop,
	options,
	web,
)
from tornado.options import options as options_data

from chunked_handler import ChunkedHandler
import naive


class SampleChunkedHandler(ChunkedHandler):
	def post(self):
		if not self._handle_chunked():
			raise web.HTTPError(500, "non-chunked request")

	def _on_chunks(self, all_chunks):
		super(SampleChunkedHandler, self)._on_chunks(all_chunks)

		print "got all chunks, total size=%d" % all_chunks.tell()


if __name__ == '__main__':
    options.define("port", default=8000, help="run on the given port", type=int)
    options.parse_command_line()

    application = web.Application([
        ('/chunked_naive$', naive.ChunkedHandler, ),
        ('/chunked$', SampleChunkedHandler, ),
    ])
    http_server = httpserver.HTTPServer(application)
    http_server.listen(options_data.port)
    ioloop.IOLoop.instance().start()

naive.py

from tornado import ioloop, web

from cStringIO import StringIO
import functools


class ChunkedHandler(web.RequestHandler):
    def post(self):
        # we assume that the wrapping server has not sent/flushed the
        # 100 (Continue) response
        if self.request.headers.get('Expect', None) == '100-continue' and \
            not 'Content-Length' in self.request.headers and \
            self.request.headers.get('Transfer-Encoding', None) == 'chunked':

            self._auto_finish = False

            self.chunks = StringIO()
            self._stream = self.request.connection.stream

            # cache callback proxies
            self._length_callback = lambda: \
                self._stream.read_until('\r\n', self._on_chunk_length)
            self._data_callback = lambda len: \
                lambda: self._stream.read_bytes(len + 2, self._on_chunk_data)

            self._stream.io_loop.add_callback(self._length_callback)

            self.request.write("HTTP/1.1 100 (Continue)\r\n\r\n")
        else:
            raise web.HTTPError(500, "non-chunked request")

    def _on_chunks(self, all_chunks):
        self.finish()
        print "got all chunks, total size=%d" % all_chunks.tell()

    def _on_chunk_length(self, data):
        assert data[-2:] == '\r\n', "chunk size ends with CRLF"
        chunk_length = int(data[:-2], 16)

        if chunk_length:
            self._stream.io_loop.add_callback(self._data_callback(chunk_length))
        else:
            self._on_chunks(self.chunks)

    def _on_chunk_data(self, data):
        assert data[-2:] == '\r\n', "chunk data ends with CRLF"
        self.chunks.write(data[:-2])

        self._stream.io_loop.add_callback(self._length_callback)