Handle file upload by Tornado and Nginx Upload Module

main.py

#!/usr/bin/env python
# encoding: utf-8

import tornado.ioloop
import tornado.web
import tornado.log
import tornado.httpserver
from tornado.options import define, options
import logging
import tornado.gen
import tornado.process
import os
import os.path
import magic
import mimetypes

define('debug', type=bool, default=False, help="enable debug, default False")
define('host', type=str, default="127.0.0.1", help="http listen host, default 127.0.0.1")
define('port', type=int, default=8080, help="http listen port, default 8080")
define('storage_path', type=str, default="storage", help="file storage path")
options.parse_command_line()

logger = logging.getLogger('fileserver')

project_dir_path = os.path.abspath(os.path.dirname(__file__))

if not os.path.exists(options.storage_path):
    os.mkdir(options.storage_path)


class LinuxUtils(object):

    @staticmethod
    @tornado.gen.coroutine
    def mv(src, dest):
        cmd = ["mv", src, dest]
        proc = tornado.process.Subprocess(cmd)
        ret = yield proc.wait_for_exit(raise_error=False)
        raise tornado.gen.Return(ret == 0)

    @staticmethod
    @tornado.gen.coroutine
    def rm(file):
        cmd = ["rm", file]
        proc = tornado.process.Subprocess(cmd)
        ret = yield proc.wait_for_exit(raise_error=False)
        raise tornado.gen.Return(ret == 0)



class UploadHandler(tornado.web.RequestHandler):

    @tornado.gen.coroutine
    def post(self):
        keys = self.request.arguments.keys()
        if "file.path" not in keys:
            self.set_status(status_code=400, reason="file field not exist.")
            self.write("400")
            return
        if filter(lambda x: not x.startswith("file."), keys):
            self.set_status(status_code=400, reason="only allow file field upload")
            self.write("400")
            return
        files = list()
        file_path = self.request.arguments['file.path']
        for index in xrange(len(file_path)):
            file = {}
            file['name'] = self.request.arguments['file.name'][index]
            file['content_type'] = self.request.arguments['file.content_type'][index]
            file['path'] = self.request.arguments['file.path'][index]
            file['md5'] = self.request.arguments['file.md5'][index]
            file['size'] = self.request.arguments['file.size'][index]
            files.append(file)
        # mv tmp file to save store storage
        for file in files:
            # debug
            src_file = file['path']
            mime = magic.from_file(src_file, mime=True)
            ext = mimetypes.guess_extension(mime, False)
            dest_file = os.path.join(
                options.storage_path,
                file['md5'] + ext
            )
            if not os.path.exists(dest_file):
                yield LinuxUtils.mv(
                    src_file,
                    dest_file
                )
            else:
                yield LinuxUtils.rm(
                    src_file
                )
            file['path'] = file['md5'] + ext
        self.write({"data": files})


class Application(tornado.web.Application):

    def __init__(self):
        handlers = [
            (r'/upload', UploadHandler),
        ]

        settings = dict()
        settings['debug'] = True
        super(Application, self).__init__(handlers, **settings)

if __name__ == '__main__':
    application = Application()
    http_server = tornado.httpserver.HTTPServer(application)
    http_server.listen(
        options.port,
        address=options.host
    )
    logger.info("http server listen on %s:%d", options.host, options.port)
    tornado.ioloop.IOLoop.current().start()

nginx.conf

server {
    listen       80;
    server_name  localhost;

    #charset koi8-r;

    #access_log  logs/host.access.log  main;

    location / {
        root   html;
        index  index.html index.htm;
    }
    location /storage {
        root   /vagrant/fileserver;
        index  index.html index.htm;
    }
    # Upload form should be submitted to this location
    location /upload {
      # Pass altered request body to this location
      upload_pass   @backend;
      upload_pass_args on;

      # Store files to this directory
      # The directory is hashed, subdirectories 0 1 2 3 4 5 6 7 8 9 should exist
      upload_store /tmp 1;

      # Allow uploaded files to be read only by user
      upload_store_access group:rw;

      # Set specified fields in request body
      upload_set_form_field $upload_field_name.name "$upload_file_name";
      upload_set_form_field $upload_field_name.content_type "$upload_content_type";
      upload_set_form_field $upload_field_name.path "$upload_tmp_path";
      #upload_set_form_field $upload_field_name.md5 "$upload_file_md5";
      #upload_set_form_field $upload_field_name.size "$upload_file_size";

      # Inform backend about hash and size of a file
      upload_aggregate_form_field "$upload_field_name.md5" "$upload_file_md5";
      upload_aggregate_form_field "$upload_field_name.size" "$upload_file_size";

      upload_pass_form_field "^submit$|^description$";

      upload_cleanup 400 404 499 500-505;
    }

    # Pass altered request body to a backend
    location @backend {
      proxy_pass   http://11.11.11.1:8080;
    }
}