Instantly share code, notes, and snippets.

Embed
What would you like to do?
Handle file upload by Tornado and Nginx Upload Module
#!/usr/bin/env python
# encoding: utf-8
import tornado.ioloop
import tornado.web
import tornado.log
import tornado.httpserver
from tornado.options import define, options
import logging
import tornado.gen
import tornado.process
import os
import os.path
import magic
import mimetypes
define('debug', type=bool, default=False, help="enable debug, default False")
define('host', type=str, default="127.0.0.1", help="http listen host, default 127.0.0.1")
define('port', type=int, default=8080, help="http listen port, default 8080")
define('storage_path', type=str, default="storage", help="file storage path")
options.parse_command_line()
logger = logging.getLogger('fileserver')
project_dir_path = os.path.abspath(os.path.dirname(__file__))
if not os.path.exists(options.storage_path):
os.mkdir(options.storage_path)
class LinuxUtils(object):
@staticmethod
@tornado.gen.coroutine
def mv(src, dest):
cmd = ["mv", src, dest]
proc = tornado.process.Subprocess(cmd)
ret = yield proc.wait_for_exit(raise_error=False)
raise tornado.gen.Return(ret == 0)
@staticmethod
@tornado.gen.coroutine
def rm(file):
cmd = ["rm", file]
proc = tornado.process.Subprocess(cmd)
ret = yield proc.wait_for_exit(raise_error=False)
raise tornado.gen.Return(ret == 0)
class UploadHandler(tornado.web.RequestHandler):
@tornado.gen.coroutine
def post(self):
keys = self.request.arguments.keys()
if "file.path" not in keys:
self.set_status(status_code=400, reason="file field not exist.")
self.write("400")
return
if filter(lambda x: not x.startswith("file."), keys):
self.set_status(status_code=400, reason="only allow file field upload")
self.write("400")
return
files = list()
file_path = self.request.arguments['file.path']
for index in xrange(len(file_path)):
file = {}
file['name'] = self.request.arguments['file.name'][index]
file['content_type'] = self.request.arguments['file.content_type'][index]
file['path'] = self.request.arguments['file.path'][index]
file['md5'] = self.request.arguments['file.md5'][index]
file['size'] = self.request.arguments['file.size'][index]
files.append(file)
# mv tmp file to save store storage
for file in files:
# debug
src_file = file['path']
mime = magic.from_file(src_file, mime=True)
ext = mimetypes.guess_extension(mime, False)
dest_file = os.path.join(
options.storage_path,
file['md5'] + ext
)
if not os.path.exists(dest_file):
yield LinuxUtils.mv(
src_file,
dest_file
)
else:
yield LinuxUtils.rm(
src_file
)
file['path'] = file['md5'] + ext
self.write({"data": files})
class Application(tornado.web.Application):
def __init__(self):
handlers = [
(r'/upload', UploadHandler),
]
settings = dict()
settings['debug'] = True
super(Application, self).__init__(handlers, **settings)
if __name__ == '__main__':
application = Application()
http_server = tornado.httpserver.HTTPServer(application)
http_server.listen(
options.port,
address=options.host
)
logger.info("http server listen on %s:%d", options.host, options.port)
tornado.ioloop.IOLoop.current().start()
server {
listen 80;
server_name localhost;
#charset koi8-r;
#access_log logs/host.access.log main;
location / {
root html;
index index.html index.htm;
}
location /storage {
root /vagrant/fileserver;
index index.html index.htm;
}
# Upload form should be submitted to this location
location /upload {
# Pass altered request body to this location
upload_pass @backend;
upload_pass_args on;
# Store files to this directory
# The directory is hashed, subdirectories 0 1 2 3 4 5 6 7 8 9 should exist
upload_store /tmp 1;
# Allow uploaded files to be read only by user
upload_store_access group:rw;
# Set specified fields in request body
upload_set_form_field $upload_field_name.name "$upload_file_name";
upload_set_form_field $upload_field_name.content_type "$upload_content_type";
upload_set_form_field $upload_field_name.path "$upload_tmp_path";
#upload_set_form_field $upload_field_name.md5 "$upload_file_md5";
#upload_set_form_field $upload_field_name.size "$upload_file_size";
# Inform backend about hash and size of a file
upload_aggregate_form_field "$upload_field_name.md5" "$upload_file_md5";
upload_aggregate_form_field "$upload_field_name.size" "$upload_file_size";
upload_pass_form_field "^submit$|^description$";
upload_cleanup 400 404 499 500-505;
}
# Pass altered request body to a backend
location @backend {
proxy_pass http://11.11.11.1:8080;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment