Handle file upload by Tornado and Nginx Upload Module
#!/usr/bin/env python | |
# encoding: utf-8 | |
import tornado.ioloop | |
import tornado.web | |
import tornado.log | |
import tornado.httpserver | |
from tornado.options import define, options | |
import logging | |
import tornado.gen | |
import tornado.process | |
import os | |
import os.path | |
import magic | |
import mimetypes | |
define('debug', type=bool, default=False, help="enable debug, default False") | |
define('host', type=str, default="127.0.0.1", help="http listen host, default 127.0.0.1") | |
define('port', type=int, default=8080, help="http listen port, default 8080") | |
define('storage_path', type=str, default="storage", help="file storage path") | |
options.parse_command_line() | |
logger = logging.getLogger('fileserver') | |
project_dir_path = os.path.abspath(os.path.dirname(__file__)) | |
if not os.path.exists(options.storage_path): | |
os.mkdir(options.storage_path) | |
class LinuxUtils(object): | |
@staticmethod | |
@tornado.gen.coroutine | |
def mv(src, dest): | |
cmd = ["mv", src, dest] | |
proc = tornado.process.Subprocess(cmd) | |
ret = yield proc.wait_for_exit(raise_error=False) | |
raise tornado.gen.Return(ret == 0) | |
@staticmethod | |
@tornado.gen.coroutine | |
def rm(file): | |
cmd = ["rm", file] | |
proc = tornado.process.Subprocess(cmd) | |
ret = yield proc.wait_for_exit(raise_error=False) | |
raise tornado.gen.Return(ret == 0) | |
class UploadHandler(tornado.web.RequestHandler): | |
@tornado.gen.coroutine | |
def post(self): | |
keys = self.request.arguments.keys() | |
if "file.path" not in keys: | |
self.set_status(status_code=400, reason="file field not exist.") | |
self.write("400") | |
return | |
if filter(lambda x: not x.startswith("file."), keys): | |
self.set_status(status_code=400, reason="only allow file field upload") | |
self.write("400") | |
return | |
files = list() | |
file_path = self.request.arguments['file.path'] | |
for index in xrange(len(file_path)): | |
file = {} | |
file['name'] = self.request.arguments['file.name'][index] | |
file['content_type'] = self.request.arguments['file.content_type'][index] | |
file['path'] = self.request.arguments['file.path'][index] | |
file['md5'] = self.request.arguments['file.md5'][index] | |
file['size'] = self.request.arguments['file.size'][index] | |
files.append(file) | |
# mv tmp file to save store storage | |
for file in files: | |
# debug | |
src_file = file['path'] | |
mime = magic.from_file(src_file, mime=True) | |
ext = mimetypes.guess_extension(mime, False) | |
dest_file = os.path.join( | |
options.storage_path, | |
file['md5'] + ext | |
) | |
if not os.path.exists(dest_file): | |
yield LinuxUtils.mv( | |
src_file, | |
dest_file | |
) | |
else: | |
yield LinuxUtils.rm( | |
src_file | |
) | |
file['path'] = file['md5'] + ext | |
self.write({"data": files}) | |
class Application(tornado.web.Application): | |
def __init__(self): | |
handlers = [ | |
(r'/upload', UploadHandler), | |
] | |
settings = dict() | |
settings['debug'] = True | |
super(Application, self).__init__(handlers, **settings) | |
if __name__ == '__main__': | |
application = Application() | |
http_server = tornado.httpserver.HTTPServer(application) | |
http_server.listen( | |
options.port, | |
address=options.host | |
) | |
logger.info("http server listen on %s:%d", options.host, options.port) | |
tornado.ioloop.IOLoop.current().start() |
server { | |
listen 80; | |
server_name localhost; | |
#charset koi8-r; | |
#access_log logs/host.access.log main; | |
location / { | |
root html; | |
index index.html index.htm; | |
} | |
location /storage { | |
root /vagrant/fileserver; | |
index index.html index.htm; | |
} | |
# Upload form should be submitted to this location | |
location /upload { | |
# Pass altered request body to this location | |
upload_pass @backend; | |
upload_pass_args on; | |
# Store files to this directory | |
# The directory is hashed, subdirectories 0 1 2 3 4 5 6 7 8 9 should exist | |
upload_store /tmp 1; | |
# Allow uploaded files to be read only by user | |
upload_store_access group:rw; | |
# Set specified fields in request body | |
upload_set_form_field $upload_field_name.name "$upload_file_name"; | |
upload_set_form_field $upload_field_name.content_type "$upload_content_type"; | |
upload_set_form_field $upload_field_name.path "$upload_tmp_path"; | |
#upload_set_form_field $upload_field_name.md5 "$upload_file_md5"; | |
#upload_set_form_field $upload_field_name.size "$upload_file_size"; | |
# Inform backend about hash and size of a file | |
upload_aggregate_form_field "$upload_field_name.md5" "$upload_file_md5"; | |
upload_aggregate_form_field "$upload_field_name.size" "$upload_file_size"; | |
upload_pass_form_field "^submit$|^description$"; | |
upload_cleanup 400 404 499 500-505; | |
} | |
# Pass altered request body to a backend | |
location @backend { | |
proxy_pass http://11.11.11.1:8080; | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment