The example to how to shutdown tornado web server gracefully...

graceful_shutdown_tornado_web_server.py

#!/usr/bin/env python

"""
How to use it:
1. Just `kill -2 PROCESS_ID` or `kill -15 PROCESS_ID`,
   The Tornado Web Server Will shutdown after process all the request.
2. When you run it behind Nginx, it can graceful reboot your production server.
"""

import time
import signal
import logging
from functools import partial

import tornado.httpserver
import tornado.ioloop
import tornado.options
import tornado.web

from tornado.options import define, options

define("port", default=8888, help="run on the given port", type=int)

MAX_WAIT_SECONDS_BEFORE_SHUTDOWN = 3


class MainHandler(tornado.web.RequestHandler):
    def get(self):
        self.write("Hello, world")


def sig_handler(server, sig, frame):
    io_loop = tornado.ioloop.IOLoop.instance()

    def stop_loop(deadline):
        now = time.time()
        if now < deadline and (io_loop._callbacks or io_loop._timeouts):
            logging.info('Waiting for next tick')
            io_loop.add_timeout(now + 1, stop_loop, deadline)
        else:
            io_loop.stop()
            logging.info('Shutdown finally')

    def shutdown():
        logging.info('Stopping http server')
        server.stop()
        logging.info('Will shutdown in %s seconds ...',
                     MAX_WAIT_SECONDS_BEFORE_SHUTDOWN)
        stop_loop(time.time() + MAX_WAIT_SECONDS_BEFORE_SHUTDOWN)

    logging.warning('Caught signal: %s', sig)
    io_loop.add_callback_from_signal(shutdown)


def main():
    tornado.options.parse_command_line()
    application = tornado.web.Application([
        (r"/", MainHandler),
    ])

    server = tornado.httpserver.HTTPServer(application)
    server.listen(options.port)

    signal.signal(signal.SIGTERM, partial(sig_handler, server))
    signal.signal(signal.SIGINT, partial(sig_handler, server))

    tornado.ioloop.IOLoop.instance().start()

    logging.info("Exit...")


if __name__ == "__main__":
    main()

You definitely have to handle it. We do something like:

class ReadyState(IntEnum):
    READY = 0
    NOT_READY = 1


class ReadyAwareMixin(object):
    """
    Mixin for Tornado's Application whichs tracks whether the application is in a ready state.
    """

    _ready_state: ReadyState = ReadyState.READY

    def __init__(self) -> None:
        super().__init__()
        self._ready_state = ReadyState.READY

    @property
    def ready_state(self):
        return self._ready_state

    @ready_state.setter
    def ready_state(self, state: ReadyState):
        self._ready_state = state


class ReadyHandler(RequestHandler):
    """
    Handler for use with Kubernetes readiness probes.
    https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#define-readiness-probes
    """

    def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any) -> None:
        super().__init__(application, request, **kwargs)

        if not isinstance(self.application, ReadyAwareMixin):
            raise ValueError('The application must inherit from ReadyAwareMixin')

    async def get(self):
        """
        Follows the rules of Kubernetes HTTP probes:
        https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#define-a-liveness-http-request
        """
        if cast(ReadyAwareMixin, self.application).ready_state == ReadyState.READY:
            self.write('OK')
            self.set_status(200)
        else:
            self.write('NOK')
            self.set_status(500)

def sig_handler(server, application, sig, frame):
    if isinstance(application, ReadyAwareMixin):
        logger.info(f'Setting Readiness to NOT_READY')
        application.ready_state = ReadyState.NOT_READY
   ... similar to above ...

@ewhauser thanks, I'll try that!
Not sure what I am missing.
As far as I understand, in a rolling-update, the pod state should change to "Terminating" state and stop receiving new traffic.
That's why my current /ready path simply returns 200 and my sig_handler for SIGTERM just prints a log and that it (to prevent the app from being closed and let the currently processing requests time to finish.

@ewhauser, what is FLAGS.TORNADO_SHUTDOWN_WAIT? Where did you get this from? Is it the same as MAX_WAIT_SECONDS_BEFORE_SHUTDOWN?