gevent_subprocess.py

"""Example code for using gevent and subprocess to run a command
with shell while streaming stdout and stderr to logging.
"""


import logging
from gevent import Greenlet, iwait, joinall
from gevent.subprocess import Popen, PIPE

logging.basicConfig(level=logging.DEBUG)


def gevent_logged_command(command):
    logging.debug("running gevent logged subprocess command: %s", command)

    proc = Popen(command, stdout=PIPE, stderr=PIPE, shell=True)

    stdout_g = Greenlet(proc.stdout.readline)
    stdout_g._name = "stdout"
    stdout_g.start()

    stderr_g = Greenlet(proc.stderr.readline)
    stderr_g._name = "stderr"
    stderr_g.start()

    running_greenlets = [stderr_g, stdout_g]

    while running_greenlets or proc.poll() is None:
        for glet in iwait(running_greenlets):
            running_greenlets.remove(glet)
            if glet.value:
                content = glet.value.strip().decode()
                if content:
                    logging.info("%s: %s", glet._name, content)

                next_line = Greenlet(getattr(proc, glet._name).readline)
                next_line._name = glet._name
                next_line.start()
                running_greenlets.append(next_line)

            break

    joinall(running_greenlets)


if __name__ == "__main__":
    logging.debug("main start")
    gevent_logged_command("./test_output.sh")
    logging.debug("main finish")