Skip to content

Instantly share code, notes, and snippets.

@solvingj
Last active March 18, 2022 14:34
Show Gist options
  • Save solvingj/275702a72812b5e459bad3fe82563f12 to your computer and use it in GitHub Desktop.
Save solvingj/275702a72812b5e459bad3fe82563f12 to your computer and use it in GitHub Desktop.
Redirect stdout of all subprocess invocations in realtime implicitly without passing subprocess.PIPE or capture_output.
import os
import subprocess
import sys
from contextlib import ExitStack
if __name__ == '__main__':
# For each process, there is a table of file descriptors (pfd).
# Each entry contains an ID number, and a reference to an ID in the
# system open file table (soft)
# Ref: https://eli.thegreenplace.net/images/2015/fd-inode-diagram.png
# Here is a notation for describing the layers of abstraction
# for a file descriptor in Python:
# (py_var) -> (pfd_id) -> (pfd_data) -> (soft_id)
# pyvar : The name of a python var containing a number.
# pfd_id : The index number of the process-level file descriptor.
# pfd_data : The index number in the system open file table which the pfd points to.
# soft_id : The number of the entry in the system open file table which will be used.
# "Get a handle to the stdout file descriptor which our terminal reads from."
# Store pfd_id for stdout in py_var.
# For illustration, we'll assume the typical value of 1, pointing to some entry "x".
# (current_stdout_fd) -> (1) -> (id_of_x) -> (x)
current_stdout_fd = sys.stdout.fileno()
# "Save the handle to the stdout our terminal reads from for later."
# Allocate a new pfd_id and pfd_data, and store pfd_id in new py_var.
# Copy the pfd_data from orig_stdout_fd.
# For illustration, we'll assume an arbitrary value of 23
# (saved_stdout_fd) -> (23) -> (id_of_x) -> (x)
saved_stdout_fd = os.dup(current_stdout_fd)
# "Get handles to new descriptors we can fully control the read/write for."
# Create a pipe in python, which in-turn:
# Allocates 1 new soft_id.
# Allocates 2 new pfd_id's, both pointing to the new sfd_id.
# For illustration, we'll assume arbitrary values of 8 and 9 pointing to some entry "y":
# (reader_fd) -> (8) -> (id_of_y) -> (y)
# (writer_fd) -> (9) -> (id_of_y) -> (y)
reader_fd, writer_fd = os.pipe()
# "Redirect stdout to our new descriptor by replacing the pointer at pfd 1"
# Overwrite id_of_x with id_of_y at pfd1, resulting in:
# (current_stdout_fd) -> (1) -> (id_of_y) -> (y)
os.dup2(writer_fd, current_stdout_fd)
# Make a function that reads one descriptor and writes to another.
def forward_data(_reader_fd: int, _writer_fd: int):
with ExitStack() as stack:
reader = stack.enter_context(os.fdopen(_reader_fd))
writer = stack.enter_context(os.fdopen(_writer_fd, mode="w"))
while msg := reader.read():
writer.write(msg)
# "Start a thread which reads the new descriptor and writes to the terminal."
# Of note, we can now write to as many destination file descriptors as we want.
# Read : (reader_fd) -> (8) -> (id_of_y) -> (y)
# Write : (saved_stdout_fd) -> (23) -> (id_of_x) -> (x)
import threading
t = threading.Thread(target=forward_data, args=(reader_fd, saved_stdout_fd))
t.start()
# "Run subprocess which inherits it's file descriptors from the current process."
# subprocess.run will see:
# (current_stdout_fd) -> (1) -> (id_of_y) -> (y)
subprocess.run(["cmd", "/c", "set"], text=True)
# Cleanup after exiting context
# __exit__
# "Redirect stdout back to our terminal by replacing the data at the ID it uses."
# For pfd 1, overwrite id_of_y with id_of_x, resulting in:
# (current_stdout_fd) -> (1) -> (id_of_x) -> (x)
os.dup2(saved_stdout_fd, current_stdout_fd)
# More cleanup
os.close(writer_fd)
t.join()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment