Python and C ctypes stdout captured in IPython Notebook - https://stackoverflow.com/questions/35745541

Python and C ctypes stdout captured in IPython Notebook.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:aa469ab66f8112985d4badf57a9852d1a16f9620bb82fd708c22300ad9899cc1"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# How to get printed output from ctypes C functions into Jupyter/IPython notebook?\n",
      "\n",
      "This Notebook is a solution/answer to my own question: <https://stackoverflow.com/questions/35745541>"
     ]
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "foo.c"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# The code in this cell will create/overwrite a file\n",
      "# 'foo.c' and will call GCC to compile it as 'foo.so'.\n",
      "#\n",
      "# The code is behind a flag to prevent people from\n",
      "# accidentally creating files in their system.\n",
      "\n",
      "YES_I_WANT_TO_WRITE_FILES_TO_MY_SYSTEM = False\n",
      "\n",
      "if YES_I_WANT_TO_WRITE_FILES_TO_MY_SYSTEM:\n",
      "    import textwrap\n",
      "    import shlex\n",
      "    import subprocess\n",
      "\n",
      "    with open('foo.c', 'wt') as foo:\n",
      "        foo.write(textwrap.dedent('''\\\n",
      "            #include <stdio.h>\n",
      "\n",
      "            void printout() {\n",
      "                puts(\"C printout puts\");\n",
      "            }\n",
      "            void printhere(FILE* f) {\n",
      "                fputs(\"C printhere fputs\\\\n\", f);\n",
      "            }        \n",
      "        '''))\n",
      "\n",
      "    subprocess.check_call(shlex.split(\n",
      "        'gcc foo.c -o foo.so -Wall -pipe -shared -fPIC'\n",
      "    ))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "ctypes - Preparing the C library"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import ctypes\n",
      "\n",
      "# use_errno parameter is optional, because I'm not checking errno anyway.\n",
      "libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True)\n",
      "\n",
      "class FILE(ctypes.Structure):\n",
      "    pass\n",
      "\n",
      "FILE_p = ctypes.POINTER(FILE)\n",
      "\n",
      "# Alternatively, we can just use:\n",
      "# FILE_p = ctypes.c_void_p\n",
      "\n",
      "# These variables, defined inside the C library, are readonly.\n",
      "cstdin = FILE_p.in_dll(libc, 'stdin')\n",
      "cstdout = FILE_p.in_dll(libc, 'stdout')\n",
      "cstderr = FILE_p.in_dll(libc, 'stderr')\n",
      "\n",
      "# C function to disable buffering.\n",
      "csetbuf = libc.setbuf\n",
      "csetbuf.argtypes = (FILE_p, ctypes.c_char_p)\n",
      "csetbuf.restype = None\n",
      "\n",
      "# C function to flush the C library buffer.\n",
      "cfflush = libc.fflush\n",
      "cfflush.argtypes = (FILE_p,)\n",
      "cfflush.restype = ctypes.c_int"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "ctypes - Loading our own shared library (DLL)"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "libfoo = ctypes.CDLL('./foo.so')\n",
      "\n",
      "printout = libfoo.printout\n",
      "printout.argtypes = ()\n",
      "printout.restype = None\n",
      "\n",
      "printhere = libfoo.printhere\n",
      "printhere.argtypes = (FILE_p,)\n",
      "printhere.restype = None"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Building our own context manager to capture stdout"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import io\n",
      "import os\n",
      "import sys\n",
      "import tempfile\n",
      "from contextlib import contextmanager\n",
      "\n",
      "@contextmanager\n",
      "def capture_c_stdout(encoding='utf8'):\n",
      "    # Flushing, it's a good practice.\n",
      "    sys.stdout.flush()\n",
      "    cfflush(cstdout)\n",
      "\n",
      "    # We need to use a actual file because we need the file descriptor number.\n",
      "    with tempfile.TemporaryFile(buffering=0) as temp:\n",
      "        # Saving a copy of the original stdout.\n",
      "        prev_sys_stdout = sys.stdout\n",
      "        prev_stdout_fd = os.dup(1)\n",
      "        os.close(1)\n",
      "\n",
      "        # Duplicating the temporary file fd into the stdout fd.\n",
      "        # In other words, replacing the stdout.\n",
      "        os.dup2(temp.fileno(), 1)\n",
      "\n",
      "        # Replacing sys.stdout for Python code.\n",
      "        #\n",
      "        # IPython Notebook version of sys.stdout is actually an\n",
      "        # in-memory OutStream, so it does not have a file descriptor.\n",
      "        # We need to replace sys.stdout so that interleaved Python\n",
      "        # and C output gets captured in the correct order.\n",
      "        #\n",
      "        # We enable line_buffering to force a flush after each line.\n",
      "        # And write_through to force all data to be passed through the\n",
      "        # wrapper directly into the binary temporary file.\n",
      "        temp_wrapper = io.TextIOWrapper(\n",
      "            temp, encoding=encoding, line_buffering=True, write_through=True)\n",
      "        sys.stdout = temp_wrapper\n",
      "\n",
      "        # Disabling buffering of C stdout.\n",
      "        csetbuf(cstdout, None)\n",
      "\n",
      "        yield\n",
      "\n",
      "        # Must flush to clear the C library buffer.\n",
      "        cfflush(cstdout)\n",
      "\n",
      "        # Restoring stdout.\n",
      "        os.dup2(prev_stdout_fd, 1)\n",
      "        os.close(prev_stdout_fd)\n",
      "        sys.stdout = prev_sys_stdout\n",
      "\n",
      "        # Printing the captured output.\n",
      "        temp_wrapper.seek(0)\n",
      "        print(temp_wrapper.read(), end='')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Testing the code!"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print('Python Before capturing')\n",
      "printout()  # Not captured, goes to the terminal\n",
      "\n",
      "with capture_c_stdout():\n",
      "    print('Python First')\n",
      "    printout()\n",
      "    print('Python Second')\n",
      "    printhere(cstdout)\n",
      "    print('Python Third')\n",
      "\n",
      "print('Python After capturing')\n",
      "printout()  # Not captured, goes to the terminal"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Python Before capturing\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Python First\n",
        "C printout puts\n",
        "Python Second\n",
        "C printhere fputs\n",
        "Python Third\n",
        "Python After capturing\n"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# Further work\n",
      "\n",
      "* This code currently has no exception handling. In fact, it has no kind of error handling. This should be improved.\n",
      "* This code only redirects `stdout`. An improvement would be to redirect both `stdout` and `stderr`."
     ]
    }
   ],
   "metadata": {}
  }
 ]
}

When running the notebook it gets as far as the section:

ctypes - Preparing the C library

whereupon I get the following:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-10-684349b68d2d> in <module>()
      5 
      6 # These variables, defined inside the C library, are readonly.
----> 7 cstdin = FILE_p.in_dll(libc, 'stdin')
      8 cstdout = FILE_p.in_dll(libc, 'stdout')
      9 cstderr = FILE_p.in_dll(libc, 'stderr')

ValueError: dlsym(0x11b5c0e50, stdin): symbol not found

on MacOS Mojave 10.14.1 with Python 3.7.0 and Jupyter 5.6.0