wurlitzer-line-logger.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4401fcfb-026e-454c-bda5-9bac47dbcc87",
   "metadata": {},
   "source": [
    "# Continuous capture of output from wurlitzer to a logger\n",
    "\n",
    "Illustration for [minrk/wurlitzer#76](https://github.com/minrk/wurlitzer/issues/76).\n",
    "                  \n",
    "First, set up a logger. This one logs to stdout and includes the timestamp,\n",
    "so we can see how often log lines are produced."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "417a3f69-3e31-441b-80ae-36a60171fa22",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test-log-pipe 2024-04-23 11:25:12,338] test message\n"
     ]
    }
   ],
   "source": [
    "import logging\n",
    "\n",
    "logger = logging.getLogger(\"test-log-pipe\")\n",
    "logger.handlers[:] = []\n",
    "logger.setLevel(logging.INFO)\n",
    "h = logging.StreamHandler(sys.stdout)\n",
    "h.setLevel(logging.INFO)\n",
    "h.setFormatter(logging.Formatter(\"[{name} {asctime}] {msg}\", style=\"{\"))\n",
    "logger.addHandler(h)\n",
    "logger.info(\"test message\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39fbb990-1bf2-49a8-ab4d-3f6ba8e3b9ef",
   "metadata": {},
   "source": [
    "Next, we need to create a Writable (anything with a `write` method),\n",
    "which will take lines and put them onto the logger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "7a7a5e20-d32b-4597-8ace-0fc83245f3dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import io\n",
    "\n",
    "class LogPipe(io.BufferedWriter):\n",
    "    \"\"\"Writeable that writes lines to a Logger object as they are written\"\"\"\n",
    "    def __init__(self, logger):\n",
    "        self.logger = logger\n",
    "        self.buf = \"\"\n",
    "    \n",
    "    def write(self, chunk):\n",
    "        \"\"\"Given chunk, split into lines\n",
    "\n",
    "        Log each line as a discrete message\n",
    "\n",
    "        If it ends with a partial line, save it until the next one\n",
    "        \"\"\"\n",
    "        if self.buf:\n",
    "            chunk = self.buf + chunk\n",
    "        lines = chunk.splitlines(True)\n",
    "        if not lines[-1].endswith(\"\\n\"):\n",
    "            self.buf = lines[-1]\n",
    "            lines = lines[:-1]\n",
    "        else:\n",
    "            self.buf = \"\"\n",
    "            \n",
    "        for line in lines:\n",
    "            self.logger.info(line.rstrip())\n",
    "\n",
    "pipe = LogPipe(logger)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "61437dbc-6729-4857-84d4-32f571da7562",
   "metadata": {},
   "source": [
    "Test it out: pipe anything written to low-level stderr to our logger (which writes to stdout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "659ff182-c50a-4cf3-b5da-2de9c2759b28",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[test-log-pipe 2024-04-23 11:27:15,554] start\n",
      "[test-log-pipe 2024-04-23 11:27:15,578] step 0\n",
      "[test-log-pipe 2024-04-23 11:27:15,679] step 1\n",
      "[test-log-pipe 2024-04-23 11:27:15,781] step 2\n",
      "[test-log-pipe 2024-04-23 11:27:15,881] step 3\n",
      "[test-log-pipe 2024-04-23 11:27:15,982] step 4\n",
      "[test-log-pipe 2024-04-23 11:27:16,083] step 5\n",
      "[test-log-pipe 2024-04-23 11:27:16,190] step 6\n",
      "[test-log-pipe 2024-04-23 11:27:16,291] step 7\n",
      "[test-log-pipe 2024-04-23 11:27:16,395] step 8\n",
      "[test-log-pipe 2024-04-23 11:27:16,495] step 9\n",
      "[test-log-pipe 2024-04-23 11:27:16,600] step 10\n",
      "[test-log-pipe 2024-04-23 11:27:16,700] step 11\n",
      "[test-log-pipe 2024-04-23 11:27:16,801] step 12\n",
      "[test-log-pipe 2024-04-23 11:27:16,906] step 13\n",
      "[test-log-pipe 2024-04-23 11:27:17,011] step 14\n",
      "[test-log-pipe 2024-04-23 11:27:17,116] end\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import time\n",
    "\n",
    "import wurlitzer\n",
    "\n",
    "logger.info(\"start\")\n",
    "with wurlitzer.pipes(stdout=None, stderr=pipe):\n",
    "    for i in range(15):\n",
    "        sys.__stderr__.write(f\"step {i}\\n\")\n",
    "        time.sleep(0.1)\n",
    "logger.info(\"end\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}