ipp-test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c58cfb40-4292-4da6-a79f-9734a44a2865",
   "metadata": {},
   "outputs": [],
   "source": [
    "import ipyparallel as ipp"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fcce0836-625f-4acc-9ba2-9e5c3bc96e23",
   "metadata": {},
   "source": [
    "## Specifying hashseed and other environment variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "72b40b6e-e496-416f-9899-7e3ab4f15729",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting 4 engines with <class 'ipyparallel.cluster.launcher.MPIEngineSetLauncher'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<DirectView all>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cluster = ipp.Cluster(engines=\"mpi\", n=4)\n",
    "\n",
    "# this could be set in your ipcluster_config.py as c.MPIEngine...\n",
    "cluster.config.MPIEngineSetLauncher.environment = {\"PYTHONHASHSEED\": \"1\"}\n",
    "\n",
    "cluster.start_cluster_sync()\n",
    "rc = cluster.connect_client_sync()\n",
    "rc.activate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "630a481b-b4bb-4eaa-a3c0-7db0874a4d65",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[stdout:0] hashseed: \n",
       "hash: -8758283693454389936\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[stdout:1] hashseed: \n",
       "hash: 7726345156143226909\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[stdout:2] hashseed: \n",
       "hash: -2044745521382537417\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[stdout:3] hashseed: \n",
       "hash: -8296317318440886908\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%px\n",
    "import os\n",
    "import random\n",
    "print(f\"hashseed: {os.environ['PYTHONHASHSEED']}\")\n",
    "print(f\"hash: {hash('abc')}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0443386b-a2ca-47d8-a12f-e90c27ffffb7",
   "metadata": {},
   "source": [
    "## Passing output file\n",
    "\n",
    "`Launcher.output_file` is where the output is captured for subprocess launchers.\n",
    "It cannot be set via configuration, but it can be specified if you launch engines separately:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "262e35c5-d5ee-4d16-a5fa-fea84a34266b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting 4 engines with <class 'ipyparallel.cluster.launcher.MPIEngineSetLauncher'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'1648192308'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "cluster = ipp.Cluster(engines=\"mpi\", n=4)\n",
    "cluster.start_controller_sync()\n",
    "cluster.start_engines_sync(output_file=f\"/tmp/engines-{os.getpid()}.log\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "a80f1f39-c888-4322-86cb-f21e19f584c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/tmp/engines-28200.log'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cluster.engine_set.output_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "6a5a1f0c-8e03-4ed2-ba0c-44f86646e833",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-03-25 08:11:49.817 [IPEngine.2] Running code in user namespace: a=5\n",
      "2022-03-25 08:11:49.818 [IPEngine.2] WARNING | debugpy_stream undefined, debugging will not be enabled\n",
      "2022-03-25 08:11:49.821 [IPEngine.0] Starting to monitor the heartbeat signal from the hub every 750 ms.\n",
      "2022-03-25 08:11:49.824 [IPEngine.0] Completed registration with id 0\n",
      "2022-03-25 08:11:49.834 [IPEngine.2] Starting to monitor the heartbeat signal from the hub every 750 ms.\n",
      "2022-03-25 08:11:49.834 [IPEngine.1] Starting to monitor the heartbeat signal from the hub every 750 ms.\n",
      "2022-03-25 08:11:49.835 [IPEngine.2] Completed registration with id 2\n",
      "2022-03-25 08:11:49.835 [IPEngine.3] Starting to monitor the heartbeat signal from the hub every 750 ms.\n",
      "2022-03-25 08:11:49.835 [IPEngine.1] Completed registration with id 1\n",
      "2022-03-25 08:11:49.835 [IPEngine.3] Completed registration with id 3\n"
     ]
    }
   ],
   "source": [
    "!tail {cluster.engine_set.output_file}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45809b03-01fe-4e7c-a2c2-1d28115a3e4b",
   "metadata": {},
   "source": [
    "## Overriding in a custom class\n",
    "\n",
    "Both of these can be accomplished with a custom launcher class\n",
    "that overrides the default environment and output file:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "12c10bca-5602-44e5-aecd-f88a80e9f896",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting 4 engines with <class '__main__.CustomLauncher'>\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:01<00:00,  2.93engine/s]\n",
      "engine output file: /var/folders/qr/3vxfnp1x2t1fw55dr288mphc0000gn/T/tmp0719m8_0/ipengine-1648193904-ix9w-1648193905-28200.log\n",
      "hashseed: ['12345', '12345', '12345', '12345']\n",
      "Stopping engine(s): 1648193905\n",
      "engine set stopped 1648193905: {'exit_code': 0, 'pid': 30966, 'identifier': 'ipengine-1648193904-ix9w-1648193905-28200'}\n",
      "Stopping controller\n",
      "Controller stopped: {'exit_code': 0, 'pid': 30952, 'identifier': 'ipcontroller-1648193904-ix9w-28200'}\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import tempfile\n",
    "\n",
    "from ipyparallel.cluster.launcher import MPIEngineSetLauncher\n",
    "from traitlets import default\n",
    "\n",
    "output_dir = tempfile.mkdtemp()\n",
    "\n",
    "class CustomLauncher(MPIEngineSetLauncher):\n",
    "    @default(\"output_file\")\n",
    "    def _default_output_dir(self):\n",
    "        return os.path.join(output_dir, f\"{self.identifier}.log\")\n",
    "    \n",
    "    def get_env(self):\n",
    "        env = {}\n",
    "\n",
    "        # inherit these variables from the parent by default\n",
    "        for var in [\"PYTHONHASHSEED\"]:\n",
    "            if var in os.environ:\n",
    "                env[var] = os.environ[var]\n",
    "\n",
    "        env.update(super().get_env())\n",
    "        return env\n",
    "\n",
    "os.environ[\"PYTHONHASHSEED\"] = \"12345\"\n",
    "\n",
    "with ipp.Cluster(engines=CustomLauncher, n=4) as rc:\n",
    "    hash_seeds = rc[:].apply_sync(os.getenv, \"PYTHONHASHSEED\")\n",
    "    print(f\"engine output file: {rc.cluster.engine_set.output_file}\")\n",
    "    print(f\"hashseed: {hash_seeds}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}