DirectML Troubleshooting Example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### System Info"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "AMD Ryzen 7 3800XT 8-Core\n",
    "\n",
    "Nvidia GeForce RTX 3090, 24GB\n",
    "\n",
    "128 GB Ram\n",
    "\n",
    "Win 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name: directMLtroubleshoot\n",
      "channels:\n",
      "  - aaronzs\n",
      "  - defaults\n",
      "  - conda-forge\n",
      "  - menpo\n",
      "dependencies:\n",
      "  - anyio=3.5.0=py39haa95532_0\n",
      "  - appdirs=1.4.4=pyhd3eb1b0_0\n",
      "  - argon2-cffi=21.3.0=pyhd3eb1b0_0\n",
      "  - argon2-cffi-bindings=21.2.0=py39h2bbff1b_0\n",
      "  - asttokens=2.0.5=pyhd3eb1b0_0\n",
      "  - attrs=22.1.0=py39haa95532_0\n",
      "  - babel=2.11.0=py39haa95532_0\n",
      "  - backcall=0.2.0=pyhd3eb1b0_0\n",
      "  - beautifulsoup4=4.12.2=py39haa95532_0\n",
      "  - blas=1.0=mkl\n",
      "  - bleach=4.1.0=pyhd3eb1b0_0\n",
      "  - bottleneck=1.3.5=py39h080aedc_0\n",
      "  - brotlipy=0.7.0=py39h2bbff1b_1003\n",
      "  - bzip2=1.0.8=he774522_0\n",
      "  - ca-certificates=2023.01.10=haa95532_0\n",
      "  - certifi=2022.12.7=py39haa95532_0\n",
      "  - cffi=1.15.1=py39h2bbff1b_3\n",
      "  - chardet=4.0.0=py39haa95532_1003\n",
      "  - colorama=0.4.6=py39haa95532_0\n",
      "  - comm=0.1.2=py39haa95532_0\n",
      "  - cryptography=39.0.1=py39h21b164f_0\n",
      "  - debugpy=1.5.1=py39hd77b12b_0\n",
      "  - decorator=5.1.1=pyhd3eb1b0_0\n",
      "  - defusedxml=0.7.1=pyhd3eb1b0_0\n",
      "  - entrypoints=0.4=py39haa95532_0\n",
      "  - executing=0.8.3=pyhd3eb1b0_0\n",
      "  - giflib=5.2.1=h8cc25b3_3\n",
      "  - glib=2.69.1=h5dc1a3c_2\n",
      "  - gst-plugins-base=1.18.5=h9e645db_0\n",
      "  - gstreamer=1.18.5=hd78058f_0\n",
      "  - icc_rt=2022.1.0=h6049295_2\n",
      "  - icu=58.2=ha925a31_3\n",
      "  - idna=3.4=py39haa95532_0\n",
      "  - importlib_metadata=6.0.0=hd3eb1b0_0\n",
      "  - intel-openmp=2021.4.0=haa95532_3556\n",
      "  - ipykernel=6.19.2=py39hd4e2768_0\n",
      "  - ipython=8.12.0=py39haa95532_0\n",
      "  - ipython_genutils=0.2.0=pyhd3eb1b0_1\n",
      "  - ipywidgets=8.0.4=py39haa95532_0\n",
      "  - jedi=0.18.1=py39haa95532_1\n",
      "  - jinja2=3.1.2=py39haa95532_0\n",
      "  - jpeg=9e=h2bbff1b_1\n",
      "  - json5=0.9.6=pyhd3eb1b0_0\n",
      "  - jsonschema=4.17.3=py39haa95532_0\n",
      "  - jupyter=1.0.0=py39haa95532_8\n",
      "  - jupyter_client=8.1.0=py39haa95532_0\n",
      "  - jupyter_console=6.6.3=py39haa95532_0\n",
      "  - jupyter_core=5.3.0=py39haa95532_0\n",
      "  - jupyter_server=1.23.4=py39haa95532_0\n",
      "  - jupyterlab=3.5.3=py39haa95532_0\n",
      "  - jupyterlab_pygments=0.1.2=py_0\n",
      "  - jupyterlab_server=2.22.0=py39haa95532_0\n",
      "  - jupyterlab_widgets=3.0.5=py39haa95532_0\n",
      "  - krb5=1.19.4=h5b6d351_0\n",
      "  - lerc=3.0=hd77b12b_0\n",
      "  - libclang13=14.0.6=default_h8e68704_1\n",
      "  - libdeflate=1.17=h2bbff1b_0\n",
      "  - libffi=3.4.2=hd77b12b_6\n",
      "  - libiconv=1.16=h2bbff1b_2\n",
      "  - libogg=1.3.5=h2bbff1b_1\n",
      "  - libpng=1.6.39=h8cc25b3_0\n",
      "  - libsodium=1.0.18=h62dcd97_0\n",
      "  - libtiff=4.5.0=h6c2663c_2\n",
      "  - libvorbis=1.3.7=he774522_0\n",
      "  - libwebp=1.2.4=hbc33d0d_1\n",
      "  - libwebp-base=1.2.4=h2bbff1b_1\n",
      "  - libxml2=2.10.3=h0ad7f3c_0\n",
      "  - libxslt=1.1.37=h2bbff1b_0\n",
      "  - lxml=4.9.2=py39h2bbff1b_0\n",
      "  - lz4-c=1.9.4=h2bbff1b_0\n",
      "  - matplotlib-inline=0.1.6=py39haa95532_0\n",
      "  - mistune=0.8.4=py39h2bbff1b_1000\n",
      "  - mkl=2021.4.0=haa95532_640\n",
      "  - mkl-service=2.4.0=py39h2bbff1b_0\n",
      "  - mkl_fft=1.3.1=py39h277e83a_0\n",
      "  - mkl_random=1.2.2=py39hf11a4ad_0\n",
      "  - nbclassic=0.5.5=py39haa95532_0\n",
      "  - nbclient=0.5.13=py39haa95532_0\n",
      "  - nbconvert=6.5.4=py39haa95532_0\n",
      "  - nbformat=5.7.0=py39haa95532_0\n",
      "  - nest-asyncio=1.5.6=py39haa95532_0\n",
      "  - notebook=6.5.4=py39haa95532_0\n",
      "  - notebook-shim=0.2.2=py39haa95532_0\n",
      "  - numexpr=2.8.4=py39h5b0cc5e_0\n",
      "  - numpy-base=1.23.5=py39h4da318b_0\n",
      "  - openssl=1.1.1t=h2bbff1b_0\n",
      "  - pandas=1.5.3=py39hf11a4ad_0\n",
      "  - pandocfilters=1.5.0=pyhd3eb1b0_0\n",
      "  - parso=0.8.3=pyhd3eb1b0_0\n",
      "  - pcre=8.45=hd77b12b_0\n",
      "  - pickleshare=0.7.5=pyhd3eb1b0_1003\n",
      "  - platformdirs=2.5.2=py39haa95532_0\n",
      "  - ply=3.11=py39haa95532_0\n",
      "  - pooch=1.4.0=pyhd3eb1b0_0\n",
      "  - prometheus_client=0.14.1=py39haa95532_0\n",
      "  - prompt-toolkit=3.0.36=py39haa95532_0\n",
      "  - prompt_toolkit=3.0.36=hd3eb1b0_0\n",
      "  - psutil=5.9.0=py39h2bbff1b_0\n",
      "  - pure_eval=0.2.2=pyhd3eb1b0_0\n",
      "  - pycparser=2.21=pyhd3eb1b0_0\n",
      "  - pygments=2.11.2=pyhd3eb1b0_0\n",
      "  - pyopenssl=23.0.0=py39haa95532_0\n",
      "  - pyqt=5.15.7=py39hd77b12b_0\n",
      "  - pyqt5-sip=12.11.0=py39hd77b12b_0\n",
      "  - pyrsistent=0.18.0=py39h196d8e1_0\n",
      "  - pysocks=1.7.1=py39haa95532_0\n",
      "  - python=3.9.16=h6244533_2\n",
      "  - python-dateutil=2.8.2=pyhd3eb1b0_0\n",
      "  - python-fastjsonschema=2.16.2=py39haa95532_0\n",
      "  - pytz=2022.7=py39haa95532_0\n",
      "  - pywin32=305=py39h2bbff1b_0\n",
      "  - pywinpty=2.0.10=py39h5da7b33_0\n",
      "  - pyzmq=23.2.0=py39hd77b12b_0\n",
      "  - qt-main=5.15.2=he8e5bd7_8\n",
      "  - qt-webengine=5.15.9=hb9a9bb5_5\n",
      "  - qtconsole=5.4.2=py39haa95532_0\n",
      "  - qtpy=2.2.0=py39haa95532_0\n",
      "  - qtwebkit=5.212=h2bbfb41_5\n",
      "  - scipy=1.10.1=py39h321e85e_0\n",
      "  - send2trash=1.8.0=pyhd3eb1b0_1\n",
      "  - setuptools=66.0.0=py39haa95532_0\n",
      "  - sip=6.6.2=py39hd77b12b_0\n",
      "  - six=1.16.0=pyhd3eb1b0_1\n",
      "  - sniffio=1.2.0=py39haa95532_1\n",
      "  - soupsieve=2.4=py39haa95532_0\n",
      "  - sqlite=3.41.2=h2bbff1b_0\n",
      "  - stack_data=0.2.0=pyhd3eb1b0_0\n",
      "  - terminado=0.17.1=py39haa95532_0\n",
      "  - tinycss2=1.2.1=py39haa95532_0\n",
      "  - tk=8.6.12=h2bbff1b_0\n",
      "  - toml=0.10.2=pyhd3eb1b0_0\n",
      "  - tomli=2.0.1=py39haa95532_0\n",
      "  - tornado=6.2=py39h2bbff1b_0\n",
      "  - traitlets=5.7.1=py39haa95532_0\n",
      "  - typing-extensions=4.5.0=py39haa95532_0\n",
      "  - typing_extensions=4.5.0=py39haa95532_0\n",
      "  - tzdata=2023c=h04d1e81_0\n",
      "  - urllib3=1.26.15=py39haa95532_0\n",
      "  - vc=14.2=h21ff451_1\n",
      "  - vs2015_runtime=14.27.29016=h5e58377_2\n",
      "  - wcwidth=0.2.5=pyhd3eb1b0_0\n",
      "  - webencodings=0.5.1=py39haa95532_1\n",
      "  - websocket-client=0.58.0=py39haa95532_4\n",
      "  - wheel=0.38.4=py39haa95532_0\n",
      "  - widgetsnbextension=4.0.5=py39haa95532_0\n",
      "  - win_inet_pton=1.1.0=py39haa95532_0\n",
      "  - winpty=0.4.3=4\n",
      "  - xz=5.2.10=h8cc25b3_1\n",
      "  - zeromq=4.3.4=hd77b12b_0\n",
      "  - zlib=1.2.13=h8cc25b3_0\n",
      "  - zstd=1.5.5=hd43e919_0\n",
      "  - pip:\n",
      "      - absl-py==1.4.0\n",
      "      - astunparse==1.6.3\n",
      "      - cachetools==5.3.0\n",
      "      - charset-normalizer==3.1.0\n",
      "      - flatbuffers==23.3.3\n",
      "      - gast==0.4.0\n",
      "      - google-auth==2.17.3\n",
      "      - google-auth-oauthlib==0.4.6\n",
      "      - google-pasta==0.2.0\n",
      "      - grpcio==1.54.0\n",
      "      - h5py==3.8.0\n",
      "      - importlib-metadata==6.6.0\n",
      "      - keras==2.10.0\n",
      "      - keras-preprocessing==1.1.2\n",
      "      - libclang==16.0.0\n",
      "      - markdown==3.4.3\n",
      "      - markupsafe==2.1.2\n",
      "      - numpy==1.24.3\n",
      "      - oauthlib==3.2.2\n",
      "      - opt-einsum==3.3.0\n",
      "      - packaging==23.1\n",
      "      - pip==23.1.1\n",
      "      - protobuf==3.19.6\n",
      "      - pyasn1==0.5.0\n",
      "      - pyasn1-modules==0.3.0\n",
      "      - requests==2.28.2\n",
      "      - requests-oauthlib==1.3.1\n",
      "      - rsa==4.9\n",
      "      - tensorboard==2.10.1\n",
      "      - tensorboard-data-server==0.6.1\n",
      "      - tensorboard-plugin-wit==1.8.1\n",
      "      - tensorflow-cpu==2.10.0\n",
      "      - tensorflow-directml-plugin==0.4.0.dev230202\n",
      "      - tensorflow-estimator==2.10.0\n",
      "      - tensorflow-intel==2.10.0\n",
      "      - tensorflow-io-gcs-filesystem==0.31.0\n",
      "      - termcolor==2.3.0\n",
      "      - werkzeug==2.3.0\n",
      "      - wrapt==1.15.0\n",
      "      - zipp==3.15.0\n",
      "prefix: C:\\Users\\Izzy\\Anaconda3\\envs\\directMLtroubleshoot\n"
     ]
    }
   ],
   "source": [
    "!conda env export"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reproducible Example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This example works *without error* on WSL2 + Docker but *fails* on DirectML plugin."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.sparse import csr_matrix\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.config.list_physical_devices('GPU')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nn_model_create():\n",
    "    inputs = tf.keras.layers.Input(shape=(800,), sparse=True)\n",
    "    dense_layer = tf.keras.layers.Dense(5000, activation=tf.nn.relu)(inputs)   \n",
    "    outputs = tf.keras.layers.Dense(1, activation=None)(dense_layer)\n",
    "    \n",
    "    model = tf.keras.Model(inputs=inputs, outputs=outputs)\n",
    "    model.compile(\n",
    "        optimizer=tf.keras.optimizers.Nadam(\n",
    "            learning_rate=4e-4\n",
    "        ),\n",
    "        loss=tf.keras.losses.MeanSquaredError(),\n",
    "    )\n",
    "    return model\n",
    "\n",
    "nn_model = nn_model_create()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_1 (InputLayer)        [(None, 800)]             0         \n",
      "                                                                 \n",
      " dense (Dense)               (None, 5000)              4005000   \n",
      "                                                                 \n",
      " dense_1 (Dense)             (None, 1)                 5001      \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 4,010,001\n",
      "Trainable params: 4,010,001\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "nn_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_sparse_matrix_to_sparse_tensor(X):\n",
    "    coo = X.tocoo()\n",
    "    indices = np.mat([coo.row, coo.col]).transpose()\n",
    "    return tf.SparseTensor(indices, coo.data, coo.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "rng = np.random.default_rng()\n",
    "X_train = pd.DataFrame(rng.binomial(2, p=0.1, size=(10, 800)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = pd.DataFrame(rng.normal(100, 30, size=(10, 1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_input = convert_sparse_matrix_to_sparse_tensor(csr_matrix(X_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Izzy\\Anaconda3\\envs\\directMLtroubleshoot\\lib\\site-packages\\tensorflow\\python\\framework\\indexed_slices.py:444: UserWarning: Converting sparse IndexedSlices(IndexedSlices(indices=Tensor(\"gradient_tape/model/dense/embedding_lookup_sparse/Reshape_1:0\", shape=(None,), dtype=int32), values=Tensor(\"gradient_tape/model/dense/embedding_lookup_sparse/Reshape:0\", shape=(None, 5000), dtype=float32), dense_shape=Tensor(\"gradient_tape/model/dense/embedding_lookup_sparse/Cast:0\", shape=(2,), dtype=int32))) to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "nn_model.fit(X_train_input, y_train.to_numpy(), epochs=100, batch_size=128)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Terminal Output\n",
    "<pre>\n",
    "2023-04-25 19:34:16.934946: I tensorflow/c/logging.cc:34] Successfully opened dynamic library C:\\Users\\Izzy\\Anaconda3\\envs\\directMLtroubleshoot\\lib\\site-packages\\tensorflow-plugins/directml/directml.d6f03b303ac3c4f2eeb8ca631688c9757\n",
    "b361310.dll\n",
    "2023-04-25 19:34:16.935597: I tensorflow/c/logging.cc:34] Successfully opened dynamic library dxgi.dll\n",
    "2023-04-25 19:34:16.937558: I tensorflow/c/logging.cc:34] Successfully opened dynamic library d3d12.dll\n",
    "2023-04-25 19:34:17.631450: I tensorflow/c/logging.cc:34] DirectML device enumeration: found 1 compatible adapters.\n",
    "2023-04-25 19:34:20.253635: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical oper\n",
    "ations:  AVX AVX2\n",
    "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
    "2023-04-25 19:34:20.254726: I tensorflow/c/logging.cc:34] DirectML: creating device on adapter 0 (NVIDIA GeForce RTX 3090)\n",
    "2023-04-25 19:34:20.366504: I tensorflow/c/logging.cc:34] Successfully opened dynamic library Kernel32.dll\n",
    "2023-04-25 19:34:20.367742: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:306] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support\n",
    ".\n",
    "2023-04-25 19:34:20.367920: W tensorflow/core/common_runtime/pluggable_device/pluggable_device_bfc_allocator.cc:28] Overriding allow_growth setting because force_memory_growth was requested by the device.\n",
    "2023-04-25 19:34:20.368034: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:272] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 82996 MB memory) -> physical PluggableDevice\n",
    " (device: 0, name: DML, pci bus id: <undefined>)\n",
    "2023-04-25 19:34:28.197753: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n",
    "2023-04-25 19:34:28.301033: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:306] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support\n",
    ".\n",
    "2023-04-25 19:34:28.301248: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:272] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 82996 MB memory) -> physical PluggableDevice\n",
    " (device: 0, name: DML, pci bus id: <undefined>)\n",
    "2023-04-25 19:34:28.302831: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:306] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support\n",
    ".\n",
    "2023-04-25 19:34:28.303030: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:272] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 82996 MB memory) -> physical PluggableDevice\n",
    " (device: 0, name: DML, pci bus id: <undefined>)\n",
    "2023-04-25 19:34:28.304440: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:306] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support\n",
    ".\n",
    "2023-04-25 19:34:28.304616: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:272] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 82996 MB memory) -> physical PluggableDevice\n",
    " (device: 0, name: DML, pci bus id: <undefined>)\n",
    "2023-04-25 19:34:28.306100: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:306] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support\n",
    ".\n",
    "2023-04-25 19:34:28.306258: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:272] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 82996 MB memory) -> physical PluggableDevice\n",
    " (device: 0, name: DML, pci bus id: <undefined>)\n",
    "2023-04-25 19:34:28.308751: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:306] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support\n",
    ".\n",
    "2023-04-25 19:34:28.308938: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:272] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 82996 MB memory) -> physical PluggableDevice\n",
    " (device: 0, name: DML, pci bus id: <undefined>)\n",
    "2023-04-25 19:34:28.310555: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:306] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support\n",
    ".\n",
    "2023-04-25 19:34:28.310723: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:272] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 82996 MB memory) -> physical PluggableDevice\n",
    " (device: 0, name: DML, pci bus id: <undefined>)\n",
    "2023-04-25 19:34:29.116881: F tensorflow/c/logging.cc:43] HRESULT failed with 0x887a0001: dml_device_->GetDeviceRemovedReason()\n",
    "</pre>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}