Quantization error simulation of SiLU (Swish) activation

quantization_error_simulation_of_silu.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fake_quantize(x):\n",
    "    min_ = np.percentile(x, 0)\n",
    "    max_ = np.percentile(x, 100)\n",
    "    x_clip = np.clip(x, a_min=min_, a_max=max_)\n",
    "    x_int8 = ((x_clip - min_) / (max_ - min_) * 255).astype(np.uint8)\n",
    "    return (x_int8.astype(float) / 255) * (max_ - min_) + min_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def relu(x, quantize=False):\n",
    "    quantize_fn = fake_quantize if quantize else (lambda x: x)\n",
    "    return quantize_fn(x.clip(min=0))\n",
    "\n",
    "\n",
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "\n",
    "def silu(x, quantize=False):\n",
    "    quantize_fn = fake_quantize if quantize else (lambda x: x)\n",
    "    h1 = quantize_fn(x)\n",
    "    h2 = quantize_fn(sigmoid(h1))\n",
    "    return quantize_fn(h1 * h2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_sqnr(x, q):\n",
    "    return 20 * np.log10(np.linalg.norm(x) / np.linalg.norm(x - q))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ReLU SQNR[dB]: 41.68666755974384\n",
      "SiLU SQNR[dB]: 30.710681151668233\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEHCAYAAABV4gY/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAaKElEQVR4nO3df5hU1Z3n8feHBgGDEfmhi2KkR3ETjIqBcXVMDMZRGWOUyWokT37gJDsk0cnoOG5WJ8/MmKxs3BmfrJroRHTyQPwZTMxIyI+RIUFjNJIGQURkwYjSkRXEOAkZBcHv/nFPY9FWdVd3VXV1Hz6v56mn7o9z7z33dPWnbp2qOqWIwMzM8jKo2RUwM7P6c7ibmWXI4W5mliGHu5lZhhzuZmYZGtzsCgCMGTMmJkyY0OxqmJkNKMuXL38pIsaWW9cvwn3ChAm0tbU1uxpmZgOKpOcqrXO3jJlZhhzuZmYZcribmWWoX/S5m5n1xOuvv057ezuvvfZas6vSJ4YNG8b48eMZMmRI1ds43M1swGlvb+eAAw5gwoQJSGp2dRoqIti2bRvt7e20trZWvZ27ZcxswHnttdcYPXp09sEOIInRo0f3+FWKw93MBqR9Idg79OZcHe5mZhlyn7uZDXhX3be6rvv7yoeP7bZMS0sLxx57LLt27aK1tZXbb7+dkSNHVix/9dVXM2LECK644oo9yy666CLOOecczj///D3LRowYwfbt22uqP2QS7pX+sNX8gczMemP48OGsXLkSgFmzZnHTTTfxxS9+sbmVKuFuGTOzGp188sn8+te/BuCZZ55h+vTpTJkyhfe97308/fTTTamTw93MrAa7d+9myZIlnHvuuQDMnj2br33tayxfvpzrrruOiy++uCn1yqJbxsysr7366qtMnjyZjRs3MmXKFM444wy2b9/OI488wgUXXLCn3I4dOyruo9ynYOr1KSBfuZuZ9UJHn/tzzz3Hzp07uemmm3jjjTcYOXIkK1eu3HNbu3ZtxX2MHj2a3/zmN3vmX375ZcaMGVOX+jnczcxqcOCBB3LjjTdy3XXXMXz4cFpbW7n33nuB4tulq1atqrjttGnT+Pa3v83OnTsBmDdvHqeddlpd6uVuGTMb8Jr9ybgTTjiB448/nnvuuYc777yTz33uc1xzzTW8/vrrzJw5k+OPPx6Aa665huuvv37Pdu3t7SxfvpwpU6bQ0tLCkUceyTe+8Y261EkRUZcd1WLq1KlRy491+KOQZvuWtWvX8q53vavZ1ehT5c5Z0vKImFquvLtlzMwyVFW4S9ooabWklZLa0rJRkhZLWp/uDyopf5WkDZLWSTqrUZU3M7PyenLlflpETC55CXAlsCQiJgJL0jySJgEzgWOA6cDNklrqWGczM+tGLd0y5wHz0/R8YEbJ8nsiYkdEPAtsAE6s4ThmZtZD1YZ7AA9IWi5pdlp2SERsBkj3B6flhwGbSrZtT8v2Imm2pDZJbVu3bu1d7c3MrKxqPwp5SkS8IOlgYLGkrgZLKPf1qrd8JCci5gJzofi0TJX1MDOzKlQV7hHxQrrfIul7FN0sL0oaFxGbJY0DtqTi7cDhJZuPB16oY53NzPb2/Uvru78P3VBVsTlz5nDXXXfR0tLCoEGDuOWWW7j11lu5/PLLmTRpEhMmTKCtrW2vb512HtJ33rx5tLW18fWvf72up9BtuEt6GzAoIn6Xps8EvgwsBGYB16b7+9MmC4G7JH0VOBSYCCyra63NzJrs0UcfZdGiRaxYsYKhQ4fy0ksvsXPnTm677bZmVw2ors/9EOBhSasoQvoHEfFjilA/Q9J64Iw0T0SsARYATwE/Bi6JiN2NqLyZWbNs3ryZMWPGMHToUADGjBnDoYceyrRp06jlS5n10m24R8SvIuL4dDsmIuak5dsi4vSImJjuXy7ZZk5EHBkR/zkiftTIEzAza4YzzzyTTZs2cfTRR3PxxRfz4IMPNrtKe/E3VM3MemHEiBEsX76cuXPnMnbsWC688ELmzZvXq3014se+PXCYmVkvtbS0MG3aNKZNm8axxx7L/Pnzu91m+PDh7Ny5k/322w+o7zC/pXzlbmbWC+vWrWP9+vV75leuXMkRRxzR7Xbvf//7ueOOO4DiBz8WLFhQt2F+S/nK3cwGvio/ulhP27dv5/Of/zyvvPIKgwcP5qijjmLu3Lmcf/75e5U77rjjGDSouI7+yEc+wg033MBnPvMZbrzxRiKCT37yk5x66ql1r5/D3cysF6ZMmcIjjzzyluVLly7dM71x48ay2y5atKhBtXqTu2XMzDLkcDczy5DD3cwGpP7wK3J9pTfn6nA3swFn2LBhbNu2bZ8I+Ihg27ZtDBs2rEfb+Q1VMxtwxo8fT3t7O/vKcOHDhg1j/PjxPdrG4W5mA86QIUNobW1tdjX6NXfLmJllyOFuZpYhh7uZWYYc7mZmGXK4m5llyOFuZpYhh7uZWYYc7mZmGXK4m5llyOFuZpYhh7uZWYYc7mZmGXK4m5llyOFuZpYhh7uZWYYc7mZmGXK4m5llyOFuZpYhh7uZWYaqDndJLZIel7QozY+StFjS+nR/UEnZqyRtkLRO0lmNqLiZmVXWkyv3S4G1JfNXAksiYiKwJM0jaRIwEzgGmA7cLKmlPtU1M7NqVBXuksYDHwRuK1l8HjA/Tc8HZpQsvycidkTEs8AG4MS61NbMzKpS7ZX79cAXgDdKlh0SEZsB0v3BaflhwKaScu1p2V4kzZbUJqlt69atPa23mZl1odtwl3QOsCUille5T5VZFm9ZEDE3IqZGxNSxY8dWuWszM6vG4CrKnAKcK+lsYBjwdkl3AC9KGhcRmyWNA7ak8u3A4SXbjwdeqGelzcysa91euUfEVRExPiImULxR+pOI+DiwEJiVis0C7k/TC4GZkoZKagUmAsvqXnMzM6uomiv3Sq4FFkj6NPA8cAFARKyRtAB4CtgFXBIRu2uuqZmZVa1H4R4RS4GlaXobcHqFcnOAOTXWzczMesnfUDUzy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQ92Gu6RhkpZJWiVpjaQvpeWjJC2WtD7dH1SyzVWSNkhaJ+msRp6AmZm9VTVX7juAD0TE8cBkYLqkk4ArgSURMRFYkuaRNAmYCRwDTAdultTSgLqbmVkF3YZ7FLan2SHpFsB5wPy0fD4wI02fB9wTETsi4llgA3BiPSttZmZdq6rPXVKLpJXAFmBxRDwGHBIRmwHS/cGp+GHAppLN29OyzvucLalNUtvWrVtrOAUzM+usqnCPiN0RMRkYD5wo6d1dFFe5XZTZ59yImBoRU8eOHVtVZc3MrDo9+rRMRLwCLKXoS39R0jiAdL8lFWsHDi/ZbDzwQq0VNTOz6lXzaZmxkkam6eHAHwNPAwuBWanYLOD+NL0QmClpqKRWYCKwrM71NjOzLgyuosw4YH76xMsgYEFELJL0KLBA0qeB54ELACJijaQFwFPALuCSiNjdmOqbmVk53YZ7RDwBnFBm+Tbg9ArbzAHm1Fw7MzPrFX9D1cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMdRvukg6X9FNJayWtkXRpWj5K0mJJ69P9QSXbXCVpg6R1ks5q5AmYmdlbVXPlvgv464h4F3AScImkScCVwJKImAgsSfOkdTOBY4DpwM2SWhpReTMzK6/bcI+IzRGxIk3/DlgLHAacB8xPxeYDM9L0ecA9EbEjIp4FNgAn1rneZmbWhR71uUuaAJwAPAYcEhGboXgCAA5OxQ4DNpVs1p6Wdd7XbEltktq2bt3ai6qbmVklVYe7pBHAd4HLIuK3XRUtsyzesiBibkRMjYipY8eOrbYaZmZWharCXdIQimC/MyLuS4tflDQurR8HbEnL24HDSzYfD7xQn+qamVk1qvm0jIB/BtZGxFdLVi0EZqXpWcD9JctnShoqqRWYCCyrX5XNzKw7g6socwrwCWC1pJVp2d8A1wILJH0aeB64ACAi1khaADxF8UmbSyJid70rbmZmlXUb7hHxMOX70QFOr7DNHGBODfUyM7Ma+BuqZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWWo23CX9E1JWyQ9WbJslKTFktan+4NK1l0laYOkdZLOalTFzcyssmqu3OcB0zstuxJYEhETgSVpHkmTgJnAMWmbmyW11K22ZmZWlW7DPSIeAl7utPg8YH6ang/MKFl+T0TsiIhngQ3AifWpqpmZVau3fe6HRMRmgHR/cFp+GLCppFx7WvYWkmZLapPUtnXr1l5Ww8zMyqn3G6oqsyzKFYyIuRExNSKmjh07ts7VMDPbt/U23F+UNA4g3W9Jy9uBw0vKjQde6H31zMysN3ob7guBWWl6FnB/yfKZkoZKagUmAstqq6KZmfXU4O4KSLobmAaMkdQO/D1wLbBA0qeB54ELACJijaQFwFPALuCSiNjdoLqbmVkF3YZ7RHy0wqrTK5SfA8yppVJmZlYbf0PVzCxDDnczsww53M3MMuRwNzPLkMPdzCxDDnczsww53M3MMuRwNzPLkMPdzCxDDnczsww53M3MMtTt2DIDwYz2f6iw5vY+rYeZWX/hK3czsww53M3MMuRwNzPLkMPdzCxDWbyhak3w/Ut7Vv5DNzSmHmZWlq/czcwylPeVe6Wry0pXkT0t31/V67y72qYv5PL3MGuCrMP9sWdfLrv8v9TxGFfdt7rs8q98+Ng6HqU+KtZ1SB9XpFYOfbNuZR3uPZXLk0Gl85hB+S97PdbFvv4llycEs33MPhnulQJ2Rh8co6LeXI329E3NXqj47d/WUT3bUV9cbfuK3myPfTLcKw9X0P909SQxo738FfqA0gdPUGb7on0y3Huq64Ct0xNFhSvh/vpEVLELq6dX9H2hnlf0fnVgA4TDvZ+oFJY2QDj0rZ9xuFehv149WwO5u8gGOH+JycwsQ75yt31X7lfn/fXLadYnHO5mjdSbJ5DefJO4p/weQfYc7mb9TX98ReEngwGnYeEuaTpwA9AC3BYR1zbqWGZWJ/V6YnGXUNM1JNwltQA3AWcA7cAvJS2MiKcacTzrPwbU59+tdn3RVVRJX3RfDeAnokZduZ8IbIiIXwFIugc4D3C4m+3r+uLVQb0M4CeKRoX7YcCmkvl2Oo2/JWk2MDvNbpe0robjjQFeqmH7nLgt9ub2eJPbYm993B43NmKnR1Ra0ahwV5llsddMxFxgbl0OJrVFxNR67Gugc1vsze3xJrfF3nJvj0Z9iakdOLxkfjzwQoOOZWZmnTQq3H8JTJTUKmk/YCawsEHHMjOzThrSLRMRuyT9BfCvFB+F/GZErGnEsZK6dO9kwm2xN7fHm9wWe8u6PRQR3ZcyM7MBxQOHmZllyOFuZpahfh3ukqZLWidpg6Qry6yXpBvT+ickvafabQeiGtvjm5K2SHqyb2vdGL1tC0mHS/qppLWS1kjqhwO59FwN7TFM0jJJq1J7fKnva19/tfyvpPUtkh6XtKjval1nEdEvbxRvxD4D/AGwH7AKmNSpzNnAjyg+V38S8Fi12w60Wy3tkdadCrwHeLLZ59Lkx8Y44D1p+gDg/+7Lj400PyJNDwEeA05q9jk1qz1K1l8O3AUsavb59PbWn6/c9wxhEBE7gY4hDEqdB3wrCr8ARkoaV+W2A00t7UFEPATk8lt+vW6LiNgcESsAIuJ3wFqKb1QPZLW0R0TE9lRmSLoN9E9Z1PS/Imk88EHgtr6sdL3153AvN4RB53/CSmWq2XagqaU9clOXtpA0ATiB4mp1IKupPVIXxEpgC7A4Ivbp9gCuB74AvNGg+vWJ/hzu3Q5h0EWZarYdaGppj9zU3BaSRgDfBS6LiN/WsW7NUFN7RMTuiJhM8U3yEyW9u77V63O9bg9J5wBbImJ5/avVt/pzuFczhEGlMjkOf1BLe+SmpraQNIQi2O+MiPsaWM++UpfHRkS8AiwFpte9hn2rlvY4BThX0kaK7pwPSLqjcVVtoGZ3+le6UXx79ldAK2++KXJMpzIfZO83RZZVu+1Au9XSHiXrJ5DHG6q1PDYEfAu4vtnn0U/aYywwMk0PB34GnNPsc2pWe3QqM40B/IZqv/2ZvagwhIGkz6b13wB+SPGu9wbgP4A/62rbJpxG3dTSHgCS7qZ4sI6R1A78fUT8c9+eRX3U2BanAJ8AVqd+ZoC/iYgf9uEp1FWN7TEOmJ9+YGcQsCAiBu7H/6j9fyUXHn7AzCxD/bnP3czMesnhbmaWIYe7mVmGHO5mZhlyuJuZZcjhbmaWIYf7ACfpMkn7l8z/UNLIZu2nv5M0oafDHqe2+WSafqeklWk42CMlbe9u+wr7HCnp4l5ue3capvavuigzT9L5vdl/X2rE407StI6heiWdk8swxj3lcB/4LgP2/HNExNlRfI28WfvpMUmDu5qvdrtGSMf4FMXwrwAzgPsj4oSIeKaGXY8Eehzukv4T8EcRcVxE/J8ajt9fXEZjH3c/oBhOYP9uS2bG4d5Akr6YfjDg39LV1hVp+VJJU9P0mDSORcdV5c8krUi3P0rLp6VtviPpaUl3ph8b+EvgUOCnkn6aym5M+/xsusJcKenZkvX/JKmt9IcZutpPmr5c0pPpdllJXddKujXt6wFJw8u0wVhJ35X0y3Q7JS2/WtJcSQ8A3yozf4SkJekKdYmkd6Tt5kn6aqrn/+6i7Uek7VZIWi2pdMjXwZLmp31/p+MfX9K1kp5Ky69LZT8ArEjfejybIoz+W0c7lRxPkv4xtdFqSRd2U49rgSPT3+cfK51HGQ8AB6ft3ifpz1O7rkrt/JYQk/Q/U7sNkvTfU/knurui7aeP3zMlPZr2f6+KAeA6fpzjaUkPAx/uOIcovqW5FDinB22ch2aPf5DrDZgCrKa4Knk7xdecr0jrlgJT0/QYYGOa3h8YlqYnAm1pehrw7xSDGw0CHgXem9ZtBMaUHLfz/BCK8UI+lOZHpfuWVI/jutpPyXm8DRgBrKEYJncCsAuYnMovAD5eph3uKqnrO4C1afpqYDkwvML894FZafpTwL+k6XnAIqAlzZ8LfLnMcQcDby9p4w0U44hMoBgh8JS07pvAFcAoYB1vfmt7ZLr/EvD5kv1e3fF3TPPb0/1/BRandj0EeJ7iq/1d1aPH4/x03g4YXTJ9TUddUzudD/wDcEs65pnA3DQ9KLXjqQPl8ZuO9RDwtrTufwB/BwyjGL53Yjq3BZSMCQN8DPhaszOhr2++cm+c9wHfi4j/iGJI2YVVbDMEuFXSauBeYFLJumUR0R4RbwArKf7Jq3ED8JOI+H6a/4ikFcDjwDGdjlHOe9N5/D6KH3W4j+LcAJ6NiJVpenmFOv0x8HUV47gsBN4u6YC0bmFEvFpStnT+ZN7sCrk91aPDvRGxGyAiFkbE35U5roD/JekJ4N8oxuo+JK3bFBE/T9N3pH3/FngNuE3ShynGG4EioLeW2X9n7wXujmL43BeBB4E/7KYe9fDudLW8miLEjilZ97cUT1KfiSLlzky3x4EVwDspArGc/vj4PSnt8+fp8TQLOCKdx7MRsT6dZ+dRHLdQvELYp/TbgcMyUWngnl282SU2rGT5XwEvAsen9a+VrNtRMr2bKv52ki6iePD/RZpvpbhK/cOI+I2keZ2OX3Y3XazrXKe3dMtQnMfJnUIcSQC/71S283yp0rbsqlyHj1GMeDglIl5PXQcd59r57xJRdLucCJwOzKRosw8Ar9J9G0HlduqqHuV3JF0C/HmaPTsiuhq2eR4wIyJWpb/3tJJ1vwSmSBoVES+nOn4lIm7p5lw69KvHL0X9F0fERzuVm9xFXTvq+GoX67PkK/fGeQj4U0nD05Xqh0rWbaR42QvFS+cOBwKb09XNJyhe4nfndxS/BboXSVMogvzjaX9QvLz+PfDvkg4B/qS7/aTzmCFpf0lvA/6U4mVytR7gzX/Ojn/EajxCEbJQBOTDPTgmFG25JQXqaRQh0eEdkk5O0x8FHk59twdGMTrkZUBHPdcCR1VxvIeAC1X8qtFYit+sXdZFPSq1NxFxU0RMTrfuxuM/ANisYoz6j3Va92OKvv0fpMfgvwKfKumnPkzSwV2cT397/P4COEXSUanM/pKOBp4GWiUdmcp9tNPujgay+GH4nnC4N0gUv9P5bYqXoN9l70C8DvicpEco+hE73AzMkvQLigdkNVeoc4EfdX6DjyJQR1G8WbVS0m0RsYriJfkair7mn3e3n3Qe8yiC6jHgtoh4vIp6dfhLYGp6A+8p4LM92O7PUnfGJ4BLyxWSdK6kL5dZdWc6bhtF6D1dsm4tRTs/QdFG/0QRMIvSsgcprkKhGPP71Crq+z3gCYqxw38CfCEi/l+lekTENoruhSd7+IZqZ39L8XdZ3OkcSce5F7iVolvlZxRdXY+mrpPvUPkJpj8+frcCFwF3p7/TL4B3RsRrwGyKJ7GHgec67es0ik/N7FM85G8fkXQ1xZtv13VX1voXSd+jCOv1za5LswzUx296hXpXRJze7Lr0NV+5m3XvSoo3Vm3geQfw182uRDP4yt1sHyZpNLCkzKrTU9eRDVAOdzOzDLlbxswsQw53M7MMOdzNzDLkcDczy9D/BxwwTu1PeiOAAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "mu = 0\n",
    "sigma = 1.0\n",
    "x = np.random.normal(mu, sigma, 1000)\n",
    "\n",
    "bins = np.linspace(0, 0.045, 50)\n",
    "plt.hist(np.abs(relu(x) - relu(x, quantize=True)), bins, label='ReLU', alpha=0.6)\n",
    "plt.hist(np.abs(silu(x) - silu(x, quantize=True)), bins, label='SiLU', alpha=0.6)\n",
    "plt.legend()\n",
    "plt.xlabel('quantization error: abs(float - fake_quantized)');\n",
    "\n",
    "print('ReLU SQNR[dB]:', compute_sqnr(relu(x), relu(x, quantize=True)))\n",
    "print('SiLU SQNR[dB]:', compute_sqnr(silu(x), silu(x, quantize=True)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}