Depthwise benchmark.ipynb

depthwise-benchmark.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Depthwise benchmark.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyN908+jnS1y/VFh//SdxYZp",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/aidan-fitz/c48ec1b0b106aa8dc90425337fdbe0e4/depthwise-benchmark.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EI2zS2bGBlDR"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2lew01rfEI9Z",
        "outputId": "9d0e8518-80ca-43a9-e888-ecfa19a6036d"
      },
      "source": [
        "torch.backends.cudnn.version()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "7603"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aNqsZmR7CvR6"
      },
      "source": [
        "device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JoPh8IoRB2AP"
      },
      "source": [
        "class SeparableConv2d(nn.Module):\n",
        "    '''Implements a depthwise separable 2D convolution as described\n",
        "    in MobileNet (https://arxiv.org/abs/1704.04861)\n",
        "    \n",
        "    See: [SeparableConv2D in Keras](https://www.tensorflow.org/api_docs/python/tf/keras/layers/SeparableConv2D)\n",
        "    \n",
        "    Implementation due to https://discuss.pytorch.org/t/how-to-modify-a-conv2d-to-depthwise-separable-convolution/15843/7'''\n",
        "\n",
        "    def __init__(self, in_channels, out_channels, kernel_size):\n",
        "        super(SeparableConv2d, self).__init__()\n",
        "        # Apply the same filter to every channel\n",
        "        self.depthwise = nn.Conv2d(in_channels, in_channels, kernel_size, padding=1, groups=in_channels)\n",
        "        # Combine channels using a 1*1 kernel\n",
        "        self.pointwise = nn.Conv2d(in_channels, out_channels, 1)\n",
        "    \n",
        "    def forward(self, x):\n",
        "        out = self.depthwise(x)\n",
        "        out = self.pointwise(out)\n",
        "        return out"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QDktrjoOB4E5"
      },
      "source": [
        "n, c_in, c_out, w, h = 48, 10, 20, 400, 600"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g5M-t-XDFyFr"
      },
      "source": [
        "dtype = torch.float16"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4CKq4JtYDIfM"
      },
      "source": [
        "input = torch.rand((n, c_in, h, w), device=device, dtype=dtype)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dGIyrYkIDOU0"
      },
      "source": [
        "conv2d = nn.Conv2d(c_in, c_out, 3, padding=1).to(device, dtype)\n",
        "conv2d_dsc = SeparableConv2d(c_in, c_out, 3).to(device, dtype)\n",
        "pointwise = nn.Conv2d(c_in, c_out, 1).to(device, dtype)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "di0ugBqODd6r",
        "outputId": "f6fcca6a-abbe-418c-c27c-cec125080477"
      },
      "source": [
        "%timeit conv2d(input)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1000 loops, best of 5: 26.9 ms per loop\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G8XMroV1DwJF",
        "outputId": "fde9505f-a53e-4d94-aade-ac35225da05a"
      },
      "source": [
        "%timeit conv2d_dsc(input)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "10 loops, best of 5: 39.4 ms per loop\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "raSVZ85uT011",
        "outputId": "6a9c9628-5733-49a2-afb9-ae6a6c417aff"
      },
      "source": [
        "%timeit pointwise(input)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1000 loops, best of 5: 15.2 ms per loop\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}