vt_demo1.ipynb

fordemo_visiontransformerattention.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNeCsnPp8IOIbD9f+GZ29R6",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/dicer2000/1bbf305afbf80e67de05b4c94da0bdd3/fordemo_visiontransformerattention.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Transformer Demo 1\n",
        "(c)2024 Brett Huffman"
      ],
      "metadata": {
        "id": "uuGbAfQJarpC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install vit-keras\n",
        "!pip install tensorflow_addons"
      ],
      "metadata": {
        "id": "OIq5Hht0OR6z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from vit_keras import vit, utils, visualize"
      ],
      "metadata": {
        "id": "YBx9IckXPd88"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tazNa6fBOEWi"
      },
      "outputs": [],
      "source": [
        "from vit_keras import vit, utils\n",
        "\n",
        "image_size = 384\n",
        "classes = utils.get_imagenet_classes()\n",
        "model = vit.vit_b16(\n",
        "    image_size=image_size,\n",
        "    activation='sigmoid',\n",
        "    pretrained=True,\n",
        "    include_top=True,\n",
        "    pretrained_top=True\n",
        ")\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "url = \"https://www.southernliving.com/thmb/tXa6uF93OgesMpTj8UVX6HfMNZw=/750x0/filters:no_upscale():max_bytes(150000):strip_icc():format(webp)/GettyImages-185743593-2000-507c6c8883a44851885ea4fbc10a2c9e.jpg\"\n",
        "#url = \"https://www.southernliving.com/thmb/gDSg2Aw5imKm_SA-WtEpMHvoNxI=/750x0/filters:no_upscale():max_bytes(150000):strip_icc():format(webp)/GettyImages-1176203433-2000-d65f1dbf87a24f5f9072f7fd50ab62dd.jpg\"\n",
        "image = utils.read(url, image_size)\n",
        "X = vit.preprocess_inputs(image).reshape(1, image_size, image_size, 3)\n",
        "y = model.predict(X)\n"
      ],
      "metadata": {
        "id": "aX5yuGZBPmDB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Show the image\n",
        "image"
      ],
      "metadata": {
        "id": "6-A6u837PP2y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get the top prediction\n",
        "print(classes[y[0].argmax()]) # Get the top prediction"
      ],
      "metadata": {
        "id": "T2sbk9f2bDy0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for l in y[0].argsort()[-5:][::-1]: #Top 5 classes\n",
        "  print(classes[l])"
      ],
      "metadata": {
        "id": "9OAwB0phT8sT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get the attention values\n",
        "attention_map = visualize.attention_map(model=model, image=image)\n",
        "\n",
        "# Plot results\n",
        "fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(100, 100))\n",
        "ax1.axis('off')\n",
        "ax2.axis('off')\n",
        "ax1.set_title('Original')\n",
        "ax2.set_title('Attention Map')\n",
        "_ = ax1.imshow(image)\n",
        "_ = ax2.imshow(attention_map)"
      ],
      "metadata": {
        "id": "d18jymNMQaqP"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}