train_dalle_pytorch.ipynb

train_dalle_pytorch.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "train_dalle_pytorch.ipynb",
      "private_outputs": true,
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/afiaka87/b29213684a1dd633df20cab49d05209d/train_dalle_pytorch.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1vyuFJyZ0WrA"
      },
      "source": [
        "# Train [DALLE-pytorch](github.com/lucidrains/DALLE-pytorch) from [lucidrains](https:/github.com/lucidrains/)\n",
        "\n",
        "Notebook by [afiaka87](https://github.com/afiaka87) \n",
        "\n",
        "With help from:\n",
        "\n",
        "valteralfred, robvanvolt, rom1504, janEbert, mehdidc, gabriel_syme, robvanvolt, mega b\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VA1PHoJrRiK9",
        "cellView": "form"
      },
      "source": [
        "# @title Licensed under the MIT License\n",
        " \n",
        "# Copyright (c) 2021 Clayton Mullis\n",
        " \n",
        "# Permission is hereby granted, free of charge, to any person obtaining a copy\n",
        "# of this software and associated documentation files (the \"Software\"), to deal\n",
        "# in the Software without restriction, including without limitation the rights\n",
        "# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n",
        "# copies of the Software, and to permit persons to whom the Software is\n",
        "# furnished to do so, subject to the following conditions:\n",
        " \n",
        "# The above copyright notice and this permission notice shall be included in\n",
        "# all copies or substantial portions of the Software.\n",
        " \n",
        "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n",
        "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n",
        "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n",
        "# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n",
        "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n",
        "# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n",
        "# THE SOFTWARE."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t6fuICXrLCuT"
      },
      "source": [
        "# Installation + Weights & Biases Login\n",
        "\n",
        "A free tier account for `https://wandb.ai` is required for experiment tracking.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5JDx0pvsvqou"
      },
      "source": [
        "%%writefile download_cah.sh\n",
        "mkdir /content/CAH\n",
        "cd /content/CAH\n",
        "wget --continue https://the-eye.eu/public/AI/cah/laion400m-dat-release.torrent\n",
        "aria2c --continue --seed-time=0 --file-allocation='trunc' --select-file=1-100 laion400m-dat-release.torrent\n",
        "cd /content"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W-oLBCkYwhqQ"
      },
      "source": [
        "!source download_cah.sh"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NCmYTQI6TmMV",
        "cellView": "form"
      },
      "source": [
        "#@title (Advanced) - DALLE-pytorch Repository and Branch Name\n",
        "\n",
        "#@markdown Install `DALLE-pytorch` from source.\n",
        "\n",
        "#@markdown Defaults to latest branch.\n",
        "repository_url = \"https://github.com/lucidrains/DALLE-pytorch.git\" #@param {type: \"string\"}\n",
        "branch = \"main\" #@param {type: \"string\"}\n",
        "\n",
        "!git clone -b \"$branch\" --single-branch \"$repository_url\"\n",
        "!git clone https://github.com/mehdidc/DALLE-pytorch\n",
        "%pip install wandb\n",
        "%cd /content/DALLE-pytorch\n",
        "!python setup.py install\n",
        "%cd /content/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Irvm6u9mW8dw",
        "cellView": "form"
      },
      "source": [
        "#@title Log in to Weights & Biasess\n",
        "#@markdown DALLE-pytorch uses W&B (https://www.wandb.ai) for experiment tracking.\n",
        "\n",
        "#@markdown If you don't want to make a free account; you can find generations in `/content/wandb` in the folder for your specific run.\n",
        "\n",
        "#@markdown **It's recommended to use W&B.**\n",
        "\n",
        "!wandb login"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r7XIByl3J8fB"
      },
      "source": [
        "# Train DALLE-pytorch"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2iN2p_nIzQqh"
      },
      "source": [
        "%pip uninstall taming-transformers-rom1504"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kM1PZdrpyuSA"
      },
      "source": [
        "%pip install omegaconf\n",
        "!git clone https://github.com/CompVis/taming-transformers\n",
        "%pip install -e taming-transformers\n",
        "%pip install pytorch-lightning"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N7KKK3aVUy4E"
      },
      "source": [
        "!python /content/DALLE-pytorch/train_dalle.py --dim 512 --truncate_captions --depth 8 --shift_tokens --random_resize_crop_lower_ratio 1.0 --wandb_name \"train_dalle_colab\" --rotary_emb --wds=jpg,txt --image_text_folder=/content/CAH/laion400m-dat-release"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}