Generate videos using Stable Diffusion Videos on Google Colaboratory. Since Google Drive is very slow compared to Dropbox and other services, it is inefficient to synchronize with it to add files locally every time a large number of images are generated on Google Colaboratory (Colab). Thus, this script describes the codes that only video files a…

stable-diffusion-videos-on-colab.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Preparation"
      ],
      "metadata": {
        "id": "z5qFZaRweOPl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q accelerate\n",
        "!pip install -q stable_diffusion_videos\n",
        "import os\n",
        "import shutil\n",
        "import torch\n",
        "import time\n",
        "import numpy as np\n",
        "from stable_diffusion_videos import StableDiffusionWalkPipeline\n",
        "from diffusers import DPMSolverMultistepScheduler\n",
        "from numpy.random import randint\n",
        "from IPython.display import HTML\n",
        "from base64 import b64encode"
      ],
      "metadata": {
        "id": "abaljnMweNgF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Setup the model"
      ],
      "metadata": {
        "id": "4g14GS3-ePyD"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zSfF74RNeJ5e"
      },
      "outputs": [],
      "source": [
        "# Specify the model\n",
        "# - model on Google Drive\n",
        "model_id = '/content/drive/MyDrive/StableDiffusion/models/NeverEnding-Dream' # Google Drive\n",
        "# - model on HuggingFace\n",
        "model_id = \"Lykon/NeverEnding-Dream\"\n",
        "\n",
        "pipeline = StableDiffusionWalkPipeline.from_pretrained(\n",
        "    model_id,\n",
        "    feature_extractor=None,\n",
        "    safety_checker=None,\n",
        "    #revision=\"fp16\",\n",
        "    torch_dtype=torch.float16,\n",
        ").to(\"cuda\")\n",
        "pipeline.scheduler = DPMSolverMultistepScheduler.from_config(pipeline.scheduler.config)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Functions"
      ],
      "metadata": {
        "id": "azeAQ2mWepyh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def generate_videos(text, text2=False, seed=None, fps=24, iter=72, ninf=100):\n",
        "    seed = int(randint(100000000000, 10000000000000) if not seed else seed)\n",
        "    print(text)\n",
        "    print(seed)\n",
        "    time1 = time.time()\n",
        "    video_path = pipeline.walk(\n",
        "        [text]+[text2 if text2 else text],\n",
        "        [seed, seed+1],\n",
        "        num_inference_steps=ninf,      # number of inference to generate one image, 50 for default\n",
        "        fps=fps,                      # use 5 for testing, 25 or 30 for better quality\n",
        "        num_interpolation_steps=iter, # use 3-5 for testing, 30 or more for better results\n",
        "        height = int(512),                   # use multiples of 64 if > 512. Multiples of 8 if < 512.\n",
        "        width  = int(768),                   # use multiples of 64 if > 512. Multiples of 8 if < 512.\n",
        "        output_dir = '/content/_videos/'\n",
        "    )\n",
        "    time2 = time.time()\n",
        "    print(video_path)\n",
        "    duration = np.round(time2 - time1, 1)\n",
        "    print('Time: ', duration, '秒')\n",
        "\n",
        "def move_and_remove(categ='video1'):\n",
        "    flist = []\n",
        "    tempdir = '/content/_videos/'\n",
        "    subdirs = os.listdir(tempdir)\n",
        "    for subdir in subdirs:\n",
        "        files = os.listdir(tempdir+subdir)\n",
        "        for f in files:\n",
        "            if f.endswith('.mp4'):\n",
        "                fullpath = tempdir + subdir + '/' +f\n",
        "                flist.append(fullpath)\n",
        "    print('生成ファイル一覧')\n",
        "    print(flist)\n",
        "\n",
        "    newdir = '/content/drive/MyDrive/StableDiffusion/video/'+categ\n",
        "    print('移行先ディレクトリ')\n",
        "    print(newdir)\n",
        "\n",
        "    if not os.path.isdir(newdir): # 移行先ディレクトリの存在確認 / なければ作成\n",
        "        os.mkdir(newdir)\n",
        "\n",
        "    for f in flist: # ファイルを一つずつ移動\n",
        "        shutil.move(f, newdir)\n",
        "\n",
        "    shutil.rmtree(tempdir)\n",
        "    os.mkdir(tempdir)"
      ],
      "metadata": {
        "id": "oCOsYyWzeq04"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Generate"
      ],
      "metadata": {
        "id": "yzgw8e3ket76"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "text = 'An uninhabited high-rise buildings with a large moon in the midnight, with a lonely atmosphere, masterpiece, 4K, HDR'\n",
        "generate_videos(text, fps=24, iter=72, outdir=outdir)\n",
        "move_and_remove(categ='building')"
      ],
      "metadata": {
        "id": "ka0ZpedQeu7b"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}