bulk_log_stuff_wandb_controlnet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b2303e67-e6dc-425d-9e79-978de1f37739",
   "metadata": {},
   "outputs": [],
   "source": [
    "BASE_CKPT = \"stabilityai/stable-diffusion-xl-base-1.0\"\n",
    "CONTROLNET_CKPTS = [\n",
    "    \"valhalla/canny-no-attn\", \"valhalla/c-small\", \"diffusers/controlnet-canny-sdxl-1.0\"\n",
    "]\n",
    "IMG_URLS = {\n",
    "    \"disco dancer with colorful lights\": \"https://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/input_image_vermeer.png\",\n",
    "    \"aerial view, a futuristic research complex in a bright foggy jungle, hard lighting\": \"https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/hf-logo.png\",\n",
    "    \"ultrarealistic shot of a furry blue bird\": \"https://huggingface.co/lllyasviel/sd-controlnet-canny/resolve/main/images/bird.png\",\n",
    "}\n",
    "NEGATIVE_PROMPT = \"bad quality, low quality, worse quality\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f7035677-0dd4-4c81-85ff-3bc1f581aba7",
   "metadata": {},
   "outputs": [],
   "source": [
    "SEED = 23\n",
    "STEPS = 30\n",
    "LOW_THRES = 100\n",
    "HIGH_THRES = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5fcc2dd4-eed8-47bb-a9c6-209b1bdf675c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "from PIL import Image\n",
    "import numpy as np\n",
    "from diffusers.utils import load_image\n",
    "\n",
    "def compute_canny(image_url):\n",
    "    image = load_image(image_url)\n",
    "    image = np.array(image)\n",
    "    image = cv2.Canny(image, LOW_THRES, HIGH_THRES)\n",
    "    image = image[:, :, None]\n",
    "    image = np.concatenate([image, image, image], axis=2)\n",
    "    canny_image = Image.fromarray(image)\n",
    "    return canny_image.resize((1024, 1024))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a165ace8-965c-414e-8634-f10b16c9048b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from diffusers import StableDiffusionXLControlNetPipeline, ControlNetModel\n",
    "import torch\n",
    "\n",
    "\n",
    "def load_pipeline(controlnet_id):\n",
    "    controlnet_model = ControlNetModel.from_pretrained(controlnet_id, torch_dtype=torch.float16)\n",
    "    pipe = StableDiffusionXLControlNetPipeline.from_pretrained(\n",
    "        BASE_CKPT,\n",
    "        controlnet=controlnet_model,\n",
    "        torch_dtype=torch.float16,\n",
    "    )\n",
    "    pipe.enable_model_cpu_offload()\n",
    "    return pipe\n",
    "\n",
    "def infer(pipeline, image, prompt):\n",
    "    generator = torch.manual_seed(SEED)\n",
    "    images = pipeline(\n",
    "        prompt, negative_prompt=NEGATIVE_PROMPT, image=image, num_inference_steps=STEPS, generator=generator,\n",
    "        num_images_per_prompt=4\n",
    "    ).images\n",
    "    return images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "87bcd930-8ade-4416-8ae6-a11729801b41",
   "metadata": {},
   "outputs": [],
   "source": [
    "import wandb\n",
    "\n",
    "def run_for_prompt_and_log(controlnet_id):\n",
    "    wandb.init(\n",
    "        project=\"controlnet-sdxl-distill-results\",\n",
    "        name=controlnet_id.split(\"/\")[-1],\n",
    "        config={\n",
    "            \"controlnet_ckpt\": controlnet_id, \"seed\": SEED,\n",
    "            \"canny_low_threshold\": LOW_THRES, \"canny_high_threshold\": HIGH_THRES\n",
    "        },\n",
    "    )\n",
    "    table = wandb.Table(\n",
    "        columns=[\n",
    "            \"Initial Image\",\n",
    "            \"Prompt\",\n",
    "            \"Negative Prompt\",\n",
    "            \"Number of Inference Steps\",\n",
    "            \"Generated Image\"\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    pipeline = load_pipeline(controlnet_id)\n",
    "    for prompt, image_url in IMG_URLS.items():\n",
    "        canny_image = compute_canny(image_url)\n",
    "        images = infer(pipeline, canny_image, prompt)\n",
    "        for image in images:\n",
    "            table.add_data(\n",
    "                wandb.Image(canny_image),\n",
    "                prompt,\n",
    "                NEGATIVE_PROMPT,\n",
    "                STEPS,\n",
    "                wandb.Image(image),\n",
    "            )\n",
    "    \n",
    "    wandb.log({\"results\": table})\n",
    "    wandb.finish()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ffd53d73-284f-45fc-bdfd-055ef47119f2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33msayakpaul\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "wandb version 0.15.8 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.15.5"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/home/sayak/wandb/run-20230814_100852-8gdltbp9</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results/runs/8gdltbp9' target=\"_blank\">golden-shape-1</a></strong> to <a href='https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results' target=\"_blank\">https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results/runs/8gdltbp9' target=\"_blank\">https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results/runs/8gdltbp9</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "86f6443f964c4b44bd242dc423453f53",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading pipeline components...:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f5bf40cefeb4c4baac58c552e21e47a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/30 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00b0e6f281a94fd19cfac287aff010a3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/30 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c0c35585a5f47f5b57713e4006db586",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/30 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">golden-shape-1</strong> at: <a href='https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results/runs/8gdltbp9' target=\"_blank\">https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results/runs/8gdltbp9</a><br/>Synced 4 W&B file(s), 1 media file(s), 20 artifact file(s) and 1 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20230814_100852-8gdltbp9/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f5165f9ab8d34f859d9af406ef237439",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value='Waiting for wandb.init()...\\r'), FloatProgress(value=0.01666908940145125, max=1.0)…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "wandb version 0.15.8 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.15.5"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/home/sayak/wandb/run-20230814_101030-64ukw77n</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results/runs/64ukw77n' target=\"_blank\">volcanic-dream-2</a></strong> to <a href='https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results' target=\"_blank\">https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results/runs/64ukw77n' target=\"_blank\">https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results/runs/64ukw77n</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e2d9b0a15d1e44c882ca9c836615accc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/1.23k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68d7213680c24506b0e15c9bcfb74511",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)on_pytorch_model.bin:   0%|          | 0.00/1.59G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da430d4e0b224c5584c9551d2d92b4b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading pipeline components...:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d10b4148e6b24c43950549ec06ac04ea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/30 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a8b898818024268a9277d296ad1fabd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/30 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ed002de273b4a9fa30a60cff84f50a5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/30 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">volcanic-dream-2</strong> at: <a href='https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results/runs/64ukw77n' target=\"_blank\">https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results/runs/64ukw77n</a><br/>Synced 4 W&B file(s), 1 media file(s), 20 artifact file(s) and 1 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20230814_101030-64ukw77n/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8698456dff6e4fd5a1b7e08fc31ec830",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value='Waiting for wandb.init()...\\r'), FloatProgress(value=0.01666847582285603, max=1.0)…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "wandb version 0.15.8 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.15.5"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/home/sayak/wandb/run-20230814_101251-afj06lzz</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results/runs/afj06lzz' target=\"_blank\">flowing-universe-3</a></strong> to <a href='https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results' target=\"_blank\">https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results/runs/afj06lzz' target=\"_blank\">https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results/runs/afj06lzz</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e447f3ca046646e998d18ea6e03cd161",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading pipeline components...:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b4693e26d9dd4eed95f8b66a6f1c1e25",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/30 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f32916ae33334aec8181244dcde137bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/30 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8f1300320df3492ca5a11ad7ab747b6f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/30 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">flowing-universe-3</strong> at: <a href='https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results/runs/afj06lzz' target=\"_blank\">https://wandb.ai/sayakpaul/controlnet-sdxl-distill-results/runs/afj06lzz</a><br/>Synced 4 W&B file(s), 1 media file(s), 20 artifact file(s) and 1 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20230814_101251-afj06lzz/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for ckpt in CONTROLNET_CKPTS:\n",
    "    run_for_prompt_and_log(ckpt)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}