Skip to content

Instantly share code, notes, and snippets.

View varunshenoy's full-sized avatar

Varun Shenoy varunshenoy

388 followers · 24 following
View GitHub Profile
@varunshenoy
varunshenoy / gist:718045f08e27fa9c1cae4b5c9d850d5f
Created August 16, 2023 04:19
pix2pix repro
This file has been truncated, but you can view the full file.
{"0":{"op":"dream","image":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAIAAAB7GkOtAAEAAElEQVR4nEz9abC1bXYWhl1ruJ9n733Oecdv6LlbLXVrAAkQUxmTwk6Riqtsl7EpA3H5Bz/iuFIxSSWVSlKQSqpiUMAMZsYgsJGQBaIEdhQwFGYICIGQhSRAAoOARlJP6u5vfN9z9n6ee6115cfab+NT37/v+87Z+xnWvdY1LfkjP/xjM2y62mK1TTETprkXpViulpHqyioRJWlmlUkRdcs5zQZzqmoVWaVDWRARgJkUM1SqSlXZMvbLrmPk3IYPQmLGWEbsU0RFAUgVx7DIJGnDaw9zZwWKap4RvgwCGakqEGp/GFL7c6qJCtg/0GFzu4xlAVFJd6sZIkqpAm2M/bIvy8LMqlR3VKpYkszy4bGHrSOTQhEhC6w0M5JCqmhWiQBCEEnCx9zzuBweH8cf+y+/+y//tt8ETPSPGAhF/sff+K988+1rr5vd6Hi63BxPt6svxnD1/XK5XC6woTRTlBCQ2M9VMmeJ68OLd5axZOK4rqpitkSFicl6/Nw77/70+y9+6LM/+QPbz/wLvNgBgPjqj8DoYmQxyX/zf/d//Pf+N7/BlpqzYuiZJKAiApUiC2ZSLBXNDBbdPWMXMbKMgCAjhxtAQOa2r4dBlkCqIAZV4UxVjSozE9PKEpWaqW4zppopBSpCEIBoZppqsWKmr0tlgVgWyz1UDWRmiQqK6ppFCFQgpplZVWZWSXPNmVAVAUmIZKYNqyQEqoYqkiBUkZHqXhHmXpn9LGWEqrEIUt2YKapqOmeaKVGAVFJFVYVZMBOWqFIQEQqFQE1y0swoxaIvY85gQoejQiCmIoSIRFJUwDJVCuY2fV0qp7nNPWyMmNN8gFlZ14/nVlVVdPdtu/jwjFJTc81IU2emuucMc2WmimWVqohIRonKsljsUaCYZpapQ1jVLy0FyAKqVERUIEJCRCisLB
@varunshenoy
varunshenoy / baseten_stable_diffusion.py
Last active July 29, 2023 20:44
This is an example of an Opendream serverless extension using Baseten that overrides the default `dream` function.
import baseten
import base64
from opendream import opendream
from opendream.layer import ImageLayer, Layer
@opendream.define_op
def dream(prompt: str):
# https://app.baseten.co/explore/stable_diffusion
@varunshenoy
varunshenoy / superresolution.py
Created June 21, 2023 01:52
An extension for Opendream that provides an operation for super-resolution via ControlNet with Tile preprocessing. Read more here: https://huggingface.co/lllyasviel/control_v11f1e_sd15_tile
import torch
from PIL import Image
from diffusers import ControlNetModel, StableDiffusionControlNetImg2ImgPipeline, UniPCMultistepScheduler
from opendream import opendream
from opendream.layer import ImageLayer, Layer
def resize_for_condition_image(input_image: Image, resolution: int):
input_image = input_image.convert("RGB")
W, H = input_image.size
@varunshenoy
varunshenoy / photoshop_gpt.py
Created June 21, 2023 01:28
PhotoshopGPT: Describe photo edits in natural language (via a prompt) and automatically apply corresponding enhancements using a combination of Instagram filters and PIL functions.
from opendream import opendream
from opendream.layer import Layer, ImageLayer, MaskLayer
import openai
import os
import json
from PIL import Image, ImageEnhance, ImageFilter
import pilgram
@opendream.define_op
@varunshenoy
varunshenoy / superresolution_with_modal.py
Created June 21, 2023 01:24
This is an example of an Opendream serverless extension using Modal. Combined with Opendream, an extension using Modal allows you to run functions on GPUs while accessing Opendream through consumer hardware. See: https://modal.com/docs/guide/trigger-deployed-functions
@opendream.define_op
def superresolution(image_layer: ImageLayer, model_ckpt: str = "runwayml/stable-diffusion-v1-5", batch_size = 1, seed = 42, selected = 0, num_steps = 20, **kwargs):
import modal
# Use ControlNet + Tile preprocessor
f = modal.Function.lookup("diffuseqr", "ControlNet.run_inference")
image = f.call(Layer.pil_to_b64(image_layer.get_image()))[0]
image = Layer.b64_to_pil(image)
return ImageLayer(image=image)
@varunshenoy
varunshenoy / sam.py
Created June 21, 2023 01:20
An extension for Opendream that provides an operation for Segment Anything. Read more here: https://segment-anything.com/
import os
import numpy as np
from PIL import Image
from urllib.request import urlretrieve
from segment_anything import SamAutomaticMaskGenerator, sam_model_registry
from opendream import opendream
from opendream.layer import ImageLayer, MaskLayer
def convert_mask_to_layer(mask):
@varunshenoy
varunshenoy / controlnet_openpose.py
Last active July 29, 2023 20:43
An extension for Opendream that provides an operation for ControlNet with OpenPose preprocessing. Read more here: https://huggingface.co/lllyasviel/sd-controlnet-openpose
import torch
from diffusers import UniPCMultistepScheduler, ControlNetModel, StableDiffusionControlNetPipeline
from opendream import opendream
from opendream.layer import ImageLayer, Layer
from controlnet_aux import OpenposeDetector
@opendream.define_op
def controlnet_openpose(control_image_layer: ImageLayer, prompt, device: str = "cpu", model_ckpt: str = "runwayml/stable-diffusion-v1-5", batch_size = 1, seed = 42, selected = 0, num_steps = 20, **kwargs):
openpose = OpenposeDetector.from_pretrained("lllyasviel/Annotators")
@varunshenoy
varunshenoy / controlnet_canny.py
Last active July 29, 2023 20:43
An extension for Opendream that provides an operation for ControlNet with Canny preprocessing. Read more here: https://huggingface.co/lllyasviel/sd-controlnet-canny
import torch
from diffusers import UniPCMultistepScheduler, ControlNetModel, StableDiffusionControlNetPipeline
from opendream import opendream
from opendream.layer import ImageLayer, Layer
from controlnet_aux import CannyDetector
@opendream.define_op
def controlnet_canny(control_image_layer: ImageLayer, prompt, device: str = "cpu", model_ckpt: str = "runwayml/stable-diffusion-v1-5", batch_size = 1, seed = 42, selected = 0, num_steps = 20, **kwargs):
canny = CannyDetector()
@varunshenoy
varunshenoy / instruct_pix2pix.py
Created June 21, 2023 01:16
An extension for Opendream that adds Instruct Pix2Pix as an operation. Read more here: https://github.com/timothybrooks/instruct-pix2pix.
import torch
from diffusers import StableDiffusionInstructPix2PixPipeline, EulerAncestralDiscreteScheduler
from opendream import opendream
from opendream.layer import Layer, ImageLayer
@opendream.define_op
def instruct_pix2pix(image_layer: ImageLayer, prompt, device = "mps"):
model_id = "timbrooks/instruct-pix2pix"
pipe = StableDiffusionInstructPix2PixPipeline.from_pretrained(model_id, torch_dtype=torch.float32, safety_checker=None)
@varunshenoy
varunshenoy / dalle.py
Last active July 29, 2023 20:52
An extension for Opendream that overrides the default `dream` function (that uses Stable Diffusion) with OpenAI's DALL-E.
from opendream import opendream
from opendream.layer import Layer
import openai
import os
@opendream.define_op
def dream(prompt: str):
openai.api_key = os.environ["OPENAI_API_KEY"]