This snippet demonstrates the failure to save multi controlnet using Modal.com, a serverless function service

pipeline_with_multicontrolnet_fails_to_save.py

import modal

stub = modal.Stub("demo_multicontrolnet_failure")
modelsVolume = modal.SharedVolume()

# https://github.com/huggingface/diffusers/commit/f7b4f51cc2a423c96cb2a4c2282e55feba0be506
GIT_SHA = "f7b4f51cc2a423c96cb2a4c2282e55feba0be506"

sd_image = (
    modal.Image.debian_slim(python_version="3.10")
    .apt_install([
        "git",
        "libgl1-mesa-glx",
        "libglib2.0-0",
    ])
    .pip_install(
        "accelerate",
        "ftfy",
        "transformers",
        "torch==2.0.0", # xformer requires torch==2.0.0
        "torchvision~=0.15.0", # to pair with torch==2.0.0
        "pytorch-lightning",
        "triton",
        "safetensors",
        "opencv-python",
        "controlnet_aux",
        "pillow",
        "einops~=0.3.0",
    )
    .pip_install("xformers")
    # Perform a shallow fetch of just the target `diffusers` commit, checking out
    # the commit in the container's current working directory, /root. Then install
    # the `diffusers` package.
    .run_commands(
        "cd /root && git init .",
        "cd /root && git remote add origin https://github.com/huggingface/diffusers",
        f"cd /root && git fetch --depth=1 origin {GIT_SHA} && git checkout {GIT_SHA}",
        "cd /root && pip install -e .",
    )
)

MODELS_DIR = "/models"

@stub.function(
    image=sd_image,
        shared_volumes={
        MODELS_DIR: modelsVolume
    },
    secrets=[modal.Secret.from_name("huggingface")],
    gpu="A10G",
    timeout=1800,
)
def pipeline_with_multicontrolnet():
    import diffusers
    import torch
    import os

    hugging_face_token = os.environ["HUGGINGFACE_TOKEN"]
    controlnet_dir = f"{MODELS_DIR}/edit-anything-v0-4-sd15/"
    controlnet_inpaint_dir = f"{MODELS_DIR}/control_v11p_sd15_inpaint/"
    sd_dir = f"{MODELS_DIR}/stable-diffusion-v1-5/"

    edit_anything_controlnet = diffusers.ControlNetModel.from_pretrained(
        "shgao/edit-anything-v0-4-sd15",
        cache_dir=controlnet_dir,
        torch_dtype=torch.float16,
    )
    edit_anything_controlnet.save_pretrained(controlnet_dir, safe_serialization=True)

    inpaint_controlnet = diffusers.ControlNetModel.from_pretrained(
        "lllyasviel/control_v11p_sd15_inpaint",
        cache_dir=controlnet_inpaint_dir,
        torch_dtype=torch.float16,
    )

    inpaint_controlnet.save_pretrained(controlnet_inpaint_dir, safe_serialization=True)

    pipe = diffusers.StableDiffusionControlNetPipeline.from_pretrained(
        "runwayml/stable-diffusion-v1-5",
        controlnet=[edit_anything_controlnet, inpaint_controlnet], # this turns the controlnet into MultiControlNet
        cache_dir=sd_dir,
        torch_dtype=torch.float16,
        use_auth_token=hugging_face_token,
    )
    pipe.scheduler = diffusers.UniPCMultistepScheduler.from_config(pipe.scheduler.config)
    pipe.save_pretrained(sd_dir, safe_serialization=True)