convert sd15 ckpt + ic-light to ip2p model

readme.md

convert_ic-light.py will take a regular 1.5 ckpt and the ic light model and combine them into a single diffusers format ip2p model. Then use the normal convert_diffuser_to_original_stable_diffusion.py script from diffusers to convert it back to a ckpt for comfy.

The results are slightly different but I think that may be due to ELLA vs normal CLIP. The lighting latent might also need to be scaled depending on the light pattern/colors, not sure yet why there's a difference there.

https://github.com/huggingface/diffusers/blob/main/scripts/convert_diffusers_to_original_stable_diffusion.py

convert_ic-light.py

import torch
import safetensors
from diffusers.pipelines.stable_diffusion.convert_from_ckpt import download_from_original_stable_diffusion_ckpt

ckpt_path = "./realisticVisionV5_1.safetensors"

ic_path = "./iclight_sd15_fc.safetensors"

dump_path = "./ic-light"

pipe = download_from_original_stable_diffusion_ckpt(
    checkpoint_path_or_dict=ckpt_path,
    original_config_file=None,
    config_files=None,
    image_size=None,
    prediction_type="epsilon",
    model_type=None,
    extract_ema=False,
    scheduler_type="pndm",
    num_in_channels=None,
    upcast_attention=False,
    from_safetensors=True,
    device="cuda",
    controlnet=None,
    load_safety_checker=False,
    )


in_channels = 8
out_channels = pipe.unet.conv_in.out_channels
pipe.unet.register_to_config(in_channels=in_channels)


with torch.no_grad():
    new_conv_in = torch.nn.Conv2d(
        in_channels, 
        out_channels, 
        pipe.unet.conv_in.kernel_size, 
        pipe.unet.conv_in.stride, 
        pipe.unet.conv_in.padding
        )
    
    new_conv_in.weight.zero_()
    new_conv_in.weight[:, :4, :, :].copy_(pipe.unet.conv_in.weight)
    new_conv_in.bias = pipe.unet.conv_in.bias
    pipe.unet.conv_in = new_conv_in


sd_offset = safetensors.torch.load_file(ic_path)
sd_origin = pipe.unet.state_dict()
keys = sd_origin.keys()
sd_merged = {k: sd_origin[k] + sd_offset[k] for k in sd_origin.keys()}
pipe.unet.load_state_dict(sd_merged, strict=True)
del sd_offset, sd_origin, sd_merged, keys


pipe.to(dtype=torch.float16)
pipe.save_pretrained(dump_path, safe_serialization=True)


'''
python convert_diffusers_to_original_stable_diffusion.py --model_path ic-light --checkpoint_path ic-light_sd15_rv51_fc_v2.safetensors --half --use_safetensors
'''