T2I-Adapter with Refiners: prerequisites

t2i_adapter_refiners.sh

# T2I-Adapter: Learning Adapters to Dig out More Controllable Ability for Text-to-Image Diffusion Models
# https://github.com/TencentARC/T2I-Adapter
# https://huggingface.co/blog/t2i-sdxl-adapters

# (1) Install (see also https://github.com/finegrain-ai/refiners#install)
git clone https://github.com/finegrain-ai/refiners.git
cd refiners
poetry install --all-extras
poetry run pip install --upgrade torch torchvision

# (2) Convert weights into Refiners format
poetry shell
python scripts/conversion/convert_transformers_clip_text_model.py --from "stabilityai/stable-diffusion-xl-base-1.0" --subfolder2 text_encoder_2 --to clip_text_xl.safetensors --half
python scripts/conversion/convert_diffusers_unet.py --from "stabilityai/stable-diffusion-xl-base-1.0" --to unet_xl.safetensors --half
python scripts/conversion/convert_diffusers_autoencoder_kl.py --from "madebyollin/sdxl-vae-fp16-fix" --subfolder "" --to lda_xl.safetensors --half
python scripts/conversion/convert_diffusers_t2i_adapter.py --from "TencentARC/t2i-adapter-canny-sdxl-1.0" --to t2i_canny_xl.safetensors --half

# (3) Download demo condition image
curl -O https://huggingface.co/Adapter/t2iadapter/resolve/main/figs_SDXLV1.0/cond_canny.png