Skip to content

Instantly share code, notes, and snippets.

@patrickgalbraith
Forked from JaySmithWpg/hack.diff
Last active May 12, 2023 00:48
Show Gist options
  • Save patrickgalbraith/74f0996d7c26c34a74fdff92d861fc2c to your computer and use it in GitHub Desktop.
Save patrickgalbraith/74f0996d7c26c34a74fdff92d861fc2c to your computer and use it in GitHub Desktop.
Hack to load custom depth map
diff --git a/modules/processing.py b/modules/processing.py
index 24c537d..0525cfd 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -156,20 +156,41 @@ class StableDiffusionProcessing():
return image_conditioning
def depth2img_image_conditioning(self, source_image):
- # Use the AddMiDaS helper to Format our source image to suit the MiDaS model
- transformer = AddMiDaS(model_type="dpt_hybrid")
- transformed = transformer({"jpg": rearrange(source_image[0], "c h w -> h w c")})
- midas_in = torch.from_numpy(transformed["midas_in"][None, ...]).to(device=shared.device)
- midas_in = repeat(midas_in, "1 ... -> n ...", n=self.batch_size)
-
- conditioning_image = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(source_image))
- conditioning = torch.nn.functional.interpolate(
- self.sd_model.depth_model(midas_in),
- size=conditioning_image.shape[2:],
- mode="bicubic",
- align_corners=False,
- )
-
+ conditioning = None
+
+ try:
+ script_dir = os.path.dirname(__file__)
+ rel_path = "../depthmap/current.png"
+ depth_img = Image.open(os.path.join(script_dir, rel_path))
+ depth_img = depth_img.convert("L")
+ depth_img = np.expand_dims(depth_img, axis=0)
+ depth_img = np.expand_dims(depth_img, axis=0).repeat(self.batch_size, axis=0)
+ depth_img = torch.from_numpy(depth_img)
+ depth_img = 2. * depth_img - 1.
+ depth_img = depth_img.to(shared.device)
+
+ conditioning_image = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(source_image))
+ conditioning = torch.nn.functional.interpolate(
+ depth_img,
+ size=conditioning_image.shape[2:],
+ mode="bicubic",
+ align_corners=False,
+ )
+ except:
+ # Use the AddMiDaS helper to Format our source image to suit the MiDaS model
+ transformer = AddMiDaS(model_type="dpt_hybrid")
+ transformed = transformer({"jpg": rearrange(source_image[0], "c h w -> h w c")})
+ midas_in = torch.from_numpy(transformed["midas_in"][None, ...]).to(device=shared.device)
+ midas_in = repeat(midas_in, "1 ... -> n ...", n=self.batch_size)
+
+ conditioning_image = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(source_image))
+ conditioning = torch.nn.functional.interpolate(
+ self.sd_model.depth_model(midas_in),
+ size=conditioning_image.shape[2:],
+ mode="bicubic",
+ align_corners=False,
+ )
+
(depth_min, depth_max) = torch.aminmax(conditioning)
conditioning = 2. * (conditioning - depth_min) / (depth_max - depth_min) - 1.
return conditioning
@patrickgalbraith
Copy link
Author

patrickgalbraith commented Jan 13, 2023

Checks if there is a file at "../depthmap/current.png" and uses that otherwise falls back to MiDaS depth map.

Apply to webui using:

git apply hack.diff

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment