Skip to content

Instantly share code, notes, and snippets.

@jschoormans
Created December 19, 2022 22:37
Show Gist options
  • Save jschoormans/06c763fe832dca1c24efc875610b5b6f to your computer and use it in GitHub Desktop.
Save jschoormans/06c763fe832dca1c24efc875610b5b6f to your computer and use it in GitHub Desktop.
HEIGHT=768
WIDTH=1024
OVERLAP = 256
PROMPT = "360 degree equirectangular panorama photograph, alps, mountains, landscape,\
trending on artstation, cinematic composition, beautiful lighting, hyper detailed, 8 k, photo, photography"
INPAINT_FIRST = False # use inpainting to fill in the first frame (force blue, green etc)
USE_OPEN_JOURNEY = False # open journey is nice, but inpainting works with vanilla SD - sometimes the difference is too big.
#Also open journey does not work with all resolution sizes
# First image - no overlap here.
if INPAINT_FIRST == False:
if USE_OPEN_JOURNEY:
model = replicate.models.get("prompthero/openjourney")
version = model.versions.get("9936c2001faa2194a261c01381f90e65261879985476014a0a37a334593a05eb")
else:
model = replicate.models.get("stability-ai/stable-diffusion")
version = model.versions.get("6359a0cab3ca6e4d3320c33d79096161208e9024d174b2311e5a21b6c7e1131c")
output = version.predict(prompt=PROMPT, width=WIDTH, height=HEIGHT)
else:
# use inpainting and force the bottom and top pixels to be blue, green, respectively
model = replicate.models.get("andreasjansson/stable-diffusion-inpainting")
version = model.versions.get("8eb2da8345bee796efcd925573f077e36ed5fb4ea3ba240ef70c23cf33f0d848")
img_initial = Image.new('RGB', (WIDTH, HEIGHT), color = (0, 0, 255))
# make bottom half black, top half blue
img_initial = np.array(img_initial)
img_initial[HEIGHT//2:,:] = (0,255,0)
img_initial[:HEIGHT//2,:] = (0,0,255)
img_initial = Image.fromarray(img_initial)
img_initial.save("initial.png")
mask_initial = Image.new('L', (WIDTH, HEIGHT), color = 255)
mask_initial = np.array(mask_initial)
mask_initial[0:5,:] = 0
mask_initial[-5:,:] = 0
mask_initial = Image.fromarray(mask_initial)
mask_initial.save("mask_initial.png")
input_object = open("initial.png", "rb")
mask_object = open("mask_initial.png", "rb")
output = version.predict(prompt=PROMPT, width=WIDTH, height=HEIGHT, image=input_object, mask=mask_object)
print(output)
# download the iamge from the url at output[0]
image = requests.get(output[0]).content
image_file = Image.open(io.BytesIO(image))
#save image as a file
tempfile = open("output.jpg", "wb")
tempfile.write(image)
tempfile.close()
# create a mask image of size 1024 x 512 that is all black
mask_img = Image.new('L', (WIDTH, HEIGHT), color = 'white')
# add white borders of width 128 on either side
mask_img.paste(0, (0, 0, OVERLAP, HEIGHT))
mask_img.paste(0, (WIDTH-OVERLAP, 0, WIDTH, HEIGHT))
mask_img.save('mask.jpg')
# do second model
model = replicate.models.get("andreasjansson/stable-diffusion-inpainting")
version = model.versions.get("8eb2da8345bee796efcd925573f077e36ed5fb4ea3ba240ef70c23cf33f0d848")
# open output.jpg
file_object = open("output.jpg", "rb")
# generate image from file_object
input_image = Image.open(file_object)
# save left_most 128 pixels to temp image
temp_image = input_image.crop((0, 0, OVERLAP, HEIGHT))
# for input_image, put right-most 128 pixels to the left side
input_image.paste(input_image.crop((WIDTH-OVERLAP, 0, WIDTH, HEIGHT)), (0, 0, OVERLAP, HEIGHT))
# put temp_image to the right side
input_image.paste(temp_image, (WIDTH-OVERLAP, 0, WIDTH, HEIGHT))
# save the image
input_image.save('input.jpg')
input_object = open("input.jpg", "rb")
mask_object = open("mask.jpg", "rb")
output = version.predict(prompt=PROMPT, width=WIDTH, height=HEIGHT, image=input_object, mask=mask_object)
image = requests.get(output[0]).content
tempfile = open("final_output.jpg", "wb")
tempfile.write(image)
tempfile.close()
# merge output.jpg and final_output.jpg
output_image = Image.open("output.jpg")
final_output_image = Image.open("final_output.jpg")
combined_image = Image.new('RGB', (2*WIDTH-2*OVERLAP, HEIGHT))
combined_image.paste(output_image, (0, 0))
combined_image.paste(final_output_image, (WIDTH-OVERLAP, 0))
combined_image.save('combined.jpg')
# send combined image to esran model to upsample it
img_to_upsample = open("combined.jpg", "rb")
model = replicate.models.get("nightmareai/real-esrgan")
version = model.versions.get("42fed1c4974146d4d2414e2be2c5277c7fcf05fcc3a73abf41610695738c1d7b")
output = version.predict(image=img_to_upsample)
# save image
img = requests.get(output).content
tempfile = open("final_output_upsampled.jpg", "wb")
tempfile.write(img)
tempfile.close()
# compress image
img = Image.open("final_output_upsampled.jpg")
img.save("final_output_upsampled.jpg", optimize=True, quality=50)
# open panellum site
url_pannellum = "https://cdn.pannellum.org/2.5/pannellum.htm#panorama={}&author=Jasper%20Schoormans&autoLoad=true".format(output)
@kartiksah406
Copy link

jrj

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment