varunshenoy

Some remarks on Large Language Models

Yoav Goldberg, January 2023

Audience: I assume you heard of chatGPT, maybe played with it a little, and was imressed by it (or tried very hard not to be). And that you also heard that it is "a large language model". And maybe that it "solved natural language understanding". Here is a short personal perspective of my thoughts of this (and similar) models, and where we stand with respect to language understanding.

Intro

Around 2014-2017, right within the rise of neural-network based methods for NLP, I was giving a semi-academic-semi-popsci lecture, revolving around the story that achieving perfect language modeling is equivalent to being as intelligent as a human. Somewhere around the same time I was also asked in an academic panel "what would you do if you were given infinite compute and no need to worry about labour costs" to which I cockily responded "I would train a really huge language model, just to show that it doesn't solve everything!". We

varunshenoy

{
 "A": [
  "18268853773021316895982238355395060664828982807259981678425657768314674662830",
  "1269919676167564086404416777616470309339349656354574148805654990228601892337",
  "1"
 ],
 "B": [
  "9941144731428033919790422095082577639772753191930357251553645255233803162606",
  "12764776385902448502054937720756409916845580452557850472317265293931882470897",
  "1"

varunshenoy

from opendream import opendream
from opendream.layer import Layer

import openai
import os

@opendream.define_op
def dream(prompt: str):
    
    openai.api_key = os.environ["OPENAI_API_KEY"]

varunshenoy

import torch
from diffusers import StableDiffusionInstructPix2PixPipeline, EulerAncestralDiscreteScheduler

from opendream import opendream
from opendream.layer import Layer, ImageLayer

@opendream.define_op
def instruct_pix2pix(image_layer: ImageLayer, prompt, device = "mps"):
    model_id = "timbrooks/instruct-pix2pix"
    pipe = StableDiffusionInstructPix2PixPipeline.from_pretrained(model_id, torch_dtype=torch.float32, safety_checker=None)

varunshenoy

import torch
from diffusers import UniPCMultistepScheduler, ControlNetModel, StableDiffusionControlNetPipeline

from opendream import opendream
from opendream.layer import ImageLayer, Layer
from controlnet_aux import CannyDetector

@opendream.define_op
def controlnet_canny(control_image_layer: ImageLayer, prompt, device: str = "cpu", model_ckpt: str = "runwayml/stable-diffusion-v1-5", batch_size = 1, seed = 42, selected = 0, num_steps = 20, **kwargs):
    canny = CannyDetector()

varunshenoy

import torch
from diffusers import UniPCMultistepScheduler, ControlNetModel, StableDiffusionControlNetPipeline

from opendream import opendream
from opendream.layer import ImageLayer, Layer
from controlnet_aux import OpenposeDetector

@opendream.define_op
def controlnet_openpose(control_image_layer: ImageLayer, prompt, device: str = "cpu", model_ckpt: str = "runwayml/stable-diffusion-v1-5", batch_size = 1, seed = 42, selected = 0, num_steps = 20, **kwargs):
    openpose = OpenposeDetector.from_pretrained("lllyasviel/Annotators")

varunshenoy

import os
import numpy as np
from PIL import Image
from urllib.request import urlretrieve
from segment_anything import SamAutomaticMaskGenerator, sam_model_registry

from opendream import opendream
from opendream.layer import ImageLayer, MaskLayer

def convert_mask_to_layer(mask):

varunshenoy

@opendream.define_op
def superresolution(image_layer: ImageLayer, model_ckpt: str = "runwayml/stable-diffusion-v1-5", batch_size = 1, seed = 42, selected = 0, num_steps = 20, **kwargs):
    import modal
    
    # Use ControlNet + Tile preprocessor
    f = modal.Function.lookup("diffuseqr", "ControlNet.run_inference")
    image = f.call(Layer.pil_to_b64(image_layer.get_image()))[0]
    image = Layer.b64_to_pil(image)
    
    return ImageLayer(image=image)

varunshenoy

from opendream import opendream
from opendream.layer import Layer, ImageLayer, MaskLayer

import openai
import os
import json
from PIL import Image, ImageEnhance, ImageFilter
import pilgram

@opendream.define_op

varunshenoy

import torch
from PIL import Image
from diffusers import ControlNetModel, StableDiffusionControlNetImg2ImgPipeline, UniPCMultistepScheduler

from opendream import opendream
from opendream.layer import ImageLayer, Layer

def resize_for_condition_image(input_image: Image, resolution: int):
    input_image = input_image.convert("RGB")
    W, H = input_image.size