README.md

Deploying CLIP on Fly

I got this to work. With the Dockerfile and fly.toml in the current directory run this:

flyctl apps create clip-datasette-on-fly
flyctl deploy

(You'll need a different app name, which you will have to update in fly.toml too)

This won't quite work out of the box though, it needs more CPU and memory. This should get it working:

flyctl scale vm shared-cpu-2x
flyctl scale memory 4096

It's running datasette-scale-to-zero - without it that would cost about $22/month.

Even that wasn't enough memory, I ended up bumping it to:

flyctl scale vm performance-1x
flyctl scale memory 8192

Which is $61.02/month if it runs all the time! So I added datasette-block-robots to hopefully stop it from being crawled.

Dockerfile

FROM python:3.11.0-slim-bullseye
COPY . /app
WORKDIR /app

ENV DATASETTE_SECRET '4c3032ea74b4c8354e5476f49095e2cf690494500b4254968bb9f2af84b22afc'
RUN pip install -U torch torchvision --extra-index-url https://download.pytorch.org/whl/cpu
RUN pip install -U datasette datasette-scale-to-zero datasette-block-robots datasette-llm-embed llm-clip
# Download the CLIP model
RUN llm embed -m clip -c 'hello world'

# Create metadata.json file
RUN echo '{"plugins": {"datasette-scale-to-zero": {"duration": "10m"}}}' > /app/metadata.json

# Expose and start the service
ENV PORT 8080
EXPOSE 8080
CMD datasette serve --host 0.0.0.0 --cors --port $PORT -m /app/metadata.json

fly.toml

app = "clip-datasette-on-fly"

[[services]]
  internal_port = 8080
  protocol = "tcp"

  [services.concurrency]
    hard_limit = 25
    soft_limit = 20

  [[services.ports]]
    handlers = ["http"]
    port = 80

  [[services.ports]]
    handlers = ["tls", "http"]
    port = 443

  [[services.tcp_checks]]
    interval = 10000
    timeout = 2000

https://clip-datasette-on-fly.fly.dev/_memory?sql=select+hex%28llm_embed%28%27clip%27%2C+%3Aq%29%29&q=this+is+a+longer+sentence

@simonw since flyctl v0.1.125 it is possible to set Machine sizes in fly.toml

app = "clip-datasette-on-fly"

[[vm]]
  size = "shared-cpu-2x"
  memory = "4gb"

[http_service]
  internal_port = 8080

[checks.up]
  port = 8080
  type = "tcp"
  interval = "10s"
  timeout = "2s"