Mastodon for fly.io with SendGrid + Cloudflare R2

1__fly.rails.toml

app = "don-example-com-rails"
kill_signal = "SIGTERM"
kill_timeout = 60
primary_region = "nrt"

[build]
  image = "tootsuite/mastodon:v4.2.0-rc1"
  [build.args]
    RAILS_ENV = "production"

[deploy]
  release_command = "bundle exec rails db:prepare"

[env]
  DB_HOST = "don-example-com-postgres.internal"
  DB_PORT = 5432
  DB_NAME= "mastodon_production"
  DEFAULT_LOCALE = "ja"
  LOCAL_DOMAIN = "don.example.com"
  RAILS_ENV = "production"
  RAILS_SERVE_STATIC_FILES = "true"
  REDIS_HOST = "fly-don-example-com-redis.upstash.io"
  REDIS_PORT = 6379
  RUBY_YJIT_ENABLE = 1
  S3_ALIAS_HOST = "don-files.example.com"
  S3_BUCKET = "donexamplecom"
  S3_ENABLED = "true"
  S3_ENDPOINT = "https://xxxxx.r2.cloudflarestorage.com" # Cloudflare R2 is cheap than AWS S3
  #S3_FORCE_SINGLE_REQUEST=true
  S3_HOSTNAME = "xxxxx.r2.cloudflarestorage.com"
  S3_PERMISSION = "private"
  S3_PROTOCOL = "https"
  S3_REGION = "auto"
  #SIDEKIQ_CONCURRENCY=2
  SINGLE_USER_MODE = "false"
  SMTP_AUTH_METHOD = "plain"
  SMTP_FROM_ADDRESS = "don.example.com Mastodon <notifications@don.example.com>"
  SMTP_OPENSSL_VERIFY_MODE = "peer"
  SMTP_PORT = 587
  SMTP_SERVER = "smtp.sendgrid.net"
  STREAMING_API_BASE_URL = "wss://don-stream.example.com"
  TRUSTED_PROXY_IP = "66.241.112.0/20,2a09:8280::/29"

[processes]
  sidekiq = "bundle exec sidekiq"
  app     = "bundle exec rails s -p 3000"

[http_service]
  processes = ["app"]
  internal_port = 3000
  # force_https = true
  # auto_stop_machines = true
  auto_start_machines = true
  min_machines_running = 1
  [http_service.concurrency]
    type = "requests"
    soft_limit = 200
    hard_limit = 250
  [[http_service.checks]]
    grace_period = "15s"
    interval = "10s"
    method = "GET"
    timeout = "5s"
    path = "/health"

[[statics]]
  guest_path = "/opt/mastodon/public"
  url_prefix = "/"

2__fly.stream.toml

app = "don-example-com-stream"
kill_signal = "SIGINT"
kill_timeout = 5
primary_region = "nrt"

[build]
  image = "tootsuite/mastodon:v4.2.0-rc1"
  [build.args]
    NODE_ENV = "production"

[env]
  DB_HOST = "xxx-postgres.internal"
  DB_PORT = 5432
  DB_NAME= "mastodon_production"
  LOCAL_DOMAIN = "don.example.com"
  NODE_ENV = "production"
  REDIS_HOST = "xxx.upstash.io" # hostname or local ipv6
  REDIS_PORT = 6379
  STREAMING_API_BASE_URL = "wss://don-stream.example.com"
  TRUSTED_PROXY_IP = "66.241.112.0/20,2a09:8280::/29" # fly.io's reverse proxies

[processes]
  streaming = "node ./streaming"

[http_service]
  processes = ["streaming"]
  internal_port = 4000
  # auto_stop_machines = true
  auto_start_machines = true
  min_machines_running = 1
  [http_service.concurrency]
    type = "connections"
    soft_limit = 100
    hard_limit = 150
  [[http_service.checks]]
    grace_period = "5s"
    interval = "10s"
    method = "GET"
    timeout = "5s"
    path = "/api/v1/streaming/health"

3__example.env.secret

AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=

DB_USER=postgres
DB_PASS=[copy from `flyctl postgres create`]

OTP_SECRET=

REDIS_PASSWORD=[copy from `flyctl redis create`]

SECRET_KEY_BASE=
SMTP_LOGIN=apikey
SMTP_PASSWORD=SG.xxx

VAPID_PRIVATE_KEY=
VAPID_PUBLIC_KEY=

Low cost Mastodon instance on Fly.io

# create apps in organization "don-example-com"
flyctl apps create don-example-com-rails -o "don-example-com"
flyctl apps create don-example-com-stream -o "don-example-com"
fly postgres create # 1 instance is enough for personal use.
flyctl redis create # you need to buy $10 paid plan. broad bandwidth is needed for mastodon.

# import secrets from .env.secret
flyctl secrets import -c fly.rails.toml < .env.secret
flyctl secrets import -c fly.stream.toml < .env.secret

# scale cpu & memory
# at least 1GB memory is needed.
# sidekiq worker needs lots of cpu & memory resources. 
# if you use slower cpus, media convert takes much longer. I recommend at least shared-cpu-4x.
flyctl scale vm shared-cpu-1x --vm-memory 2048 --process-group app -a don-example-com-rails
flyctl scale vm shared-cpu-4x --vm-memory 3072 --process-group sidekiq -a don-example-com-rails

# streaming server is very light app, so you can keep initial setup of shared-cpu-1x & 256MB RAMs.

# deploying
# you may need to disable http_service.checks for the first time because health checking will be failed if no database is created
flyctl deploy -c fly.stream.toml
flyctl deploy -c fly.rails.toml

If you use Cloudflare for front-facing CDN, you don't need to allocate static IPv4. Point your domain to the fly.io app's ipv6 address as AAAA with Cloudflare proxy. (no need to set A record)

I decided to expose the WebSocket stream server to the Internet because the free Cloudflare plan can't have many WebSocket streams. So, you need to issue a TLS certificate in fly.io. Set acme-challenge to your DNS provider. Set CNAME for the streaming server and Uncheck the Proxy in Cloudflare DNS settings (DNS only).

just like this:

These commands are based on my memory, and maybe some lacking commands are needed 🫠.
domains are:

• don.example.com for mastodon app
• don-stream.example.com for mastodon streaming server
• don-sg.example.com for SendGrid
• don-files.example.com for media files (AWS S3 or Cloudflare R2)

if you use a slower CPU machine, you may need to set longer http_service.checks.grace_period because container boot time will take much longer on slow machines.