docker-compose.yml

---
version: '3.4'

services:
  postgresql:
    image: docker.io/library/postgres:12-alpine
    restart: unless-stopped
    network_mode: host
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"]
      start_period: 20s
      interval: 30s
      retries: 5
      timeout: 5s
    volumes:
      - database:/var/lib/postgresql/data
    environment:
      - POSTGRES_PASSWORD=${PG_PASS:?database password required}
      - POSTGRES_USER=${PG_USER:-authentik}
      - POSTGRES_DB=${PG_DB:-authentik}
    env_file:
      - .env
  redis:
    image: docker.io/library/redis:6.2.6-alpine
    command: --save 60 1 --loglevel warning
    restart: unless-stopped
    network_mode: host
    healthcheck:
      test: ["CMD-SHELL", "redis-cli ping | grep PONG"]
      start_period: 20s
      interval: 30s
      retries: 5
      timeout: 3s
    volumes:
      - redis:/data
  server:
    image: ${AUTHENTIK_IMAGE:-ghcr.io/goauthentik/server}:${AUTHENTIK_TAG:-2022.8.2}
    restart: unless-stopped
    command: server
    network_mode: host
    environment:
      AUTHENTIK_REDIS__HOST: ${REDIS_HOST:-localhost}
      AUTHENTIK_REDIS__PORT: ${REDIS_PORT:-6379}
      AUTHENTIK_REDIS__PASSWORD: ${REDIS_PASSWORD:-""}
      AUTHENTIK_REDIS__CACHE_DB: "0"
      AUTHENTIK_REDIS__MESSAGE_QUEUE_DB: "0"
      AUTHENTIK_REDIS__WS_DB: "0"
      AUTHENTIK_POSTGRESQL__HOST: localhost
      AUTHENTIK_POSTGRESQL__USER: ${PG_USER:-authentik}
      AUTHENTIK_POSTGRESQL__NAME: ${PG_DB:-authentik}
      AUTHENTIK_POSTGRESQL__PASSWORD: ${PG_PASS}
    volumes:
      - ./media:/media
      - ./custom-templates:/templates
      - geoip:/geoip
    env_file:
      - .env
  worker:
    image: ${AUTHENTIK_IMAGE:-ghcr.io/goauthentik/server}:${AUTHENTIK_TAG:-2022.8.2}
    restart: unless-stopped
    command: worker
    network_mode: host
    environment:
      AUTHENTIK_REDIS__HOST: ${REDIS_HOST:-localhost}
      AUTHENTIK_REDIS__PORT: ${REDIS_PORT:-6379}
      AUTHENTIK_REDIS__PASSWORD: ${REDIS_PASSWORD:-""}
      AUTHENTIK_REDIS__CACHE_DB: "0"
      AUTHENTIK_REDIS__MESSAGE_QUEUE_DB: "0"
      AUTHENTIK_REDIS__WS_DB: "0"
      AUTHENTIK_POSTGRESQL__HOST: localhost
      AUTHENTIK_POSTGRESQL__USER: ${PG_USER:-authentik}
      AUTHENTIK_POSTGRESQL__NAME: ${PG_DB:-authentik}
      AUTHENTIK_POSTGRESQL__PASSWORD: ${PG_PASS}
    user: root
    volumes:
      - ./media:/media
      - ./certs:/certs
      - /var/run/docker.sock:/var/run/docker.sock
      - ./custom-templates:/templates
      - geoip:/geoip
    env_file:
      - .env
  geoipupdate:
    image: "maxmindinc/geoipupdate:latest"
    volumes:
      - "geoip:/usr/share/GeoIP"
    environment:
      GEOIPUPDATE_EDITION_IDS: "GeoLite2-City"
      GEOIPUPDATE_FREQUENCY: "8"
    env_file:
      - .env

volumes:
  database:
    driver: local
  redis:
    driver: local
  geoip:
    driver: local

example-error-output.log

worker_1       | {"event": "Task failure", "exc": "ResponseError('Wrong number of response items from pipeline execution')", "level": "warning", "logger": "authentik.root.celery", "pid": 30, "timestamp": "2022-09-12T20:11:55.271860"}
worker_1       | {"event": "/usr/local/lib/python3.10/site-packages/celery/app/trace.py:660: RuntimeWarning: Exception raised outside body: ResponseError('Wrong number of response items from pipeline execution'):\nTraceback (most recent call last):\n  File \"/usr/local/lib/python3.10/site-packages/celery/app/trace.py\", line 518, in trace_task\n    task.backend.mark_as_done(\n  File \"/usr/local/lib/python3.10/site-packages/celery/backends/base.py\", line 162, in mark_as_done\n    self.store_result(task_id, result, state, request=request)\n  File \"/usr/local/lib/python3.10/site-packages/celery/backends/base.py\", line 528, in store_result\n    self._store_result(task_id, result, state, traceback,\n  File \"/usr/local/lib/python3.10/site-packages/celery/backends/base.py\", line 962, in _store_result\n    self._set_with_state(self.get_key_for_task(task_id), self.encode(meta), state)\n  File \"/usr/local/lib/python3.10/site-packages/celery/backends/base.py\", line 839, in _set_with_state\n    return self.set(key, value)\n  File \"/usr/local/lib/python3.10/site-packages/celery/backends/redis.py\", line 392, in set\n    return self.ensure(self._set, (key, value), **retry_policy)\n  File \"/usr/local/lib/python3.10/site-packages/celery/backends/redis.py\", line 376, in ensure\n    return retry_over_time(\n  File \"/usr/local/lib/python3.10/site-packages/kombu/utils/functional.py\", line 312, in retry_over_time\n    return fun(*args, **kwargs)\n  File \"/usr/local/lib/python3.10/site-packages/celery/backends/redis.py\", line 401, in _set\n    pipe.execute()\n  File \"/usr/local/lib/python3.10/site-packages/redis/client.py\", line 2078, in execute\n    return conn.retry.call_with_retry(\n  File \"/usr/local/lib/python3.10/site-packages/redis/retry.py\", line 46, in call_with_retry\n    return do()\n  File \"/usr/local/lib/python3.10/site-packages/redis/client.py\", line 2079, in <lambda>\n    lambda: execute(conn, stack, raise_on_error),\n  File \"/usr/local/lib/python3.10/site-packages/redis/client.py\", line 1968, in _execute_transaction\n    raise ResponseError(\nredis.exceptions.ResponseError: Wrong number of response items from pipeline execution\n\n  warn(RuntimeWarning(\n", "level": "warning", "logger": "py.warnings", "timestamp": 1663013515.2726576}
worker_1       | {"event": "Task failure", "exc": "ResponseError('Wrong number of response items from pipeline execution')", "level": "warning", "logger": "authentik.root.celery", "pid": 30, "timestamp": "2022-09-12T20:11:55.333918"}
worker_1       | {"event": "Task authentik.outposts.tasks.outpost_controller_all[5651a109-82b0-4fab-85c7-4ce3246d0710] raised unexpected: ResponseError('Wrong number of response items from pipeline execution')", "exc_info": ["<class 'redis.exceptions.ResponseError'>", "ResponseError('Wrong number of response items from pipeline execution')", "<billiard.einfo.Traceback object at 0x7f6dfe03be50>"], "level": "error", "logger": "celery.app.trace", "timestamp": 1663013515.334216}
worker_1       | {"event": "Task finished", "level": "info", "logger": "authentik.root.celery", "pid": 32, "state": "SUCCESS", "task_id": "da7a294d-f3af-4f5b-8d2d-0c7515d47ea2", "task_name": "proxy_set_defaults", "timestamp": "2022-09-12T20:11:55.334688"}
worker_1       | {"event": "Task failure", "exc": "ResponseError('Wrong number of response items from pipeline execution')", "level": "warning", "logger": "authentik.root.celery", "pid": 32, "timestamp": "2022-09-12T20:11:55.337344"}
worker_1       | {"event": "/usr/local/lib/python3.10/site-packages/celery/app/trace.py:660: RuntimeWarning: Exception raised outside body: ResponseError('Wrong number of response items from pipeline execution'):\nTraceback (most recent call last):\n  File \"/usr/local/lib/python3.10/site-packages/celery/app/trace.py\", line 518, in trace_task\n    task.backend.mark_as_done(\n  File \"/usr/local/lib/python3.10/site-packages/celery/backends/base.py\", line 162, in mark_as_done\n    self.store_result(task_id, result, state, request=request)\n  File \"/usr/local/lib/python3.10/site-packages/celery/backends/base.py\", line 528, in store_result\n    self._store_result(task_id, result, state, traceback,\n  File \"/usr/local/lib/python3.10/site-packages/celery/backends/base.py\", line 962, in _store_result\n    self._set_with_state(self.get_key_for_task(task_id), self.encode(meta), state)\n  File \"/usr/local/lib/python3.10/site-packages/celery/backends/base.py\", line 839, in _set_with_state\n    return self.set(key, value)\n  File \"/usr/local/lib/python3.10/site-packages/celery/backends/redis.py\", line 392, in set\n    return self.ensure(self._set, (key, value), **retry_policy)\n  File \"/usr/local/lib/python3.10/site-packages/celery/backends/redis.py\", line 376, in ensure\n    return retry_over_time(\n  File \"/usr/local/lib/python3.10/site-packages/kombu/utils/functional.py\", line 312, in retry_over_time\n    return fun(*args, **kwargs)\n  File \"/usr/local/lib/python3.10/site-packages/celery/backends/redis.py\", line 401, in _set\n    pipe.execute()\n  File \"/usr/local/lib/python3.10/site-packages/redis/client.py\", line 2078, in execute\n    return conn.retry.call_with_retry(\n  File \"/usr/local/lib/python3.10/site-packages/redis/retry.py\", line 46, in call_with_retry\n    return do()\n  File \"/usr/local/lib/python3.10/site-packages/redis/client.py\", line 2079, in <lambda>\n    lambda: execute(conn, stack, raise_on_error),\n  File \"/usr/local/lib/python3.10/site-packages/redis/client.py\", line 1968, in _execute_transaction\n    raise ResponseError(\nredis.exceptions.ResponseError: Wrong number of response items from pipeline execution\n\n  warn(RuntimeWarning(\n", "level": "warning", "logger": "py.warnings", "timestamp": 1663013515.337992}
worker_1       | {"event": "Task failure", "exc": "ResponseError('Wrong number of response items from pipeline execution')", "level": "warning", "logger": "authentik.root.celery", "pid": 32, "timestamp": "2022-09-12T20:11:55.451948"}
worker_1       | {"event": "Task authentik.providers.proxy.tasks.proxy_set_defaults[da7a294d-f3af-4f5b-8d2d-0c7515d47ea2] raised unexpected: ResponseError('Wrong number of response items from pipeline execution')", "exc_info": ["<class 'redis.exceptions.ResponseError'>", "ResponseError('Wrong number of response items from pipeline execution')", "<billiard.einfo.Traceback object at 0x7f6dfe0c8550>"], "level": "error", "logger": "celery.app.trace", "timestamp": 1663013515.4522877}

reproduction-steps.sh

# Create an empty folder somewhere, and cd into it.
mkdir /tmp/authentik-celery-stock-redis-vs-upstash
cd /tmp/authentik-celery-stock-redis-vs-upstash

# You can also use openssl instead: `openssl rand -base64 36`
sudo apt-get install -y pwgen

# This just generates some keys the application needs.
echo "PG_PASS=$(pwgen -s 40 1)" >> .env
echo "AUTHENTIK_SECRET_KEY=$(pwgen -s 50 1)" >> .env

# Now grab the Docker Compose manifest from this gist and download it as docker-compose.yml.
curl ...

# The Docker Compose manifest defines Postgres (needed by the app, Authentik) and Redis as external dependencies. It uses
# host networking because that made it possible for me to more easily do packet captures, but you can obviously tweak this
# for whatever works best for you.

# The first run, if you want a baseline, would be to use the provided Redis container. We'll run the "server" service first
# to pre-populate the database and make sure it's happy before running the "worker" service, which is the component that
# actually experiences errors when using Upstash Redis.

# You should see a bunch of output from the "server" service, including a bunch of applying migrations and so on. Just wait
# for this to complete and the output to quiesce, which should take around 30-45 seconds.
docker-compose up postgresql redis server

# Now you can just Ctrl-C out of that, and run the actual worker. You should see a bunch of output streaming in the console
# talking about "Task started" and "Task finished". If you're seeing that, and you're not seeing gnarly Python stacktraces,
# then things are working as expected.
docker-compose up postgresql redis worker

# Now, let's run against Upstash Redis. We'll just clean up all the volumes first, so that we can repeat the above steps.
#
# As well, this time we'll override some environment variables to specify the host/port/password for Upstart Redis. You
# can obviously see the defaults in docker-compose.yml, but I was emulating this all using `fly redis connect`, so my
# example instructions here will reflect that fact:
docker-compose down -v

# Again, do our first run which bootstraps the database. Wait for the output to eventually quiesce, etc, and then you
# can just Ctrl-C to move on.
REDIS_HOST="localhost" REDIS_PORT="16379" REDIS_PASSWORD="asjdlkadlkasjdalk" docker-compose up postgresql server

# Now we'll run the worker pointed at Upstash Redis which should, after starting up and doing some bootstrap activities,
# start spewing a bunch of errors related to the "wrong number of response items from pipeline execution".
REDIS_HOST="localhost" REDIS_PORT="16379" REDIS_PASSWORD="asjdlkadlkasjdalk" docker-compose up postgresql worker