Nginx config for scaling matrix synapse server via workers

nginx.conf

# Matrix Synapse workers example config 

# backends
upstream synapse_master {
	server 192.19.18.12:38008;
}

upstream synapse_federation {
	server 192.19.18.12:8083;
}

upstream synapse_client {
	server 192.19.18.12:8084;
}

# map urls to backend
# based on list from 
# https://github.com/matrix-org/synapse/blob/develop/docs/workers.md
map $request_uri $synapse_backend {
	default synapse_master;

	# Sync requests
	~*^/_matrix/client/(r0|v3)/sync$ synapse_client;
	~*^/_matrix/client/(api/v1|r0|v3)/events$ synapse_client;
	~*^/_matrix/client/(api/v1|r0|v3)/initialSync$ synapse_client;
	~*^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$ synapse_client;

	# Federation requests
	~*^/_matrix/federation/v1/event/ synapse_federation;
	~*^/_matrix/federation/v1/state/ synapse_federation;
	~*^/_matrix/federation/v1/state_ids/ synapse_federation;
	~*^/_matrix/federation/v1/backfill/ synapse_federation;
	~*^/_matrix/federation/v1/get_missing_events/ synapse_federation;
	~*^/_matrix/federation/v1/publicRooms synapse_federation;
	~*^/_matrix/federation/v1/query/ synapse_federation;
	~*^/_matrix/federation/v1/make_join/ synapse_federation;
	~*^/_matrix/federation/v1/make_leave/ synapse_federation;
	~*^/_matrix/federation/(v1|v2)/send_join/ synapse_federation;
	~*^/_matrix/federation/(v1|v2)/send_leave/ synapse_federation;
	~*^/_matrix/federation/(v1|v2)/invite/ synapse_federation;
	~*^/_matrix/federation/v1/event_auth/ synapse_federation;
	~*^/_matrix/federation/v1/exchange_third_party_invite/ synapse_federation;
	~*^/_matrix/federation/v1/user/devices/ synapse_federation;
	~*^/_matrix/key/v2/query synapse_federation;
	~*^/_matrix/federation/v1/hierarchy/ synapse_federation;

	# Inbound federation transaction request
	~*^/_matrix/federation/v1/send/ synapse_federation;

	# Client API requests
	~*^/_matrix/client/(api/v1|r0|v3|unstable)/createRoom$ synapse_client;
	~*^/_matrix/client/(api/v1|r0|v3|unstable)/publicRooms$ synapse_client;
	~*^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/joined_members$ synapse_client;
	~*^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/context/.*$ synapse_client;
	~*^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/members$ synapse_client;
	~*^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/state$ synapse_client;
	~*^/_matrix/client/v1/rooms/.*/hierarchy$ synapse_client;
	~*^/_matrix/client/unstable/org.matrix.msc2716/rooms/.*/batch_send$ synapse_client;
	~*^/_matrix/client/unstable/im.nheko.summary/rooms/.*/summary$ synapse_client;
	~*^/_matrix/client/(r0|v3|unstable)/account/3pid$ synapse_client;
	~*^/_matrix/client/(r0|v3|unstable)/account/whoami$ synapse_client;
	~*^/_matrix/client/(r0|v3|unstable)/devices$ synapse_client;
	~*^/_matrix/client/versions$ synapse_client;
	~*^/_matrix/client/(api/v1|r0|v3|unstable)/voip/turnServer$ synapse_client;
	~*^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/event/ synapse_client;
	~*^/_matrix/client/(api/v1|r0|v3|unstable)/joined_rooms$ synapse_client;
	~*^/_matrix/client/(api/v1|r0|v3|unstable)/search$ synapse_client;

	# Encryption requests
	~*^/_matrix/client/(r0|v3|unstable)/keys/query$ synapse_client;
	~*^/_matrix/client/(r0|v3|unstable)/keys/changes$ synapse_client;
	~*^/_matrix/client/(r0|v3|unstable)/keys/claim$ synapse_client;
	~*^/_matrix/client/(r0|v3|unstable)/room_keys/ synapse_client;

	# Registration/login requests
	~*^/_matrix/client/(api/v1|r0|v3|unstable)/login$ synapse_client;
	~*^/_matrix/client/(r0|v3|unstable)/register$ synapse_client;
	~*^/_matrix/client/v1/register/m.login.registration_token/validity$ synapse_client;

	# Event sending requests
	~*^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/redact synapse_client;
	~*^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/send synapse_client;
	~*^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/state/ synapse_client;
	~*^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$ synapse_client;
	~*^/_matrix/client/(api/v1|r0|v3|unstable)/join/ synapse_client;
	~*^/_matrix/client/(api/v1|r0|v3|unstable)/profile/ synapse_client;

	# Account data requests
	~*^/_matrix/client/(r0|v3|unstable)/.*/tags synapse_client;
	~*^/_matrix/client/(r0|v3|unstable)/.*/account_data synapse_client;

	# Receipts requests
	#~*^/_matrix/client/(r0|v3|unstable)/rooms/.*/receipt synapse_client;
	#~*^/_matrix/client/(r0|v3|unstable)/rooms/.*/read_markers synapse_client;

	# Presence requests
	~*^/_matrix/client/(api/v1|r0|v3|unstable)/presence/ synapse_client;

	# User directory search requests
	~*^/_matrix/client/(r0|v3|unstable)/user_directory/search$ synapse_client;
}

server {
	listen 443 ssl http2;
	listen [::]:443 ssl http2;
	server_name example.com;

	# SSL
	ssl_certificate /etc/letsencrypt/live/example.com/fullchain.pem;
	ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem;
	ssl_trusted_certificate /etc/letsencrypt/live/example.com/chain.pem;

	# security
	#include snippets/security.conf;

	location /_matrix {
		# use whatever backend our map gives us
		proxy_pass http://$synapse_backend;
		include snippets/proxy.conf;
		client_max_body_size 5g;
		# Fuck no
		access_log off;
		log_not_found off;
		add_header Backend-Server $synapse_backend;
	}

	# This is also provided by synapse server
	location /.well-known/matrix/client {
		return 200 '{"m.homeserver": {"base_url": "https://example.com/"}}';
		default_type application/json;
		add_header Access-Control-Allow-Origin *;
		access_log off;
		log_not_found off;
	}

	location /.well-known/matrix/server {
		return 200 '{"m.server": "example.com:443"}';
		default_type application/json;
		add_header Access-Control-Allow-Origin *;
		access_log off;
		log_not_found off;
	}

	# additional config
	#include snippets/general.conf;
}

# HTTP redirect
server {
	listen 80;
	listen [::]:80;

	server_name example.com;

	include snippets/letsencrypt.conf;

	location / {
		return 301 https://example.com$request_uri;
	}
}

Where is $synapse_backend defined?

Where is $synapse_backend defined?

In line 19. This nginx config uses map feature to set synapse_backend based on request_uri regex match.

If you can post the worker yaml files, that will be great.

@rajil I use config files from the documentation https://matrix-org.github.io/synapse/latest/systemd-with-workers/index.html

Is there any particular reason why Receipts requests are commented out?

@waclaw66 afaik something broke if I tried to route them to a worker. You can try to enable those lines and see if it works now.

@waclaw66 afaik something broke if I tried to route them to a worker. You can try to enable those lines and see if it works now.

You're right, seems it has to be proccessed on stream writer instead of client worker.

that makes sense, i have not setup seperate stream writes on my server.

Sync requests and some others do not route to synapse_client (but to synapse_master) because $request_uri variable is full uri with args. Try add add_header Backend-Server $upstream_addr; to see real backend address.
regex must be fixed (without $ at the end) or use $uri instead of $request_uri

@udovichenkoAlexander Thank's for the feedback! nginx variables magic has always been a bit of a confusion point for me. I will try to implement the fixes when I get the time.