HAProxy reverse proxy set for EMQX cluster

haproxy.cfg

global
  log stdout format raw local0 info
  maxconn 10240
  tune.ssl.default-dh-param 2048

# ---------------------------------------------------------------------------
# Railway private networking uses its own DNS resolver available inside the
# container via the system resolver (populated in /etc/resolv.conf at runtime).
# We tell HAProxy to use it so server addresses like emqx1.railway.internal
# are resolved at runtime rather than at config parse time, which would fail
# because private networking is unavailable during the build phase.
# ---------------------------------------------------------------------------
resolvers railway_dns
  # The system resolver address is injected by Railway into /etc/resolv.conf.
  # 127.0.0.11 is the default DNS gateway for Railway (and Docker) containers.
  nameserver dns1 127.0.0.11:53
  accepted_payload_size 8192   # support large DNS responses with many A/AAAA records
  timeout resolve 2s
  timeout retry   2s
  hold valid      10s          # re-query every 10s to pick up pod IP changes
  hold other      30s
  hold refused    30s
  hold nx         30s
  hold timeout    30s

defaults
  log global
  mode tcp
  option tcplog
  option dontlognull
  timeout connect 10s
  timeout queue   30s
  # Must exceed MQTT keepalive * 1.2
  timeout client  240s
  timeout server  240s
  retries 3
  option redispatch
  maxconn 10240

# ---------------------------------------------------------------------------
# MQTT — plain TCP (port 1883)
# ---------------------------------------------------------------------------
frontend mqtt_servers
  bind *:1883
  mode tcp
  tcp-request inspect-delay 10s
  # Rate-limit: max 20 new connections/s per source IP
  # table mqtt_backend — the stick-table lives there, not in this frontend
  tcp-request connection track-sc0 src table mqtt_backend
  tcp-request connection reject if { sc_conn_rate(0) gt 20 }
  # Reject non-MQTT traffic after inspecting the buffer
  tcp-request content reject unless { req.payload(0,0),mqtt_is_valid }
  default_backend mqtt_backend

backend mqtt_backend
  mode tcp
  balance roundrobin
  # Sticky session: pin a client_id to the same broker node for 30 min
  stick-table type string len 32 size 1000k expire 30m
  stick on req.payload(0,0),mqtt_field_value(connect,client_identifier)
  option tcp-check
  # init-addr none — do NOT fail if hostname is unresolvable at startup;
  # HAProxy will retry via the resolver once the network is up.
  default-server resolvers railway_dns init-addr none check
  server emqx1 ${EMQX1_HOST}:${EMQX_MQTT_PORT}
  server emqx2 ${EMQX2_HOST}:${EMQX_MQTT_PORT}
  server emqx3 ${EMQX3_HOST}:${EMQX_MQTT_PORT}

# ---------------------------------------------------------------------------
# MQTTS — TLS passthrough (port 8883)
# ---------------------------------------------------------------------------
frontend mqtts_servers
  bind *:8883
  mode tcp
  tcp-request inspect-delay 10s
  tcp-request content reject unless { req.payload(0,0),mqtt_is_valid }
  default_backend mqtts_backend

backend mqtts_backend
  mode tcp
  balance roundrobin
  default-server resolvers railway_dns init-addr none check
  server emqx1 ${EMQX1_HOST}:${EMQX_MQTTS_PORT} weight ${EMQX1_WEIGHT}
  server emqx2 ${EMQX2_HOST}:${EMQX_MQTTS_PORT} weight ${EMQX2_WEIGHT}
  server emqx3 ${EMQX3_HOST}:${EMQX_MQTTS_PORT} weight ${EMQX3_WEIGHT}

# ---------------------------------------------------------------------------
# MQTT over WebSocket — raw TCP passthrough (port 8083)
# ---------------------------------------------------------------------------
frontend mqtt_ws_frontend
  bind *:8083
  mode tcp
  tcp-request inspect-delay 5s
  default_backend mqtt_ws_backend

backend mqtt_ws_backend
  mode tcp
  balance roundrobin
  default-server resolvers railway_dns init-addr none check
  server emqx1 ${EMQX1_HOST}:${EMQX_WS_PORT}
  server emqx2 ${EMQX2_HOST}:${EMQX_WS_PORT}
  server emqx3 ${EMQX3_HOST}:${EMQX_WS_PORT}

# ---------------------------------------------------------------------------
# Public HTTP entrypoint — WebSocket upgrade + stats + dashboard proxy
# Exposed on ${PORT} (Railway dynamic port)
# ---------------------------------------------------------------------------
frontend http_public
  bind *:${PORT}
  mode http
  option httplog
  option forwardfor

  # -- ACLs --
  acl is_websocket hdr(Connection) -i upgrade
  acl is_websocket hdr(Upgrade)    -i websocket
  acl mqtt_ws_path path_beg /mqtt
  acl is_stats     path_beg /stats
  acl is_local_net src 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 fd00::/8 ::1

  # http-request rules MUST come before use_backend directives
  http-request deny if is_stats !is_local_net

  # -- Routing --
  use_backend mqtt_ws_http_backend if is_websocket mqtt_ws_path
  use_backend stats_backend if is_stats is_local_net

  default_backend emqx_dashboard_backend

backend mqtt_ws_http_backend
  mode http
  balance roundrobin
  option forwardfor
  http-response set-header X-Content-Type-Options nosniff
  default-server resolvers railway_dns init-addr none check
  server emqx1 ${EMQX1_HOST}:${EMQX_WS_PORT}
  server emqx2 ${EMQX2_HOST}:${EMQX_WS_PORT}
  server emqx3 ${EMQX3_HOST}:${EMQX_WS_PORT}

# ---------------------------------------------------------------------------
# EMQX Dashboard — internal only (bound to loopback)
# ---------------------------------------------------------------------------
frontend emqx_dashboard_frontend
  bind 127.0.0.1:18083
  mode http
  option httplog
  default_backend emqx_dashboard_backend

backend emqx_dashboard_backend
  mode http
  balance roundrobin
  option forwardfor
  option httpchk GET /api/v5/status
  http-response set-header X-Frame-Options SAMEORIGIN
  http-response set-header X-Content-Type-Options nosniff
  http-response set-header Referrer-Policy strict-origin-when-cross-origin
  default-server resolvers railway_dns init-addr none check
  server emqx1 ${EMQX1_HOST}:${EMQX_DASHBOARD_PORT}
  server emqx2 ${EMQX2_HOST}:${EMQX_DASHBOARD_PORT}
  server emqx3 ${EMQX3_HOST}:${EMQX_DASHBOARD_PORT}

# ---------------------------------------------------------------------------
# HAProxy Stats — internal ONLY, bound to loopback
# ---------------------------------------------------------------------------
frontend stats_internal
  bind 127.0.0.1:8888
  mode http
  stats enable
  stats uri /stats
  stats refresh 10s
  stats auth ${HAPROXY_STATS_USER}:${HAPROXY_STATS_PASSWORD}
  stats show-legends
  stats show-node

backend stats_backend
  mode http
  stats enable
  stats uri /stats
  stats refresh 10s
  stats auth ${HAPROXY_STATS_USER}:${HAPROXY_STATS_PASSWORD}