secret
Last active

nginx-push-stream-module configuration for >1MM concurrent subscribers

  • Download Gist
nginx -V
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40
nginx version: nginx/1.4.5
TLS SNI support enabled
configure arguments:
--prefix=/usr/share/nginx
--conf-path=/etc/nginx/nginx.conf
--error-log-path=/var/log/nginx/error.log
--http-client-body-temp-path=/var/lib/nginx/body
--http-fastcgi-temp-path=/var/lib/nginx/fastcgi
--http-log-path=/var/log/nginx/access.log
--http-proxy-temp-path=/var/lib/nginx/proxy
--http-scgi-temp-path=/var/lib/nginx/scgi
--http-uwsgi-temp-path=/var/lib/nginx/uwsgi
--lock-path=/var/lock/nginx.lock
--pid-path=/run/nginx.pid
--with-pcre-jit
--with-debug
--with-file-aio
--with-http_addition_module
--with-http_geoip_module
--with-http_gzip_static_module
--with-http_image_filter_module
--with-http_realip_module
--with-http_secure_link_module
--with-http_spdy_module
--with-http_stub_status_module
--with-http_ssl_module
--with-http_sub_module
--with-http_xslt_module
--with-ipv6
--add-module=/home/vagrant/nginx/debian/modules/nginx-development-kit
--add-module=/home/vagrant/nginx/debian/modules/set-misc-nginx-module
--add-module=/home/vagrant/nginx/debian/modules/headers-more-nginx-module
--add-module=/home/vagrant/nginx/debian/modules/echo-nginx-module
--add-module=/home/vagrant/nginx/debian/modules/lua-nginx-module
--add-module=/home/vagrant/nginx/debian/modules/memc-nginx-module
--add-module=/home/vagrant/nginx/debian/modules/srcache-nginx-module
--add-module=/home/vagrant/nginx/debian/modules/nginx-push-stream-module
--add-module=/home/vagrant/nginx/debian/modules/ngx_http_redis
--add-module=/home/vagrant/nginx/debian/modules/nginx-upstream-fair
--add-module=/home/vagrant/nginx/debian/modules/nginx-stats
nginx.conf
Nginx
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81
user www-data www-data;
worker_processes 8;
worker_cpu_affinity 00000001 00000010 00000100 00001000 00010000 00100000 01000000 10000000;
worker_rlimit_nofile 262143;
 
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
 
events {
worker_connections 262143;
accept_mutex on;
accept_mutex_delay 50ms;
multi_accept off;
use epoll;
epoll_events 512;
}
 
http {
# Doubling per http://mailman.nginx.org/pipermail/nginx/2013-July/039995.html
variables_hash_max_size 1024;
 
# Lets not log anything by default
access_log off;
 
include /etc/nginx/mime.types;
default_type application/octet-stream;
 
# Enable gzip by default
gzip on;
gzip_proxied expired no-cache no-store private auth;
gzip_vary on;
 
# https://github.com/h5bp/server-configs-nginx/blob/fa41e580d/nginx.conf#L101-L116
gzip_types
application/atom+xml
application/javascript
application/json
application/rss+xml
application/vnd.ms-fontobject
application/x-font-ttf
application/x-web-app-manifest+json
application/xhtml+xml
application/xml
font/opentype
image/svg+xml
image/x-icon
text/css
text/plain
text/x-component
 
# Legacy for Disqus
application/x-javascript
text/javascript
text/xml
;
 
# Networking optimizations
sendfile on;
tcp_nodelay on;
tcp_nopush off;
 
# Try not to keep conns around too long
lingering_time 5s;
lingering_timeout 1s;
client_header_timeout 5s;
reset_timedout_connection on;
send_timeout 9s;
 
# Don't advert version
server_tokens off;
 
# SSL Hardening (if ever get ECC cert remove !ECDSA from end)
ssl_stapling on;
ssl_stapling_verify on;
ssl_ciphers EECDH+AESGCM:ECDH+RC4:EECDH:EDH+AESGCM:EDH+AES:RC4:!aNULL:!eNULL:!MD5:!3DES:!PSK:!DSS:!ECDSA;
ssl_prefer_server_ciphers on;
ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
ssl_session_cache shared:SSL:128m
 
include /etc/nginx/sites-enabled/*;
}
pushstream.conf
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68
push_stream_shared_memory_size 512M;
 
push_stream_message_ttl 60s;
push_stream_max_messages_stored_per_channel 127;
push_stream_channel_deleted_message_text "";
push_stream_ping_message_text "\n";
 
server {
listen 80 backlog=32768 rcvbuf=4194304 sndbuf=16777216;
listen 443 ssl default_server backlog=32768 rcvbuf=4194304 sndbuf=16777216;
 
location = /publish {
allow 10.25.8.2;
allow 10.25.8.3;
deny all;
 
push_stream_publisher admin;
push_stream_channels_path $arg_channel;
push_stream_channel_info_on_publish off;
push_stream_store_messages on;
keepalive_timeout 60s;
keepalive_requests 4294967294;
}
 
location ^~ /subscribe/ {
location ~ /subscribe/([\d]+)$ {
set $channel $1;
push_stream_channels_path $channel;
 
push_stream_last_received_message_time $arg_since;
 
push_stream_subscriber streaming;
push_stream_subscriber_connection_ttl 90s;
push_stream_ping_message_interval 10s;
 
default_type application/json;
 
gzip_no_buffer on;
gzip_window 2k;
gzip_hash 4k;
}
}
 
location ^~ /ws/ {
location ~ /ws/([\d]+)$ {
set $channel $1;
push_stream_channels_path $channel;
 
push_stream_subscriber websocket;
push_stream_websocket_allow_publish off;
push_stream_ping_message_interval 10s;
}
}
 
location = /push-stream-status {
access_log off;
 
allow 127.0.0.1;
deny all;
 
push_stream_channels_statistics;
push_stream_channels_path $arg_channel;
}
 
location / {
return 403;
}
}
sysctl.conf
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
net.core.netdev_max_backlog=65535
net.core.optmem_max=20480
net.core.rmem_default=16384
net.core.rmem_max=16777216
net.core.somaxconn=65535
net.core.wmem_default=32768
net.core.wmem_max=16777216
net.ipv4.tcp_fin_timeout=15
net.ipv4.tcp_keepalive_intvl=30
net.ipv4.tcp_keepalive_probes=3
net.ipv4.tcp_keepalive_time=120
net.ipv4.tcp_max_orphans=262144
net.ipv4.tcp_max_syn_backlog=524288
net.ipv4.tcp_max_tw_buckets=524288
net.ipv4.tcp_mem=1572864 1835008 2097152
net.ipv4.tcp_no_metrics_save=1
net.ipv4.tcp_orphan_retries=0
net.ipv4.tcp_rmem=4096 16384 16777216
net.ipv4.tcp_synack_retries=2
net.ipv4.tcp_syncookies=1
net.ipv4.tcp_syn_retries=2
net.ipv4.tcp_wmem=4096 32768 16777216

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.