nginx-push-stream-module configuration for >1MM concurrent subscribers

nginx -V

nginx version: nginx/1.4.5
TLS SNI support enabled
configure arguments:
  --prefix=/usr/share/nginx
  --conf-path=/etc/nginx/nginx.conf
  --error-log-path=/var/log/nginx/error.log
  --http-client-body-temp-path=/var/lib/nginx/body
  --http-fastcgi-temp-path=/var/lib/nginx/fastcgi
  --http-log-path=/var/log/nginx/access.log
  --http-proxy-temp-path=/var/lib/nginx/proxy
  --http-scgi-temp-path=/var/lib/nginx/scgi
  --http-uwsgi-temp-path=/var/lib/nginx/uwsgi
  --lock-path=/var/lock/nginx.lock
  --pid-path=/run/nginx.pid
  --with-pcre-jit
  --with-debug
  --with-file-aio
  --with-http_addition_module
  --with-http_geoip_module
  --with-http_gzip_static_module
  --with-http_image_filter_module
  --with-http_realip_module
  --with-http_secure_link_module
  --with-http_spdy_module
  --with-http_stub_status_module
  --with-http_ssl_module
  --with-http_sub_module
  --with-http_xslt_module
  --with-ipv6
  --add-module=/home/vagrant/nginx/debian/modules/nginx-development-kit
  --add-module=/home/vagrant/nginx/debian/modules/set-misc-nginx-module
  --add-module=/home/vagrant/nginx/debian/modules/headers-more-nginx-module
  --add-module=/home/vagrant/nginx/debian/modules/echo-nginx-module
  --add-module=/home/vagrant/nginx/debian/modules/lua-nginx-module
  --add-module=/home/vagrant/nginx/debian/modules/memc-nginx-module
  --add-module=/home/vagrant/nginx/debian/modules/srcache-nginx-module
  --add-module=/home/vagrant/nginx/debian/modules/nginx-push-stream-module
  --add-module=/home/vagrant/nginx/debian/modules/ngx_http_redis
  --add-module=/home/vagrant/nginx/debian/modules/nginx-upstream-fair
  --add-module=/home/vagrant/nginx/debian/modules/nginx-stats

nginx.conf

user www-data www-data;
worker_processes 8;
worker_cpu_affinity 00000001 00000010 00000100 00001000 00010000 00100000 01000000 10000000;
worker_rlimit_nofile 262143;

error_log /var/log/nginx/error.log;
pid /run/nginx.pid;

events {
  worker_connections 262143;
  accept_mutex on;
  accept_mutex_delay 50ms;
  multi_accept off;
  use epoll;
  epoll_events 512;
}

http {
  # Doubling per http://mailman.nginx.org/pipermail/nginx/2013-July/039995.html
  variables_hash_max_size 1024;

  # Lets not log anything by default
  access_log off;

  include /etc/nginx/mime.types;
  default_type application/octet-stream;

  # Enable gzip by default
  gzip on;
  gzip_proxied expired no-cache no-store private auth;
  gzip_vary on;

  # https://github.com/h5bp/server-configs-nginx/blob/fa41e580d/nginx.conf#L101-L116
  gzip_types
    application/atom+xml
    application/javascript
    application/json
    application/rss+xml
    application/vnd.ms-fontobject
    application/x-font-ttf
    application/x-web-app-manifest+json
    application/xhtml+xml
    application/xml
    font/opentype
    image/svg+xml
    image/x-icon
    text/css
    text/plain
    text/x-component

    # Legacy for Disqus
    application/x-javascript
    text/javascript
    text/xml
    ;

  # Networking optimizations
  sendfile on;
  tcp_nodelay on;
  tcp_nopush off;

  # Try not to keep conns around too long
  lingering_time 5s;
  lingering_timeout 1s;
  client_header_timeout 5s;
  reset_timedout_connection on;
  send_timeout 9s;

  # Don't advert version
  server_tokens off;

  # SSL Hardening (if ever get ECC cert remove !ECDSA from end)
  ssl_stapling on;
  ssl_stapling_verify on;
  ssl_ciphers EECDH+AESGCM:ECDH+RC4:EECDH:EDH+AESGCM:EDH+AES:RC4:!aNULL:!eNULL:!MD5:!3DES:!PSK:!DSS:!ECDSA;
  ssl_prefer_server_ciphers on;
  ssl_protocols TLSv1.2 TLSv1.1 TLSv1;
  ssl_session_cache shared:SSL:128m

  include /etc/nginx/sites-enabled/*;
}

pushstream.conf

push_stream_shared_memory_size 512M;

push_stream_message_ttl 60s;
push_stream_max_messages_stored_per_channel 127;
push_stream_channel_deleted_message_text "";
push_stream_ping_message_text "\n";

server {
    listen 80 backlog=32768 rcvbuf=4194304 sndbuf=16777216;
    listen 443 ssl default_server backlog=32768 rcvbuf=4194304 sndbuf=16777216;

    location = /publish {
        allow 10.25.8.2;
        allow 10.25.8.3;
        deny all;

        push_stream_publisher admin;
        push_stream_channels_path $arg_channel;
        push_stream_channel_info_on_publish off;
        push_stream_store_messages on;
        keepalive_timeout 60s;
        keepalive_requests 4294967294;
    }

    location ^~ /subscribe/ {
        location ~ /subscribe/([\d]+)$ {
            set $channel $1;
            push_stream_channels_path $channel;

            push_stream_last_received_message_time $arg_since;

            push_stream_subscriber streaming;
            push_stream_subscriber_connection_ttl 90s;
            push_stream_ping_message_interval 10s;

            default_type application/json;

            gzip_no_buffer on;
            gzip_window 2k;
            gzip_hash 4k;
        }
    }

    location ^~ /ws/ {
        location ~ /ws/([\d]+)$ {
            set $channel $1;
            push_stream_channels_path $channel;

            push_stream_subscriber websocket;
            push_stream_websocket_allow_publish off;
            push_stream_ping_message_interval 10s;
        }
    }

    location = /push-stream-status {
        access_log off;

        allow 127.0.0.1;
        deny all;

        push_stream_channels_statistics;
        push_stream_channels_path $arg_channel;
    }

    location / {
        return 403;
    }
}

sysctl.conf

net.core.netdev_max_backlog=65535
net.core.optmem_max=20480
net.core.rmem_default=16384
net.core.rmem_max=16777216
net.core.somaxconn=65535
net.core.wmem_default=32768
net.core.wmem_max=16777216
net.ipv4.tcp_fin_timeout=15
net.ipv4.tcp_keepalive_intvl=30
net.ipv4.tcp_keepalive_probes=3
net.ipv4.tcp_keepalive_time=120
net.ipv4.tcp_max_orphans=262144
net.ipv4.tcp_max_syn_backlog=524288
net.ipv4.tcp_max_tw_buckets=524288
net.ipv4.tcp_mem=1572864 1835008 2097152
net.ipv4.tcp_no_metrics_save=1
net.ipv4.tcp_orphan_retries=0
net.ipv4.tcp_rmem=4096 16384 16777216
net.ipv4.tcp_synack_retries=2
net.ipv4.tcp_syncookies=1
net.ipv4.tcp_syn_retries=2
net.ipv4.tcp_wmem=4096 32768 16777216