nginx.conf

# vi: ft=nginx

user www-data www-data;

# automatically determine the optimal worker number
worker_processes 8;

# limit on the maximum number of open files for worker processes
worker_rlimit_nofile 100000;

events {
  # can be that high because worker_rlimit_nofile is high
  worker_connections 10000;

  # accept as many connections as possible after getting a notification about a new connection
  multi_accept on;

  use epoll;
}

error_log /var/log/nginx/error.log error;

http {
  server_tokens off;
  sendfile on;
  tcp_nopush on;
  tcp_nodelay on;

  log_format json_cached escape=json
    '{'
      '"cached_status":"$sent_http_x_proxy_cache",'
      '"host":"$host",'
      '"time_local":"$time_local",'
      '"request":"$request",'
      '"status": "$status",'
      '"body_bytes_sent":"$body_bytes_sent",'
      '"request_time":"$request_time",'
      '"http_referrer":"$http_referer",'
      '"http_user_agent":"$http_user_agent"'
    '}';

  access_log /var/log/nginx/access.log json_cached;

  reset_timedout_connection on;

  client_body_timeout 10;
  client_header_timeout 10;
  keepalive_timeout 30;
  send_timeout 30;

  client_body_buffer_size 10K;
  client_header_buffer_size 1k;
  client_max_body_size 8m;
  large_client_header_buffers 4 8k;

  # limit connections
  # http://nginx.org/en/docs/http/ngx_http_limit_conn_module.html
  limit_conn_zone $binary_remote_addr zone=conn_limit_per_ip:10m;
  limit_conn conn_limit_per_ip 30;

  # limit requests
  # http://nginx.org/en/docs/http/ngx_http_limit_req_module.html
  limit_req_zone $binary_remote_addr zone=req_limit_per_ip:10m rate=25r/s;
  limit_req zone=req_limit_per_ip burst=50 nodelay;

  include mime.types;
  default_type text/html;
  charset UTF-8;

  gzip on;
  gzip_proxied expired no-cache no-store private auth;
  gzip_min_length 1000;
  gzip_comp_level 6;
  gzip_types text/plain text/xml application/xml text/css text/javascript application/javascript application/x-javascript text/x-component application/json application/xhtml+xml application/rss+xml application/atom+xml application/vnd.ms-fontobject image/svg+xml application/x-font-ttf font/opentype;

  open_file_cache max=100000 inactive=20s;
  open_file_cache_valid 30s;
  open_file_cache_min_uses 2;
  open_file_cache_errors on;

  server_names_hash_bucket_size 128;
  types_hash_max_size 2048;

  resolver 8.8.8.8;

  # The "auto_ssl" shared dict should be defined with enough storage space to
  # hold your certificate data. 1MB of storage holds certificates for
  # approximately 100 separate domains.
  lua_shared_dict auto_ssl 1m;
  # The "auto_ssl_settings" shared dict is used to temporarily store various settings
  # like the secret used by the hook server on port 8999. Do not change or
  # omit it.
  lua_shared_dict auto_ssl_settings 64k;

  lua_shared_dict prometheus_metrics 100M;
  lua_package_path "/opt/openresty/lualib/?.lua;/opt/openresty/luajit/share/lua/5.1/?.lua;/opt/openresty/luajit/share/lua/5.1/nginx/?.lua";

  # Initial setup tasks.
  init_by_lua_block {
    auto_ssl = (require "resty.auto-ssl").new()

    -- Define a function to determine which SNI domains to automatically handle
    -- and register new certificates for. Defaults to not allowing any domains,
    -- so this must be configured.
    auto_ssl:set("allow_domain", function(domain)
      if ngx.re.match(domain, "network.example.org$", "ijo") then
        return true
      elseif ngx.re.match(domain, "example.org$", "ijo") then
        return false
      else
        return true
      end
    end)

    auto_ssl:set("storage_adapter", "resty.auto-ssl.storage_adapters.redis")

    auto_ssl:set("redis", {
      host = "redis.host",
      port = 1234,
      prefix = "example_org_letsencrypt",
      auth = "redis.auth"
    })

    auto_ssl:init()

    prometheus = require("prometheus").init("prometheus_metrics")
    metric_requests = prometheus:counter(
      "nginx_http_requests_total", "Number of HTTP requests", {"host", "status"})
    metric_latency = prometheus:histogram(
      "nginx_http_request_duration_seconds", "HTTP request latency", {"host"})
    metric_connections = prometheus:gauge(
      "nginx_http_connections", "Number of HTTP connections", {"state"})
  }

  init_worker_by_lua_block {
    auto_ssl:init_worker()
  }

  # Internal server running on port 8999 for handling certificate tasks.
  server {
    listen 127.0.0.1:8999;

    # Increase the body buffer size, to ensure the internal POSTs can always
    # parse the full POST contents into memory.
    client_body_buffer_size 128k;
    client_max_body_size 128k;

    location / {
      content_by_lua_block {
        auto_ssl:hook_server()
      }
    }
  }

  log_by_lua_block {
    local host = ngx.var.host:gsub("^www.", "")
    metric_requests:inc(1, {host, ngx.var.status})
    metric_latency:observe(tonumber(ngx.var.request_time), {host})
  }

  server {
    listen 9145;
    location /metrics {
      content_by_lua '
        metric_connections:set(ngx.var.connections_reading, {"reading"})
        metric_connections:set(ngx.var.connections_waiting, {"waiting"})
        metric_connections:set(ngx.var.connections_writing, {"writing"})
        prometheus:collect()
      ';
    }
  }
  include sites-enabled/*.conf;
}

sites-enabled-example.org.conf

proxy_cache_path /opt/example-org-cache levels=1:2 keys_zone=example-org-cache:5m max_size=1g inactive=1d use_temp_path=off;

server {
  listen 80;
  listen 443 ssl http2;
  server_name _;

  # You must still define a static ssl_certificate file for nginx to start.
  ssl_certificate /certs/secret.crt;
  ssl_certificate_key /certs/secret.key;
  ssl_session_timeout 1d;
  ssl_session_cache shared:SSL:50m;
  ssl_session_tickets off;

  ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
  ssl_ciphers 'ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS';
  ssl_prefer_server_ciphers on;

  # Dynamic handler for issuing or returning certs for SNI domains.
  ssl_certificate_by_lua_block {
    auto_ssl:ssl_certificate()
  }

  set $upstream_endpoint app.service.com;
  location / {
    proxy_pass $scheme://$upstream_endpoint;

    # normalize all accept-encoding headers to just gzip
    set $ae "";
    if ($http_accept_encoding ~* gzip) {
      set $ae "gzip";
    }

    # define own cache key with actual host and gzip info
    proxy_cache_key $ae$scheme://$http_host$request_uri;

    # pass Accept-Encoding header to backend
    proxy_set_header Accept-Encoding $ae;

    # pass Authorization header to backend
    proxy_set_header Authorization $http_authorization;

    # set correct header for ssl requests to work
    proxy_set_header Host $upstream_endpoint;

    # set host so rails app delivers the correct content
    proxy_set_header X-Forwarded-Host $http_host;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header X-Forwarded-Proto $scheme;

    # which cache to use
    proxy_cache example-org-cache;

    # use conditional requests to query upstream server
    proxy_cache_revalidate on;

    # when to use stale responses
    proxy_cache_use_stale error timeout invalid_header http_502;

    # 200 responses will be cached for 1m max
    proxy_cache_valid 200 1m;

    # 301 responses will be cached for 1m max
    proxy_cache_valid 301 1m;

    # 404 responses will be cached for 10s max
    proxy_cache_valid 404 10s;

    # only one request will populate the cache, others will get stale response
    proxy_cache_lock on;

    # enable byte-range requests
    proxy_force_ranges on;

    # pass on the server name to allow TLS SNI
    proxy_ssl_server_name on;

    # allow to bypass the cache by force-reloading
    proxy_cache_bypass $arg_nocache;

    # we need to ignore these, otherwise the response will never be cached
    proxy_ignore_headers vary;

    # hide heroku header
    proxy_hide_header via;

    # add cache status to response headers
    add_header X-Proxy-Cache $upstream_cache_status;
  }

  # Endpoint used for performing domain verification with Let's Encrypt.
  location /.well-known/acme-challenge/ {
    content_by_lua_block {
      auto_ssl:challenge_server()
    }
  }
}