A refreshed 'nginx__dynamic_tls_records.patch' patch for nginx 1.11.5
What we do now: | |
We use a static record size of 4K. This gives a good balance of latency and | |
throughput. | |
Optimize latency: | |
By initialy sending small (1 TCP segment) sized records, we are able to avoid | |
HoL blocking of the first byte. This means TTFB is sometime lower by a whole | |
RTT. | |
Optimizing throughput: | |
By sending increasingly larger records later in the connection, when HoL is not | |
a problem, we reduce the overhead of TLS record (29 bytes per record with | |
GCM/CHACHA-POLY). | |
Logic: | |
Start each connection with small records (1369 byte default, change with | |
ssl_dyn_rec_size_lo). After a given number of records (40, change with | |
ssl_dyn_rec_threshold) start sending larger records (4229, ssl_dyn_rec_size_hi). | |
Eventually after the same number of records, start sending the largest records | |
(ssl_buffer_size). | |
In case the connection idles for a given amount of time (1s, | |
ssl_dyn_rec_timeout), the process repeats itself (i.e. begin sending small | |
records again). | |
Upstream source: | |
https://github.com/cloudflare/sslconfig/blob/master/patches/nginx__dynamic_tls_records.patch | |
--- a/src/event/ngx_event_openssl.c | |
+++ b/src/event/ngx_event_openssl.c | |
@@ -1131,6 +1131,7 @@ | |
sc->buffer = ((flags & NGX_SSL_BUFFER) != 0); | |
sc->buffer_size = ssl->buffer_size; | |
+ sc->dyn_rec = ssl->dyn_rec; | |
sc->session_ctx = ssl->ctx; | |
@@ -1669,6 +1670,41 @@ | |
for ( ;; ) { | |
+ /* Dynamic record resizing: | |
+ We want the initial records to fit into one TCP segment | |
+ so we don't get TCP HoL blocking due to TCP Slow Start. | |
+ A connection always starts with small records, but after | |
+ a given amount of records sent, we make the records larger | |
+ to reduce header overhead. | |
+ After a connection has idled for a given timeout, begin | |
+ the process from the start. The actual parameters are | |
+ configurable. If dyn_rec_timeout is 0, we assume dyn_rec is off. */ | |
+ | |
+ if (c->ssl->dyn_rec.timeout > 0 ) { | |
+ | |
+ if (ngx_current_msec - c->ssl->dyn_rec_last_write > | |
+ c->ssl->dyn_rec.timeout) | |
+ { | |
+ buf->end = buf->start + c->ssl->dyn_rec.size_lo; | |
+ c->ssl->dyn_rec_records_sent = 0; | |
+ | |
+ } else { | |
+ if (c->ssl->dyn_rec_records_sent > | |
+ c->ssl->dyn_rec.threshold * 2) | |
+ { | |
+ buf->end = buf->start + c->ssl->buffer_size; | |
+ | |
+ } else if (c->ssl->dyn_rec_records_sent > | |
+ c->ssl->dyn_rec.threshold) | |
+ { | |
+ buf->end = buf->start + c->ssl->dyn_rec.size_hi; | |
+ | |
+ } else { | |
+ buf->end = buf->start + c->ssl->dyn_rec.size_lo; | |
+ } | |
+ } | |
+ } | |
+ | |
while (in && buf->last < buf->end && send < limit) { | |
if (in->buf->last_buf || in->buf->flush) { | |
flush = 1; | |
@@ -1770,6 +1806,9 @@ | |
if (n > 0) { | |
+ c->ssl->dyn_rec_records_sent++; | |
+ c->ssl->dyn_rec_last_write = ngx_current_msec; | |
+ | |
if (c->ssl->saved_read_handler) { | |
c->read->handler = c->ssl->saved_read_handler; | |
--- a/src/event/ngx_event_openssl.h | |
+++ b/src/event/ngx_event_openssl.h | |
@@ -54,10 +54,19 @@ | |
#define ngx_ssl_conn_t SSL | |
+typedef struct { | |
+ ngx_msec_t timeout; | |
+ ngx_uint_t threshold; | |
+ size_t size_lo; | |
+ size_t size_hi; | |
+} ngx_ssl_dyn_rec_t; | |
+ | |
+ | |
struct ngx_ssl_s { | |
SSL_CTX *ctx; | |
ngx_log_t *log; | |
size_t buffer_size; | |
+ ngx_ssl_dyn_rec_t dyn_rec; | |
}; | |
@@ -80,6 +89,10 @@ | |
unsigned no_wait_shutdown:1; | |
unsigned no_send_shutdown:1; | |
unsigned handshake_buffer_set:1; | |
+ | |
+ ngx_ssl_dyn_rec_t dyn_rec; | |
+ ngx_msec_t dyn_rec_last_write; | |
+ ngx_uint_t dyn_rec_records_sent; | |
}; | |
@@ -89,7 +102,7 @@ | |
#define NGX_SSL_DFLT_BUILTIN_SCACHE -5 | |
-#define NGX_SSL_MAX_SESSION_SIZE 4096 | |
+#define NGX_SSL_MAX_SESSION_SIZE 16384 | |
typedef struct ngx_ssl_sess_id_s ngx_ssl_sess_id_t; | |
--- a/src/http/modules/ngx_http_ssl_module.c | |
+++ b/src/http/modules/ngx_http_ssl_module.c | |
@@ -233,6 +233,41 @@ | |
offsetof(ngx_http_ssl_srv_conf_t, stapling_verify), | |
NULL }, | |
+ { ngx_string("ssl_dyn_rec_enable"), | |
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, | |
+ ngx_conf_set_flag_slot, | |
+ NGX_HTTP_SRV_CONF_OFFSET, | |
+ offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_enable), | |
+ NULL }, | |
+ | |
+ { ngx_string("ssl_dyn_rec_timeout"), | |
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, | |
+ ngx_conf_set_msec_slot, | |
+ NGX_HTTP_SRV_CONF_OFFSET, | |
+ offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_timeout), | |
+ NULL }, | |
+ | |
+ { ngx_string("ssl_dyn_rec_size_lo"), | |
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, | |
+ ngx_conf_set_size_slot, | |
+ NGX_HTTP_SRV_CONF_OFFSET, | |
+ offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_lo), | |
+ NULL }, | |
+ | |
+ { ngx_string("ssl_dyn_rec_size_hi"), | |
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, | |
+ ngx_conf_set_size_slot, | |
+ NGX_HTTP_SRV_CONF_OFFSET, | |
+ offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_hi), | |
+ NULL }, | |
+ | |
+ { ngx_string("ssl_dyn_rec_threshold"), | |
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, | |
+ ngx_conf_set_num_slot, | |
+ NGX_HTTP_SRV_CONF_OFFSET, | |
+ offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_threshold), | |
+ NULL }, | |
+ | |
ngx_null_command | |
}; | |
@@ -533,6 +568,11 @@ | |
sscf->session_ticket_keys = NGX_CONF_UNSET_PTR; | |
sscf->stapling = NGX_CONF_UNSET; | |
sscf->stapling_verify = NGX_CONF_UNSET; | |
+ sscf->dyn_rec_enable = NGX_CONF_UNSET; | |
+ sscf->dyn_rec_timeout = NGX_CONF_UNSET_MSEC; | |
+ sscf->dyn_rec_size_lo = NGX_CONF_UNSET_SIZE; | |
+ sscf->dyn_rec_size_hi = NGX_CONF_UNSET_SIZE; | |
+ sscf->dyn_rec_threshold = NGX_CONF_UNSET_UINT; | |
return sscf; | |
} | |
@@ -598,6 +638,20 @@ | |
ngx_conf_merge_str_value(conf->stapling_responder, | |
prev->stapling_responder, ""); | |
+ ngx_conf_merge_value(conf->dyn_rec_enable, prev->dyn_rec_enable, 0); | |
+ ngx_conf_merge_msec_value(conf->dyn_rec_timeout, prev->dyn_rec_timeout, | |
+ 1000); | |
+ /* Default sizes for the dynamic record sizes are defined to fit maximal | |
+ TLS + IPv6 overhead in a single TCP segment for lo and 3 segments for hi: | |
+ 1369 = 1500 - 40 (IP) - 20 (TCP) - 10 (Time) - 61 (Max TLS overhead) */ | |
+ ngx_conf_merge_size_value(conf->dyn_rec_size_lo, prev->dyn_rec_size_lo, | |
+ 1369); | |
+ /* 4229 = (1500 - 40 - 20 - 10) * 3 - 61 */ | |
+ ngx_conf_merge_size_value(conf->dyn_rec_size_hi, prev->dyn_rec_size_hi, | |
+ 4229); | |
+ ngx_conf_merge_uint_value(conf->dyn_rec_threshold, prev->dyn_rec_threshold, | |
+ 40); | |
+ | |
conf->ssl.log = cf->log; | |
if (conf->enable) { | |
@@ -778,6 +832,28 @@ | |
} | |
+ if (conf->dyn_rec_enable) { | |
+ conf->ssl.dyn_rec.timeout = conf->dyn_rec_timeout; | |
+ conf->ssl.dyn_rec.threshold = conf->dyn_rec_threshold; | |
+ | |
+ if (conf->buffer_size > conf->dyn_rec_size_lo) { | |
+ conf->ssl.dyn_rec.size_lo = conf->dyn_rec_size_lo; | |
+ | |
+ } else { | |
+ conf->ssl.dyn_rec.size_lo = conf->buffer_size; | |
+ } | |
+ | |
+ if (conf->buffer_size > conf->dyn_rec_size_hi) { | |
+ conf->ssl.dyn_rec.size_hi = conf->dyn_rec_size_hi; | |
+ | |
+ } else { | |
+ conf->ssl.dyn_rec.size_hi = conf->buffer_size; | |
+ } | |
+ | |
+ } else { | |
+ conf->ssl.dyn_rec.timeout = 0; | |
+ } | |
+ | |
return NGX_CONF_OK; | |
} | |
--- a/src/http/modules/ngx_http_ssl_module.h | |
+++ b/src/http/modules/ngx_http_ssl_module.h | |
@@ -57,6 +57,12 @@ | |
u_char *file; | |
ngx_uint_t line; | |
+ | |
+ ngx_flag_t dyn_rec_enable; | |
+ ngx_msec_t dyn_rec_timeout; | |
+ size_t dyn_rec_size_lo; | |
+ size_t dyn_rec_size_hi; | |
+ ngx_uint_t dyn_rec_threshold; | |
} ngx_http_ssl_srv_conf_t; | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment