[PATCH] Upstream keepalive: keepalive_pool_key directive.
Thibault Charbonnier
thibaultcha at fastmail.com
Fri Aug 30 19:43:40 UTC 2019
# HG changeset patch
# User Thibault Charbonnier <thibaultcha at me.com>
# Date 1567193727 25200
# Fri Aug 30 12:35:27 2019 -0700
# Node ID 40abd582aafbd22c2435afed5fd3311333ca99bd
# Parent a31ec2b79f9f208fecbc92020d12edb2a5207480
Upstream keepalive: keepalive_pool_key directive.
This directive gives control to the user as to how they wish upstream
connection pools be created and used.
By default, each upstream block maintains a unique connection pool per
IP/port tuple specified via the `server` directive.
With this change, if the `keepalive_pool_key` directive is specified in
an upstream block, the connection pools will be further segmented
according to the IP/port/pool_key combination. Each connection pool will
still have up to as many connections as specified in the `keepalive`
directive.
* When `keepalive_pool_key` evaluates to an empty string, the pool_key
behavior is disabled (as if only `keepalive` was specified).
* When the pool_key behavior is in effect, SSL session reuse will be
disabled out of the box (as if `proxy_ssl_session_reuse off;` was
specified).
* Two new variables `$keepalive_pool_key` and `$keepalive_pool_key_raw`
can be accessed to read the value of the current pool_key (if set).
The former will compile the value, while the later will return it as
specified by the user.
This solves existing limitations of the upstream block such as
detailed in the following tickets:
* https://trac.nginx.org/nginx/ticket/1340
* https://trac.nginx.org/nginx/ticket/1593
For example, let's consider a use-case similar to #1340, in which
different SNIs are intended to open different connections:
upstream upstream.domain.com {
server 1.1.1.1:443;
keepalive 60;
keepalive_pool_key $upstream_sni;
}
http {
server {
...
location /proxy {
set $upstream_sni $host;
proxy_ssl_name $upstream_sni;
proxy_ssl_server_name on;
proxy_http_version 1.1;
proxy_set_header Connection '';
proxy_pass https://upstream.domain.com;
}
}
}
Without the above `keepalive_pool_key`, the same connection would be
reused even when different SNIs should be used.
Since `keepalive_pool_key` supports variables, users can use more
granular connection pooling rules, such as distinguishing client
certificates:
keepalive_pool_key $ssl_client_cert_cn;
or preferred protocols:
keepalive_pool_key $ssl_client_protocols;
or other properties of the request, connection, or TLS handshake.
Note that while the above examples are built with variables that must be
initialized by the users themselves as of today, nothing prevents future
releases from adding these variables (or others), and supporting such
use-cases out of the box.
diff -r a31ec2b79f9f -r 40abd582aafb
src/http/modules/ngx_http_upstream_keepalive_module.c
--- a/src/http/modules/ngx_http_upstream_keepalive_module.c Tue Apr 02
14:45:52 2019 -0700
+++ b/src/http/modules/ngx_http_upstream_keepalive_module.c Fri Aug 30
12:35:27 2019 -0700
@@ -21,6 +21,8 @@
ngx_http_upstream_init_pt original_init_upstream;
ngx_http_upstream_init_peer_pt original_init_peer;
+ ngx_http_complex_value_t pool_key;
+
} ngx_http_upstream_keepalive_srv_conf_t;
@@ -33,6 +35,8 @@
socklen_t socklen;
ngx_sockaddr_t sockaddr;
+ uint32_t pool_key_crc32;
+
} ngx_http_upstream_keepalive_cache_t;
@@ -51,6 +55,8 @@
ngx_event_save_peer_session_pt original_save_session;
#endif
+ uint32_t pool_key_crc32;
+
} ngx_http_upstream_keepalive_peer_data_t;
@@ -75,6 +81,11 @@
static void *ngx_http_upstream_keepalive_create_conf(ngx_conf_t *cf);
static char *ngx_http_upstream_keepalive(ngx_conf_t *cf, ngx_command_t
*cmd,
void *conf);
+static char *ngx_http_upstream_keepalive_pool_key(ngx_conf_t *cf,
+ ngx_command_t *cmd, void *conf);
+static ngx_int_t ngx_http_upstream_keepalive_add_variables(ngx_conf_t *cf);
+static ngx_int_t ngx_http_upstream_keepalive_pool_key_var_get(
+ ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data);
static ngx_command_t ngx_http_upstream_keepalive_commands[] = {
@@ -100,12 +111,33 @@
offsetof(ngx_http_upstream_keepalive_srv_conf_t, requests),
NULL },
+ { ngx_string("keepalive_pool_key"),
+ NGX_HTTP_UPS_CONF|NGX_CONF_TAKE1,
+ ngx_http_upstream_keepalive_pool_key,
+ NGX_HTTP_SRV_CONF_OFFSET,
+ 0,
+ NULL },
+
ngx_null_command
};
+static ngx_http_variable_t ngx_http_upstream_keepalive_vars[] = {
+
+ { ngx_string("keepalive_pool_key"), NULL,
+ ngx_http_upstream_keepalive_pool_key_var_get,
+ 1, NGX_HTTP_VAR_NOCACHEABLE, 0 },
+
+ { ngx_string("keepalive_pool_key_raw"), NULL,
+ ngx_http_upstream_keepalive_pool_key_var_get,
+ 0, NGX_HTTP_VAR_NOCACHEABLE, 0 },
+
+ ngx_http_null_variable
+};
+
+
static ngx_http_module_t ngx_http_upstream_keepalive_module_ctx = {
- NULL, /* preconfiguration */
+ ngx_http_upstream_keepalive_add_variables, /* preconfiguration */
NULL, /* postconfiguration */
NULL, /* create main configuration */
@@ -186,6 +218,7 @@
{
ngx_http_upstream_keepalive_peer_data_t *kp;
ngx_http_upstream_keepalive_srv_conf_t *kcf;
+ ngx_str_t pool_key;
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"init keepalive peer");
@@ -202,6 +235,25 @@
return NGX_ERROR;
}
+ if (kcf->pool_key.value.data) {
+ if (ngx_http_complex_value(r, &kcf->pool_key, &pool_key) !=
NGX_OK) {
+ return NGX_ERROR;
+ }
+
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "keepalive peer pool_key: \"%V\"", &pool_key);
+
+ } else {
+ pool_key.len = 0;
+ }
+
+ if (pool_key.len > 0) {
+ kp->pool_key_crc32 = ngx_crc32_long(pool_key.data, pool_key.len);
+
+ } else {
+ kp->pool_key_crc32 = 0;
+ }
+
kp->conf = kcf;
kp->upstream = r->upstream;
kp->data = r->upstream->peer.data;
@@ -256,8 +308,9 @@
c = item->connection;
if (ngx_memn2cmp((u_char *) &item->sockaddr, (u_char *)
pc->sockaddr,
- item->socklen, pc->socklen)
- == 0)
+ item->socklen, pc->socklen) == 0
+ && (item->pool_key_crc32 == 0
+ || item->pool_key_crc32 == kp->pool_key_crc32))
{
ngx_queue_remove(q);
ngx_queue_insert_head(&kp->conf->free, q);
@@ -386,6 +439,8 @@
item->socklen = pc->socklen;
ngx_memcpy(&item->sockaddr, pc->sockaddr, pc->socklen);
+ item->pool_key_crc32 = kp->pool_key_crc32;
+
if (c->read->ready) {
ngx_http_upstream_keepalive_close_handler(c->read);
}
@@ -477,6 +532,10 @@
{
ngx_http_upstream_keepalive_peer_data_t *kp = data;
+ if (kp->pool_key_crc32 != 0) {
+ return NGX_OK;
+ }
+
return kp->original_set_session(pc, kp->data);
}
@@ -486,7 +545,10 @@
{
ngx_http_upstream_keepalive_peer_data_t *kp = data;
- kp->original_save_session(pc, kp->data);
+ if (kp->pool_key_crc32 == 0) {
+ kp->original_save_session(pc, kp->data);
+ }
+
return;
}
@@ -509,6 +571,7 @@
*
* conf->original_init_upstream = NULL;
* conf->original_init_peer = NULL;
+ * conf->pool_key = NULL;
* conf->max_cached = 0;
*/
@@ -559,3 +622,94 @@
return NGX_CONF_OK;
}
+
+
+static char *
+ngx_http_upstream_keepalive_pool_key(ngx_conf_t *cf, ngx_command_t *cmd,
+ void *conf)
+{
+ ngx_http_upstream_keepalive_srv_conf_t *kcf = conf;
+ ngx_str_t *value;
+ ngx_http_compile_complex_value_t ccv;
+
+ value = cf->args->elts;
+
+ if (kcf->pool_key.value.data) {
+ return "is duplicate";
+ }
+
+ ngx_memzero(&ccv, sizeof(ngx_http_compile_complex_value_t));
+
+ ccv.cf = cf;
+ ccv.value = &value[1];
+ ccv.complex_value = &kcf->pool_key;
+
+ if (ngx_http_compile_complex_value(&ccv) != NGX_OK) {
+ return NGX_CONF_ERROR;
+ }
+
+ return NGX_CONF_OK;
+}
+
+
+static ngx_int_t
+ngx_http_upstream_keepalive_add_variables(ngx_conf_t *cf)
+{
+ ngx_http_variable_t *var, *v;
+
+ for (v = ngx_http_upstream_keepalive_vars; v->name.len; v++) {
+ var = ngx_http_add_variable(cf, &v->name, v->flags);
+ if (var == NULL) {
+ return NGX_ERROR;
+ }
+
+ *var = *v;
+ }
+
+ return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_http_upstream_keepalive_pool_key_var_get(ngx_http_request_t *r,
+ ngx_http_variable_value_t *v, uintptr_t data)
+{
+ ngx_flag_t compile = data;
+ ngx_str_t pool_key;
+ ngx_http_upstream_t *u;
+ ngx_http_upstream_srv_conf_t *uscf;
+ ngx_http_upstream_keepalive_srv_conf_t *kcf;
+
+ u = r->upstream;
+ if (u == NULL) {
+ goto not_found;
+ }
+
+ uscf = u->conf->upstream;
+
+ kcf = ngx_http_conf_upstream_srv_conf(uscf,
+
ngx_http_upstream_keepalive_module);
+
+ if (compile) {
+ if (ngx_http_complex_value(r, &kcf->pool_key, &pool_key) !=
NGX_OK) {
+ goto not_found;
+ }
+
+ } else {
+ pool_key = kcf->pool_key.value;
+ }
+
+ v->valid = 1;
+ v->not_found = 0;
+ v->no_cacheable = 1;
+ v->data = pool_key.data;
+ v->len = pool_key.len;
+
+ return NGX_OK;
+
+not_found:
+
+ v->not_found = 1;
+
+ return NGX_OK;
+}
-------------- next part --------------
A non-text attachment was scrubbed...
Name: upstream_keepalive_pool_key.patch
Type: text/x-patch
Size: 10017 bytes
Desc: not available
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20190830/f8983409/attachment-0001.bin>
More information about the nginx-devel
mailing list