[PATCH] Support of proxy v2 protocol for NGINX stream module
Vasiliy Soshnikov
vasiliy.soshnikov at gmail.com
Fri Apr 9 13:26:52 UTC 2021
diff -r 82e174e47663 src/core/ngx_proxy_protocol.c
--- a/src/core/ngx_proxy_protocol.c Thu Apr 08 00:16:30 2021 +0300
+++ b/src/core/ngx_proxy_protocol.c Fri Apr 09 16:10:29 2021 +0300
@@ -13,6 +13,34 @@
#define NGX_PROXY_PROTOCOL_AF_INET6 2
+#define NGX_PROXY_PROTOCOL_V2_SIG
"\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A"
+#define NGX_PROXY_PROTOCOL_V2_SIG_LEN 12
+#define NGX_PROXY_PROTOCOL_V2_HDR_LEN 16
+#define NGX_PROXY_PROTOCOL_V2_HDR_LEN_INET \
+ (NGX_PROXY_PROTOCOL_V2_HDR_LEN + (4 + 4 + 2 + 2))
+#define NGX_PROXY_PROTOCOL_V2_HDR_LEN_INET6 \
+ (NGX_PROXY_PROTOCOL_V2_HDR_LEN + (16 + 16 + 2 + 2))
+
+#define NGX_PROXY_PROTOCOL_V2_CMD_PROXY (0x20 | 0x01)
+
+#define NGX_PROXY_PROTOCOL_V2_TRANS_STREAM 0x01
+
+#define NGX_PROXY_PROTOCOL_V2_FAM_UNSPEC 0x00
+#define NGX_PROXY_PROTOCOL_V2_FAM_INET 0x10
+#define NGX_PROXY_PROTOCOL_V2_FAM_INET6 0x20
+
+#define NGX_PROXY_PROTOCOL_V2_TYPE_ALPN 0x01
+#define NGX_PROXY_PROTOCOL_V2_TYPE_SSL 0x20
+#define NGX_PROXY_PROTOCOL_V2_SUBTYPE_SSL_VERSION 0x21
+#define NGX_PROXY_PROTOCOL_V2_SUBTYPE_SSL_CIPHER 0x23
+#define NGX_PROXY_PROTOCOL_V2_SUBTYPE_SSL_SIG_ALG 0x24
+#define NGX_PROXY_PROTOCOL_V2_SUBTYPE_SSL_KEY_ALG 0x25
+
+#define NGX_PROXY_PROTOCOL_V2_CLIENT_SSL 0x01
+#define NGX_PROXY_PROTOCOL_V2_CLIENT_CERT_CONN 0x02
+#define NGX_PROXY_PROTOCOL_V2_CLIENT_CERT_SESS 0x04
+
+
#define ngx_proxy_protocol_parse_uint16(p) ((p)[0] << 8 | (p)[1])
@@ -40,12 +68,68 @@
} ngx_proxy_protocol_inet6_addrs_t;
+typedef union {
+ struct {
+ uint32_t src_addr;
+ uint32_t dst_addr;
+ uint16_t src_port;
+ uint16_t dst_port;
+ } ip4;
+ struct {
+ uint8_t src_addr[16];
+ uint8_t dst_addr[16];
+ uint16_t src_port;
+ uint16_t dst_port;
+ } ip6;
+} ngx_proxy_protocol_addrs_t;
+
+
+typedef struct {
+ u_char signature[12];
+ uint8_t version_command;
+ uint8_t family_transport;
+ uint16_t len;
+ ngx_proxy_protocol_addrs_t addr;
+} ngx_proxy_protocol_v2_header_t;
+
+
+struct ngx_tlv_s {
+ uint8_t type;
+ uint8_t length_hi;
+ uint8_t length_lo;
+ uint8_t value[0];
+} __attribute__((packed));
+
+typedef struct ngx_tlv_s ngx_tlv_t;
+
+
+#if (NGX_STREAM_SSL)
+struct ngx_tlv_ssl_s {
+ ngx_tlv_t tlv;
+ uint8_t client;
+ uint32_t verify;
+ uint8_t sub_tlv[];
+} __attribute__((packed));
+
+typedef struct ngx_tlv_ssl_s ngx_tlv_ssl_t;
+#endif
+
+
static u_char *ngx_proxy_protocol_read_addr(ngx_connection_t *c, u_char *p,
u_char *last, ngx_str_t *addr);
static u_char *ngx_proxy_protocol_read_port(u_char *p, u_char *last,
in_port_t *port, u_char sep);
static u_char *ngx_proxy_protocol_v2_read(ngx_connection_t *c, u_char *buf,
u_char *last);
+static u_char *ngx_proxy_protocol_v2_write(ngx_connection_t *c, u_char
*buf,
+ u_char *last);
+#if (NGX_HAVE_INET6)
+static void ngx_v4tov6(struct in6_addr *sin6_addr, struct sockaddr *addr);
+#endif
+#if (NGX_STREAM_SSL)
+static u_char *ngx_copy_tlv(u_char *pos, u_char *last, u_char type,
+ u_char *value, uint16_t value_len);
+#endif
u_char *
@@ -223,7 +307,8 @@
u_char *
-ngx_proxy_protocol_write(ngx_connection_t *c, u_char *buf, u_char *last)
+ngx_proxy_protocol_write(ngx_connection_t *c, u_char *buf, u_char *last,
+ ngx_uint_t pp_version)
{
ngx_uint_t port, lport;
@@ -235,6 +320,10 @@
return NULL;
}
+ if (pp_version == 2) {
+ return ngx_proxy_protocol_v2_write(c, buf, last);
+ }
+
switch (c->sockaddr->sa_family) {
case AF_INET:
@@ -420,3 +509,344 @@
return end;
}
+
+
+static u_char *
+ngx_proxy_protocol_v2_write(ngx_connection_t *c, u_char *buf, u_char *last)
+{
+ struct sockaddr *src, *dst;
+ ngx_proxy_protocol_v2_header_t *header;
+#if (NGX_HAVE_INET6)
+ struct in6_addr v6_tmp;
+ ngx_int_t v6_used;
+#endif
+#if (NGX_STREAM_SSL)
+ ngx_tlv_ssl_t *tlv;
+ u_char *value, *pos;
+ u_char kbuf[100];
+ const unsigned char *data;
+ unsigned int data_len;
+
+ X509 *crt;
+ EVP_PKEY *key;
+ const ASN1_OBJECT *algorithm;
+ const char *s;
+
+ long rc;
+ size_t tlv_len;
+#endif
+ size_t len;
+
+ header = (ngx_proxy_protocol_v2_header_t *) buf;
+
+ header->len = 0;
+
+ src = c->sockaddr;
+ dst = c->local_sockaddr;
+
+ len = 0;
+
+#if (NGX_HAVE_INET6)
+ v6_used = 0;
+#endif
+
+ ngx_memcpy(header->signature, NGX_PROXY_PROTOCOL_V2_SIG,
+ NGX_PROXY_PROTOCOL_V2_SIG_LEN);
+
+ header->version_command = NGX_PROXY_PROTOCOL_V2_CMD_PROXY;
+ header->family_transport = NGX_PROXY_PROTOCOL_V2_TRANS_STREAM;
+
+ /** Addrs */
+
+ switch (src->sa_family) {
+
+ case AF_INET:
+
+ if (dst->sa_family == AF_INET) {
+
+ header->addr.ip4.src_addr =
+ ((struct sockaddr_in *) src)->sin_addr.s_addr;
+ header->addr.ip4.src_port = ((struct sockaddr_in *)
src)->sin_port;
+ }
+#if (NGX_HAVE_INET6)
+ else /** dst == AF_INET6 */{
+
+ ngx_v4tov6(&v6_tmp, src);
+ ngx_memcpy(header->addr.ip6.src_addr, &v6_tmp, 16);
+ header->addr.ip6.src_port = ((struct sockaddr_in *)
src)->sin_port;
+ }
+#endif
+ break;
+
+#if (NGX_HAVE_INET6)
+ case AF_INET6:
+ v6_used = 1;
+
+ ngx_memcpy(header->addr.ip6.src_addr,
+ &((struct sockaddr_in6 *) src)->sin6_addr, 16);
+ header->addr.ip6.src_port = ((struct sockaddr_in6 *)
src)->sin6_port;
+
+ break;
+#endif
+
+ default:
+ ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0,
+ "PROXY protocol v2 unsupported src address family %ui",
+ src->sa_family);
+ goto unspec;
+ };
+
+ switch (dst->sa_family) {
+ case AF_INET:
+
+ if (src->sa_family == AF_INET) {
+
+ header->addr.ip4.dst_addr =
+ ((struct sockaddr_in *) dst)->sin_addr.s_addr;
+ header->addr.ip4.dst_port = ((struct sockaddr_in *)
dst)->sin_port;
+ }
+#if (NGX_HAVE_INET6)
+ else /** src == AF_INET6 */{
+
+ ngx_v4tov6(&v6_tmp, dst);
+ ngx_memcpy(header->addr.ip6.dst_addr, &v6_tmp, 16);
+ header->addr.ip6.dst_port = ((struct sockaddr_in *)
dst)->sin_port;
+
+ }
+#endif
+ break;
+
+#if (NGX_HAVE_INET6)
+ case AF_INET6:
+ v6_used = 1;
+
+ ngx_memcpy(header->addr.ip6.dst_addr,
+ &((struct sockaddr_in6 *) dst)->sin6_addr, 16);
+ header->addr.ip6.dst_port = ((struct sockaddr_in6 *)
dst)->sin6_port;
+
+ break;
+#endif
+
+ default:
+ ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0,
+ "PROXY protocol v2 unsupported dest address family
%ui",
+ dst->sa_family);
+ goto unspec;
+ }
+
+#if (NGX_HAVE_INET6)
+ if (!v6_used) {
+ header->family_transport |= NGX_PROXY_PROTOCOL_V2_FAM_INET;
+ len = NGX_PROXY_PROTOCOL_V2_HDR_LEN_INET;
+
+ } else {
+ header->family_transport |= NGX_PROXY_PROTOCOL_V2_FAM_INET6;
+ len = NGX_PROXY_PROTOCOL_V2_HDR_LEN_INET6;
+
+ }
+#else
+ header->family_transport |= NGX_PROXY_PROTOCOL_V2_FAM_INET;
+ len = NGX_PROXY_PROTOCOL_V2_HDR_LEN_INET;
+#endif
+
+ /** SSL TLVs */
+
+#if (NGX_STREAM_SSL)
+
+ data = NULL;
+ data_len = 0;
+
+ tlv = (ngx_tlv_ssl_t *) (buf + len);
+ ngx_memzero(tlv, sizeof(ngx_tlv_ssl_t));
+
+ tlv->tlv.type = NGX_PROXY_PROTOCOL_V2_TYPE_SSL;
+ pos = buf + len + sizeof(ngx_tlv_ssl_t);
+
+ tlv->client |= NGX_PROXY_PROTOCOL_V2_CLIENT_SSL;
+
+ if (c->ssl != NULL) {
+
+#ifdef TLSEXT_TYPE_application_layer_protocol_negotiation
+ SSL_get0_alpn_selected(c->ssl->connection, &data, &data_len);
+
+#ifdef TLSEXT_TYPE_next_proto_neg
+ if (data_len == 0) {
+ SSL_get0_next_proto_negotiated(c->ssl->connection,
+ &data, &data_len);
+ }
+#endif
+
+#else /* TLSEXT_TYPE_next_proto_neg */
+ SSL_get0_next_proto_negotiated(c->ssl->connection, &data,
&data_len);
+#endif
+
+ if (data_len) {
+
+ pos = ngx_copy_tlv(pos, last,
+ NGX_PROXY_PROTOCOL_V2_TYPE_ALPN,
+ (u_char *) data, (uint16_t) data_len);
+ if (pos == NULL) {
+ return NULL;
+ }
+ }
+
+ value = (u_char *) SSL_get_version(c->ssl->connection);
+ if (value != NULL) {
+
+ pos = ngx_copy_tlv(pos, last,
+ NGX_PROXY_PROTOCOL_V2_SUBTYPE_SSL_VERSION,
+ value, ngx_strlen(value));
+ if (pos == NULL) {
+ return NULL;
+ }
+ }
+
+ crt = SSL_get_peer_certificate(c->ssl->connection);
+ if (crt != NULL) {
+
+ tlv->client |= NGX_PROXY_PROTOCOL_V2_CLIENT_CERT_SESS;
+
+ rc = SSL_get_verify_result(c->ssl->connection);
+ tlv->verify = htonl(rc);
+
+ if (rc == X509_V_OK) {
+
+ if (ngx_ssl_ocsp_get_status(c, &s) == NGX_OK) {
+ tlv->client |= NGX_PROXY_PROTOCOL_V2_CLIENT_CERT_CONN;
+ }
+ }
+
+ X509_free(crt);
+ }
+
+ crt = SSL_get_certificate(c->ssl->connection);
+ if (crt != NULL) {
+
+ key = X509_get_pubkey(crt);
+
+ /** Key */
+ if (key != NULL) {
+
+ switch (EVP_PKEY_base_id(key)) {
+ case EVP_PKEY_RSA:
+ value = (u_char *) "RSA";
+ break;
+ case EVP_PKEY_EC:
+ value = (u_char *) "EC";
+ break;
+ case EVP_PKEY_DSA:
+ value = (u_char *) "DSA";
+ break;
+ default:
+ value = NULL;
+ break;
+ }
+
+ if (value != NULL) {
+
+ value = ngx_snprintf(kbuf, sizeof(kbuf) - 1, "%s%d%Z",
+ value, EVP_PKEY_bits(key));
+
+ pos = ngx_copy_tlv(pos, last,
+ NGX_PROXY_PROTOCOL_V2_SUBTYPE_SSL_KEY_ALG,
+ kbuf, ngx_strlen(kbuf));
+ }
+
+ EVP_PKEY_free(key);
+
+ if (pos == NULL) {
+ return NULL;
+ }
+ }
+
+ /* ALG */
+ X509_ALGOR_get0(&algorithm, NULL, NULL,
X509_get0_tbs_sigalg(crt));
+ value = (u_char *) OBJ_nid2sn(OBJ_obj2nid(algorithm));
+
+ if (value != NULL) {
+
+ pos = ngx_copy_tlv(pos, last,
+ NGX_PROXY_PROTOCOL_V2_SUBTYPE_SSL_SIG_ALG,
+ value, ngx_strlen(value));
+ if (pos == NULL) {
+ return NULL;
+ }
+ }
+ }
+
+ value = (u_char *) SSL_get_cipher_name(c->ssl->connection);
+ if (value != NULL) {
+
+ pos = ngx_copy_tlv(pos, last,
+ NGX_PROXY_PROTOCOL_V2_SUBTYPE_SSL_CIPHER,
+ value, ngx_strlen(value));
+ if (pos == NULL) {
+ return NULL;
+ }
+ }
+ }
+
+ tlv_len = pos - (buf + len);
+
+ tlv->tlv.length_hi = (uint16_t) (tlv_len - sizeof(ngx_tlv_t)) >> 8;
+ tlv->tlv.length_lo = (uint16_t) (tlv_len - sizeof(ngx_tlv_t)) & 0x00ff;
+
+ len = len + tlv_len;
+
+#endif
+
+ header->len = htons(len - NGX_PROXY_PROTOCOL_V2_HDR_LEN);
+ return buf + len;
+
+unspec:
+ header->family_transport |= NGX_PROXY_PROTOCOL_V2_FAM_UNSPEC;
+ header->len = 0;
+
+ return buf + NGX_PROXY_PROTOCOL_V2_HDR_LEN;
+}
+
+
+#if (NGX_HAVE_INET6)
+static void
+ngx_v4tov6(struct in6_addr *sin6_addr, struct sockaddr *addr)
+{
+ static const char rfc4291[] = { 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0xFF, 0xFF };
+
+ struct in_addr tmp_addr, *sin_addr;
+
+ sin_addr = &((struct sockaddr_in *) addr)->sin_addr;
+
+ tmp_addr.s_addr = sin_addr->s_addr;
+ ngx_memcpy(sin6_addr->s6_addr, rfc4291, sizeof(rfc4291));
+ ngx_memcpy(sin6_addr->s6_addr + 12, &tmp_addr.s_addr, 4);
+}
+#endif
+
+
+#if (NGX_STREAM_SSL)
+
+static u_char *
+ngx_copy_tlv(u_char *pos, u_char *last, u_char type,
+ u_char *value, uint16_t value_len)
+{
+ ngx_tlv_t *tlv;
+
+ if (last - pos < (long) sizeof(*tlv)) {
+ return NULL;
+ }
+
+ tlv = (ngx_tlv_t *) pos;
+
+ tlv->type = type;
+ tlv->length_hi = (uint16_t) value_len >> 8;
+ tlv->length_lo = (uint16_t) value_len & 0x00ff;
+ ngx_memcpy(tlv->value, value, value_len);
+
+ return pos + (value_len + sizeof(*tlv));
+}
+
+#endif
+
+
diff -r 82e174e47663 src/core/ngx_proxy_protocol.h
--- a/src/core/ngx_proxy_protocol.h Thu Apr 08 00:16:30 2021 +0300
+++ b/src/core/ngx_proxy_protocol.h Fri Apr 09 16:10:29 2021 +0300
@@ -13,7 +13,7 @@
#include <ngx_core.h>
-#define NGX_PROXY_PROTOCOL_MAX_HEADER 107
+#define NGX_PROXY_PROTOCOL_MAX_HEADER 214
struct ngx_proxy_protocol_s {
@@ -27,7 +27,7 @@
u_char *ngx_proxy_protocol_read(ngx_connection_t *c, u_char *buf,
u_char *last);
u_char *ngx_proxy_protocol_write(ngx_connection_t *c, u_char *buf,
- u_char *last);
+ u_char *last, ngx_uint_t pp_version);
#endif /* _NGX_PROXY_PROTOCOL_H_INCLUDED_ */
diff -r 82e174e47663 src/stream/ngx_stream_proxy_module.c
--- a/src/stream/ngx_stream_proxy_module.c Thu Apr 08 00:16:30 2021 +0300
+++ b/src/stream/ngx_stream_proxy_module.c Fri Apr 09 16:10:29 2021 +0300
@@ -30,7 +30,7 @@
ngx_uint_t responses;
ngx_uint_t next_upstream_tries;
ngx_flag_t next_upstream;
- ngx_flag_t proxy_protocol;
+ ngx_uint_t proxy_protocol;
ngx_stream_upstream_local_t *local;
ngx_flag_t socket_keepalive;
@@ -121,6 +121,14 @@
#endif
+static ngx_conf_enum_t ngx_stream_proxy_protocol[] = {
+ { ngx_string("off"), 0 },
+ { ngx_string("on"), 1 },
+ { ngx_string("v2"), 2 },
+ { ngx_null_string, 0 }
+};
+
+
static ngx_conf_deprecated_t ngx_conf_deprecated_proxy_downstream_buffer
= {
ngx_conf_deprecated, "proxy_downstream_buffer", "proxy_buffer_size"
};
@@ -239,10 +247,10 @@
{ ngx_string("proxy_protocol"),
NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_FLAG,
- ngx_conf_set_flag_slot,
+ ngx_conf_set_enum_slot,
NGX_STREAM_SRV_CONF_OFFSET,
offsetof(ngx_stream_proxy_srv_conf_t, proxy_protocol),
- NULL },
+ &ngx_stream_proxy_protocol },
#if (NGX_STREAM_SSL)
@@ -891,7 +899,8 @@
cl->buf->pos = p;
- p = ngx_proxy_protocol_write(c, p, p +
NGX_PROXY_PROTOCOL_MAX_HEADER);
+ p = ngx_proxy_protocol_write(c, p, p +
NGX_PROXY_PROTOCOL_MAX_HEADER,
+ u->proxy_protocol);
if (p == NULL) {
ngx_stream_proxy_finalize(s, NGX_STREAM_INTERNAL_SERVER_ERROR);
return;
@@ -942,14 +951,15 @@
ngx_log_debug0(NGX_LOG_DEBUG_STREAM, c->log, 0,
"stream proxy send PROXY protocol header");
- p = ngx_proxy_protocol_write(c, buf, buf +
NGX_PROXY_PROTOCOL_MAX_HEADER);
+ u = s->upstream;
+
+ p = ngx_proxy_protocol_write(c, buf, buf +
NGX_PROXY_PROTOCOL_MAX_HEADER,
+ u->proxy_protocol);
if (p == NULL) {
ngx_stream_proxy_finalize(s, NGX_STREAM_INTERNAL_SERVER_ERROR);
return NGX_ERROR;
}
- u = s->upstream;
-
pc = u->peer.connection;
size = p - buf;
@@ -1998,7 +2008,7 @@
conf->responses = NGX_CONF_UNSET_UINT;
conf->next_upstream_tries = NGX_CONF_UNSET_UINT;
conf->next_upstream = NGX_CONF_UNSET;
- conf->proxy_protocol = NGX_CONF_UNSET;
+ conf->proxy_protocol = NGX_CONF_UNSET_UINT;
conf->local = NGX_CONF_UNSET_PTR;
conf->socket_keepalive = NGX_CONF_UNSET;
@@ -2053,7 +2063,7 @@
ngx_conf_merge_value(conf->next_upstream, prev->next_upstream, 1);
- ngx_conf_merge_value(conf->proxy_protocol, prev->proxy_protocol, 0);
+ ngx_conf_merge_uint_value(conf->proxy_protocol, prev->proxy_protocol,
0);
ngx_conf_merge_ptr_value(conf->local, prev->local, NULL);
diff -r 82e174e47663 src/stream/ngx_stream_upstream.h
--- a/src/stream/ngx_stream_upstream.h Thu Apr 08 00:16:30 2021 +0300
+++ b/src/stream/ngx_stream_upstream.h Fri Apr 09 16:10:29 2021 +0300
@@ -141,7 +141,7 @@
ngx_stream_upstream_resolved_t *resolved;
ngx_stream_upstream_state_t *state;
unsigned connected:1;
- unsigned proxy_protocol:1;
+ unsigned proxy_protocol:2;
} ngx_stream_upstream_t;
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx-devel/attachments/20210409/4d337b40/attachment-0001.htm>
More information about the nginx-devel
mailing list