[PATCH] Mail: added PROXY PROTOCOL support
Murad Mamedov
mail at muradm.net
Sat Jan 16 11:38:03 UTC 2021
Hi,
We are trying to use Nginx as frontend for SMTP and IMAP servers
inside docker swarm cluster. Nginx allows us to validate clients
and TLS client certificates confidently. More over, we are able
to post validate client certificates with "auth_http" and
"auth_http_pass_client_cert" options. Everything working fine,
except the fact that Nginx it self resides behind load balancer.
The only reliable way to acquire client and real server addresses
is to support PROXY PROTOCOL header coming from load balancer.
Below patch adds PROXY PROTOCOL support for both downstream (header
received from load balancer) and upstream (header sent to real
servers like Postfix, Exim, Dovecot).
Why this is done:
- while there seems to be support for "XCLIENT" in SMTP, it
first works only for SMTP protocol and not for others,
second it is meaningless when Nginx itself within docker
swarm container and network mesh. More over, configuring
"xclient" for SMTP even, requires to use "trusted" mode
of configuration in Postfix at least, to accept XCLIENT,
which complicates the things.
- Mail software like Postfix, Exim, Dovecot (probably others)
support PROXY PROTOCOL out of the box.
How it is done:
- using Nginx core existing "ngx_proxy_protocol_read" and
"ngx_proxy_protocol_write" functions.
- if configured, first thing done is header reading, remaining
processing is untouched. For upstream, if configured, first
thing done is header sending, remaining processing in untouched.
Tests are passing, and in reply to this email I will be sending another
patch with more tests that cover testing this patch.
--
muradm
# HG changeset patch
# User muradm <mail at muradm.net>
# Date 1610795056 -10800
# Sat Jan 16 14:04:16 2021 +0300
# Node ID 5f6f4a627b889e5c6438601d8ff07a940f12df3c
# Parent 83c4622053b02821a12d522d08eaff3ac27e65e3
Mail: added PROXY PROTOCOL support.
This implements propxy protocol support for both upstream and downstream.
Downstream proxy protocol support:
mail {
server {
listen <port> [ssl] proxy_protocol;
protocol <imap|pop3|smtp>;
}
}
This will properly handle incoming connections from load balancer sending
PROXY protocol header. Without this, it is impossible to run nginx mail
proxy behind such balancer. Header reading is done with existing function
"ngx_proxy_protocol_read", so it should support both v1 and v2 headers.
This will also set "sockaddr" and "local_sockaddr" addresses from received
header, mimicing "set_realip". While "realip_module" deals with variables
etc., which is necessary for HTTP protocol, mail protocols are pretty
strict, so there is no need for flexible handling of real addresses
received.
Upstream proxy protocol support:
mail {
server {
listen <port> [ssl];
protocol <imap|pop3|smtp>;
proxy_protocol on;
}
}
With this, upstream server (like Postfix, Exim, Dovecot) will have PROXY
protocol header. Mentioned programs do support proxy protocol out of the
box. Header is written with existing function "ngx_proxy_protocol_write"
which supports only v1 header writing. Contents of header are written
from "sockaddr" and "local_sockaddr".
Downstream and upstream proxy protocol support:
mail {
server {
listen <port> [ssl] proxy_protocol;
protocol <imap|pop3|smtp>;
proxy_protocol on;
}
}
This will combine both receiving PROXY header and sending PROXY header. With
this, upstream server (like Postfix, Exim, Dovecot) will receive the same
header as was sent by downstream load balancer.
Above configurations work for SSL as well and should be transparent to other
mail related configurations.
diff -r 83c4622053b0 -r 5f6f4a627b88 src/mail/ngx_mail.c
--- a/src/mail/ngx_mail.c Tue Jan 12 16:59:31 2021 +0300
+++ b/src/mail/ngx_mail.c Sat Jan 16 14:04:16 2021 +0300
@@ -402,6 +402,7 @@
addrs[i].addr = sin->sin_addr.s_addr;
addrs[i].conf.ctx = addr[i].opt.ctx;
+ addrs[i].conf.proxy_protocol = addr[i].opt.proxy_protocol;
#if (NGX_MAIL_SSL)
addrs[i].conf.ssl = addr[i].opt.ssl;
#endif
@@ -436,6 +437,7 @@
addrs6[i].addr6 = sin6->sin6_addr;
addrs6[i].conf.ctx = addr[i].opt.ctx;
+ addrs6[i].conf.proxy_protocol = addr[i].opt.proxy_protocol;
#if (NGX_MAIL_SSL)
addrs6[i].conf.ssl = addr[i].opt.ssl;
#endif
diff -r 83c4622053b0 -r 5f6f4a627b88 src/mail/ngx_mail.h
--- a/src/mail/ngx_mail.h Tue Jan 12 16:59:31 2021 +0300
+++ b/src/mail/ngx_mail.h Sat Jan 16 14:04:16 2021 +0300
@@ -37,6 +37,7 @@
unsigned bind:1;
unsigned wildcard:1;
unsigned ssl:1;
+ unsigned proxy_protocol:1;
#if (NGX_HAVE_INET6)
unsigned ipv6only:1;
#endif
@@ -56,6 +57,7 @@
ngx_mail_conf_ctx_t *ctx;
ngx_str_t addr_text;
ngx_uint_t ssl; /* unsigned ssl:1; */
+ unsigned proxy_protocol:1;
} ngx_mail_addr_conf_t;
typedef struct {
@@ -190,6 +192,7 @@
void **ctx;
void **main_conf;
void **srv_conf;
+ ngx_mail_addr_conf_t *addr_conf;
ngx_resolver_ctx_t *resolver_ctx;
@@ -197,6 +200,7 @@
ngx_uint_t mail_state;
+ unsigned proxy_protocol:1;
unsigned protocol:3;
unsigned blocked:1;
unsigned quit:1;
diff -r 83c4622053b0 -r 5f6f4a627b88 src/mail/ngx_mail_core_module.c
--- a/src/mail/ngx_mail_core_module.c Tue Jan 12 16:59:31 2021 +0300
+++ b/src/mail/ngx_mail_core_module.c Sat Jan 16 14:04:16 2021 +0300
@@ -548,6 +548,11 @@
#endif
}
+ if (ngx_strcmp(value[i].data, "proxy_protocol") == 0) {
+ ls->proxy_protocol = 1;
+ continue;
+ }
+
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"the invalid \"%V\" parameter", &value[i]);
return NGX_CONF_ERROR;
diff -r 83c4622053b0 -r 5f6f4a627b88 src/mail/ngx_mail_handler.c
--- a/src/mail/ngx_mail_handler.c Tue Jan 12 16:59:31 2021 +0300
+++ b/src/mail/ngx_mail_handler.c Sat Jan 16 14:04:16 2021 +0300
@@ -12,6 +12,8 @@
static void ngx_mail_init_session(ngx_connection_t *c);
+static void ngx_mail_init_connection_complete(ngx_connection_t *c);
+static void ngx_mail_proxy_protocol_handler(ngx_event_t *rev);
#if (NGX_MAIL_SSL)
static void ngx_mail_ssl_init_connection(ngx_ssl_t *ssl, ngx_connection_t *c);
@@ -128,6 +130,7 @@
s->main_conf = addr_conf->ctx->main_conf;
s->srv_conf = addr_conf->ctx->srv_conf;
+ s->addr_conf = addr_conf;
s->addr_text = &addr_conf->addr_text;
@@ -159,13 +162,161 @@
c->log_error = NGX_ERROR_INFO;
+ /*
+ * Before all process proxy protocol
+ */
+
+ if (addr_conf->proxy_protocol) {
+ s->proxy_protocol = 1;
+ c->log->action = "reading PROXY protocol header";
+ c->read->handler = ngx_mail_proxy_protocol_handler;
+
+ ngx_add_timer(c->read, cscf->timeout);
+
+ if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
+ ngx_mail_close_connection(c);
+ }
+
+ return;
+ }
+
+ ngx_mail_init_connection_complete(c);
+}
+
+
+ngx_int_t
+ngx_mail_proxy_protoco_set_addrs(ngx_connection_t *c)
+{
+ ngx_addr_t addr_peer, addr_local;
+ u_char *p, text[NGX_SOCKADDR_STRLEN];
+ size_t len;
+
+ if (ngx_parse_addr(c->pool, &addr_peer,
+ c->proxy_protocol->src_addr.data,
+ c->proxy_protocol->src_addr.len) != NGX_OK)
+ {
+ return NGX_ERROR;
+ }
+
+ ngx_inet_set_port(addr_peer.sockaddr, c->proxy_protocol->src_port);
+
+ if (ngx_parse_addr(c->pool, &addr_local,
+ c->proxy_protocol->dst_addr.data,
+ c->proxy_protocol->dst_addr.len) != NGX_OK)
+ {
+ return NGX_ERROR;
+ }
+
+ ngx_inet_set_port(addr_local.sockaddr, c->proxy_protocol->dst_port);
+
+ len = ngx_sock_ntop(addr_peer.sockaddr, addr_peer.socklen, text,
+ NGX_SOCKADDR_STRLEN, 0);
+ if (len == 0) {
+ return NGX_ERROR;
+ }
+
+ p = ngx_pnalloc(c->pool, len);
+ if (p == NULL) {
+ return NGX_ERROR;
+ }
+
+ ngx_memcpy(p, text, len);
+
+ c->sockaddr = addr_peer.sockaddr;
+ c->socklen = addr_peer.socklen;
+ c->addr_text.len = len;
+ c->addr_text.data = p;
+
+ len = ngx_sock_ntop(addr_local.sockaddr, addr_local.socklen, text,
+ NGX_SOCKADDR_STRLEN, 0);
+ if (len == 0) {
+ return NGX_ERROR;
+ }
+
+ p = ngx_pnalloc(c->pool, len);
+ if (p == NULL) {
+ return NGX_ERROR;
+ }
+
+ ngx_memcpy(p, text, len);
+
+ c->local_sockaddr = addr_local.sockaddr;
+ c->local_socklen = addr_local.socklen;
+
+ return NGX_OK;
+}
+
+
+void
+ngx_mail_proxy_protocol_handler(ngx_event_t *rev)
+{
+ ngx_connection_t *c;
+ u_char *p, buf[NGX_PROXY_PROTOCOL_MAX_HEADER];
+ size_t size;
+ ssize_t n;
+
+ c = rev->data;
+
+ if (rev->timedout) {
+ ngx_log_error(NGX_LOG_INFO, c->log, NGX_ETIMEDOUT,
+ "mail PROXY protocol header timed out");
+ c->timedout = 1;
+ ngx_mail_close_connection(c);
+ return;
+ }
+
+ ngx_log_debug0(NGX_LOG_DEBUG_MAIL, c->log, 0,
+ "mail PROXY protocol handler");
+
+ size = NGX_PROXY_PROTOCOL_MAX_HEADER;
+
+ n = recv(c->fd, (char *) buf, size, MSG_PEEK);
+
+ ngx_log_debug1(NGX_LOG_DEBUG, c->log, 0, "mail recv(): %z", n);
+
+ p = ngx_proxy_protocol_read(c, buf, buf + n);
+
+ if (p == NULL) {
+ ngx_mail_close_connection(c);
+ return;
+ }
+
+ ngx_log_error(NGX_LOG_NOTICE, c->log, 0,
+ "PROXY protocol %V:%d => %V:%d",
+ &c->proxy_protocol->src_addr,
+ c->proxy_protocol->src_port,
+ &c->proxy_protocol->dst_addr,
+ c->proxy_protocol->dst_port);
+
+ size = p - buf;
+
+ if (c->recv(c, buf, size) != (ssize_t) size) {
+ ngx_mail_close_connection(c);
+ return;
+ }
+
+ if (ngx_mail_proxy_protoco_set_addrs(c) != NGX_OK) {
+ ngx_mail_close_connection(c);
+ return;
+ }
+
+ ngx_mail_init_connection_complete(c);
+}
+
+
+void
+ngx_mail_init_connection_complete(ngx_connection_t *c)
+{
#if (NGX_MAIL_SSL)
{
- ngx_mail_ssl_conf_t *sslcf;
+ ngx_mail_session_t *s;
+ ngx_mail_ssl_conf_t *sslcf;
+
+ s = c->data;
sslcf = ngx_mail_get_module_srv_conf(s, ngx_mail_ssl_module);
- if (sslcf->enable || addr_conf->ssl) {
+ if (sslcf->enable || s->addr_conf->ssl) {
c->log->action = "SSL handshaking";
ngx_mail_ssl_init_connection(&sslcf->ssl, c);
@@ -348,6 +499,7 @@
return;
}
+ c->log->action = "sending client greeting line";
c->write->handler = ngx_mail_send;
cscf->protocol->init_session(s, c);
diff -r 83c4622053b0 -r 5f6f4a627b88 src/mail/ngx_mail_proxy_module.c
--- a/src/mail/ngx_mail_proxy_module.c Tue Jan 12 16:59:31 2021 +0300
+++ b/src/mail/ngx_mail_proxy_module.c Sat Jan 16 14:04:16 2021 +0300
@@ -19,6 +19,7 @@
ngx_flag_t smtp_auth;
size_t buffer_size;
ngx_msec_t timeout;
+ ngx_flag_t proxy_protocol;
} ngx_mail_proxy_conf_t;
@@ -36,7 +37,7 @@
static void *ngx_mail_proxy_create_conf(ngx_conf_t *cf);
static char *ngx_mail_proxy_merge_conf(ngx_conf_t *cf, void *parent,
void *child);
-
+static ngx_int_t ngx_mail_proxy_send_proxy_protocol(ngx_mail_session_t *s);
static ngx_command_t ngx_mail_proxy_commands[] = {
@@ -82,6 +83,13 @@
offsetof(ngx_mail_proxy_conf_t, smtp_auth),
NULL },
+ { ngx_string("proxy_protocol"),
+ NGX_MAIL_MAIN_CONF|NGX_MAIL_SRV_CONF|NGX_CONF_FLAG,
+ ngx_conf_set_flag_slot,
+ NGX_MAIL_SRV_CONF_OFFSET,
+ offsetof(ngx_mail_proxy_conf_t, proxy_protocol),
+ NULL },
+
ngx_null_command
};
@@ -169,6 +177,12 @@
s->out.len = 0;
+ if (pcf->proxy_protocol == 1) {
+ if (ngx_mail_proxy_send_proxy_protocol(s) != NGX_OK) {
+ ngx_mail_proxy_internal_server_error(s);
+ }
+ }
+
switch (s->protocol) {
case NGX_MAIL_POP3_PROTOCOL:
@@ -189,6 +203,60 @@
}
+ngx_int_t
+ngx_mail_proxy_send_proxy_protocol(ngx_mail_session_t *s)
+{
+ u_char *p;
+ ssize_t n, size;
+ ngx_connection_t *c, *pc;
+ ngx_peer_connection_t *u;
+ u_char buf[NGX_PROXY_PROTOCOL_MAX_HEADER];
+
+ c = s->connection;
+
+ ngx_log_debug0(NGX_LOG_DEBUG_MAIL, c->log, 0,
+ "mail proxy send PROXY protocol header");
+
+ p = ngx_proxy_protocol_write(c, buf, buf + NGX_PROXY_PROTOCOL_MAX_HEADER);
+ if (p == NULL) {
+ ngx_mail_proxy_internal_server_error(s);
+ return NGX_ERROR;
+ }
+
+ u = &s->proxy->upstream;
+
+ pc = u->connection;
+
+ size = p - buf;
+
+ n = pc->send(pc, buf, size);
+
+ if (n < NGX_OK) {
+ ngx_mail_proxy_internal_server_error(s);
+ return NGX_ERROR;
+ }
+
+ if (n != size) {
+
+ /*
+ * PROXY protocol specification:
+ * The sender must always ensure that the header
+ * is sent at once, so that the transport layer
+ * maintains atomicity along the path to the receiver.
+ */
+
+ ngx_log_error(NGX_LOG_ERR, c->log, 0,
+ "could not send PROXY protocol header at once");
+
+ ngx_mail_proxy_internal_server_error(s);
+
+ return NGX_ERROR;
+ }
+
+ return NGX_OK;
+}
+
+
static void
ngx_mail_proxy_block_read(ngx_event_t *rev)
{
@@ -1184,6 +1252,7 @@
pcf->smtp_auth = NGX_CONF_UNSET;
pcf->buffer_size = NGX_CONF_UNSET_SIZE;
pcf->timeout = NGX_CONF_UNSET_MSEC;
+ pcf->proxy_protocol = NGX_CONF_UNSET;
return pcf;
}
More information about the nginx-devel
mailing list