[PATCH] Mail: added PROXY PROTOCOL support

muradm mail at muradm.net
Sat Jan 16 11:56:16 UTC 2021


# HG changeset patch
# User muradm <mail at muradm.net>
# Date 1610795056 -10800
#      Sat Jan 16 14:04:16 2021 +0300
# Node ID 5f6f4a627b889e5c6438601d8ff07a940f12df3c
# Parent  83c4622053b02821a12d522d08eaff3ac27e65e3
Mail: added PROXY PROTOCOL support.

This implements propxy protocol support for both upstream and downstream.

Downstream proxy protocol support:

mail {
    server {
        listen <port> [ssl] proxy_protocol;
	protocol <imap|pop3|smtp>;
    }
}

This will properly handle incoming connections from load balancer sending
PROXY protocol header. Without this, it is impossible to run nginx mail
proxy behind such balancer. Header reading is done with existing function
"ngx_proxy_protocol_read", so it should support both v1 and v2 headers.
This will also set "sockaddr" and "local_sockaddr" addresses from received
header, mimicing "set_realip". While "realip_module" deals with variables
etc., which is necessary for HTTP protocol, mail protocols are pretty
strict, so there is no need for flexible handling of real addresses
received.

Upstream proxy protocol support:

mail {
    server {
        listen <port> [ssl];
	protocol <imap|pop3|smtp>;
	proxy_protocol on;
    }
}

With this, upstream server (like Postfix, Exim, Dovecot) will have PROXY
protocol header. Mentioned programs do support proxy protocol out of the
box. Header is written with existing function "ngx_proxy_protocol_write"
which supports only v1 header writing. Contents of header are written
from "sockaddr" and "local_sockaddr".

Downstream and upstream proxy protocol support:

mail {
    server {
        listen <port> [ssl] proxy_protocol;
	protocol <imap|pop3|smtp>;
	proxy_protocol on;
    }
}

This will combine both receiving PROXY header and sending PROXY header. With
this, upstream server (like Postfix, Exim, Dovecot) will receive the same
header as was sent by downstream load balancer.

Above configurations work for SSL as well and should be transparent to other
mail related configurations.

diff -r 83c4622053b0 -r 5f6f4a627b88 src/mail/ngx_mail.c
--- a/src/mail/ngx_mail.c	Tue Jan 12 16:59:31 2021 +0300
+++ b/src/mail/ngx_mail.c	Sat Jan 16 14:04:16 2021 +0300
@@ -402,6 +402,7 @@
         addrs[i].addr = sin->sin_addr.s_addr;
 
         addrs[i].conf.ctx = addr[i].opt.ctx;
+        addrs[i].conf.proxy_protocol = addr[i].opt.proxy_protocol;
 #if (NGX_MAIL_SSL)
         addrs[i].conf.ssl = addr[i].opt.ssl;
 #endif
@@ -436,6 +437,7 @@
         addrs6[i].addr6 = sin6->sin6_addr;
 
         addrs6[i].conf.ctx = addr[i].opt.ctx;
+        addrs6[i].conf.proxy_protocol = addr[i].opt.proxy_protocol;
 #if (NGX_MAIL_SSL)
         addrs6[i].conf.ssl = addr[i].opt.ssl;
 #endif
diff -r 83c4622053b0 -r 5f6f4a627b88 src/mail/ngx_mail.h
--- a/src/mail/ngx_mail.h	Tue Jan 12 16:59:31 2021 +0300
+++ b/src/mail/ngx_mail.h	Sat Jan 16 14:04:16 2021 +0300
@@ -37,6 +37,7 @@
     unsigned                bind:1;
     unsigned                wildcard:1;
     unsigned                ssl:1;
+    unsigned                proxy_protocol:1;
 #if (NGX_HAVE_INET6)
     unsigned                ipv6only:1;
 #endif
@@ -56,6 +57,7 @@
     ngx_mail_conf_ctx_t    *ctx;
     ngx_str_t               addr_text;
     ngx_uint_t              ssl;    /* unsigned   ssl:1; */
+    unsigned                proxy_protocol:1;
 } ngx_mail_addr_conf_t;
 
 typedef struct {
@@ -190,6 +192,7 @@
     void                  **ctx;
     void                  **main_conf;
     void                  **srv_conf;
+    ngx_mail_addr_conf_t   *addr_conf;
 
     ngx_resolver_ctx_t     *resolver_ctx;
 
@@ -197,6 +200,7 @@
 
     ngx_uint_t              mail_state;
 
+    unsigned                proxy_protocol:1;
     unsigned                protocol:3;
     unsigned                blocked:1;
     unsigned                quit:1;
diff -r 83c4622053b0 -r 5f6f4a627b88 src/mail/ngx_mail_core_module.c
--- a/src/mail/ngx_mail_core_module.c	Tue Jan 12 16:59:31 2021 +0300
+++ b/src/mail/ngx_mail_core_module.c	Sat Jan 16 14:04:16 2021 +0300
@@ -548,6 +548,11 @@
 #endif
         }
 
+        if (ngx_strcmp(value[i].data, "proxy_protocol") == 0) {
+            ls->proxy_protocol = 1;
+            continue;
+        }
+
         ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
                            "the invalid \"%V\" parameter", &value[i]);
         return NGX_CONF_ERROR;
diff -r 83c4622053b0 -r 5f6f4a627b88 src/mail/ngx_mail_handler.c
--- a/src/mail/ngx_mail_handler.c	Tue Jan 12 16:59:31 2021 +0300
+++ b/src/mail/ngx_mail_handler.c	Sat Jan 16 14:04:16 2021 +0300
@@ -12,6 +12,8 @@
 
 
 static void ngx_mail_init_session(ngx_connection_t *c);
+static void ngx_mail_init_connection_complete(ngx_connection_t *c);
+static void ngx_mail_proxy_protocol_handler(ngx_event_t *rev);
 
 #if (NGX_MAIL_SSL)
 static void ngx_mail_ssl_init_connection(ngx_ssl_t *ssl, ngx_connection_t *c);
@@ -128,6 +130,7 @@
 
     s->main_conf = addr_conf->ctx->main_conf;
     s->srv_conf = addr_conf->ctx->srv_conf;
+    s->addr_conf = addr_conf;
 
     s->addr_text = &addr_conf->addr_text;
 
@@ -159,13 +162,161 @@
 
     c->log_error = NGX_ERROR_INFO;
 
+    /*
+     * Before all process proxy protocol
+     */
+
+    if (addr_conf->proxy_protocol) {
+        s->proxy_protocol = 1;
+        c->log->action = "reading PROXY protocol header";
+        c->read->handler = ngx_mail_proxy_protocol_handler;
+
+        ngx_add_timer(c->read, cscf->timeout);
+
+        if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
+            ngx_mail_close_connection(c);
+        }
+
+        return;
+    }
+
+    ngx_mail_init_connection_complete(c);
+}
+
+
+ngx_int_t
+ngx_mail_proxy_protoco_set_addrs(ngx_connection_t *c)
+{
+    ngx_addr_t                addr_peer, addr_local;
+    u_char                   *p, text[NGX_SOCKADDR_STRLEN];
+    size_t                    len;
+
+    if (ngx_parse_addr(c->pool, &addr_peer,
+                       c->proxy_protocol->src_addr.data,
+                       c->proxy_protocol->src_addr.len) != NGX_OK)
+    {
+        return NGX_ERROR;
+    }
+
+    ngx_inet_set_port(addr_peer.sockaddr, c->proxy_protocol->src_port);
+
+    if (ngx_parse_addr(c->pool, &addr_local,
+                       c->proxy_protocol->dst_addr.data,
+                       c->proxy_protocol->dst_addr.len) != NGX_OK)
+    {
+        return NGX_ERROR;
+    }
+
+    ngx_inet_set_port(addr_local.sockaddr, c->proxy_protocol->dst_port);
+
+    len = ngx_sock_ntop(addr_peer.sockaddr, addr_peer.socklen, text,
+                        NGX_SOCKADDR_STRLEN, 0);
+    if (len == 0) {
+        return NGX_ERROR;
+    }
+
+    p = ngx_pnalloc(c->pool, len);
+    if (p == NULL) {
+        return NGX_ERROR;
+    }
+
+    ngx_memcpy(p, text, len);
+
+    c->sockaddr = addr_peer.sockaddr;
+    c->socklen = addr_peer.socklen;
+    c->addr_text.len = len;
+    c->addr_text.data = p;
+
+    len = ngx_sock_ntop(addr_local.sockaddr, addr_local.socklen, text,
+                        NGX_SOCKADDR_STRLEN, 0);
+    if (len == 0) {
+        return NGX_ERROR;
+    }
+
+    p = ngx_pnalloc(c->pool, len);
+    if (p == NULL) {
+        return NGX_ERROR;
+    }
+
+    ngx_memcpy(p, text, len);
+
+    c->local_sockaddr = addr_local.sockaddr;
+    c->local_socklen = addr_local.socklen;
+
+    return NGX_OK;
+}
+
+
+void
+ngx_mail_proxy_protocol_handler(ngx_event_t *rev)
+{
+    ngx_connection_t          *c;
+    u_char                    *p, buf[NGX_PROXY_PROTOCOL_MAX_HEADER];
+    size_t                     size;
+    ssize_t                    n;
+
+    c = rev->data;
+
+    if (rev->timedout) {
+        ngx_log_error(NGX_LOG_INFO, c->log, NGX_ETIMEDOUT,
+                      "mail PROXY protocol header timed out");
+        c->timedout = 1;
+        ngx_mail_close_connection(c);
+        return;
+    }
+
+    ngx_log_debug0(NGX_LOG_DEBUG_MAIL, c->log, 0,
+                   "mail PROXY protocol handler");
+
+    size = NGX_PROXY_PROTOCOL_MAX_HEADER;
+
+    n = recv(c->fd, (char *) buf, size, MSG_PEEK);
+
+    ngx_log_debug1(NGX_LOG_DEBUG, c->log, 0, "mail recv(): %z", n);
+
+    p = ngx_proxy_protocol_read(c, buf, buf + n);
+
+    if (p == NULL) {
+        ngx_mail_close_connection(c);
+        return;
+    }
+
+    ngx_log_error(NGX_LOG_NOTICE, c->log, 0,
+                  "PROXY protocol %V:%d => %V:%d",
+                  &c->proxy_protocol->src_addr,
+                  c->proxy_protocol->src_port,
+                  &c->proxy_protocol->dst_addr,
+                  c->proxy_protocol->dst_port);
+
+    size = p - buf;
+
+    if (c->recv(c, buf, size) != (ssize_t) size) {
+        ngx_mail_close_connection(c);
+        return;
+    }
+
+    if (ngx_mail_proxy_protoco_set_addrs(c) != NGX_OK) {
+        ngx_mail_close_connection(c);
+        return;
+    }
+
+    ngx_mail_init_connection_complete(c);
+}
+
+
+void
+ngx_mail_init_connection_complete(ngx_connection_t *c)
+{
 #if (NGX_MAIL_SSL)
     {
-    ngx_mail_ssl_conf_t  *sslcf;
+    ngx_mail_session_t        *s;
+    ngx_mail_ssl_conf_t       *sslcf;
+
+    s = c->data;
 
     sslcf = ngx_mail_get_module_srv_conf(s, ngx_mail_ssl_module);
 
-    if (sslcf->enable || addr_conf->ssl) {
+    if (sslcf->enable || s->addr_conf->ssl) {
         c->log->action = "SSL handshaking";
 
         ngx_mail_ssl_init_connection(&sslcf->ssl, c);
@@ -348,6 +499,7 @@
         return;
     }
 
+    c->log->action = "sending client greeting line";
     c->write->handler = ngx_mail_send;
 
     cscf->protocol->init_session(s, c);
diff -r 83c4622053b0 -r 5f6f4a627b88 src/mail/ngx_mail_proxy_module.c
--- a/src/mail/ngx_mail_proxy_module.c	Tue Jan 12 16:59:31 2021 +0300
+++ b/src/mail/ngx_mail_proxy_module.c	Sat Jan 16 14:04:16 2021 +0300
@@ -19,6 +19,7 @@
     ngx_flag_t  smtp_auth;
     size_t      buffer_size;
     ngx_msec_t  timeout;
+    ngx_flag_t  proxy_protocol;
 } ngx_mail_proxy_conf_t;
 
 
@@ -36,7 +37,7 @@
 static void *ngx_mail_proxy_create_conf(ngx_conf_t *cf);
 static char *ngx_mail_proxy_merge_conf(ngx_conf_t *cf, void *parent,
     void *child);
-
+static ngx_int_t ngx_mail_proxy_send_proxy_protocol(ngx_mail_session_t *s);
 
 static ngx_command_t  ngx_mail_proxy_commands[] = {
 
@@ -82,6 +83,13 @@
       offsetof(ngx_mail_proxy_conf_t, smtp_auth),
       NULL },
 
+    { ngx_string("proxy_protocol"),
+      NGX_MAIL_MAIN_CONF|NGX_MAIL_SRV_CONF|NGX_CONF_FLAG,
+      ngx_conf_set_flag_slot,
+      NGX_MAIL_SRV_CONF_OFFSET,
+      offsetof(ngx_mail_proxy_conf_t, proxy_protocol),
+      NULL },
+
       ngx_null_command
 };
 
@@ -169,6 +177,12 @@
 
     s->out.len = 0;
 
+    if (pcf->proxy_protocol == 1) {
+        if (ngx_mail_proxy_send_proxy_protocol(s) != NGX_OK) {
+            ngx_mail_proxy_internal_server_error(s);
+        }
+    }
+
     switch (s->protocol) {
 
     case NGX_MAIL_POP3_PROTOCOL:
@@ -189,6 +203,60 @@
 }
 
 
+ngx_int_t
+ngx_mail_proxy_send_proxy_protocol(ngx_mail_session_t *s)
+{
+    u_char                       *p;
+    ssize_t                       n, size;
+    ngx_connection_t             *c, *pc;
+    ngx_peer_connection_t        *u;
+    u_char                        buf[NGX_PROXY_PROTOCOL_MAX_HEADER];
+
+    c = s->connection;
+
+    ngx_log_debug0(NGX_LOG_DEBUG_MAIL, c->log, 0,
+                   "mail proxy send PROXY protocol header");
+
+    p = ngx_proxy_protocol_write(c, buf, buf + NGX_PROXY_PROTOCOL_MAX_HEADER);
+    if (p == NULL) {
+        ngx_mail_proxy_internal_server_error(s);
+        return NGX_ERROR;
+    }
+
+    u = &s->proxy->upstream;
+
+    pc = u->connection;
+
+    size = p - buf;
+
+    n = pc->send(pc, buf, size);
+
+    if (n < NGX_OK) {
+        ngx_mail_proxy_internal_server_error(s);
+        return NGX_ERROR;
+    }
+
+    if (n != size) {
+
+        /*
+         * PROXY protocol specification:
+         * The sender must always ensure that the header
+         * is sent at once, so that the transport layer
+         * maintains atomicity along the path to the receiver.
+         */
+
+        ngx_log_error(NGX_LOG_ERR, c->log, 0,
+                      "could not send PROXY protocol header at once");
+
+        ngx_mail_proxy_internal_server_error(s);
+
+        return NGX_ERROR;
+    }
+
+    return NGX_OK;
+}
+
+
 static void
 ngx_mail_proxy_block_read(ngx_event_t *rev)
 {
@@ -1184,6 +1252,7 @@
     pcf->smtp_auth = NGX_CONF_UNSET;
     pcf->buffer_size = NGX_CONF_UNSET_SIZE;
     pcf->timeout = NGX_CONF_UNSET_MSEC;
+    pcf->proxy_protocol = NGX_CONF_UNSET;
 
     return pcf;
 }


More information about the nginx-devel mailing list