[PATCH] Mail: add the "reuseport" option of the "listen" directive

[Robert Mueller]

# HG changeset patch
# User Rob Mueller <robm at fastmail.fm>
# Date 1629171218 14400
#      Mon Aug 16 23:33:38 2021 -0400
# Node ID 89ff95b268e9817b344447b7e6785354229a4bab
# Parent  dda421871bc213dd2eb3da0015d6228839323583
Mail: add the "reuseport" option of the "listen" directive

The "reuseport" option was added to the "listen" directive of the http
and stream modules in 1.9.1, but it wasn't added to the mail module. This
adds the option to the mail module to make it consistent with the http
and stream modules.

In newer linux kernel versions (somewhere between 4.9 and 5.10) this
option seems much more important. On production debian servers, not having
or using this option caused processes to become very unbalanced. With 8
worker processes, we would see one worker process accepting 70%+ of all
connections, a second process with about 10% or so, and the remaining
20% of connections spread over the other 6 processes.  This obviously
started causing problems as the worker process accepting the majority
of connections would end up being 100% CPU bound well before the servers
overall capacity.

Adding and enabling this option fixed this entirely, and now all
worker processes seem to accept and even spread of connections.

diff -r dda421871bc2 -r 89ff95b268e9 src/mail/ngx_mail.c
--- a/src/mail/ngx_mail.c	Tue Aug 10 23:43:17 2021 +0300
+++ b/src/mail/ngx_mail.c	Mon Aug 16 23:33:38 2021 -0400
@@ -347,6 +347,10 @@
             ls->ipv6only = addr[i].opt.ipv6only;
 #endif
 
+#if (NGX_HAVE_REUSEPORT)
+            ls->reuseport = addr[i].opt.reuseport;
+#endif
+
             mport = ngx_palloc(cf->pool, sizeof(ngx_mail_port_t));
             if (mport == NULL) {
                 return NGX_CONF_ERROR;
diff -r dda421871bc2 -r 89ff95b268e9 src/mail/ngx_mail.h
--- a/src/mail/ngx_mail.h	Tue Aug 10 23:43:17 2021 +0300
+++ b/src/mail/ngx_mail.h	Mon Aug 16 23:33:38 2021 -0400
@@ -40,6 +40,7 @@
 #if (NGX_HAVE_INET6)
     unsigned                ipv6only:1;
 #endif
+    unsigned                reuseport:1;
     unsigned                so_keepalive:2;
     unsigned                proxy_protocol:1;
 #if (NGX_HAVE_KEEPALIVE_TUNABLE)
diff -r dda421871bc2 -r 89ff95b268e9 src/mail/ngx_mail_core_module.c
--- a/src/mail/ngx_mail_core_module.c	Tue Aug 10 23:43:17 2021 +0300
+++ b/src/mail/ngx_mail_core_module.c	Mon Aug 16 23:33:38 2021 -0400
@@ -447,6 +447,18 @@
 #endif
         }
 
+        if (ngx_strcmp(value[i].data, "reuseport") == 0) {
+#if (NGX_HAVE_REUSEPORT)
+            ls->reuseport = 1;
+            ls->bind = 1;
+#else
+            ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+                               "reuseport is not supported "
+                               "on this platform, ignored");
+#endif
+            continue;
+        }
+
         if (ngx_strcmp(value[i].data, "ssl") == 0) {
 #if (NGX_MAIL_SSL)
             ngx_mail_ssl_conf_t  *sslcf;


Rob Mueller
robm at fastmail.fm