[PATCH] SO_REUSEPORT support from master process

Yingqi Lu Yingqi.Lu at intel.com
Sat Sep 20 23:28:58 UTC 2014


 src/core/ngx_connection.c |  21 ++++++++++++++-
 src/core/ngx_cycle.c      |  52 +++++++++++++++++++++++++++++++++----
 src/core/ngx_cycle.h      |   2 +
 src/http/ngx_http.c       |  64 ++++++++++++++++++++++++----------------------
 4 files changed, 101 insertions(+), 38 deletions(-)


# HG changeset patch
# User Yingqi Lu <Yingqi.Lu at intel.com>
# Date 1411255543 25200
#      Sat Sep 20 16:25:43 2014 -0700
# Node ID cf28c5af69b500ae368158df5fa4ad68f90170cc
# Parent  43512a33e8f232fb6f256b90e76ac4f1472a0c1f
SO_REUSEPORT support from master process

It duplicates and configures certain number of listen sockets in the master
process with SO_REUSEPORT enabled. All the worker processes can inherit them.
The number of the listen sockets to be duplicated is calculated based on the
number of total CPU threads. With big system that has more CPU threads, more
duplicated listen sockets created to improve the throughput and scalability.
With system that has only 8 or less CPU threads, there will be only 1 listen
socket. Duplicated listen sockets are only being created when necessary. In
case that SO_REUSEPORT is not supported by the OS, it falls back default
behavior.

diff -r 43512a33e8f2 -r cf28c5af69b5 src/core/ngx_connection.c
--- a/src/core/ngx_connection.c	Wed Aug 13 15:11:45 2014 +0400
+++ b/src/core/ngx_connection.c	Sat Sep 20 16:25:43 2014 -0700
@@ -304,7 +304,7 @@
 ngx_int_t
 ngx_open_listening_sockets(ngx_cycle_t *cycle)
 {
-    int               reuseaddr;
+    int               reuseaddr, reuseport;
     ngx_uint_t        i, tries, failed;
     ngx_err_t         err;
     ngx_log_t        *log;
@@ -312,6 +312,7 @@
     ngx_listening_t  *ls;
 
     reuseaddr = 1;
+    reuseport = 1;
 #if (NGX_SUPPRESS_WARN)
     failed = 0;
 #endif
@@ -370,6 +371,24 @@
                 return NGX_ERROR;
             }
 
+            if (ngx_so_reuseport_enabled)
+            {
+                if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT,
+                               (const void *) &reuseport, sizeof(int))
+                    == -1) {
+                    ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
+                                  "setsockopt(SO_REUSEPORT) %V failed",
+                                  &ls[i].addr_text);
+                    if (ngx_close_socket(s) == -1) {
+                        ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
+                                      ngx_close_socket_n " %V failed",
+                                      &ls[i].addr_text);
+                    }
+
+                    return NGX_ERROR;
+                }
+            }
+
 #if (NGX_HAVE_INET6 && defined IPV6_V6ONLY)
 
             if (ls[i].sockaddr->sa_family == AF_INET6) {
diff -r 43512a33e8f2 -r cf28c5af69b5 src/core/ngx_cycle.c
--- a/src/core/ngx_cycle.c	Wed Aug 13 15:11:45 2014 +0400
+++ b/src/core/ngx_cycle.c	Sat Sep 20 16:25:43 2014 -0700
@@ -26,6 +26,9 @@
 ngx_uint_t             ngx_test_config;
 ngx_uint_t             ngx_quiet_mode;
 
+ngx_uint_t             ngx_so_reuseport_enabled;
+ngx_uint_t             ngx_num_dup_sockets;
+
 #if (NGX_THREADS)
 ngx_tls_key_t          ngx_core_tls_key;
 #endif
@@ -54,7 +57,39 @@
     ngx_core_conf_t     *ccf, *old_ccf;
     ngx_core_module_t   *module;
     char                 hostname[NGX_MAXHOSTNAMELEN];
+    ngx_uint_t           num_cores, taken;
+    ngx_socket_t         temp_s;
+    int                  one = 1;
 
+    /* check if SO_REUSEPORT feature is enabled */
+    temp_s = ngx_socket(AF_INET, SOCK_STREAM, 0);
+
+#ifndef SO_REUSEPORT
+#define SO_REUSEPORT 15
+#endif
+    if (setsockopt(temp_s, SOL_SOCKET, SO_REUSEPORT,
+                  (const void *) &one, sizeof(int)) == 0) {
+        ngx_so_reuseport_enabled = 1;
+    } else {
+        ngx_so_reuseport_enabled = 0;
+    }
+    ngx_close_socket(temp_s);
+
+    if (ngx_so_reuseport_enabled) {
+#ifdef _SC_NPROCESSORS_ONLN
+        num_cores = sysconf(_SC_NPROCESSORS_CONF);
+#else
+        num_cores = 1;
+#endif
+        if (num_cores > 8) {
+            ngx_num_dup_sockets = num_cores/8;
+        } else {
+            ngx_num_dup_sockets = 1;
+        }
+    } else {
+        ngx_num_dup_sockets = 1;
+    }
+ 
     ngx_timezone_update();
 
     /* force localtime update with a new timezone */
@@ -114,7 +149,8 @@
     }
 
 
-    n = old_cycle->paths.nelts ? old_cycle->paths.nelts : 10;
+    n = old_cycle->paths.nelts ? old_cycle->paths.nelts :
+        10 * ngx_num_dup_sockets;
 
     cycle->paths.elts = ngx_pcalloc(pool, n * sizeof(ngx_path_t *));
     if (cycle->paths.elts == NULL) {
@@ -164,7 +200,8 @@
         return NULL;
     }
 
-    n = old_cycle->listening.nelts ? old_cycle->listening.nelts : 10;
+    n = old_cycle->listening.nelts ? old_cycle->listening.nelts :
+        10 * ngx_num_dup_sockets;
 
     cycle->listening.elts = ngx_pcalloc(pool, n * sizeof(ngx_listening_t));
     if (cycle->listening.elts == NULL) {
@@ -231,7 +268,8 @@
 
     ngx_memzero(&conf, sizeof(ngx_conf_t));
     /* STUB: init array ? */
-    conf.args = ngx_array_create(pool, 10, sizeof(ngx_str_t));
+    conf.args = ngx_array_create(pool,
+                                 10 * ngx_num_dup_sockets, sizeof(ngx_str_t));
     if (conf.args == NULL) {
         ngx_destroy_pool(pool);
         return NULL;
@@ -486,6 +524,7 @@
         }
 
         nls = cycle->listening.elts;
+        taken = 0;
         for (n = 0; n < cycle->listening.nelts; n++) {
 
             for (i = 0; i < old_cycle->listening.nelts; i++) {
@@ -493,9 +532,9 @@
                     continue;
                 }
 
-                if (ngx_cmp_sockaddr(nls[n].sockaddr, nls[n].socklen,
+                if ((ngx_cmp_sockaddr(nls[n].sockaddr, nls[n].socklen,
                                      ls[i].sockaddr, ls[i].socklen, 1)
-                    == NGX_OK)
+                    == NGX_OK) && i >= taken)
                 {
                     nls[n].fd = ls[i].fd;
                     nls[n].previous = &ls[i];
@@ -540,6 +579,7 @@
                         nls[n].add_deferred = 1;
                     }
 #endif
+                    taken = i + 1;
                     break;
                 }
             }
@@ -747,7 +787,7 @@
             exit(1);
         }
 
-        n = 10;
+        n = 10 * ngx_num_dup_sockets;
         ngx_old_cycles.elts = ngx_pcalloc(ngx_temp_pool,
                                           n * sizeof(ngx_cycle_t *));
         if (ngx_old_cycles.elts == NULL) {
diff -r 43512a33e8f2 -r cf28c5af69b5 src/core/ngx_cycle.h
--- a/src/core/ngx_cycle.h	Wed Aug 13 15:11:45 2014 +0400
+++ b/src/core/ngx_cycle.h	Sat Sep 20 16:25:43 2014 -0700
@@ -136,6 +136,8 @@
 extern ngx_module_t           ngx_core_module;
 extern ngx_uint_t             ngx_test_config;
 extern ngx_uint_t             ngx_quiet_mode;
+extern ngx_uint_t             ngx_so_reuseport_enabled;
+extern ngx_uint_t             ngx_num_dup_sockets;
 #if (NGX_THREADS)
 extern ngx_tls_key_t          ngx_core_tls_key;
 #endif
diff -r 43512a33e8f2 -r cf28c5af69b5 src/http/ngx_http.c
--- a/src/http/ngx_http.c	Wed Aug 13 15:11:45 2014 +0400
+++ b/src/http/ngx_http.c	Sat Sep 20 16:25:43 2014 -0700
@@ -1671,7 +1671,7 @@
 static ngx_int_t
 ngx_http_init_listening(ngx_conf_t *cf, ngx_http_conf_port_t *port)
 {
-    ngx_uint_t                 i, last, bind_wildcard;
+    ngx_uint_t                 i, j, last, bind_wildcard;
     ngx_listening_t           *ls;
     ngx_http_port_t           *hport;
     ngx_http_conf_addr_t      *addr;
@@ -1703,40 +1703,42 @@
             continue;
         }
 
-        ls = ngx_http_add_listening(cf, &addr[i]);
-        if (ls == NULL) {
-            return NGX_ERROR;
-        }
-
-        hport = ngx_pcalloc(cf->pool, sizeof(ngx_http_port_t));
-        if (hport == NULL) {
-            return NGX_ERROR;
-        }
-
-        ls->servers = hport;
-
-        if (i == last - 1) {
-            hport->naddrs = last;
-
-        } else {
-            hport->naddrs = 1;
-            i = 0;
-        }
-
-        switch (ls->sockaddr->sa_family) {
-
-#if (NGX_HAVE_INET6)
-        case AF_INET6:
-            if (ngx_http_add_addrs6(cf, hport, addr) != NGX_OK) {
+        for(j = 0; j < ngx_num_dup_sockets; j++) {
+            ls = ngx_http_add_listening(cf, &addr[i]);
+            if (ls == NULL) {
                 return NGX_ERROR;
             }
-            break;
-#endif
-        default: /* AF_INET */
-            if (ngx_http_add_addrs(cf, hport, addr) != NGX_OK) {
+
+            hport = ngx_pcalloc(cf->pool, sizeof(ngx_http_port_t));
+            if (hport == NULL) {
                 return NGX_ERROR;
             }
-            break;
+
+            ls->servers = hport;
+
+            if (i == last - 1) {
+                hport->naddrs = last;
+
+            } else {
+                hport->naddrs = 1;
+                i = 0;
+            }
+
+            switch (ls->sockaddr->sa_family) {
+
+#if (NGX_HAVE_INET6)
+            case AF_INET6:
+                if (ngx_http_add_addrs6(cf, hport, addr) != NGX_OK) {
+                    return NGX_ERROR;
+                }
+                break;
+#endif
+            default: /* AF_INET */
+                if (ngx_http_add_addrs(cf, hport, addr) != NGX_OK) {
+                    return NGX_ERROR;
+                }
+                break;
+            }
         }
 
         addr++;



More information about the nginx-devel mailing list