[PATCH] SO_REUSEPORT support from master process
Yingqi Lu
Yingqi.Lu at intel.com
Sat Sep 20 23:28:58 UTC 2014
src/core/ngx_connection.c | 21 ++++++++++++++-
src/core/ngx_cycle.c | 52 +++++++++++++++++++++++++++++++++----
src/core/ngx_cycle.h | 2 +
src/http/ngx_http.c | 64 ++++++++++++++++++++++++----------------------
4 files changed, 101 insertions(+), 38 deletions(-)
# HG changeset patch
# User Yingqi Lu <Yingqi.Lu at intel.com>
# Date 1411255543 25200
# Sat Sep 20 16:25:43 2014 -0700
# Node ID cf28c5af69b500ae368158df5fa4ad68f90170cc
# Parent 43512a33e8f232fb6f256b90e76ac4f1472a0c1f
SO_REUSEPORT support from master process
It duplicates and configures certain number of listen sockets in the master
process with SO_REUSEPORT enabled. All the worker processes can inherit them.
The number of the listen sockets to be duplicated is calculated based on the
number of total CPU threads. With big system that has more CPU threads, more
duplicated listen sockets created to improve the throughput and scalability.
With system that has only 8 or less CPU threads, there will be only 1 listen
socket. Duplicated listen sockets are only being created when necessary. In
case that SO_REUSEPORT is not supported by the OS, it falls back default
behavior.
diff -r 43512a33e8f2 -r cf28c5af69b5 src/core/ngx_connection.c
--- a/src/core/ngx_connection.c Wed Aug 13 15:11:45 2014 +0400
+++ b/src/core/ngx_connection.c Sat Sep 20 16:25:43 2014 -0700
@@ -304,7 +304,7 @@
ngx_int_t
ngx_open_listening_sockets(ngx_cycle_t *cycle)
{
- int reuseaddr;
+ int reuseaddr, reuseport;
ngx_uint_t i, tries, failed;
ngx_err_t err;
ngx_log_t *log;
@@ -312,6 +312,7 @@
ngx_listening_t *ls;
reuseaddr = 1;
+ reuseport = 1;
#if (NGX_SUPPRESS_WARN)
failed = 0;
#endif
@@ -370,6 +371,24 @@
return NGX_ERROR;
}
+ if (ngx_so_reuseport_enabled)
+ {
+ if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT,
+ (const void *) &reuseport, sizeof(int))
+ == -1) {
+ ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
+ "setsockopt(SO_REUSEPORT) %V failed",
+ &ls[i].addr_text);
+ if (ngx_close_socket(s) == -1) {
+ ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
+ ngx_close_socket_n " %V failed",
+ &ls[i].addr_text);
+ }
+
+ return NGX_ERROR;
+ }
+ }
+
#if (NGX_HAVE_INET6 && defined IPV6_V6ONLY)
if (ls[i].sockaddr->sa_family == AF_INET6) {
diff -r 43512a33e8f2 -r cf28c5af69b5 src/core/ngx_cycle.c
--- a/src/core/ngx_cycle.c Wed Aug 13 15:11:45 2014 +0400
+++ b/src/core/ngx_cycle.c Sat Sep 20 16:25:43 2014 -0700
@@ -26,6 +26,9 @@
ngx_uint_t ngx_test_config;
ngx_uint_t ngx_quiet_mode;
+ngx_uint_t ngx_so_reuseport_enabled;
+ngx_uint_t ngx_num_dup_sockets;
+
#if (NGX_THREADS)
ngx_tls_key_t ngx_core_tls_key;
#endif
@@ -54,7 +57,39 @@
ngx_core_conf_t *ccf, *old_ccf;
ngx_core_module_t *module;
char hostname[NGX_MAXHOSTNAMELEN];
+ ngx_uint_t num_cores, taken;
+ ngx_socket_t temp_s;
+ int one = 1;
+ /* check if SO_REUSEPORT feature is enabled */
+ temp_s = ngx_socket(AF_INET, SOCK_STREAM, 0);
+
+#ifndef SO_REUSEPORT
+#define SO_REUSEPORT 15
+#endif
+ if (setsockopt(temp_s, SOL_SOCKET, SO_REUSEPORT,
+ (const void *) &one, sizeof(int)) == 0) {
+ ngx_so_reuseport_enabled = 1;
+ } else {
+ ngx_so_reuseport_enabled = 0;
+ }
+ ngx_close_socket(temp_s);
+
+ if (ngx_so_reuseport_enabled) {
+#ifdef _SC_NPROCESSORS_ONLN
+ num_cores = sysconf(_SC_NPROCESSORS_CONF);
+#else
+ num_cores = 1;
+#endif
+ if (num_cores > 8) {
+ ngx_num_dup_sockets = num_cores/8;
+ } else {
+ ngx_num_dup_sockets = 1;
+ }
+ } else {
+ ngx_num_dup_sockets = 1;
+ }
+
ngx_timezone_update();
/* force localtime update with a new timezone */
@@ -114,7 +149,8 @@
}
- n = old_cycle->paths.nelts ? old_cycle->paths.nelts : 10;
+ n = old_cycle->paths.nelts ? old_cycle->paths.nelts :
+ 10 * ngx_num_dup_sockets;
cycle->paths.elts = ngx_pcalloc(pool, n * sizeof(ngx_path_t *));
if (cycle->paths.elts == NULL) {
@@ -164,7 +200,8 @@
return NULL;
}
- n = old_cycle->listening.nelts ? old_cycle->listening.nelts : 10;
+ n = old_cycle->listening.nelts ? old_cycle->listening.nelts :
+ 10 * ngx_num_dup_sockets;
cycle->listening.elts = ngx_pcalloc(pool, n * sizeof(ngx_listening_t));
if (cycle->listening.elts == NULL) {
@@ -231,7 +268,8 @@
ngx_memzero(&conf, sizeof(ngx_conf_t));
/* STUB: init array ? */
- conf.args = ngx_array_create(pool, 10, sizeof(ngx_str_t));
+ conf.args = ngx_array_create(pool,
+ 10 * ngx_num_dup_sockets, sizeof(ngx_str_t));
if (conf.args == NULL) {
ngx_destroy_pool(pool);
return NULL;
@@ -486,6 +524,7 @@
}
nls = cycle->listening.elts;
+ taken = 0;
for (n = 0; n < cycle->listening.nelts; n++) {
for (i = 0; i < old_cycle->listening.nelts; i++) {
@@ -493,9 +532,9 @@
continue;
}
- if (ngx_cmp_sockaddr(nls[n].sockaddr, nls[n].socklen,
+ if ((ngx_cmp_sockaddr(nls[n].sockaddr, nls[n].socklen,
ls[i].sockaddr, ls[i].socklen, 1)
- == NGX_OK)
+ == NGX_OK) && i >= taken)
{
nls[n].fd = ls[i].fd;
nls[n].previous = &ls[i];
@@ -540,6 +579,7 @@
nls[n].add_deferred = 1;
}
#endif
+ taken = i + 1;
break;
}
}
@@ -747,7 +787,7 @@
exit(1);
}
- n = 10;
+ n = 10 * ngx_num_dup_sockets;
ngx_old_cycles.elts = ngx_pcalloc(ngx_temp_pool,
n * sizeof(ngx_cycle_t *));
if (ngx_old_cycles.elts == NULL) {
diff -r 43512a33e8f2 -r cf28c5af69b5 src/core/ngx_cycle.h
--- a/src/core/ngx_cycle.h Wed Aug 13 15:11:45 2014 +0400
+++ b/src/core/ngx_cycle.h Sat Sep 20 16:25:43 2014 -0700
@@ -136,6 +136,8 @@
extern ngx_module_t ngx_core_module;
extern ngx_uint_t ngx_test_config;
extern ngx_uint_t ngx_quiet_mode;
+extern ngx_uint_t ngx_so_reuseport_enabled;
+extern ngx_uint_t ngx_num_dup_sockets;
#if (NGX_THREADS)
extern ngx_tls_key_t ngx_core_tls_key;
#endif
diff -r 43512a33e8f2 -r cf28c5af69b5 src/http/ngx_http.c
--- a/src/http/ngx_http.c Wed Aug 13 15:11:45 2014 +0400
+++ b/src/http/ngx_http.c Sat Sep 20 16:25:43 2014 -0700
@@ -1671,7 +1671,7 @@
static ngx_int_t
ngx_http_init_listening(ngx_conf_t *cf, ngx_http_conf_port_t *port)
{
- ngx_uint_t i, last, bind_wildcard;
+ ngx_uint_t i, j, last, bind_wildcard;
ngx_listening_t *ls;
ngx_http_port_t *hport;
ngx_http_conf_addr_t *addr;
@@ -1703,40 +1703,42 @@
continue;
}
- ls = ngx_http_add_listening(cf, &addr[i]);
- if (ls == NULL) {
- return NGX_ERROR;
- }
-
- hport = ngx_pcalloc(cf->pool, sizeof(ngx_http_port_t));
- if (hport == NULL) {
- return NGX_ERROR;
- }
-
- ls->servers = hport;
-
- if (i == last - 1) {
- hport->naddrs = last;
-
- } else {
- hport->naddrs = 1;
- i = 0;
- }
-
- switch (ls->sockaddr->sa_family) {
-
-#if (NGX_HAVE_INET6)
- case AF_INET6:
- if (ngx_http_add_addrs6(cf, hport, addr) != NGX_OK) {
+ for(j = 0; j < ngx_num_dup_sockets; j++) {
+ ls = ngx_http_add_listening(cf, &addr[i]);
+ if (ls == NULL) {
return NGX_ERROR;
}
- break;
-#endif
- default: /* AF_INET */
- if (ngx_http_add_addrs(cf, hport, addr) != NGX_OK) {
+
+ hport = ngx_pcalloc(cf->pool, sizeof(ngx_http_port_t));
+ if (hport == NULL) {
return NGX_ERROR;
}
- break;
+
+ ls->servers = hport;
+
+ if (i == last - 1) {
+ hport->naddrs = last;
+
+ } else {
+ hport->naddrs = 1;
+ i = 0;
+ }
+
+ switch (ls->sockaddr->sa_family) {
+
+#if (NGX_HAVE_INET6)
+ case AF_INET6:
+ if (ngx_http_add_addrs6(cf, hport, addr) != NGX_OK) {
+ return NGX_ERROR;
+ }
+ break;
+#endif
+ default: /* AF_INET */
+ if (ngx_http_add_addrs(cf, hport, addr) != NGX_OK) {
+ return NGX_ERROR;
+ }
+ break;
+ }
}
addr++;
More information about the nginx-devel
mailing list