[PATCH] Added SO_REUSEPORT support
Yingqi Lu
Yingqi.Lu at intel.com
Sat Sep 20 23:03:19 UTC 2014
src/core/ngx_connection.c | 21 ++++++++++++++-
src/core/ngx_cycle.c | 52 +++++++++++++++++++++++++++++++++----
src/core/ngx_cycle.h | 2 +
src/http/ngx_http.c | 64 ++++++++++++++++++++++++----------------------
4 files changed, 101 insertions(+), 38 deletions(-)
# HG changeset patch
# User Yingqi Lu <Yingqi.Lu at intel.com>
# Date 1411248953 25200
# Sat Sep 20 14:35:53 2014 -0700
# Node ID 04abfbb10a7f5e4efd5187d1382a880483a79294
# Parent 43512a33e8f232fb6f256b90e76ac4f1472a0c1f
Added SO_REUSEPORT support.
It duplicates and configures certain number of listen sockets in the master
process with SO_REUSEPORT enabled. All the worker processes can inherit them.
The number of the listen sockets to be duplicated is calculated based on the
number of total CPU threads. With big system that has more CPU threads, more
duplicated listen sockets created to improve the throughput and scalability.
With system that has only 8 or less CPU threads, there will be only 1 listen
socket. Duplicated listen sockets are only being created when necessary. In
case that SO_REUSEPORT is not supported by the OS, it falls back default
behavior.
diff -r 43512a33e8f2 -r 04abfbb10a7f src/core/ngx_connection.c
--- a/src/core/ngx_connection.c Wed Aug 13 15:11:45 2014 +0400
+++ b/src/core/ngx_connection.c Sat Sep 20 14:35:53 2014 -0700
@@ -304,7 +304,7 @@
ngx_int_t
ngx_open_listening_sockets(ngx_cycle_t *cycle)
{
- int reuseaddr;
+ int reuseaddr, reuseport;
ngx_uint_t i, tries, failed;
ngx_err_t err;
ngx_log_t *log;
@@ -312,6 +312,7 @@
ngx_listening_t *ls;
reuseaddr = 1;
+ reuseport = 1;
#if (NGX_SUPPRESS_WARN)
failed = 0;
#endif
@@ -370,6 +371,24 @@
return NGX_ERROR;
}
+ if (ngx_so_reuseport_enabled)
+ {
+ if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT,
+ (const void *) &reuseport, sizeof(int))
+ == -1) {
+ ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
+ "setsockopt(SO_REUSEPORT) %V failed",
+ &ls[i].addr_text);
+ if (ngx_close_socket(s) == -1) {
+ ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
+ ngx_close_socket_n " %V failed",
+ &ls[i].addr_text);
+ }
+
+ return NGX_ERROR;
+ }
+ }
+
#if (NGX_HAVE_INET6 && defined IPV6_V6ONLY)
if (ls[i].sockaddr->sa_family == AF_INET6) {
diff -r 43512a33e8f2 -r 04abfbb10a7f src/core/ngx_cycle.c
--- a/src/core/ngx_cycle.c Wed Aug 13 15:11:45 2014 +0400
+++ b/src/core/ngx_cycle.c Sat Sep 20 14:35:53 2014 -0700
@@ -26,6 +26,9 @@
ngx_uint_t ngx_test_config;
ngx_uint_t ngx_quiet_mode;
+ngx_uint_t ngx_so_reuseport_enabled;
+ngx_uint_t ngx_num_dup_sockets;
+
#if (NGX_THREADS)
ngx_tls_key_t ngx_core_tls_key;
#endif
@@ -54,7 +57,39 @@
ngx_core_conf_t *ccf, *old_ccf;
ngx_core_module_t *module;
char hostname[NGX_MAXHOSTNAMELEN];
+ ngx_uint_t num_cores, taken;
+ ngx_socket_t temp_s;
+ int one = 1;
+ /* check if SO_REUSEPORT feature is enabled */
+ temp_s = ngx_socket(AF_INET, SOCK_STREAM, 0);
+
+#ifndef SO_REUSEPORT
+#define SO_REUSEPORT 15
+#endif
+ if (setsockopt(temp_s, SOL_SOCKET, SO_REUSEPORT,
+ (const void *) &one, sizeof(int)) == 0) {
+ ngx_so_reuseport_enabled = 1;
+ } else {
+ ngx_so_reuseport_enabled = 0;
+ }
+ ngx_close_socket(temp_s);
+
+ if (ngx_so_reuseport_enabled) {
+#ifdef _SC_NPROCESSORS_ONLN
+ num_cores = sysconf(_SC_NPROCESSORS_CONF);
+#else
+ num_cores = 1;
+#endif
+ if (num_cores > 8) {
+ ngx_num_dup_sockets = num_cores/8;
+ } else {
+ ngx_num_dup_sockets = 1;
+ }
+ } else {
+ ngx_num_dup_sockets = 1;
+ }
+
ngx_timezone_update();
/* force localtime update with a new timezone */
@@ -114,7 +149,8 @@
}
- n = old_cycle->paths.nelts ? old_cycle->paths.nelts : 10;
+ n = old_cycle->paths.nelts ? old_cycle->paths.nelts :
+ 10 * ngx_num_dup_sockets;
cycle->paths.elts = ngx_pcalloc(pool, n * sizeof(ngx_path_t *));
if (cycle->paths.elts == NULL) {
@@ -164,7 +200,8 @@
return NULL;
}
- n = old_cycle->listening.nelts ? old_cycle->listening.nelts : 10;
+ n = old_cycle->listening.nelts ? old_cycle->listening.nelts :
+ 10 * ngx_num_dup_sockets;
cycle->listening.elts = ngx_pcalloc(pool, n * sizeof(ngx_listening_t));
if (cycle->listening.elts == NULL) {
@@ -231,7 +268,8 @@
ngx_memzero(&conf, sizeof(ngx_conf_t));
/* STUB: init array ? */
- conf.args = ngx_array_create(pool, 10, sizeof(ngx_str_t));
+ conf.args = ngx_array_create(pool,
+ 10 * ngx_num_dup_sockets, sizeof(ngx_str_t));
if (conf.args == NULL) {
ngx_destroy_pool(pool);
return NULL;
@@ -486,6 +524,7 @@
}
nls = cycle->listening.elts;
+ taken = 0;
for (n = 0; n < cycle->listening.nelts; n++) {
for (i = 0; i < old_cycle->listening.nelts; i++) {
@@ -493,9 +532,9 @@
continue;
}
- if (ngx_cmp_sockaddr(nls[n].sockaddr, nls[n].socklen,
+ if ((ngx_cmp_sockaddr(nls[n].sockaddr, nls[n].socklen,
ls[i].sockaddr, ls[i].socklen, 1)
- == NGX_OK)
+ == NGX_OK) && i >= taken)
{
nls[n].fd = ls[i].fd;
nls[n].previous = &ls[i];
@@ -540,6 +579,7 @@
nls[n].add_deferred = 1;
}
#endif
+ taken = i + 1;
break;
}
}
@@ -747,7 +787,7 @@
exit(1);
}
- n = 10;
+ n = 10 * ngx_num_dup_sockets;
ngx_old_cycles.elts = ngx_pcalloc(ngx_temp_pool,
n * sizeof(ngx_cycle_t *));
if (ngx_old_cycles.elts == NULL) {
diff -r 43512a33e8f2 -r 04abfbb10a7f src/core/ngx_cycle.h
--- a/src/core/ngx_cycle.h Wed Aug 13 15:11:45 2014 +0400
+++ b/src/core/ngx_cycle.h Sat Sep 20 14:35:53 2014 -0700
@@ -136,6 +136,8 @@
extern ngx_module_t ngx_core_module;
extern ngx_uint_t ngx_test_config;
extern ngx_uint_t ngx_quiet_mode;
+extern ngx_uint_t ngx_so_reuseport_enabled;
+extern ngx_uint_t ngx_num_dup_sockets;
#if (NGX_THREADS)
extern ngx_tls_key_t ngx_core_tls_key;
#endif
diff -r 43512a33e8f2 -r 04abfbb10a7f src/http/ngx_http.c
--- a/src/http/ngx_http.c Wed Aug 13 15:11:45 2014 +0400
+++ b/src/http/ngx_http.c Sat Sep 20 14:35:53 2014 -0700
@@ -1671,7 +1671,7 @@
static ngx_int_t
ngx_http_init_listening(ngx_conf_t *cf, ngx_http_conf_port_t *port)
{
- ngx_uint_t i, last, bind_wildcard;
+ ngx_uint_t i, j, last, bind_wildcard;
ngx_listening_t *ls;
ngx_http_port_t *hport;
ngx_http_conf_addr_t *addr;
@@ -1703,40 +1703,42 @@
continue;
}
- ls = ngx_http_add_listening(cf, &addr[i]);
- if (ls == NULL) {
- return NGX_ERROR;
- }
-
- hport = ngx_pcalloc(cf->pool, sizeof(ngx_http_port_t));
- if (hport == NULL) {
- return NGX_ERROR;
- }
-
- ls->servers = hport;
-
- if (i == last - 1) {
- hport->naddrs = last;
-
- } else {
- hport->naddrs = 1;
- i = 0;
- }
-
- switch (ls->sockaddr->sa_family) {
-
-#if (NGX_HAVE_INET6)
- case AF_INET6:
- if (ngx_http_add_addrs6(cf, hport, addr) != NGX_OK) {
+ for(j = 0; j < ngx_num_dup_sockets; j++) {
+ ls = ngx_http_add_listening(cf, &addr[i]);
+ if (ls == NULL) {
return NGX_ERROR;
}
- break;
-#endif
- default: /* AF_INET */
- if (ngx_http_add_addrs(cf, hport, addr) != NGX_OK) {
+
+ hport = ngx_pcalloc(cf->pool, sizeof(ngx_http_port_t));
+ if (hport == NULL) {
return NGX_ERROR;
}
- break;
+
+ ls->servers = hport;
+
+ if (i == last - 1) {
+ hport->naddrs = last;
+
+ } else {
+ hport->naddrs = 1;
+ i = 0;
+ }
+
+ switch (ls->sockaddr->sa_family) {
+
+#if (NGX_HAVE_INET6)
+ case AF_INET6:
+ if (ngx_http_add_addrs6(cf, hport, addr) != NGX_OK) {
+ return NGX_ERROR;
+ }
+ break;
+#endif
+ default: /* AF_INET */
+ if (ngx_http_add_addrs(cf, hport, addr) != NGX_OK) {
+ return NGX_ERROR;
+ }
+ break;
+ }
}
addr++;
More information about the nginx-devel
mailing list