[PATCH] Events: add ability to receive sockets by SCM_RIGHTS
Sergey S. Kovalev
xray at randoman.ru
Thu Nov 3 12:47:16 UTC 2016
# HG changeset patch
# User Sergey Kovalev <xray at randoman.ru>
# Date 1478146426 -25200
# Thu Nov 03 11:13:46 2016 +0700
# Node ID 57a3036fd504a93dcbfd75a263a5d71523cdc379
# Parent cb4a4e9bba8ed7038dc423a9ad07139215db3363
Events: add ability to receive sockets by SCM_RIGHTS
With this patch ngx_event_recvmsg() handler can process SCM_RIGHTS
messages and treat received sockets as new accepted connections.
This feature can be used to improve overall performance with tricky
request routing (like sharing port for several daemons) or
can be used in some security systems to check connection data
before passing it to nginx.
To build nginx with this feature, add configuration option:
$ ./configure --with-cmsg-recv-sockets
To use it, declare listen on unix socket with "cmsg" option on it:
listen unix:/tmp/nginx-cmsg.sock cmsg default_server;
This socket is created as SOCK_DGRAM and can't be used for direct
request processing.
diff -r cb4a4e9bba8e -r 57a3036fd504 auto/modules
--- a/auto/modules Tue Nov 01 20:39:21 2016 +0300
+++ b/auto/modules Thu Nov 03 11:13:46 2016 +0700
@@ -1133,6 +1133,10 @@
fi
+if [ $CMSG_RECV_SOCKETS = YES ]; then
+ have=NGX_CMSG_RECV_SOCKETS . auto/have
+fi
+
#if [ -r $NGX_OBJS/auto ]; then
# . $NGX_OBJS/auto
#fi
diff -r cb4a4e9bba8e -r 57a3036fd504 auto/options
--- a/auto/options Tue Nov 01 20:39:21 2016 +0300
+++ b/auto/options Thu Nov 03 11:13:46 2016 +0700
@@ -149,6 +149,8 @@
ZLIB_OPT=
ZLIB_ASM=NO
+CMSG_RECV_SOCKETS=NO
+
USE_PERL=NO
NGX_PERL=perl
@@ -324,6 +326,9 @@
--without-stream_upstream_zone_module)
STREAM_UPSTREAM_ZONE=NO ;;
+ --with-cmsg-recv-sockets) CMSG_RECV_SOCKETS=YES ;;
+
+
--with-google_perftools_module) NGX_GOOGLE_PERFTOOLS=YES ;;
--with-cpp_test_module) NGX_CPP_TEST=YES ;;
@@ -532,6 +537,10 @@
--without-stream_upstream_zone_module
disable ngx_stream_upstream_zone_module
+ --with-cmsg-recv-sockets enable accepting sockets passed via
+ unix datagram socket with SCM_RIGHTS
+ ability
+
--with-google_perftools_module enable ngx_google_perftools_module
--with-cpp_test_module enable ngx_cpp_test_module
diff -r cb4a4e9bba8e -r 57a3036fd504 src/core/ngx_connection.c
--- a/src/core/ngx_connection.c Tue Nov 01 20:39:21 2016 +0300
+++ b/src/core/ngx_connection.c Thu Nov 03 11:13:46 2016 +0700
@@ -18,7 +18,7 @@
ngx_listening_t *
ngx_create_listening(ngx_conf_t *cf, struct sockaddr *sockaddr,
- socklen_t socklen)
+ socklen_t socklen, ngx_int_t socktype)
{
size_t len;
ngx_listening_t *ls;
@@ -73,7 +73,7 @@
ngx_memcpy(ls->addr_text.data, text, len);
ls->fd = (ngx_socket_t) -1;
- ls->type = SOCK_STREAM;
+ ls->type = socktype;
ls->backlog = NGX_LISTEN_BACKLOG;
ls->rcvbuf = -1;
diff -r cb4a4e9bba8e -r 57a3036fd504 src/core/ngx_connection.h
--- a/src/core/ngx_connection.h Tue Nov 01 20:39:21 2016 +0300
+++ b/src/core/ngx_connection.h Thu Nov 03 11:13:46 2016 +0700
@@ -66,6 +66,10 @@
unsigned addr_ntop:1;
unsigned wildcard:1;
+#if (NGX_HAVE_MSGHDR_MSG_CONTROL && NGX_CMSG_RECV_SOCKETS)
+ unsigned cmsg:1;
+#endif
+
#if (NGX_HAVE_INET6)
unsigned ipv6only:1;
#endif
@@ -204,7 +208,7 @@
ngx_listening_t *ngx_create_listening(ngx_conf_t *cf, struct sockaddr *sockaddr,
- socklen_t socklen);
+ socklen_t socklen, ngx_int_t socktype);
ngx_int_t ngx_clone_listening(ngx_conf_t *cf, ngx_listening_t *ls);
ngx_int_t ngx_set_inherited_sockets(ngx_cycle_t *cycle);
ngx_int_t ngx_open_listening_sockets(ngx_cycle_t *cycle);
diff -r cb4a4e9bba8e -r 57a3036fd504 src/event/ngx_event_accept.c
--- a/src/event/ngx_event_accept.c Tue Nov 01 20:39:21 2016 +0300
+++ b/src/event/ngx_event_accept.c Thu Nov 03 11:13:46 2016 +0700
@@ -19,18 +19,206 @@
#endif
+static inline ngx_int_t ngx_accept_socket(ngx_event_t *ev, ngx_socket_t s,
+ ngx_sockaddr_t *sa, socklen_t socklen)
+{
+ ngx_log_t *log;
+ ngx_event_t *rev, *wev;
+ ngx_listening_t *ls;
+ ngx_connection_t *c, *lc;
+
+ lc = ev->data;
+ ls = lc->listening;
+
+#if (NGX_STAT_STUB)
+ (void) ngx_atomic_fetch_add(ngx_stat_accepted, 1);
+#endif
+
+ ngx_accept_disabled = ngx_cycle->connection_n / 8
+ - ngx_cycle->free_connection_n;
+
+ c = ngx_get_connection(s, ev->log);
+
+ if (c == NULL) {
+ if (ngx_close_socket(s) == -1) {
+ ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno,
+ ngx_close_socket_n " failed");
+ }
+
+ return NGX_ERROR;
+ }
+
+ c->type = SOCK_STREAM;
+
+#if (NGX_STAT_STUB)
+ (void) ngx_atomic_fetch_add(ngx_stat_active, 1);
+#endif
+
+ c->pool = ngx_create_pool(ls->pool_size, ev->log);
+ if (c->pool == NULL) {
+ ngx_close_accepted_connection(c);
+ return NGX_ERROR;
+ }
+
+ c->sockaddr = ngx_palloc(c->pool, socklen);
+ if (c->sockaddr == NULL) {
+ ngx_close_accepted_connection(c);
+ return NGX_ERROR;
+ }
+
+ ngx_memcpy(c->sockaddr, sa, socklen);
+
+ log = ngx_palloc(c->pool, sizeof(ngx_log_t));
+ if (log == NULL) {
+ ngx_close_accepted_connection(c);
+ return NGX_ERROR;
+ }
+
+ /* set a blocking mode for iocp and non-blocking mode for others */
+
+ if (ngx_inherited_nonblocking) {
+ if (ngx_event_flags & NGX_USE_IOCP_EVENT) {
+ if (ngx_blocking(s) == -1) {
+ ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno,
+ ngx_blocking_n " failed");
+ ngx_close_accepted_connection(c);
+ return NGX_ERROR;
+ }
+ }
+
+ } else {
+ if (!(ngx_event_flags & NGX_USE_IOCP_EVENT)) {
+ if (ngx_nonblocking(s) == -1) {
+ ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno,
+ ngx_nonblocking_n " failed");
+ ngx_close_accepted_connection(c);
+ return NGX_ERROR;
+ }
+ }
+ }
+
+ *log = ls->log;
+
+ c->recv = ngx_recv;
+ c->send = ngx_send;
+ c->recv_chain = ngx_recv_chain;
+ c->send_chain = ngx_send_chain;
+
+ c->log = log;
+ c->pool->log = log;
+
+ c->socklen = socklen;
+ c->listening = ls;
+ c->local_sockaddr = ls->sockaddr;
+ c->local_socklen = ls->socklen;
+
+#if (NGX_HAVE_UNIX_DOMAIN)
+ if (c->sockaddr->sa_family == AF_UNIX) {
+ c->tcp_nopush = NGX_TCP_NOPUSH_DISABLED;
+ c->tcp_nodelay = NGX_TCP_NODELAY_DISABLED;
+#if (NGX_SOLARIS)
+ /* Solaris's sendfilev() supports AF_NCA, AF_INET, and AF_INET6 */
+ c->sendfile = 0;
+#endif
+ }
+#endif
+
+ rev = c->read;
+ wev = c->write;
+
+ wev->ready = 1;
+
+ if (ngx_event_flags & NGX_USE_IOCP_EVENT) {
+ rev->ready = 1;
+ }
+
+ if (ev->deferred_accept) {
+ rev->ready = 1;
+#if (NGX_HAVE_KQUEUE)
+ rev->available = 1;
+#endif
+ }
+
+ rev->log = log;
+ wev->log = log;
+
+ /*
+ * TODO: MT: - ngx_atomic_fetch_add()
+ * or protection by critical section or light mutex
+ *
+ * TODO: MP: - allocated in a shared memory
+ * - ngx_atomic_fetch_add()
+ * or protection by critical section or light mutex
+ */
+
+ c->number = ngx_atomic_fetch_add(ngx_connection_counter, 1);
+
+#if (NGX_STAT_STUB)
+ (void) ngx_atomic_fetch_add(ngx_stat_handled, 1);
+#endif
+
+ if (ls->addr_ntop) {
+ c->addr_text.data = ngx_pnalloc(c->pool, ls->addr_text_max_len);
+ if (c->addr_text.data == NULL) {
+ ngx_close_accepted_connection(c);
+ return NGX_ERROR;
+ }
+
+ c->addr_text.len = ngx_sock_ntop(c->sockaddr, c->socklen,
+ c->addr_text.data,
+ ls->addr_text_max_len, 0);
+ if (c->addr_text.len == 0) {
+ ngx_close_accepted_connection(c);
+ return NGX_ERROR;
+ }
+ }
+
+#if (NGX_DEBUG)
+ {
+ ngx_str_t addr;
+ u_char text[NGX_SOCKADDR_STRLEN];
+ ngx_event_conf_t *ecf;
+
+ ecf = ngx_event_get_conf(ngx_cycle->conf_ctx, ngx_event_core_module);
+
+ ngx_debug_accepted_connection(ecf, c);
+
+ if (log->log_level & NGX_LOG_DEBUG_EVENT) {
+ addr.data = text;
+ addr.len = ngx_sock_ntop(c->sockaddr, c->socklen, text,
+ NGX_SOCKADDR_STRLEN, 1);
+
+ ngx_log_debug3(NGX_LOG_DEBUG_EVENT, log, 0,
+ "*%uA accept: %V fd:%d", c->number, &addr, s);
+ }
+
+ }
+#endif
+
+ if (ngx_add_conn && (ngx_event_flags & NGX_USE_EPOLL_EVENT) == 0) {
+ if (ngx_add_conn(c) == NGX_ERROR) {
+ ngx_close_accepted_connection(c);
+ return NGX_ERROR;
+ }
+ }
+
+ log->data = NULL;
+ log->handler = NULL;
+
+ ls->handler(c);
+
+ return NGX_OK;
+}
+
void
ngx_event_accept(ngx_event_t *ev)
{
socklen_t socklen;
ngx_err_t err;
- ngx_log_t *log;
ngx_uint_t level;
ngx_socket_t s;
- ngx_event_t *rev, *wev;
ngx_sockaddr_t sa;
- ngx_listening_t *ls;
- ngx_connection_t *c, *lc;
+ ngx_connection_t *lc;
ngx_event_conf_t *ecf;
#if (NGX_HAVE_ACCEPT4)
static ngx_uint_t use_accept4 = 1;
@@ -51,7 +239,6 @@
}
lc = ev->data;
- ls = lc->listening;
ev->ready = 0;
ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
@@ -134,180 +321,10 @@
return;
}
-#if (NGX_STAT_STUB)
- (void) ngx_atomic_fetch_add(ngx_stat_accepted, 1);
-#endif
-
- ngx_accept_disabled = ngx_cycle->connection_n / 8
- - ngx_cycle->free_connection_n;
-
- c = ngx_get_connection(s, ev->log);
-
- if (c == NULL) {
- if (ngx_close_socket(s) == -1) {
- ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno,
- ngx_close_socket_n " failed");
- }
-
+ if (ngx_accept_socket(ev, s, &sa, socklen) != NGX_OK) {
return;
}
- c->type = SOCK_STREAM;
-
-#if (NGX_STAT_STUB)
- (void) ngx_atomic_fetch_add(ngx_stat_active, 1);
-#endif
-
- c->pool = ngx_create_pool(ls->pool_size, ev->log);
- if (c->pool == NULL) {
- ngx_close_accepted_connection(c);
- return;
- }
-
- c->sockaddr = ngx_palloc(c->pool, socklen);
- if (c->sockaddr == NULL) {
- ngx_close_accepted_connection(c);
- return;
- }
-
- ngx_memcpy(c->sockaddr, &sa, socklen);
-
- log = ngx_palloc(c->pool, sizeof(ngx_log_t));
- if (log == NULL) {
- ngx_close_accepted_connection(c);
- return;
- }
-
- /* set a blocking mode for iocp and non-blocking mode for others */
-
- if (ngx_inherited_nonblocking) {
- if (ngx_event_flags & NGX_USE_IOCP_EVENT) {
- if (ngx_blocking(s) == -1) {
- ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno,
- ngx_blocking_n " failed");
- ngx_close_accepted_connection(c);
- return;
- }
- }
-
- } else {
- if (!(ngx_event_flags & NGX_USE_IOCP_EVENT)) {
- if (ngx_nonblocking(s) == -1) {
- ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno,
- ngx_nonblocking_n " failed");
- ngx_close_accepted_connection(c);
- return;
- }
- }
- }
-
- *log = ls->log;
-
- c->recv = ngx_recv;
- c->send = ngx_send;
- c->recv_chain = ngx_recv_chain;
- c->send_chain = ngx_send_chain;
-
- c->log = log;
- c->pool->log = log;
-
- c->socklen = socklen;
- c->listening = ls;
- c->local_sockaddr = ls->sockaddr;
- c->local_socklen = ls->socklen;
-
-#if (NGX_HAVE_UNIX_DOMAIN)
- if (c->sockaddr->sa_family == AF_UNIX) {
- c->tcp_nopush = NGX_TCP_NOPUSH_DISABLED;
- c->tcp_nodelay = NGX_TCP_NODELAY_DISABLED;
-#if (NGX_SOLARIS)
- /* Solaris's sendfilev() supports AF_NCA, AF_INET, and AF_INET6 */
- c->sendfile = 0;
-#endif
- }
-#endif
-
- rev = c->read;
- wev = c->write;
-
- wev->ready = 1;
-
- if (ngx_event_flags & NGX_USE_IOCP_EVENT) {
- rev->ready = 1;
- }
-
- if (ev->deferred_accept) {
- rev->ready = 1;
-#if (NGX_HAVE_KQUEUE)
- rev->available = 1;
-#endif
- }
-
- rev->log = log;
- wev->log = log;
-
- /*
- * TODO: MT: - ngx_atomic_fetch_add()
- * or protection by critical section or light mutex
- *
- * TODO: MP: - allocated in a shared memory
- * - ngx_atomic_fetch_add()
- * or protection by critical section or light mutex
- */
-
- c->number = ngx_atomic_fetch_add(ngx_connection_counter, 1);
-
-#if (NGX_STAT_STUB)
- (void) ngx_atomic_fetch_add(ngx_stat_handled, 1);
-#endif
-
- if (ls->addr_ntop) {
- c->addr_text.data = ngx_pnalloc(c->pool, ls->addr_text_max_len);
- if (c->addr_text.data == NULL) {
- ngx_close_accepted_connection(c);
- return;
- }
-
- c->addr_text.len = ngx_sock_ntop(c->sockaddr, c->socklen,
- c->addr_text.data,
- ls->addr_text_max_len, 0);
- if (c->addr_text.len == 0) {
- ngx_close_accepted_connection(c);
- return;
- }
- }
-
-#if (NGX_DEBUG)
- {
- ngx_str_t addr;
- u_char text[NGX_SOCKADDR_STRLEN];
-
- ngx_debug_accepted_connection(ecf, c);
-
- if (log->log_level & NGX_LOG_DEBUG_EVENT) {
- addr.data = text;
- addr.len = ngx_sock_ntop(c->sockaddr, c->socklen, text,
- NGX_SOCKADDR_STRLEN, 1);
-
- ngx_log_debug3(NGX_LOG_DEBUG_EVENT, log, 0,
- "*%uA accept: %V fd:%d", c->number, &addr, s);
- }
-
- }
-#endif
-
- if (ngx_add_conn && (ngx_event_flags & NGX_USE_EPOLL_EVENT) == 0) {
- if (ngx_add_conn(c) == NGX_ERROR) {
- ngx_close_accepted_connection(c);
- return;
- }
- }
-
- log->data = NULL;
- log->handler = NULL;
-
- ls->handler(c);
-
if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) {
ev->available--;
}
@@ -335,6 +352,10 @@
#if (NGX_HAVE_MSGHDR_MSG_CONTROL)
+#if (NGX_CMSG_RECV_SOCKETS)
+ u_char msg_control_rights[CMSG_SPACE(sizeof(ngx_int_t))];
+#endif
+
#if (NGX_HAVE_IP_RECVDSTADDR)
u_char msg_control[CMSG_SPACE(sizeof(struct in_addr))];
#elif (NGX_HAVE_IP_PKTINFO)
@@ -380,7 +401,12 @@
msg.msg_iovlen = 1;
#if (NGX_HAVE_MSGHDR_MSG_CONTROL)
-
+#if (NGX_CMSG_RECV_SOCKETS)
+ if (ls->cmsg) {
+ msg.msg_control = &msg_control_rights;
+ msg.msg_controllen = sizeof(msg_control_rights);
+ } else
+#endif
if (ls->wildcard) {
#if (NGX_HAVE_IP_RECVDSTADDR || NGX_HAVE_IP_PKTINFO)
@@ -426,6 +452,44 @@
"recvmsg() truncated data");
continue;
}
+
+#if (NGX_CMSG_RECV_SOCKETS)
+ if (ls->cmsg) {
+ struct cmsghdr *cmsg;
+ for (cmsg = CMSG_FIRSTHDR(&msg);
+ cmsg != NULL;
+ cmsg = CMSG_NXTHDR(&msg, cmsg))
+ {
+ if ((cmsg->cmsg_level == SOL_SOCKET)
+ && (cmsg->cmsg_type == SCM_RIGHTS))
+ {
+ ngx_sockaddr_t sa;
+ socklen_t sa_len;
+ ngx_socket_t s;
+
+ sa_len = sizeof(sa);
+ s = *((ngx_socket_t *) CMSG_DATA(cmsg));
+
+ if (getpeername(s, (struct sockaddr *)&sa, &sa_len) == -1) {
+ ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno,
+ "getpeername() failed");
+
+ ngx_close_socket(s);
+ } else {
+ if (ngx_nonblocking(s) == -1) {
+ ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno,
+ ngx_nonblocking_n " failed");
+
+ ngx_close_socket(s);
+ } else {
+ ngx_accept_socket(ev, s, &sa, sa_len);
+ }
+ }
+ }
+
+ }
+ } else {
+#endif
#endif
ngx_accept_disabled = ngx_cycle->connection_n / 8
@@ -625,6 +689,9 @@
ls->handler(c);
+#if (NGX_HAVE_MSGHDR_MSG_CONTROL && NGX_CMSG_RECV_SOCKETS)
+ }
+#endif
if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) {
ev->available -= n;
}
diff -r cb4a4e9bba8e -r 57a3036fd504 src/http/ngx_http.c
--- a/src/http/ngx_http.c Tue Nov 01 20:39:21 2016 +0300
+++ b/src/http/ngx_http.c Thu Nov 03 11:13:46 2016 +0700
@@ -1703,9 +1703,16 @@
ngx_listening_t *ls;
ngx_http_core_loc_conf_t *clcf;
ngx_http_core_srv_conf_t *cscf;
+ ngx_int_t socktype = SOCK_STREAM;
+
+#if (NGX_HAVE_MSGHDR_MSG_CONTROL && NGX_CMSG_RECV_SOCKETS)
+ if (addr->opt.cmsg) {
+ socktype = SOCK_DGRAM;
+ }
+#endif
ls = ngx_create_listening(cf, &addr->opt.sockaddr.sockaddr,
- addr->opt.socklen);
+ addr->opt.socklen, socktype);
if (ls == NULL) {
return NULL;
}
@@ -1772,6 +1779,10 @@
ls->reuseport = addr->opt.reuseport;
#endif
+#if (NGX_HAVE_MSGHDR_MSG_CONTROL && NGX_CMSG_RECV_SOCKETS)
+ ls->cmsg = addr->opt.cmsg;
+#endif
+
return ls;
}
diff -r cb4a4e9bba8e -r 57a3036fd504 src/http/ngx_http_core_module.c
--- a/src/http/ngx_http_core_module.c Tue Nov 01 20:39:21 2016 +0300
+++ b/src/http/ngx_http_core_module.c Thu Nov 03 11:13:46 2016 +0700
@@ -4246,6 +4246,13 @@
continue;
}
+#if (NGX_HAVE_MSGHDR_MSG_CONTROL && NGX_CMSG_RECV_SOCKETS)
+ if (ngx_strcmp(value[n].data, "cmsg") == 0) {
+ lsopt.cmsg = 1;
+ continue;
+ }
+#endif
+
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"invalid parameter \"%V\"", &value[n]);
return NGX_CONF_ERROR;
diff -r cb4a4e9bba8e -r 57a3036fd504 src/http/ngx_http_core_module.h
--- a/src/http/ngx_http_core_module.h Tue Nov 01 20:39:21 2016 +0300
+++ b/src/http/ngx_http_core_module.h Thu Nov 03 11:13:46 2016 +0700
@@ -77,6 +77,10 @@
unsigned so_keepalive:2;
unsigned proxy_protocol:1;
+#if (NGX_HAVE_MSGHDR_MSG_CONTROL && NGX_CMSG_RECV_SOCKETS)
+ unsigned cmsg:1;
+#endif
+
int backlog;
int rcvbuf;
int sndbuf;
diff -r cb4a4e9bba8e -r 57a3036fd504 src/mail/ngx_mail.c
--- a/src/mail/ngx_mail.c Tue Nov 01 20:39:21 2016 +0300
+++ b/src/mail/ngx_mail.c Thu Nov 03 11:13:46 2016 +0700
@@ -317,7 +317,7 @@
}
ls = ngx_create_listening(cf, &addr[i].opt.sockaddr.sockaddr,
- addr[i].opt.socklen);
+ addr[i].opt.socklen, SOCK_STREAM);
if (ls == NULL) {
return NGX_CONF_ERROR;
}
diff -r cb4a4e9bba8e -r 57a3036fd504 src/stream/ngx_stream.c
--- a/src/stream/ngx_stream.c Tue Nov 01 20:39:21 2016 +0300
+++ b/src/stream/ngx_stream.c Thu Nov 03 11:13:46 2016 +0700
@@ -477,7 +477,7 @@
}
ls = ngx_create_listening(cf, &addr[i].opt.sockaddr.sockaddr,
- addr[i].opt.socklen);
+ addr[i].opt.socklen, SOCK_STREAM);
if (ls == NULL) {
return NGX_CONF_ERROR;
}
More information about the nginx-devel
mailing list