[nginx] Events: fixed balancing between workers with EPOLLEXCLUSIVE.

[Maxim Dounin]

details:   https://hg.nginx.org/nginx/rev/e2d07e4ec636
branches:  
changeset: 7992:e2d07e4ec636
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Thu Dec 30 01:08:46 2021 +0300
description:
Events: fixed balancing between workers with EPOLLEXCLUSIVE.

Linux with EPOLLEXCLUSIVE usually notifies only the process which was first
to add the listening socket to the epoll instance.  As a result most of the
connections are handled by the first worker process (ticket #2285).  To fix
this, we re-add the socket periodically, so other workers will get a chance
to accept connections.

diffstat:

 src/event/ngx_event.c        |   5 +++
 src/event/ngx_event.h        |   1 +
 src/event/ngx_event_accept.c |  58 ++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 64 insertions(+), 0 deletions(-)

diffs (122 lines):

diff -r 57581198e51e -r e2d07e4ec636 src/event/ngx_event.c
--- a/src/event/ngx_event.c	Wed Dec 29 22:59:53 2021 +0300
+++ b/src/event/ngx_event.c	Thu Dec 30 01:08:46 2021 +0300
@@ -55,6 +55,7 @@ ngx_uint_t            ngx_accept_events;
 ngx_uint_t            ngx_accept_mutex_held;
 ngx_msec_t            ngx_accept_mutex_delay;
 ngx_int_t             ngx_accept_disabled;
+ngx_uint_t            ngx_use_exclusive_accept;
 
 
 #if (NGX_STAT_STUB)
@@ -644,6 +645,8 @@ ngx_event_process_init(ngx_cycle_t *cycl
 
 #endif
 
+    ngx_use_exclusive_accept = 0;
+
     ngx_queue_init(&ngx_posted_accept_events);
     ngx_queue_init(&ngx_posted_next_events);
     ngx_queue_init(&ngx_posted_events);
@@ -889,6 +892,8 @@ ngx_event_process_init(ngx_cycle_t *cycl
         if ((ngx_event_flags & NGX_USE_EPOLL_EVENT)
             && ccf->worker_processes > 1)
         {
+            ngx_use_exclusive_accept = 1;
+
             if (ngx_add_event(rev, NGX_READ_EVENT, NGX_EXCLUSIVE_EVENT)
                 == NGX_ERROR)
             {
diff -r 57581198e51e -r e2d07e4ec636 src/event/ngx_event.h
--- a/src/event/ngx_event.h	Wed Dec 29 22:59:53 2021 +0300
+++ b/src/event/ngx_event.h	Thu Dec 30 01:08:46 2021 +0300
@@ -462,6 +462,7 @@ extern ngx_uint_t             ngx_accept
 extern ngx_uint_t             ngx_accept_mutex_held;
 extern ngx_msec_t             ngx_accept_mutex_delay;
 extern ngx_int_t              ngx_accept_disabled;
+extern ngx_uint_t             ngx_use_exclusive_accept;
 
 
 #if (NGX_STAT_STUB)
diff -r 57581198e51e -r e2d07e4ec636 src/event/ngx_event_accept.c
--- a/src/event/ngx_event_accept.c	Wed Dec 29 22:59:53 2021 +0300
+++ b/src/event/ngx_event_accept.c	Thu Dec 30 01:08:46 2021 +0300
@@ -11,6 +11,9 @@
 
 
 static ngx_int_t ngx_disable_accept_events(ngx_cycle_t *cycle, ngx_uint_t all);
+#if (NGX_HAVE_EPOLLEXCLUSIVE)
+static void ngx_reorder_accept_events(ngx_listening_t *ls);
+#endif
 static void ngx_close_accepted_connection(ngx_connection_t *c);
 
 
@@ -314,6 +317,10 @@ ngx_event_accept(ngx_event_t *ev)
         }
 
     } while (ev->available);
+
+#if (NGX_HAVE_EPOLLEXCLUSIVE)
+    ngx_reorder_accept_events(ls);
+#endif
 }
 
 
@@ -420,6 +427,57 @@ ngx_disable_accept_events(ngx_cycle_t *c
 }
 
 
+#if (NGX_HAVE_EPOLLEXCLUSIVE)
+
+static void
+ngx_reorder_accept_events(ngx_listening_t *ls)
+{
+    ngx_connection_t  *c;
+
+    /*
+     * Linux with EPOLLEXCLUSIVE usually notifies only the process which
+     * was first to add the listening socket to the epoll instance.  As
+     * a result most of the connections are handled by the first worker
+     * process.  To fix this, we re-add the socket periodically, so other
+     * workers will get a chance to accept connections.
+     */
+
+    if (!ngx_use_exclusive_accept) {
+        return;
+    }
+
+#if (NGX_HAVE_REUSEPORT)
+
+    if (ls->reuseport) {
+        return;
+    }
+
+#endif
+
+    c = ls->connection;
+
+    if (c->requests++ % 16 != 0
+        && ngx_accept_disabled <= 0)
+    {
+        return;
+    }
+
+    if (ngx_del_event(c->read, NGX_READ_EVENT, NGX_DISABLE_EVENT)
+        == NGX_ERROR)
+    {
+        return;
+    }
+
+    if (ngx_add_event(c->read, NGX_READ_EVENT, NGX_EXCLUSIVE_EVENT)
+        == NGX_ERROR)
+    {
+        return;
+    }
+}
+
+#endif
+
+
 static void
 ngx_close_accepted_connection(ngx_connection_t *c)
 {