[nginx] Core: idle connections now closed only once on exiting.

[Valentin Bartenev]

details:   http://hg.nginx.org/nginx/rev/5e6142609e48
branches:  
changeset: 6220:5e6142609e48
user:      Valentin Bartenev <vbart at nginx.com>
date:      Tue Aug 11 16:28:55 2015 +0300
description:
Core: idle connections now closed only once on exiting.

Iterating through all connections takes a lot of CPU time, especially
with large number of worker connections configured.  As a result
nginx processes used to consume CPU time during graceful shutdown.
To mitigate this we now only do a full scan for idle connections when
shutdown signal is received.

Transitions of connections to idle ones are now expected to be
avoided if the ngx_exiting flag is set.  The upstream keepalive module
was modified to follow this.

diffstat:

 src/core/ngx_connection.c                             |  20 +++++++++++++++++
 src/core/ngx_connection.h                             |   1 +
 src/http/modules/ngx_http_upstream_keepalive_module.c |   4 +++
 src/os/unix/ngx_process_cycle.c                       |  19 +--------------
 src/os/win32/ngx_process_cycle.c                      |  22 +++---------------
 5 files changed, 31 insertions(+), 35 deletions(-)

diffs (146 lines):

diff -r 808fd1f0b94b -r 5e6142609e48 src/core/ngx_connection.c
--- a/src/core/ngx_connection.c	Tue Aug 11 10:28:00 2015 +0300
+++ b/src/core/ngx_connection.c	Tue Aug 11 16:28:55 2015 +0300
@@ -1163,6 +1163,26 @@ ngx_drain_connections(void)
 }
 
 
+void
+ngx_close_idle_connections(ngx_cycle_t *cycle)
+{
+    ngx_uint_t         i;
+    ngx_connection_t  *c;
+
+    c = cycle->connections;
+
+    for (i = 0; i < cycle->connection_n; i++) {
+
+        /* THREAD: lock */
+
+        if (c[i].fd != -1 && c[i].idle) {
+            c[i].close = 1;
+            c[i].read->handler(c[i].read);
+        }
+    }
+}
+
+
 ngx_int_t
 ngx_connection_local_sockaddr(ngx_connection_t *c, ngx_str_t *s,
     ngx_uint_t port)
diff -r 808fd1f0b94b -r 5e6142609e48 src/core/ngx_connection.h
--- a/src/core/ngx_connection.h	Tue Aug 11 10:28:00 2015 +0300
+++ b/src/core/ngx_connection.h	Tue Aug 11 16:28:55 2015 +0300
@@ -215,6 +215,7 @@ ngx_int_t ngx_open_listening_sockets(ngx
 void ngx_configure_listening_sockets(ngx_cycle_t *cycle);
 void ngx_close_listening_sockets(ngx_cycle_t *cycle);
 void ngx_close_connection(ngx_connection_t *c);
+void ngx_close_idle_connections(ngx_cycle_t *cycle);
 ngx_int_t ngx_connection_local_sockaddr(ngx_connection_t *c, ngx_str_t *s,
     ngx_uint_t port);
 ngx_int_t ngx_connection_error(ngx_connection_t *c, ngx_err_t err, char *text);
diff -r 808fd1f0b94b -r 5e6142609e48 src/http/modules/ngx_http_upstream_keepalive_module.c
--- a/src/http/modules/ngx_http_upstream_keepalive_module.c	Tue Aug 11 10:28:00 2015 +0300
+++ b/src/http/modules/ngx_http_upstream_keepalive_module.c	Tue Aug 11 16:28:55 2015 +0300
@@ -302,6 +302,10 @@ ngx_http_upstream_free_keepalive_peer(ng
         goto invalid;
     }
 
+    if (ngx_terminate || ngx_exiting) {
+        goto invalid;
+    }
+
     if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
         goto invalid;
     }
diff -r 808fd1f0b94b -r 5e6142609e48 src/os/unix/ngx_process_cycle.c
--- a/src/os/unix/ngx_process_cycle.c	Tue Aug 11 10:28:00 2015 +0300
+++ b/src/os/unix/ngx_process_cycle.c	Tue Aug 11 16:28:55 2015 +0300
@@ -728,9 +728,6 @@ ngx_worker_process_cycle(ngx_cycle_t *cy
 {
     ngx_int_t worker = (intptr_t) data;
 
-    ngx_uint_t         i;
-    ngx_connection_t  *c;
-
     ngx_process = NGX_PROCESS_WORKER;
     ngx_worker = worker;
 
@@ -741,19 +738,6 @@ ngx_worker_process_cycle(ngx_cycle_t *cy
     for ( ;; ) {
 
         if (ngx_exiting) {
-
-            c = cycle->connections;
-
-            for (i = 0; i < cycle->connection_n; i++) {
-
-                /* THREAD: lock */
-
-                if (c[i].fd != -1 && c[i].idle) {
-                    c[i].close = 1;
-                    c[i].read->handler(c[i].read);
-                }
-            }
-
             ngx_event_cancel_timers();
 
             if (ngx_event_timer_rbtree.root == ngx_event_timer_rbtree.sentinel)
@@ -781,8 +765,9 @@ ngx_worker_process_cycle(ngx_cycle_t *cy
             ngx_setproctitle("worker process is shutting down");
 
             if (!ngx_exiting) {
+                ngx_exiting = 1;
                 ngx_close_listening_sockets(cycle);
-                ngx_exiting = 1;
+                ngx_close_idle_connections(cycle);
             }
         }
 
diff -r 808fd1f0b94b -r 5e6142609e48 src/os/win32/ngx_process_cycle.c
--- a/src/os/win32/ngx_process_cycle.c	Tue Aug 11 10:28:00 2015 +0300
+++ b/src/os/win32/ngx_process_cycle.c	Tue Aug 11 16:28:55 2015 +0300
@@ -761,10 +761,8 @@ failed:
 static ngx_thread_value_t __stdcall
 ngx_worker_thread(void *data)
 {
-    ngx_int_t          n;
-    ngx_uint_t         i;
-    ngx_cycle_t       *cycle;
-    ngx_connection_t  *c;
+    ngx_int_t     n;
+    ngx_cycle_t  *cycle;
 
     cycle = (ngx_cycle_t *) ngx_cycle;
 
@@ -780,19 +778,6 @@ ngx_worker_thread(void *data)
     while (!ngx_quit) {
 
         if (ngx_exiting) {
-
-            c = cycle->connections;
-
-            for (i = 0; i < cycle->connection_n; i++) {
-
-                /* THREAD: lock */
-
-                if (c[i].fd != (ngx_socket_t) -1 && c[i].idle) {
-                    c[i].close = 1;
-                    c[i].read->handler(c[i].read);
-                }
-            }
-
             ngx_event_cancel_timers();
 
             if (ngx_event_timer_rbtree.root
@@ -814,8 +799,9 @@ ngx_worker_thread(void *data)
             ngx_quit = 0;
 
             if (!ngx_exiting) {
+                ngx_exiting = 1;
                 ngx_close_listening_sockets(cycle);
-                ngx_exiting = 1;
+                ngx_close_idle_connections(cycle);
             }
         }