[PATCH] Introduced worker_shutdown_idle_delay
Artem Pogartsev
iam at arteomp.com
Wed Nov 8 04:51:39 UTC 2023
# HG changeset patch
# User Artem Pogartsev <iam at arteomp.com>
# Date 1699416491 -10800
# Wed Nov 08 07:08:11 2023 +0300
# Node ID eb0dd3d903431f4dd7a62d629db457ecddeadd96
# Parent 7ec761f0365f418511e30b82e9adf80bc56681df
Introduced worker_shutdown_idle_delay.
The directive configures a delay before closing idle connections to be used
when gracefully shutting down worker processes. When the timer expires, nginx
will attempt to close any remaining idle connections.
The delay is only added for connections with the "shutdown_delay" flag, and
currently it's only keepalive HTTP/1.1 connections. The behavior is not changed
for protocols that have built-in graceful shutdown mechanisms, such as GOAWAY
frames in HTTP2 and HTTP3.
Although it's perfectly fine to close an HTTP/1.1 connection at any time
according to RFC 9112, it may still be useful to delay closing a connection,
wait for the next client request, and close the connection with a
"Connection: close" header to avoid unnecessary retries by clients. This is
especially important for environments with frequent nginx reloads and large
amounts of non-idempotent requests which are quite problematic for automatic
retries.
Should be used carefully to not delay configuration reloads too much (and thus
increase nginx resource usage), and ideally in combination with properly
configured clients:
client_idle_timeout < min(server_shutdown_delay, server_idle_timeout)
diff -r 7ec761f0365f -r eb0dd3d90343 contrib/vim/syntax/nginx.vim
--- a/contrib/vim/syntax/nginx.vim Thu Oct 26 23:35:09 2023 +0300
+++ b/contrib/vim/syntax/nginx.vim Wed Nov 08 07:08:11 2023 +0300
@@ -705,6 +705,7 @@
syn keyword ngxDirective contained worker_rlimit_core
syn keyword ngxDirective contained worker_rlimit_nofile
syn keyword ngxDirective contained worker_shutdown_timeout
+syn keyword ngxDirective contained worker_shutdown_idle_delay
syn keyword ngxDirective contained working_directory
syn keyword ngxDirective contained xclient
syn keyword ngxDirective contained xml_entities
diff -r 7ec761f0365f -r eb0dd3d90343 src/core/nginx.c
--- a/src/core/nginx.c Thu Oct 26 23:35:09 2023 +0300
+++ b/src/core/nginx.c Wed Nov 08 07:08:11 2023 +0300
@@ -132,6 +132,13 @@
offsetof(ngx_core_conf_t, shutdown_timeout),
NULL },
+ { ngx_string("worker_shutdown_idle_delay"),
+ NGX_MAIN_CONF|NGX_DIRECT_CONF|NGX_CONF_TAKE1,
+ ngx_conf_set_msec_slot,
+ 0,
+ offsetof(ngx_core_conf_t, shutdown_idle_delay),
+ NULL },
+
{ ngx_string("working_directory"),
NGX_MAIN_CONF|NGX_DIRECT_CONF|NGX_CONF_TAKE1,
ngx_conf_set_str_slot,
@@ -1115,6 +1122,7 @@
ccf->master = NGX_CONF_UNSET;
ccf->timer_resolution = NGX_CONF_UNSET_MSEC;
ccf->shutdown_timeout = NGX_CONF_UNSET_MSEC;
+ ccf->shutdown_idle_delay = NGX_CONF_UNSET_MSEC;
ccf->worker_processes = NGX_CONF_UNSET;
ccf->debug_points = NGX_CONF_UNSET;
@@ -1144,6 +1152,7 @@
ngx_conf_init_value(ccf->master, 1);
ngx_conf_init_msec_value(ccf->timer_resolution, 0);
ngx_conf_init_msec_value(ccf->shutdown_timeout, 0);
+ ngx_conf_init_msec_value(ccf->shutdown_idle_delay, 0);
ngx_conf_init_value(ccf->worker_processes, 1);
ngx_conf_init_value(ccf->debug_points, 0);
diff -r 7ec761f0365f -r eb0dd3d90343 src/core/ngx_connection.c
--- a/src/core/ngx_connection.c Thu Oct 26 23:35:09 2023 +0300
+++ b/src/core/ngx_connection.c Wed Nov 08 07:08:11 2023 +0300
@@ -1427,7 +1427,7 @@
void
-ngx_close_idle_connections(ngx_cycle_t *cycle)
+ngx_close_idle_connections(ngx_cycle_t *cycle, ngx_msec_t shutdown_delay)
{
ngx_uint_t i;
ngx_connection_t *c;
@@ -1439,10 +1439,15 @@
/* THREAD: lock */
if (c[i].fd != (ngx_socket_t) -1 && c[i].idle) {
+ if (shutdown_delay && c[i].shutdown_delay) {
+ continue;
+ }
c[i].close = 1;
c[i].read->handler(c[i].read);
}
}
+
+ ngx_set_shutdown_idle_timer(cycle, shutdown_delay);
}
diff -r 7ec761f0365f -r eb0dd3d90343 src/core/ngx_connection.h
--- a/src/core/ngx_connection.h Thu Oct 26 23:35:09 2023 +0300
+++ b/src/core/ngx_connection.h Wed Nov 08 07:08:11 2023 +0300
@@ -183,6 +183,7 @@
unsigned idle:1;
unsigned reusable:1;
unsigned close:1;
+ unsigned shutdown_delay:1;
unsigned shared:1;
unsigned sendfile:1;
@@ -222,7 +223,7 @@
void ngx_configure_listening_sockets(ngx_cycle_t *cycle);
void ngx_close_listening_sockets(ngx_cycle_t *cycle);
void ngx_close_connection(ngx_connection_t *c);
-void ngx_close_idle_connections(ngx_cycle_t *cycle);
+void ngx_close_idle_connections(ngx_cycle_t *cycle, ngx_msec_t shutdown_delay);
ngx_int_t ngx_connection_local_sockaddr(ngx_connection_t *c, ngx_str_t *s,
ngx_uint_t port);
ngx_int_t ngx_tcp_nodelay(ngx_connection_t *c);
diff -r 7ec761f0365f -r eb0dd3d90343 src/core/ngx_cycle.c
--- a/src/core/ngx_cycle.c Thu Oct 26 23:35:09 2023 +0300
+++ b/src/core/ngx_cycle.c Wed Nov 08 07:08:11 2023 +0300
@@ -16,6 +16,7 @@
static ngx_int_t ngx_test_lockfile(u_char *file, ngx_log_t *log);
static void ngx_clean_old_cycles(ngx_event_t *ev);
static void ngx_shutdown_timer_handler(ngx_event_t *ev);
+static void ngx_shutdown_idle_timer_handler(ngx_event_t *ev);
volatile ngx_cycle_t *ngx_cycle;
@@ -24,6 +25,7 @@
static ngx_pool_t *ngx_temp_pool;
static ngx_event_t ngx_cleaner_event;
static ngx_event_t ngx_shutdown_event;
+static ngx_event_t ngx_shutdown_idle_event;
ngx_uint_t ngx_test_config;
ngx_uint_t ngx_dump_config;
@@ -1468,3 +1470,30 @@
c[i].read->handler(c[i].read);
}
}
+
+
+void
+ngx_set_shutdown_idle_timer(ngx_cycle_t *cycle, ngx_msec_t shutdown_delay)
+{
+ if (shutdown_delay) {
+ ngx_shutdown_idle_event.handler = ngx_shutdown_idle_timer_handler;
+ ngx_shutdown_idle_event.data = cycle;
+ ngx_shutdown_idle_event.log = cycle->log;
+ ngx_shutdown_idle_event.cancelable = 1;
+
+ ngx_add_timer(&ngx_shutdown_idle_event, shutdown_delay);
+ }
+}
+
+
+static void
+ngx_shutdown_idle_timer_handler(ngx_event_t *ev)
+{
+ ngx_cycle_t *cycle;
+
+ cycle = ev->data;
+
+ ngx_log_debug0(NGX_LOG_DEBUG_CORE, ev->log, 0, "shutdown idle delay timer");
+
+ ngx_close_idle_connections(cycle, 0);
+}
diff -r 7ec761f0365f -r eb0dd3d90343 src/core/ngx_cycle.h
--- a/src/core/ngx_cycle.h Thu Oct 26 23:35:09 2023 +0300
+++ b/src/core/ngx_cycle.h Wed Nov 08 07:08:11 2023 +0300
@@ -92,6 +92,7 @@
ngx_msec_t timer_resolution;
ngx_msec_t shutdown_timeout;
+ ngx_msec_t shutdown_idle_delay;
ngx_int_t worker_processes;
ngx_int_t debug_points;
@@ -136,6 +137,7 @@
ngx_shm_zone_t *ngx_shared_memory_add(ngx_conf_t *cf, ngx_str_t *name,
size_t size, void *tag);
void ngx_set_shutdown_timer(ngx_cycle_t *cycle);
+void ngx_set_shutdown_idle_timer(ngx_cycle_t *cycle, ngx_msec_t shutdown_delay);
extern volatile ngx_cycle_t *ngx_cycle;
diff -r 7ec761f0365f -r eb0dd3d90343 src/http/ngx_http_request.c
--- a/src/http/ngx_http_request.c Thu Oct 26 23:35:09 2023 +0300
+++ b/src/http/ngx_http_request.c Wed Nov 08 07:08:11 2023 +0300
@@ -3282,6 +3282,7 @@
r->http_state = NGX_HTTP_KEEPALIVE_STATE;
#endif
+ c->shutdown_delay = 1;
c->idle = 1;
ngx_reusable_connection(c, 1);
diff -r 7ec761f0365f -r eb0dd3d90343 src/os/unix/ngx_process_cycle.c
--- a/src/os/unix/ngx_process_cycle.c Thu Oct 26 23:35:09 2023 +0300
+++ b/src/os/unix/ngx_process_cycle.c Wed Nov 08 07:08:11 2023 +0300
@@ -698,6 +698,8 @@
static void
ngx_worker_process_cycle(ngx_cycle_t *cycle, void *data)
{
+ ngx_core_conf_t *ccf;
+
ngx_int_t worker = (intptr_t) data;
ngx_process = NGX_PROCESS_WORKER;
@@ -733,9 +735,10 @@
if (!ngx_exiting) {
ngx_exiting = 1;
+ ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);
ngx_set_shutdown_timer(cycle);
ngx_close_listening_sockets(cycle);
- ngx_close_idle_connections(cycle);
+ ngx_close_idle_connections(cycle, ccf->shutdown_idle_delay);
ngx_event_process_posted(cycle, &ngx_posted_events);
}
}
diff -r 7ec761f0365f -r eb0dd3d90343 src/os/win32/ngx_process_cycle.c
--- a/src/os/win32/ngx_process_cycle.c Thu Oct 26 23:35:09 2023 +0300
+++ b/src/os/win32/ngx_process_cycle.c Wed Nov 08 07:08:11 2023 +0300
@@ -762,9 +762,10 @@
static ngx_thread_value_t __stdcall
ngx_worker_thread(void *data)
{
- ngx_int_t n;
- ngx_time_t *tp;
- ngx_cycle_t *cycle;
+ ngx_int_t n;
+ ngx_time_t *tp;
+ ngx_cycle_t *cycle;
+ ngx_core_conf_t *ccf;
tp = ngx_timeofday();
srand((ngx_pid << 16) ^ (unsigned) tp->sec ^ tp->msec);
@@ -801,9 +802,10 @@
if (!ngx_exiting) {
ngx_exiting = 1;
+ ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);
ngx_set_shutdown_timer(cycle);
ngx_close_listening_sockets(cycle);
- ngx_close_idle_connections(cycle);
+ ngx_close_idle_connections(cycle, ccf->shutdown_idle_delay);
ngx_event_process_posted(cycle, &ngx_posted_events);
}
}
More information about the nginx-devel
mailing list