[PATCH 6 of 6] Upstream: reschedule in-progress resolve tasks at worker exit
Aleksei Bavshin
a.bavshin at nginx.com
Wed Feb 1 01:37:01 UTC 2023
# HG changeset patch
# User Aleksei Bavshin <a.bavshin at nginx.com>
# Date 1671568350 28800
# Tue Dec 20 12:32:30 2022 -0800
# Node ID 34e439843ed3a2122cba54b1f40b77ea6e874078
# Parent f8eb6b94d8f46008eb5f2f1dbc747750d5755506
Upstream: reschedule in-progress resolve tasks at worker exit.
Workers may exit at a different time, depending on the active
connections and other factors. If the peer was being resolved in one
of the workers at the moment of termination, it won't be returned to
the resolve queue and remaining workers won't be able to continue
updating it.
To address that we can disown all interrupted resolve tasks and put
them back to the queue at worker exit. The same logic runs on the
process init to recover tasks affected by a worker crash.
diff --git a/src/http/modules/ngx_http_upstream_zone_module.c b/src/http/modules/ngx_http_upstream_zone_module.c
--- a/src/http/modules/ngx_http_upstream_zone_module.c
+++ b/src/http/modules/ngx_http_upstream_zone_module.c
@@ -40,7 +40,10 @@ static ngx_command_t ngx_http_upstream_
};
+static ngx_uint_t ngx_http_upstream_zone_recover_peers(
+ ngx_http_upstream_srv_conf_t *uscf);
static ngx_int_t ngx_http_upstream_zone_init_worker(ngx_cycle_t *cycle);
+static void ngx_http_upstream_zone_exit_worker(ngx_cycle_t *cycle);
static void ngx_http_upstream_zone_resolve_timer(ngx_event_t *event);
static void ngx_http_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx);
static void ngx_http_upstream_zone_resolve_queue_insert(ngx_queue_t *queue,
@@ -77,7 +80,7 @@ ngx_module_t ngx_http_upstream_zone_mod
ngx_http_upstream_zone_init_worker, /* init process */
NULL, /* init thread */
NULL, /* exit thread */
- NULL, /* exit process */
+ ngx_http_upstream_zone_exit_worker, /* exit process */
NULL, /* exit master */
NGX_MODULE_V1_PADDING
};
@@ -573,10 +576,51 @@ ngx_http_upstream_zone_resolve_timer(ngx
}
+static ngx_uint_t
+ngx_http_upstream_zone_recover_peers(ngx_http_upstream_srv_conf_t *uscf)
+{
+ ngx_msec_t now;
+ ngx_uint_t n;
+ ngx_http_upstream_host_t *host;
+ ngx_http_upstream_rr_peer_t *peer;
+ ngx_http_upstream_rr_peers_t *peers;
+
+ n = 0;
+ now = ngx_current_msec;
+ peers = uscf->peer.data;
+
+ do {
+ ngx_http_upstream_rr_peers_wlock(peers);
+
+ for (peer = peers->resolve; peer; peer = peer->next) {
+
+ host = peer->host;
+
+ if (host->worker != ngx_worker) {
+ continue;
+ }
+
+ host->expires = now;
+ host->worker = NGX_UPSTREAM_RESOLVE_NO_WORKER;
+ ngx_queue_insert_head(&peers->resolve_queue, &host->queue);
+
+ n++;
+ }
+
+ ngx_http_upstream_rr_peers_unlock(peers);
+
+ peers = peers->next;
+
+ } while (peers);
+
+ return n;
+}
+
+
static ngx_int_t
ngx_http_upstream_zone_init_worker(ngx_cycle_t *cycle)
{
- ngx_uint_t i;
+ ngx_uint_t i, n;
ngx_event_t *event;
ngx_http_upstream_srv_conf_t *uscf, **uscfp;
ngx_http_upstream_main_conf_t *umcf;
@@ -594,6 +638,7 @@ ngx_http_upstream_zone_init_worker(ngx_c
}
uscfp = umcf->upstreams.elts;
+ n = 0;
for (i = 0; i < umcf->upstreams.nelts; i++) {
@@ -603,6 +648,8 @@ ngx_http_upstream_zone_init_worker(ngx_c
continue;
}
+ n += ngx_http_upstream_zone_recover_peers(uscf);
+
event = &uscf->event;
event->data = uscf;
event->handler = ngx_http_upstream_zone_resolve_timer;
@@ -612,11 +659,56 @@ ngx_http_upstream_zone_init_worker(ngx_c
ngx_add_timer(event, 1);
}
+ if (n) {
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, cycle->log, 0,
+ "upstream: recovered %ui resolvable peers", n);
+ }
+
return NGX_OK;
}
static void
+ngx_http_upstream_zone_exit_worker(ngx_cycle_t *cycle)
+{
+ ngx_uint_t i, n;
+ ngx_http_upstream_srv_conf_t *uscf, **uscfp;
+ ngx_http_upstream_main_conf_t *umcf;
+
+ if (ngx_process != NGX_PROCESS_WORKER
+ && ngx_process != NGX_PROCESS_SINGLE)
+ {
+ return;
+ }
+
+ umcf = ngx_http_cycle_get_module_main_conf(cycle, ngx_http_upstream_module);
+
+ if (umcf == NULL) {
+ return;
+ }
+
+ uscfp = umcf->upstreams.elts;
+ n = 0;
+
+ for (i = 0; i < umcf->upstreams.nelts; i++) {
+
+ uscf = uscfp[i];
+
+ if (uscf->shm_zone == NULL) {
+ continue;
+ }
+
+ n += ngx_http_upstream_zone_recover_peers(uscf);
+ }
+
+ if (n) {
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, cycle->log, 0,
+ "upstream: released %ui resolvable peers", n);
+ }
+}
+
+
+static void
ngx_http_upstream_zone_start_resolve(ngx_http_upstream_srv_conf_t *uscf,
ngx_http_upstream_host_t *host)
{
diff --git a/src/stream/ngx_stream_upstream_zone_module.c b/src/stream/ngx_stream_upstream_zone_module.c
--- a/src/stream/ngx_stream_upstream_zone_module.c
+++ b/src/stream/ngx_stream_upstream_zone_module.c
@@ -40,7 +40,10 @@ static ngx_command_t ngx_stream_upstrea
};
+static ngx_uint_t ngx_stream_upstream_zone_recover_peers(
+ ngx_stream_upstream_srv_conf_t *uscf);
static ngx_int_t ngx_stream_upstream_zone_init_worker(ngx_cycle_t *cycle);
+static void ngx_stream_upstream_zone_exit_worker(ngx_cycle_t *cycle);
static void ngx_stream_upstream_zone_resolve_timer(ngx_event_t *event);
static void ngx_stream_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx);
static void ngx_stream_upstream_zone_resolve_queue_insert(ngx_queue_t *queue,
@@ -74,7 +77,7 @@ ngx_module_t ngx_stream_upstream_zone_m
ngx_stream_upstream_zone_init_worker, /* init process */
NULL, /* init thread */
NULL, /* exit thread */
- NULL, /* exit process */
+ ngx_stream_upstream_zone_exit_worker, /* exit process */
NULL, /* exit master */
NGX_MODULE_V1_PADDING
};
@@ -570,10 +573,51 @@ ngx_stream_upstream_zone_resolve_timer(n
}
+static ngx_uint_t
+ngx_stream_upstream_zone_recover_peers(ngx_stream_upstream_srv_conf_t *uscf)
+{
+ ngx_msec_t now;
+ ngx_uint_t n;
+ ngx_stream_upstream_host_t *host;
+ ngx_stream_upstream_rr_peer_t *peer;
+ ngx_stream_upstream_rr_peers_t *peers;
+
+ n = 0;
+ now = ngx_current_msec;
+ peers = uscf->peer.data;
+
+ do {
+ ngx_stream_upstream_rr_peers_wlock(peers);
+
+ for (peer = peers->resolve; peer; peer = peer->next) {
+
+ host = peer->host;
+
+ if (host->worker != ngx_worker) {
+ continue;
+ }
+
+ host->expires = now;
+ host->worker = NGX_UPSTREAM_RESOLVE_NO_WORKER;
+ ngx_queue_insert_head(&peers->resolve_queue, &host->queue);
+
+ n++;
+ }
+
+ ngx_stream_upstream_rr_peers_unlock(peers);
+
+ peers = peers->next;
+
+ } while (peers);
+
+ return n;
+}
+
+
static ngx_int_t
ngx_stream_upstream_zone_init_worker(ngx_cycle_t *cycle)
{
- ngx_uint_t i;
+ ngx_uint_t i, n;
ngx_event_t *event;
ngx_stream_upstream_srv_conf_t *uscf, **uscfp;
ngx_stream_upstream_main_conf_t *umcf;
@@ -592,6 +636,7 @@ ngx_stream_upstream_zone_init_worker(ngx
}
uscfp = umcf->upstreams.elts;
+ n = 0;
for (i = 0; i < umcf->upstreams.nelts; i++) {
@@ -601,6 +646,8 @@ ngx_stream_upstream_zone_init_worker(ngx
continue;
}
+ n += ngx_stream_upstream_zone_recover_peers(uscf);
+
event = &uscf->event;
event->data = uscf;
event->handler = ngx_stream_upstream_zone_resolve_timer;
@@ -610,11 +657,57 @@ ngx_stream_upstream_zone_init_worker(ngx
ngx_add_timer(event, 1);
}
+ if (n) {
+ ngx_log_debug1(NGX_LOG_DEBUG_STREAM, cycle->log, 0,
+ "upstream: recovered %ui resolvable peers", n);
+ }
+
return NGX_OK;
}
static void
+ngx_stream_upstream_zone_exit_worker(ngx_cycle_t *cycle)
+{
+ ngx_uint_t i, n;
+ ngx_stream_upstream_srv_conf_t *uscf, **uscfp;
+ ngx_stream_upstream_main_conf_t *umcf;
+
+ if (ngx_process != NGX_PROCESS_WORKER
+ && ngx_process != NGX_PROCESS_SINGLE)
+ {
+ return;
+ }
+
+ umcf = ngx_stream_cycle_get_module_main_conf(cycle,
+ ngx_stream_upstream_module);
+
+ if (umcf == NULL) {
+ return;
+ }
+
+ uscfp = umcf->upstreams.elts;
+ n = 0;
+
+ for (i = 0; i < umcf->upstreams.nelts; i++) {
+
+ uscf = uscfp[i];
+
+ if (uscf->shm_zone == NULL) {
+ continue;
+ }
+
+ n += ngx_stream_upstream_zone_recover_peers(uscf);
+ }
+
+ if (n) {
+ ngx_log_debug1(NGX_LOG_DEBUG_STREAM, cycle->log, 0,
+ "upstream: released %ui resolvable peers", n);
+ }
+}
+
+
+static void
ngx_stream_upstream_zone_start_resolve(ngx_stream_upstream_srv_conf_t *uscf,
ngx_stream_upstream_host_t *host)
{
More information about the nginx-devel
mailing list