[PATCH] Added keepalive_async_fails command
J Carter
jordanc.carter at outlook.com
Sun Apr 2 18:31:16 UTC 2023
re-sending the patch as an attachment as the formatting is still weird,
and fixed typo I spotted..
On 02/04/2023 18:57, J Carter wrote:
> Hello,
>
> I've also attached an example nginx.conf and test script that
> simulates the asynchronous close events.
> Two different test cases can be found within that, one with path /1
> for single peer upstream and /2 for multi-peer.
>
> You should see 2 upstream addresses repeated in a row
> per-upstream-server in the access log by default, as it fails
> through the cached connections & next performs next upstream tries.
>
> Any feedback would be appreciated.
>
> # HG changeset patch
> # User jordanc.carter at outlook.com
> # Date 1680457073 -3600
> # Sun Apr 02 18:37:53 2023 +0100
> # Node ID 9ec4d7a8cdf6cdab00d09dff75fa6045f6f5533f
> # Parent 5f1d05a21287ba0290dd3a17ad501595b442a194
> Added keepalive_async_fails command to keepalive load balancer module.
> This value determines the number suspected keepalive race events
> per-upstream-try that will be tolerated before a subsequent network
> connection
> error is considered a true failure.
>
> diff -r 5f1d05a21287 -r 9ec4d7a8cdf6 src/event/ngx_event_connect.h
> --- a/src/event/ngx_event_connect.h Tue Mar 28 18:01:54 2023 +0300
> +++ b/src/event/ngx_event_connect.h Sun Apr 02 18:37:53 2023 +0100
> @@ -17,6 +17,7 @@
> #define NGX_PEER_KEEPALIVE 1
> #define NGX_PEER_NEXT 2
> #define NGX_PEER_FAILED 4
> +#define NGX_PEER_ASYNC_FAILED 8
>
>
> typedef struct ngx_peer_connection_s ngx_peer_connection_t;
> @@ -41,6 +42,7 @@
> ngx_str_t *name;
>
> ngx_uint_t tries;
> + ngx_uint_t async_fails;
> ngx_msec_t start_time;
>
> ngx_event_get_peer_pt get;
> diff -r 5f1d05a21287 -r 9ec4d7a8cdf6
> src/http/modules/ngx_http_upstream_keepalive_module.c
> --- a/src/http/modules/ngx_http_upstream_keepalive_module.c Tue Mar
> 28 18:01:54 2023 +0300
> +++ b/src/http/modules/ngx_http_upstream_keepalive_module.c Sun Apr
> 02 18:37:53 2023 +0100
> @@ -13,6 +13,7 @@
> typedef struct {
> ngx_uint_t max_cached;
> ngx_uint_t requests;
> + ngx_uint_t max_async_fails;
> ngx_msec_t time;
> ngx_msec_t timeout;
>
> @@ -108,6 +109,13 @@
> offsetof(ngx_http_upstream_keepalive_srv_conf_t, requests),
> NULL },
>
> + { ngx_string("keepalive_async_fails"),
> + NGX_HTTP_UPS_CONF|NGX_CONF_TAKE1,
> + ngx_conf_set_num_slot,
> + NGX_HTTP_SRV_CONF_OFFSET,
> + offsetof(ngx_http_upstream_keepalive_srv_conf_t, max_async_fails),
> + NULL },
> +
> ngx_null_command
> };
>
> @@ -160,6 +168,7 @@
> ngx_conf_init_msec_value(kcf->time, 3600000);
> ngx_conf_init_msec_value(kcf->timeout, 60000);
> ngx_conf_init_uint_value(kcf->requests, 1000);
> + ngx_conf_init_uint_value(kcf->max_async_fails, 2);
>
> if (kcf->original_init_upstream(cf, us) != NGX_OK) {
> return NGX_ERROR;
> @@ -320,6 +329,21 @@
> u = kp->upstream;
> c = pc->connection;
>
> + if (state & NGX_PEER_ASYNC_FAILED) {
> + pc->async_fails++;
> +
> + if (pc->async_fails == 2) {
> + pc->async_fails = 0;
> + state = NGX_PEER_FAILED;
> +
> + } else {
> + pc->tries++;
> + }
> + goto invalid;
> + }
> +
> + pc->async_fails = 0;
> +
> if (state & NGX_PEER_FAILED
> || c == NULL
> || c->read->eof
> @@ -529,6 +553,8 @@
> conf->time = NGX_CONF_UNSET_MSEC;
> conf->timeout = NGX_CONF_UNSET_MSEC;
> conf->requests = NGX_CONF_UNSET_UINT;
> + conf->max_async_fails = NGX_CONF_UNSET_UINT;
> +
>
> return conf;
> }
> diff -r 5f1d05a21287 -r 9ec4d7a8cdf6 src/http/ngx_http_upstream.c
> --- a/src/http/ngx_http_upstream.c Tue Mar 28 18:01:54 2023 +0300
> +++ b/src/http/ngx_http_upstream.c Sun Apr 02 18:37:53 2023 +0100
> @@ -4317,6 +4317,8 @@
> {
> state = NGX_PEER_NEXT;
>
> + } else if (u->peer.cached && ft_type ==
> NGX_HTTP_UPSTREAM_FT_ERROR) {
> + state = NGX_PEER_ASYNC_FAILED;
> } else {
> state = NGX_PEER_FAILED;
> }
> @@ -4330,11 +4332,6 @@
> "upstream timed out");
> }
>
> - if (u->peer.cached && ft_type == NGX_HTTP_UPSTREAM_FT_ERROR) {
> - /* TODO: inform balancer instead */
> - u->peer.tries++;
> - }
> -
> switch (ft_type) {
>
> case NGX_HTTP_UPSTREAM_FT_TIMEOUT:
> @@ -4421,7 +4418,6 @@
> return;
> }
> #endif
> -
> ngx_http_upstream_finalize_request(r, u, status);
> return;
> }
> diff -r 5f1d05a21287 -r 9ec4d7a8cdf6
> src/http/ngx_http_upstream_round_robin.c
> --- a/src/http/ngx_http_upstream_round_robin.c Tue Mar 28 18:01:54
> 2023 +0300
> +++ b/src/http/ngx_http_upstream_round_robin.c Sun Apr 02 18:37:53
> 2023 +0100
> @@ -297,6 +297,7 @@
> r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
> r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
> r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
> + r->upstream->peer.async_fails = 0;
> #if (NGX_HTTP_SSL)
> r->upstream->peer.set_session =
> ngx_http_upstream_set_round_robin_peer_session;
> @@ -418,6 +419,7 @@
> r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
> r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
> r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
> + r->upstream->peer.async_fails = 0;
> #if (NGX_HTTP_SSL)
> r->upstream->peer.set_session = ngx_http_upstream_empty_set_session;
> r->upstream->peer.save_session =
> ngx_http_upstream_empty_save_session;
> @@ -459,7 +461,10 @@
>
> rrp->current = peer;
>
> - } else {
> + } else if (pc->async_fails > 0) {
> + peer = rrp->current;
> + }
> + else {
>
> /* there are several peers */
>
> @@ -615,18 +620,7 @@
> ngx_http_upstream_rr_peers_rlock(rrp->peers);
> ngx_http_upstream_rr_peer_lock(rrp->peers, peer);
>
> - if (rrp->peers->single) {
> -
> - peer->conns--;
> -
> - ngx_http_upstream_rr_peer_unlock(rrp->peers, peer);
> - ngx_http_upstream_rr_peers_unlock(rrp->peers);
> -
> - pc->tries = 0;
> - return;
> - }
> -
> - if (state & NGX_PEER_FAILED) {
> + if (state & NGX_PEER_FAILED && !rrp->peers->single) {
> now = ngx_time();
>
> peer->fails++;
>
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> https://mailman.nginx.org/mailman/listinfo/nginx-devel
-------------- next part --------------
# HG changeset patch
# User jordanc.carter at outlook.com
# Date 1680459126 -3600
# Sun Apr 02 19:12:06 2023 +0100
# Node ID 4295bf4613e155e063914ac10f898dbe98ae4a54
# Parent 5f1d05a21287ba0290dd3a17ad501595b442a194
Added keepalive_async_fails command to keepalive load balancer module.
This value determines the number suspected keepalive race events
per-upstream-try that will be tolerated before a subsequent network connection
error is considered a true failure.
diff -r 5f1d05a21287 -r 4295bf4613e1 src/event/ngx_event_connect.h
--- a/src/event/ngx_event_connect.h Tue Mar 28 18:01:54 2023 +0300
+++ b/src/event/ngx_event_connect.h Sun Apr 02 19:12:06 2023 +0100
@@ -17,6 +17,7 @@
#define NGX_PEER_KEEPALIVE 1
#define NGX_PEER_NEXT 2
#define NGX_PEER_FAILED 4
+#define NGX_PEER_ASYNC_FAILED 8
typedef struct ngx_peer_connection_s ngx_peer_connection_t;
@@ -41,6 +42,7 @@
ngx_str_t *name;
ngx_uint_t tries;
+ ngx_uint_t async_fails;
ngx_msec_t start_time;
ngx_event_get_peer_pt get;
diff -r 5f1d05a21287 -r 4295bf4613e1 src/http/modules/ngx_http_upstream_keepalive_module.c
--- a/src/http/modules/ngx_http_upstream_keepalive_module.c Tue Mar 28 18:01:54 2023 +0300
+++ b/src/http/modules/ngx_http_upstream_keepalive_module.c Sun Apr 02 19:12:06 2023 +0100
@@ -13,6 +13,7 @@
typedef struct {
ngx_uint_t max_cached;
ngx_uint_t requests;
+ ngx_uint_t max_async_fails;
ngx_msec_t time;
ngx_msec_t timeout;
@@ -108,6 +109,13 @@
offsetof(ngx_http_upstream_keepalive_srv_conf_t, requests),
NULL },
+ { ngx_string("keepalive_async_fails"),
+ NGX_HTTP_UPS_CONF|NGX_CONF_TAKE1,
+ ngx_conf_set_num_slot,
+ NGX_HTTP_SRV_CONF_OFFSET,
+ offsetof(ngx_http_upstream_keepalive_srv_conf_t, max_async_fails),
+ NULL },
+
ngx_null_command
};
@@ -160,6 +168,7 @@
ngx_conf_init_msec_value(kcf->time, 3600000);
ngx_conf_init_msec_value(kcf->timeout, 60000);
ngx_conf_init_uint_value(kcf->requests, 1000);
+ ngx_conf_init_uint_value(kcf->max_async_fails, 2);
if (kcf->original_init_upstream(cf, us) != NGX_OK) {
return NGX_ERROR;
@@ -320,6 +329,21 @@
u = kp->upstream;
c = pc->connection;
+ if (state & NGX_PEER_ASYNC_FAILED) {
+ pc->async_fails++;
+
+ if (pc->async_fails == kp->conf->max_async_fails) {
+ pc->async_fails = 0;
+ state = NGX_PEER_FAILED;
+
+ } else {
+ pc->tries++;
+ }
+ goto invalid;
+ }
+
+ pc->async_fails = 0;
+
if (state & NGX_PEER_FAILED
|| c == NULL
|| c->read->eof
@@ -529,6 +553,8 @@
conf->time = NGX_CONF_UNSET_MSEC;
conf->timeout = NGX_CONF_UNSET_MSEC;
conf->requests = NGX_CONF_UNSET_UINT;
+ conf->max_async_fails = NGX_CONF_UNSET_UINT;
+
return conf;
}
diff -r 5f1d05a21287 -r 4295bf4613e1 src/http/ngx_http_upstream.c
--- a/src/http/ngx_http_upstream.c Tue Mar 28 18:01:54 2023 +0300
+++ b/src/http/ngx_http_upstream.c Sun Apr 02 19:12:06 2023 +0100
@@ -4317,6 +4317,8 @@
{
state = NGX_PEER_NEXT;
+ } else if (u->peer.cached && ft_type == NGX_HTTP_UPSTREAM_FT_ERROR) {
+ state = NGX_PEER_ASYNC_FAILED;
} else {
state = NGX_PEER_FAILED;
}
@@ -4330,11 +4332,6 @@
"upstream timed out");
}
- if (u->peer.cached && ft_type == NGX_HTTP_UPSTREAM_FT_ERROR) {
- /* TODO: inform balancer instead */
- u->peer.tries++;
- }
-
switch (ft_type) {
case NGX_HTTP_UPSTREAM_FT_TIMEOUT:
@@ -4421,7 +4418,6 @@
return;
}
#endif
-
ngx_http_upstream_finalize_request(r, u, status);
return;
}
diff -r 5f1d05a21287 -r 4295bf4613e1 src/http/ngx_http_upstream_round_robin.c
--- a/src/http/ngx_http_upstream_round_robin.c Tue Mar 28 18:01:54 2023 +0300
+++ b/src/http/ngx_http_upstream_round_robin.c Sun Apr 02 19:12:06 2023 +0100
@@ -297,6 +297,7 @@
r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
+ r->upstream->peer.async_fails = 0;
#if (NGX_HTTP_SSL)
r->upstream->peer.set_session =
ngx_http_upstream_set_round_robin_peer_session;
@@ -418,6 +419,7 @@
r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
+ r->upstream->peer.async_fails = 0;
#if (NGX_HTTP_SSL)
r->upstream->peer.set_session = ngx_http_upstream_empty_set_session;
r->upstream->peer.save_session = ngx_http_upstream_empty_save_session;
@@ -459,7 +461,10 @@
rrp->current = peer;
- } else {
+ } else if (pc->async_fails > 0) {
+ peer = rrp->current;
+ }
+ else {
/* there are several peers */
@@ -615,18 +620,7 @@
ngx_http_upstream_rr_peers_rlock(rrp->peers);
ngx_http_upstream_rr_peer_lock(rrp->peers, peer);
- if (rrp->peers->single) {
-
- peer->conns--;
-
- ngx_http_upstream_rr_peer_unlock(rrp->peers, peer);
- ngx_http_upstream_rr_peers_unlock(rrp->peers);
-
- pc->tries = 0;
- return;
- }
-
- if (state & NGX_PEER_FAILED) {
+ if (state & NGX_PEER_FAILED && !rrp->peers->single) {
now = ngx_time();
peer->fails++;
More information about the nginx-devel
mailing list