[PATCH] Added keepalive_async_fails command

J Carter jordanc.carter at outlook.com
Sun Apr 2 18:31:16 UTC 2023


re-sending the patch as an attachment as the formatting is still weird, 
and fixed typo I spotted..

On 02/04/2023 18:57, J Carter wrote:
> Hello,
>
> I've also attached an example nginx.conf and test script that 
> simulates the asynchronous close events.
> Two different test cases can be found within that, one with path /1 
> for single peer upstream and /2 for multi-peer.
>
> You should see 2 upstream addresses repeated in a row 
> per-upstream-server in the access log by default, as it fails
> through the cached connections & next performs next upstream tries.
>
> Any feedback would be appreciated.
>
> # HG changeset patch
> # User jordanc.carter at outlook.com
> # Date 1680457073 -3600
> #      Sun Apr 02 18:37:53 2023 +0100
> # Node ID 9ec4d7a8cdf6cdab00d09dff75fa6045f6f5533f
> # Parent  5f1d05a21287ba0290dd3a17ad501595b442a194
> Added keepalive_async_fails command to keepalive load balancer module.
> This value determines the number suspected keepalive race events
> per-upstream-try that will be tolerated before a subsequent network 
> connection
> error is considered a true failure.
>
> diff -r 5f1d05a21287 -r 9ec4d7a8cdf6 src/event/ngx_event_connect.h
> --- a/src/event/ngx_event_connect.h    Tue Mar 28 18:01:54 2023 +0300
> +++ b/src/event/ngx_event_connect.h    Sun Apr 02 18:37:53 2023 +0100
> @@ -17,6 +17,7 @@
>  #define NGX_PEER_KEEPALIVE           1
>  #define NGX_PEER_NEXT                2
>  #define NGX_PEER_FAILED              4
> +#define NGX_PEER_ASYNC_FAILED        8
>
>
>  typedef struct ngx_peer_connection_s  ngx_peer_connection_t;
> @@ -41,6 +42,7 @@
>      ngx_str_t                       *name;
>
>      ngx_uint_t                       tries;
> +    ngx_uint_t                       async_fails;
>      ngx_msec_t                       start_time;
>
>      ngx_event_get_peer_pt            get;
> diff -r 5f1d05a21287 -r 9ec4d7a8cdf6 
> src/http/modules/ngx_http_upstream_keepalive_module.c
> --- a/src/http/modules/ngx_http_upstream_keepalive_module.c    Tue Mar 
> 28 18:01:54 2023 +0300
> +++ b/src/http/modules/ngx_http_upstream_keepalive_module.c    Sun Apr 
> 02 18:37:53 2023 +0100
> @@ -13,6 +13,7 @@
>  typedef struct {
>      ngx_uint_t                         max_cached;
>      ngx_uint_t                         requests;
> +    ngx_uint_t                         max_async_fails;
>      ngx_msec_t                         time;
>      ngx_msec_t                         timeout;
>
> @@ -108,6 +109,13 @@
>        offsetof(ngx_http_upstream_keepalive_srv_conf_t, requests),
>        NULL },
>
> +     { ngx_string("keepalive_async_fails"),
> +      NGX_HTTP_UPS_CONF|NGX_CONF_TAKE1,
> +      ngx_conf_set_num_slot,
> +      NGX_HTTP_SRV_CONF_OFFSET,
> +      offsetof(ngx_http_upstream_keepalive_srv_conf_t, max_async_fails),
> +      NULL },
> +
>        ngx_null_command
>  };
>
> @@ -160,6 +168,7 @@
>      ngx_conf_init_msec_value(kcf->time, 3600000);
>      ngx_conf_init_msec_value(kcf->timeout, 60000);
>      ngx_conf_init_uint_value(kcf->requests, 1000);
> +    ngx_conf_init_uint_value(kcf->max_async_fails, 2);
>
>      if (kcf->original_init_upstream(cf, us) != NGX_OK) {
>          return NGX_ERROR;
> @@ -320,6 +329,21 @@
>      u = kp->upstream;
>      c = pc->connection;
>
> +    if (state & NGX_PEER_ASYNC_FAILED) {
> +        pc->async_fails++;
> +
> +        if (pc->async_fails == 2) {
> +            pc->async_fails = 0;
> +            state = NGX_PEER_FAILED;
> +
> +        } else {
> +            pc->tries++;
> +        }
> +        goto invalid;
> +    }
> +
> +    pc->async_fails = 0;
> +
>      if (state & NGX_PEER_FAILED
>          || c == NULL
>          || c->read->eof
> @@ -529,6 +553,8 @@
>      conf->time = NGX_CONF_UNSET_MSEC;
>      conf->timeout = NGX_CONF_UNSET_MSEC;
>      conf->requests = NGX_CONF_UNSET_UINT;
> +    conf->max_async_fails = NGX_CONF_UNSET_UINT;
> +
>
>      return conf;
>  }
> diff -r 5f1d05a21287 -r 9ec4d7a8cdf6 src/http/ngx_http_upstream.c
> --- a/src/http/ngx_http_upstream.c    Tue Mar 28 18:01:54 2023 +0300
> +++ b/src/http/ngx_http_upstream.c    Sun Apr 02 18:37:53 2023 +0100
> @@ -4317,6 +4317,8 @@
>          {
>              state = NGX_PEER_NEXT;
>
> +        } else if (u->peer.cached && ft_type == 
> NGX_HTTP_UPSTREAM_FT_ERROR) {
> +            state = NGX_PEER_ASYNC_FAILED;
>          } else {
>              state = NGX_PEER_FAILED;
>          }
> @@ -4330,11 +4332,6 @@
>                        "upstream timed out");
>      }
>
> -    if (u->peer.cached && ft_type == NGX_HTTP_UPSTREAM_FT_ERROR) {
> -        /* TODO: inform balancer instead */
> -        u->peer.tries++;
> -    }
> -
>      switch (ft_type) {
>
>      case NGX_HTTP_UPSTREAM_FT_TIMEOUT:
> @@ -4421,7 +4418,6 @@
>              return;
>          }
>  #endif
> -
>          ngx_http_upstream_finalize_request(r, u, status);
>          return;
>      }
> diff -r 5f1d05a21287 -r 9ec4d7a8cdf6 
> src/http/ngx_http_upstream_round_robin.c
> --- a/src/http/ngx_http_upstream_round_robin.c    Tue Mar 28 18:01:54 
> 2023 +0300
> +++ b/src/http/ngx_http_upstream_round_robin.c    Sun Apr 02 18:37:53 
> 2023 +0100
> @@ -297,6 +297,7 @@
>      r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
>      r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
>      r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
> +    r->upstream->peer.async_fails = 0;
>  #if (NGX_HTTP_SSL)
>      r->upstream->peer.set_session =
> ngx_http_upstream_set_round_robin_peer_session;
> @@ -418,6 +419,7 @@
>      r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
>      r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
>      r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
> +    r->upstream->peer.async_fails = 0;
>  #if (NGX_HTTP_SSL)
>      r->upstream->peer.set_session = ngx_http_upstream_empty_set_session;
>      r->upstream->peer.save_session = 
> ngx_http_upstream_empty_save_session;
> @@ -459,7 +461,10 @@
>
>          rrp->current = peer;
>
> -    } else {
> +    } else if (pc->async_fails > 0) {
> +        peer = rrp->current;
> +    }
> +    else {
>
>          /* there are several peers */
>
> @@ -615,18 +620,7 @@
>      ngx_http_upstream_rr_peers_rlock(rrp->peers);
>      ngx_http_upstream_rr_peer_lock(rrp->peers, peer);
>
> -    if (rrp->peers->single) {
> -
> -        peer->conns--;
> -
> -        ngx_http_upstream_rr_peer_unlock(rrp->peers, peer);
> -        ngx_http_upstream_rr_peers_unlock(rrp->peers);
> -
> -        pc->tries = 0;
> -        return;
> -    }
> -
> -    if (state & NGX_PEER_FAILED) {
> +    if (state & NGX_PEER_FAILED && !rrp->peers->single) {
>          now = ngx_time();
>
>          peer->fails++;
>
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> https://mailman.nginx.org/mailman/listinfo/nginx-devel
-------------- next part --------------
# HG changeset patch
# User jordanc.carter at outlook.com
# Date 1680459126 -3600
#      Sun Apr 02 19:12:06 2023 +0100
# Node ID 4295bf4613e155e063914ac10f898dbe98ae4a54
# Parent  5f1d05a21287ba0290dd3a17ad501595b442a194
Added keepalive_async_fails command to keepalive load balancer module.
This value determines the number suspected keepalive race events
per-upstream-try that will be tolerated before a subsequent network connection
error is considered a true failure.

diff -r 5f1d05a21287 -r 4295bf4613e1 src/event/ngx_event_connect.h
--- a/src/event/ngx_event_connect.h	Tue Mar 28 18:01:54 2023 +0300
+++ b/src/event/ngx_event_connect.h	Sun Apr 02 19:12:06 2023 +0100
@@ -17,6 +17,7 @@
 #define NGX_PEER_KEEPALIVE           1
 #define NGX_PEER_NEXT                2
 #define NGX_PEER_FAILED              4
+#define NGX_PEER_ASYNC_FAILED        8
 
 
 typedef struct ngx_peer_connection_s  ngx_peer_connection_t;
@@ -41,6 +42,7 @@
     ngx_str_t                       *name;
 
     ngx_uint_t                       tries;
+    ngx_uint_t                       async_fails;
     ngx_msec_t                       start_time;
 
     ngx_event_get_peer_pt            get;
diff -r 5f1d05a21287 -r 4295bf4613e1 src/http/modules/ngx_http_upstream_keepalive_module.c
--- a/src/http/modules/ngx_http_upstream_keepalive_module.c	Tue Mar 28 18:01:54 2023 +0300
+++ b/src/http/modules/ngx_http_upstream_keepalive_module.c	Sun Apr 02 19:12:06 2023 +0100
@@ -13,6 +13,7 @@
 typedef struct {
     ngx_uint_t                         max_cached;
     ngx_uint_t                         requests;
+    ngx_uint_t                         max_async_fails;
     ngx_msec_t                         time;
     ngx_msec_t                         timeout;
 
@@ -108,6 +109,13 @@
       offsetof(ngx_http_upstream_keepalive_srv_conf_t, requests),
       NULL },
 
+     { ngx_string("keepalive_async_fails"),
+      NGX_HTTP_UPS_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_num_slot,
+      NGX_HTTP_SRV_CONF_OFFSET,
+      offsetof(ngx_http_upstream_keepalive_srv_conf_t, max_async_fails),
+      NULL },
+
       ngx_null_command
 };
 
@@ -160,6 +168,7 @@
     ngx_conf_init_msec_value(kcf->time, 3600000);
     ngx_conf_init_msec_value(kcf->timeout, 60000);
     ngx_conf_init_uint_value(kcf->requests, 1000);
+    ngx_conf_init_uint_value(kcf->max_async_fails, 2);
 
     if (kcf->original_init_upstream(cf, us) != NGX_OK) {
         return NGX_ERROR;
@@ -320,6 +329,21 @@
     u = kp->upstream;
     c = pc->connection;
 
+    if (state & NGX_PEER_ASYNC_FAILED) {
+        pc->async_fails++;
+
+        if (pc->async_fails == kp->conf->max_async_fails) {
+            pc->async_fails = 0;
+            state = NGX_PEER_FAILED;
+
+        } else {
+            pc->tries++;
+        }
+        goto invalid;
+    }
+
+    pc->async_fails = 0;
+
     if (state & NGX_PEER_FAILED
         || c == NULL
         || c->read->eof
@@ -529,6 +553,8 @@
     conf->time = NGX_CONF_UNSET_MSEC;
     conf->timeout = NGX_CONF_UNSET_MSEC;
     conf->requests = NGX_CONF_UNSET_UINT;
+    conf->max_async_fails = NGX_CONF_UNSET_UINT;
+
 
     return conf;
 }
diff -r 5f1d05a21287 -r 4295bf4613e1 src/http/ngx_http_upstream.c
--- a/src/http/ngx_http_upstream.c	Tue Mar 28 18:01:54 2023 +0300
+++ b/src/http/ngx_http_upstream.c	Sun Apr 02 19:12:06 2023 +0100
@@ -4317,6 +4317,8 @@
         {
             state = NGX_PEER_NEXT;
 
+        } else if (u->peer.cached && ft_type == NGX_HTTP_UPSTREAM_FT_ERROR) {
+            state = NGX_PEER_ASYNC_FAILED;
         } else {
             state = NGX_PEER_FAILED;
         }
@@ -4330,11 +4332,6 @@
                       "upstream timed out");
     }
 
-    if (u->peer.cached && ft_type == NGX_HTTP_UPSTREAM_FT_ERROR) {
-        /* TODO: inform balancer instead */
-        u->peer.tries++;
-    }
-
     switch (ft_type) {
 
     case NGX_HTTP_UPSTREAM_FT_TIMEOUT:
@@ -4421,7 +4418,6 @@
             return;
         }
 #endif
-
         ngx_http_upstream_finalize_request(r, u, status);
         return;
     }
diff -r 5f1d05a21287 -r 4295bf4613e1 src/http/ngx_http_upstream_round_robin.c
--- a/src/http/ngx_http_upstream_round_robin.c	Tue Mar 28 18:01:54 2023 +0300
+++ b/src/http/ngx_http_upstream_round_robin.c	Sun Apr 02 19:12:06 2023 +0100
@@ -297,6 +297,7 @@
     r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
     r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
     r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
+    r->upstream->peer.async_fails = 0;
 #if (NGX_HTTP_SSL)
     r->upstream->peer.set_session =
                                ngx_http_upstream_set_round_robin_peer_session;
@@ -418,6 +419,7 @@
     r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
     r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
     r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
+    r->upstream->peer.async_fails = 0;
 #if (NGX_HTTP_SSL)
     r->upstream->peer.set_session = ngx_http_upstream_empty_set_session;
     r->upstream->peer.save_session = ngx_http_upstream_empty_save_session;
@@ -459,7 +461,10 @@
 
         rrp->current = peer;
 
-    } else {
+    } else if (pc->async_fails > 0) {
+        peer = rrp->current;
+    }
+    else {
 
         /* there are several peers */
 
@@ -615,18 +620,7 @@
     ngx_http_upstream_rr_peers_rlock(rrp->peers);
     ngx_http_upstream_rr_peer_lock(rrp->peers, peer);
 
-    if (rrp->peers->single) {
-
-        peer->conns--;
-
-        ngx_http_upstream_rr_peer_unlock(rrp->peers, peer);
-        ngx_http_upstream_rr_peers_unlock(rrp->peers);
-
-        pc->tries = 0;
-        return;
-    }
-
-    if (state & NGX_PEER_FAILED) {
+    if (state & NGX_PEER_FAILED && !rrp->peers->single) {
         now = ngx_time();
 
         peer->fails++;


More information about the nginx-devel mailing list