[nginx] Upstream: improved peer selection concurrency for hash and ip_hash.

[Ruslan Ermilov]

details:   http://hg.nginx.org/nginx/rev/ed599ea6c1f1
branches:  
changeset: 7300:ed599ea6c1f1
user:      Ruslan Ermilov <ru at nginx.com>
date:      Thu Jun 14 07:03:50 2018 +0300
description:
Upstream: improved peer selection concurrency for hash and ip_hash.

diffstat:

 src/http/modules/ngx_http_upstream_hash_module.c    |  8 +++++++-
 src/http/modules/ngx_http_upstream_ip_hash_module.c |  8 +++++++-
 src/stream/ngx_stream_upstream_hash_module.c        |  8 +++++++-
 3 files changed, 21 insertions(+), 3 deletions(-)

diffs (138 lines):

diff -r faf14dc9ab4d -r ed599ea6c1f1 src/http/modules/ngx_http_upstream_hash_module.c
--- a/src/http/modules/ngx_http_upstream_hash_module.c	Wed Jun 13 22:37:49 2018 +0300
+++ b/src/http/modules/ngx_http_upstream_hash_module.c	Thu Jun 14 07:03:50 2018 +0300
@@ -176,7 +176,7 @@ ngx_http_upstream_get_hash_peer(ngx_peer
     ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
                    "get hash peer, try: %ui", pc->tries);
 
-    ngx_http_upstream_rr_peers_wlock(hp->rrp.peers);
+    ngx_http_upstream_rr_peers_rlock(hp->rrp.peers);
 
     if (hp->tries > 20 || hp->rrp.peers->single) {
         ngx_http_upstream_rr_peers_unlock(hp->rrp.peers);
@@ -228,10 +228,13 @@ ngx_http_upstream_get_hash_peer(ngx_peer
             goto next;
         }
 
+        ngx_http_upstream_rr_peer_lock(hp->rrp.peers, peer);
+
         ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
                        "get hash peer, value:%uD, peer:%ui", hp->hash, p);
 
         if (peer->down) {
+            ngx_http_upstream_rr_peer_unlock(hp->rrp.peers, peer);
             goto next;
         }
 
@@ -239,10 +242,12 @@ ngx_http_upstream_get_hash_peer(ngx_peer
             && peer->fails >= peer->max_fails
             && now - peer->checked <= peer->fail_timeout)
         {
+            ngx_http_upstream_rr_peer_unlock(hp->rrp.peers, peer);
             goto next;
         }
 
         if (peer->max_conns && peer->conns >= peer->max_conns) {
+            ngx_http_upstream_rr_peer_unlock(hp->rrp.peers, peer);
             goto next;
         }
 
@@ -268,6 +273,7 @@ ngx_http_upstream_get_hash_peer(ngx_peer
         peer->checked = now;
     }
 
+    ngx_http_upstream_rr_peer_unlock(hp->rrp.peers, peer);
     ngx_http_upstream_rr_peers_unlock(hp->rrp.peers);
 
     hp->rrp.tried[n] |= m;
diff -r faf14dc9ab4d -r ed599ea6c1f1 src/http/modules/ngx_http_upstream_ip_hash_module.c
--- a/src/http/modules/ngx_http_upstream_ip_hash_module.c	Wed Jun 13 22:37:49 2018 +0300
+++ b/src/http/modules/ngx_http_upstream_ip_hash_module.c	Thu Jun 14 07:03:50 2018 +0300
@@ -161,7 +161,7 @@ ngx_http_upstream_get_ip_hash_peer(ngx_p
 
     /* TODO: cached */
 
-    ngx_http_upstream_rr_peers_wlock(iphp->rrp.peers);
+    ngx_http_upstream_rr_peers_rlock(iphp->rrp.peers);
 
     if (iphp->tries > 20 || iphp->rrp.peers->single) {
         ngx_http_upstream_rr_peers_unlock(iphp->rrp.peers);
@@ -201,7 +201,10 @@ ngx_http_upstream_get_ip_hash_peer(ngx_p
         ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
                        "get ip hash peer, hash: %ui %04XL", p, (uint64_t) m);
 
+        ngx_http_upstream_rr_peer_lock(iphp->rrp.peers, peer);
+
         if (peer->down) {
+            ngx_http_upstream_rr_peer_unlock(iphp->rrp.peers, peer);
             goto next;
         }
 
@@ -209,10 +212,12 @@ ngx_http_upstream_get_ip_hash_peer(ngx_p
             && peer->fails >= peer->max_fails
             && now - peer->checked <= peer->fail_timeout)
         {
+            ngx_http_upstream_rr_peer_unlock(iphp->rrp.peers, peer);
             goto next;
         }
 
         if (peer->max_conns && peer->conns >= peer->max_conns) {
+            ngx_http_upstream_rr_peer_unlock(iphp->rrp.peers, peer);
             goto next;
         }
 
@@ -238,6 +243,7 @@ ngx_http_upstream_get_ip_hash_peer(ngx_p
         peer->checked = now;
     }
 
+    ngx_http_upstream_rr_peer_unlock(iphp->rrp.peers, peer);
     ngx_http_upstream_rr_peers_unlock(iphp->rrp.peers);
 
     iphp->rrp.tried[n] |= m;
diff -r faf14dc9ab4d -r ed599ea6c1f1 src/stream/ngx_stream_upstream_hash_module.c
--- a/src/stream/ngx_stream_upstream_hash_module.c	Wed Jun 13 22:37:49 2018 +0300
+++ b/src/stream/ngx_stream_upstream_hash_module.c	Thu Jun 14 07:03:50 2018 +0300
@@ -176,7 +176,7 @@ ngx_stream_upstream_get_hash_peer(ngx_pe
     ngx_log_debug1(NGX_LOG_DEBUG_STREAM, pc->log, 0,
                    "get hash peer, try: %ui", pc->tries);
 
-    ngx_stream_upstream_rr_peers_wlock(hp->rrp.peers);
+    ngx_stream_upstream_rr_peers_rlock(hp->rrp.peers);
 
     if (hp->tries > 20 || hp->rrp.peers->single) {
         ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers);
@@ -227,10 +227,13 @@ ngx_stream_upstream_get_hash_peer(ngx_pe
             goto next;
         }
 
+        ngx_stream_upstream_rr_peer_lock(hp->rrp.peers, peer);
+
         ngx_log_debug2(NGX_LOG_DEBUG_STREAM, pc->log, 0,
                        "get hash peer, value:%uD, peer:%ui", hp->hash, p);
 
         if (peer->down) {
+            ngx_stream_upstream_rr_peer_unlock(hp->rrp.peers, peer);
             goto next;
         }
 
@@ -238,10 +241,12 @@ ngx_stream_upstream_get_hash_peer(ngx_pe
             && peer->fails >= peer->max_fails
             && now - peer->checked <= peer->fail_timeout)
         {
+            ngx_stream_upstream_rr_peer_unlock(hp->rrp.peers, peer);
             goto next;
         }
 
         if (peer->max_conns && peer->conns >= peer->max_conns) {
+            ngx_stream_upstream_rr_peer_unlock(hp->rrp.peers, peer);
             goto next;
         }
 
@@ -267,6 +272,7 @@ ngx_stream_upstream_get_hash_peer(ngx_pe
         peer->checked = now;
     }
 
+    ngx_stream_upstream_rr_peer_unlock(hp->rrp.peers, peer);
     ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers);
 
     hp->rrp.tried[n] |= m;