proxied request hang when DNS server down

[Ruslan Ermilov]

On Tue, Aug 19, 2014 at 01:43:06PM +0800, ywsample wrote:
> http {
>     resolver 127.0.0.1;
>     resolver_timeout 1s;
>     server {
>         listen 8000;
>         location /test {
>             proxy_pass http://$arg_host:8080;
>         }
>     }
>     server {
>         listen 8080;
>         .......
>     }
> }
> ps: other configure is normal
> 
> I have recently discoverd that when DNS server down, the first proxy
> request return 504 and than the follow proxy request may hang forever.
> I use two curl simulate concurrent requests
> 
> I found than the first request timeout in 1 second and exit, also it delete
> the timeout timer. because another request waiting dns response, so nginx
> retry send dns query, but dns never response(because it down), it lead to
> the follow request hang and also no timer relate to it;
> 
> PS：base on nginx-1.7.4

Please try the attached patch.  If you would like an attribution
in a commit log, please tell me your real name.
-------------- next part --------------
# HG changeset patch
# User Ruslan Ermilov <ru at nginx.com>
# Date 1408448606 -14400
#      Tue Aug 19 15:43:26 2014 +0400
# Node ID f70b61673e6d536831527f3a27d0b32753180374
# Parent  18daf56f477c7f264243937cdca2e35797d9afb9
Resolver: notify all waiting requests on timeout.

If a "resolver_timeout" occurs, only the first waiting request
was notified.  Other requests may hang forever.

diff --git a/src/core/ngx_resolver.c b/src/core/ngx_resolver.c
--- a/src/core/ngx_resolver.c
+++ b/src/core/ngx_resolver.c
@@ -664,7 +664,7 @@ ngx_resolve_name_locked(ngx_resolver_t *
         }
 
         ctx->event->handler = ngx_resolver_timeout_handler;
-        ctx->event->data = ctx;
+        ctx->event->data = rn;
         ctx->event->log = r->log;
         ctx->ident = -1;
 
@@ -857,7 +857,7 @@ ngx_resolve_addr(ngx_resolver_ctx_t *ctx
     }
 
     ctx->event->handler = ngx_resolver_timeout_handler;
-    ctx->event->data = ctx;
+    ctx->event->data = rn;
     ctx->event->log = r->log;
     ctx->ident = -1;
 
@@ -2790,13 +2790,20 @@ done:
 static void
 ngx_resolver_timeout_handler(ngx_event_t *ev)
 {
-    ngx_resolver_ctx_t  *ctx;
-
-    ctx = ev->data;
-
-    ctx->state = NGX_RESOLVE_TIMEDOUT;
-
-    ctx->handler(ctx);
+    ngx_resolver_ctx_t   *ctx, *next;
+    ngx_resolver_node_t  *rn;
+
+    rn = ev->data;
+    ctx = rn->waiting;
+
+    do {
+        ctx->state = NGX_RESOLVE_TIMEDOUT;
+        next = ctx->next;
+
+        ctx->handler(ctx);
+
+        ctx = next;
+    } while (ctx);
 }