wow, makes me want to rebuild all my server nodes to include this patch.<br><br><div class="gmail_quote">On Fri, Sep 9, 2011 at 8:32 PM, Maxim Dounin <span dir="ltr"><<a href="mailto:mdounin@mdounin.ru">mdounin@mdounin.ru</a>></span> wrote:<br>
<blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex;"># HG changeset patch<br>
# User Maxim Dounin <<a href="mailto:mdounin@mdounin.ru">mdounin@mdounin.ru</a>><br>
# Date 1315564269 -14400<br>
# Node ID b667ed67c0b9046b94291fa6f52e850006011718<br>
# Parent  014764a85840606c90317e9f44f2b9fa139cbc8b<br>
Buffers reuse in chunked filter.<br>
<br>
There were 2 buffers allocated on each buffer chain sent through chunked<br>
filter (one buffer for chunk size, another one for trailing CRLF, about<br>
120 bytes in total on 32-bit platforms).  This resulted in large memory<br>
consumption with long-lived requests sending many buffer chains.  Usual<br>
example of problematic scenario is streaming though proxy with<br>
proxy_buffering set to off.<br>
<br>
Introduced buffers reuse reduces memory consumption in the above problematic<br>
scenario.<br>
<br>
See here for initial report:<br>
<a href="http://mailman.nginx.org/pipermail/nginx/2010-April/019814.html" target="_blank">http://mailman.nginx.org/pipermail/nginx/2010-April/019814.html</a><br>
<br>
diff --git a/src/http/modules/ngx_http_chunked_filter_module.c b/src/http/modules/ngx_http_chunked_filter_module.c<br>
--- a/src/http/modules/ngx_http_chunked_filter_module.c<br>
+++ b/src/http/modules/ngx_http_chunked_filter_module.c<br>
@@ -9,6 +9,12 @@<br>
 #include <ngx_http.h><br>
<br>
<br>
+typedef struct {<br>
+    ngx_chain_t         *free;<br>
+    ngx_chain_t         *busy;<br>
+} ngx_http_chunked_filter_ctx_t;<br>
+<br>
+<br>
 static ngx_int_t ngx_http_chunked_filter_init(ngx_conf_t *cf);<br>
<br>
<br>
@@ -50,7 +56,8 @@ static ngx_http_output_body_filter_pt<br>
 static ngx_int_t<br>
 ngx_http_chunked_header_filter(ngx_http_request_t *r)<br>
 {<br>
-    ngx_http_core_loc_conf_t  *clcf;<br>
+    ngx_http_core_loc_conf_t       *clcf;<br>
+    ngx_http_chunked_filter_ctx_t  *ctx;<br>
<br>
     if (r->headers_out.status == NGX_HTTP_NOT_MODIFIED<br>
         || r->headers_out.status == NGX_HTTP_NO_CONTENT<br>
@@ -70,6 +77,14 @@ ngx_http_chunked_header_filter(ngx_http_<br>
             if (clcf->chunked_transfer_encoding) {<br>
                 r->chunked = 1;<br>
<br>
+                ctx = ngx_pcalloc(r->pool,<br>
+                                  sizeof(ngx_http_chunked_filter_ctx_t));<br>
+                if (ctx == NULL) {<br>
+                    return NGX_ERROR;<br>
+                }<br>
+<br>
+                ngx_http_set_ctx(r, ctx, ngx_http_chunked_filter_module);<br>
+<br>
             } else {<br>
                 r->keepalive = 0;<br>
             }<br>
@@ -83,17 +98,21 @@ ngx_http_chunked_header_filter(ngx_http_<br>
 static ngx_int_t<br>
 ngx_http_chunked_body_filter(ngx_http_request_t *r, ngx_chain_t *in)<br>
 {<br>
-    u_char       *chunk;<br>
-    off_t         size;<br>
-    ngx_buf_t    *b;<br>
-    ngx_chain_t   out, tail, *cl, *tl, **ll;<br>
+    u_char                         *chunk;<br>
+    off_t                           size;<br>
+    ngx_int_t                       rc;<br>
+    ngx_buf_t                      *b;<br>
+    ngx_chain_t                    *out, *cl, *tl, **ll;<br>
+    ngx_http_chunked_filter_ctx_t  *ctx;<br>
<br>
     if (in == NULL || !r->chunked || r->header_only) {<br>
         return ngx_http_next_body_filter(r, in);<br>
     }<br>
<br>
-    out.buf = NULL;<br>
-    ll = &out.next;<br>
+    ctx = ngx_http_get_module_ctx(r, ngx_http_chunked_filter_module);<br>
+<br>
+    out = NULL;<br>
+    ll = &out;<br>
<br>
     size = 0;<br>
     cl = in;<br>
@@ -127,31 +146,46 @@ ngx_http_chunked_body_filter(ngx_http_re<br>
     }<br>
<br>
     if (size) {<br>
-        b = ngx_calloc_buf(r->pool);<br>
-        if (b == NULL) {<br>
+        tl = ngx_chain_get_free_buf(r->pool, &ctx->free);<br>
+        if (tl == NULL) {<br>
             return NGX_ERROR;<br>
         }<br>
<br>
-        /* the "0000000000000000" is 64-bit hexadimal string */<br>
+        b = tl->buf;<br>
+        chunk = b->start;<br>
<br>
-        chunk = ngx_palloc(r->pool, sizeof("0000000000000000" CRLF) - 1);<br>
         if (chunk == NULL) {<br>
-            return NGX_ERROR;<br>
+            /* the "0000000000000000" is 64-bit hexadecimal string */<br>
+<br>
+            chunk = ngx_palloc(r->pool, sizeof("0000000000000000" CRLF) - 1);<br>
+            if (chunk == NULL) {<br>
+                return NGX_ERROR;<br>
+            }<br>
+<br>
+            b->start = chunk;<br>
+            b->end = chunk + sizeof("0000000000000000" CRLF) - 1;<br>
         }<br>
<br>
+        b->tag = (ngx_buf_tag_t) &ngx_http_chunked_filter_module;<br>
+        b->memory = 0;<br>
         b->temporary = 1;<br>
         b->pos = chunk;<br>
         b->last = ngx_sprintf(chunk, "%xO" CRLF, size);<br>
<br>
-        out.buf = b;<br>
+        tl->next = out;<br>
+        out = tl;<br>
     }<br>
<br>
     if (cl->buf->last_buf) {<br>
-        b = ngx_calloc_buf(r->pool);<br>
-        if (b == NULL) {<br>
+        tl = ngx_chain_get_free_buf(r->pool, &ctx->free);<br>
+        if (tl == NULL) {<br>
             return NGX_ERROR;<br>
         }<br>
+<br>
+        b = tl->buf;<br>
<br>
+        b->tag = (ngx_buf_tag_t) &ngx_http_chunked_filter_module;<br>
+        b->temporary = 0;<br>
         b->memory = 1;<br>
         b->last_buf = 1;<br>
         b->pos = (u_char *) CRLF "0" CRLF CRLF;<br>
@@ -159,35 +193,38 @@ ngx_http_chunked_body_filter(ngx_http_re<br>
<br>
         cl->buf->last_buf = 0;<br>
<br>
+        *ll = tl;<br>
+<br>
         if (size == 0) {<br>
             b->pos += 2;<br>
-            out.buf = b;<br>
-            out.next = NULL;<br>
-<br>
-            return ngx_http_next_body_filter(r, &out);<br>
         }<br>
<br>
-    } else {<br>
-        if (size == 0) {<br>
-            *ll = NULL;<br>
-            return ngx_http_next_body_filter(r, out.next);<br>
-        }<br>
-<br>
-        b = ngx_calloc_buf(r->pool);<br>
-        if (b == NULL) {<br>
+    } else if (size > 0) {<br>
+        tl = ngx_chain_get_free_buf(r->pool, &ctx->free);<br>
+        if (tl == NULL) {<br>
             return NGX_ERROR;<br>
         }<br>
<br>
+        b = tl->buf;<br>
+<br>
+        b->tag = (ngx_buf_tag_t) &ngx_http_chunked_filter_module;<br>
+        b->temporary = 0;<br>
         b->memory = 1;<br>
         b->pos = (u_char *) CRLF;<br>
         b->last = b->pos + 2;<br>
+<br>
+        *ll = tl;<br>
+<br>
+    } else {<br>
+        *ll = NULL;<br>
     }<br>
<br>
-    tail.buf = b;<br>
-    tail.next = NULL;<br>
-    *ll = &tail;<br>
+    rc = ngx_http_next_body_filter(r, out);<br>
<br>
-    return ngx_http_next_body_filter(r, &out);<br>
+    ngx_chain_update_chains(&ctx->free, &ctx->busy, &out,<br>
+                            (ngx_buf_tag_t) &ngx_http_chunked_filter_module);<br>
+<br>
+    return rc;<br>
 }<br>
<br>
<br>
<br>
_______________________________________________<br>
nginx-devel mailing list<br>
<a href="mailto:nginx-devel@nginx.org">nginx-devel@nginx.org</a><br>
<a href="http://mailman.nginx.org/mailman/listinfo/nginx-devel" target="_blank">http://mailman.nginx.org/mailman/listinfo/nginx-devel</a><br>
</blockquote></div><br>