[PATCH 1 of 2] Buffers reuse in chunked filter
Kornél Schadl
neil at krnl.hu
Fri Sep 9 12:36:55 UTC 2011
are these patches included in svn?
On Fri, Sep 9, 2011 at 2:01 PM, SplitIce <mat999 at gmail.com> wrote:
> wow, makes me want to rebuild all my server nodes to include this patch.
>
> On Fri, Sep 9, 2011 at 8:32 PM, Maxim Dounin <mdounin at mdounin.ru> wrote:
>>
>> # HG changeset patch
>> # User Maxim Dounin <mdounin at mdounin.ru>
>> # Date 1315564269 -14400
>> # Node ID b667ed67c0b9046b94291fa6f52e850006011718
>> # Parent 014764a85840606c90317e9f44f2b9fa139cbc8b
>> Buffers reuse in chunked filter.
>>
>> There were 2 buffers allocated on each buffer chain sent through chunked
>> filter (one buffer for chunk size, another one for trailing CRLF, about
>> 120 bytes in total on 32-bit platforms). This resulted in large memory
>> consumption with long-lived requests sending many buffer chains. Usual
>> example of problematic scenario is streaming though proxy with
>> proxy_buffering set to off.
>>
>> Introduced buffers reuse reduces memory consumption in the above
>> problematic
>> scenario.
>>
>> See here for initial report:
>> http://mailman.nginx.org/pipermail/nginx/2010-April/019814.html
>>
>> diff --git a/src/http/modules/ngx_http_chunked_filter_module.c
>> b/src/http/modules/ngx_http_chunked_filter_module.c
>> --- a/src/http/modules/ngx_http_chunked_filter_module.c
>> +++ b/src/http/modules/ngx_http_chunked_filter_module.c
>> @@ -9,6 +9,12 @@
>> #include <ngx_http.h>
>>
>>
>> +typedef struct {
>> + ngx_chain_t *free;
>> + ngx_chain_t *busy;
>> +} ngx_http_chunked_filter_ctx_t;
>> +
>> +
>> static ngx_int_t ngx_http_chunked_filter_init(ngx_conf_t *cf);
>>
>>
>> @@ -50,7 +56,8 @@ static ngx_http_output_body_filter_pt
>> static ngx_int_t
>> ngx_http_chunked_header_filter(ngx_http_request_t *r)
>> {
>> - ngx_http_core_loc_conf_t *clcf;
>> + ngx_http_core_loc_conf_t *clcf;
>> + ngx_http_chunked_filter_ctx_t *ctx;
>>
>> if (r->headers_out.status == NGX_HTTP_NOT_MODIFIED
>> || r->headers_out.status == NGX_HTTP_NO_CONTENT
>> @@ -70,6 +77,14 @@ ngx_http_chunked_header_filter(ngx_http_
>> if (clcf->chunked_transfer_encoding) {
>> r->chunked = 1;
>>
>> + ctx = ngx_pcalloc(r->pool,
>> + sizeof(ngx_http_chunked_filter_ctx_t));
>> + if (ctx == NULL) {
>> + return NGX_ERROR;
>> + }
>> +
>> + ngx_http_set_ctx(r, ctx, ngx_http_chunked_filter_module);
>> +
>> } else {
>> r->keepalive = 0;
>> }
>> @@ -83,17 +98,21 @@ ngx_http_chunked_header_filter(ngx_http_
>> static ngx_int_t
>> ngx_http_chunked_body_filter(ngx_http_request_t *r, ngx_chain_t *in)
>> {
>> - u_char *chunk;
>> - off_t size;
>> - ngx_buf_t *b;
>> - ngx_chain_t out, tail, *cl, *tl, **ll;
>> + u_char *chunk;
>> + off_t size;
>> + ngx_int_t rc;
>> + ngx_buf_t *b;
>> + ngx_chain_t *out, *cl, *tl, **ll;
>> + ngx_http_chunked_filter_ctx_t *ctx;
>>
>> if (in == NULL || !r->chunked || r->header_only) {
>> return ngx_http_next_body_filter(r, in);
>> }
>>
>> - out.buf = NULL;
>> - ll = &out.next;
>> + ctx = ngx_http_get_module_ctx(r, ngx_http_chunked_filter_module);
>> +
>> + out = NULL;
>> + ll = &out;
>>
>> size = 0;
>> cl = in;
>> @@ -127,31 +146,46 @@ ngx_http_chunked_body_filter(ngx_http_re
>> }
>>
>> if (size) {
>> - b = ngx_calloc_buf(r->pool);
>> - if (b == NULL) {
>> + tl = ngx_chain_get_free_buf(r->pool, &ctx->free);
>> + if (tl == NULL) {
>> return NGX_ERROR;
>> }
>>
>> - /* the "0000000000000000" is 64-bit hexadimal string */
>> + b = tl->buf;
>> + chunk = b->start;
>>
>> - chunk = ngx_palloc(r->pool, sizeof("0000000000000000" CRLF) - 1);
>> if (chunk == NULL) {
>> - return NGX_ERROR;
>> + /* the "0000000000000000" is 64-bit hexadecimal string */
>> +
>> + chunk = ngx_palloc(r->pool, sizeof("0000000000000000" CRLF) -
>> 1);
>> + if (chunk == NULL) {
>> + return NGX_ERROR;
>> + }
>> +
>> + b->start = chunk;
>> + b->end = chunk + sizeof("0000000000000000" CRLF) - 1;
>> }
>>
>> + b->tag = (ngx_buf_tag_t) &ngx_http_chunked_filter_module;
>> + b->memory = 0;
>> b->temporary = 1;
>> b->pos = chunk;
>> b->last = ngx_sprintf(chunk, "%xO" CRLF, size);
>>
>> - out.buf = b;
>> + tl->next = out;
>> + out = tl;
>> }
>>
>> if (cl->buf->last_buf) {
>> - b = ngx_calloc_buf(r->pool);
>> - if (b == NULL) {
>> + tl = ngx_chain_get_free_buf(r->pool, &ctx->free);
>> + if (tl == NULL) {
>> return NGX_ERROR;
>> }
>> +
>> + b = tl->buf;
>>
>> + b->tag = (ngx_buf_tag_t) &ngx_http_chunked_filter_module;
>> + b->temporary = 0;
>> b->memory = 1;
>> b->last_buf = 1;
>> b->pos = (u_char *) CRLF "0" CRLF CRLF;
>> @@ -159,35 +193,38 @@ ngx_http_chunked_body_filter(ngx_http_re
>>
>> cl->buf->last_buf = 0;
>>
>> + *ll = tl;
>> +
>> if (size == 0) {
>> b->pos += 2;
>> - out.buf = b;
>> - out.next = NULL;
>> -
>> - return ngx_http_next_body_filter(r, &out);
>> }
>>
>> - } else {
>> - if (size == 0) {
>> - *ll = NULL;
>> - return ngx_http_next_body_filter(r, out.next);
>> - }
>> -
>> - b = ngx_calloc_buf(r->pool);
>> - if (b == NULL) {
>> + } else if (size > 0) {
>> + tl = ngx_chain_get_free_buf(r->pool, &ctx->free);
>> + if (tl == NULL) {
>> return NGX_ERROR;
>> }
>>
>> + b = tl->buf;
>> +
>> + b->tag = (ngx_buf_tag_t) &ngx_http_chunked_filter_module;
>> + b->temporary = 0;
>> b->memory = 1;
>> b->pos = (u_char *) CRLF;
>> b->last = b->pos + 2;
>> +
>> + *ll = tl;
>> +
>> + } else {
>> + *ll = NULL;
>> }
>>
>> - tail.buf = b;
>> - tail.next = NULL;
>> - *ll = &tail;
>> + rc = ngx_http_next_body_filter(r, out);
>>
>> - return ngx_http_next_body_filter(r, &out);
>> + ngx_chain_update_chains(&ctx->free, &ctx->busy, &out,
>> + (ngx_buf_tag_t)
>> &ngx_http_chunked_filter_module);
>> +
>> + return rc;
>> }
>>
>>
>>
>> _______________________________________________
>> nginx-devel mailing list
>> nginx-devel at nginx.org
>> http://mailman.nginx.org/mailman/listinfo/nginx-devel
>
>
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx-devel
>
More information about the nginx-devel
mailing list