Buffer reuse like gzip filter module, with pre-configured number of buffers

[hanzhai]

Hi Maxim,

Thanks for your reply. Your guide made me understand thoroughly the role of
calling ngx_http_next_body_filter(r, NULL) in the gzip module which helped a
lot. The buffer now can be reused but I still got one issue that confused me
a lot.

I got curl: (18) transfer closed with outstanding read data remaining error
when I access the path the code modified. I captured packets through tcpdump
and the last packet containing the response was marked as Malformed Packet.

Here's the code:

if (ctx->nomem) {
    if (ngx_http_next_body_filter(r, NULL) == NGX_ERROR) {
        goto failed;
    }
    ngx_chain_t *cl = NULL;
    ngx_chain_update_chains(r->pool, &ctx->free, &ctx->busy, &cl,
                            (ngx_buf_tag_t) &ngx_http_my_filter_module);
    ctx->nomem = 0;
    flush = 0;
} else {
    flush = ctx->busy ? 1 : 0;
}

for (;;) {

    /* cycle while we can write to a client */

    for (;;) {

        /* cycle while there is data to insert into the beginning */

        rc = ngx_http_my_get_buf(r, ctx);

        if (rc == NGX_DECLINED) {
            break;
        }

        if (rc == NGX_ERROR) {
            goto failed;
        }

        /* there are buffers to write data */

        // rc = operation to copy 64 kb data to the ctx->out_buf;
        ctx->out_buf->last = ctx->out_buf->pos +  64 * 4096;

        ngx_chain_t *cl = ngx_alloc_chain_link(r->pool);
        if (cl == NULL) {
            goto failed;
        }

        cl->buf = ctx->out_buf;
        cl->next = NULL;

        *ctx->last_out = cl;
        ctx->last_out = &cl->next;

        if (rc == OK) {
            ctx->stage = DONE;
            break;
        }

        /* rc == NGX_AGAIN */
    }

    if (ctx->out == NULL && !flush) {
        return ctx->busy ? NGX_AGAIN : NGX_OK;
    }

    ngx_chain_t *a = ctx->out;
    while (a) {
        if (ngx_buf_size(a->buf)) {
            ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "beign: %*s,
end: %*s", 
            10, a->buf->pos, 10, a->buf->last - 10);  // add logging to make
sure the buf is complete, all buf were logged
        }
        a = a->next;
    }

    rc = ngx_http_next_body_filter(r, ctx->out);

    if (rc == NGX_ERROR) {
        goto failed;
    }

    ngx_chain_update_chains(r->pool, &ctx->free, &ctx->busy, &ctx->out,
                            (ngx_buf_tag_t) &ngx_http_my_filter_module);
    ctx->last_out = &ctx->out;

    ctx->nomem = 0;
    flush = 0;

    if (ctx->stage == DONE) {
        return rc;
    }
}

static ngx_int_t ngx_http_my_get_buf(ngx_http_request_t *r,
ngx_http_my_ctx_t *ctx) {

    ngx_chain_t *cl;
    ngx_http_my_loc_conf_t *conf = ngx_http_get_module_loc_conf(r,
ngx_http_my_filter_module);

    if (ctx->free) {

        cl = ctx->free;
        ctx->out_buf = cl->buf;
        ctx->free = cl->next;

        ngx_free_chain(r->pool, cl);

    } else if (ctx->bufs < conf->bufs.num) {

        ctx->out_buf = ngx_create_temp_buf(r->pool, conf->bufs.size);
        if (ctx->out_buf == NULL) {
            return NGX_ERROR;
        }

        ctx->out_buf->tag = (ngx_buf_tag_t) &ngx_http_my_filter_module;
        ctx->out_buf->recycled = 1;
        ctx->bufs++;

    } else {
        ctx->nomem = 1;
        return NGX_DECLINED;
    }

    return NGX_OK;
}

Posted at Nginx Forum: https://forum.nginx.org/read.php?2,294374,294385#msg-294385