memory consumption proxying a comet server

[Arrigo Zanette]

Hi,
when proxying a comet server the http write filter module keeps
allocating few bytes for each chunk of data coming from the comet
server. These data are freed only when the request terminates, that is
possibly never.

There is also a forum thread (http://forum.nginx.org/read.php?2,73923)
 that describes the problem very well.

Below I report a workaround that seems working for my application.
Basically, I guess if I can free the allocated memory and free it, by
rewinding the position in the pool.
The workaround is disabled by default and can be enabled using a new flag:

http {
....
write_filter_reuse_heuristic on;
....
}

I hope you come up with a more clean solution, eventually.

Regards,
Arrigo Zanette


--- a/nginx-0.8.38/src/http/ngx_http_write_filter_module.c
2009-06-02 16:01:50.000000000 +0200
+++ b/nginx-0.8.38/src/http/ngx_http_write_filter_module.c
2010-05-29 20:11:30.342660954 +0200
@@ -9,8 +9,25 @@
 #include <ngx_http.h>


+typedef struct {
+    ngx_flag_t                     memory_reuse_heuristic;
+} ngx_http_write_filter_loc_conf_t;
+
 static ngx_int_t ngx_http_write_filter_init(ngx_conf_t *cf);
+static void * ngx_http_write_filter_create_loc_conf(ngx_conf_t *cf);
+static char * ngx_http_write_filter_merge_loc_conf(ngx_conf_t *cf,
void *parent, void *child);
+
+static ngx_command_t  ngx_http_write_filter_commands[] = {
+
+    { ngx_string("write_filter_memory_reuse_heuristic"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
+      ngx_conf_set_flag_slot,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_write_filter_loc_conf_t, memory_reuse_heuristic),
+      NULL },

+      ngx_null_command
+};

 static ngx_http_module_t  ngx_http_write_filter_module_ctx = {
     NULL,                                  /* preconfiguration */
@@ -22,15 +39,15 @@
     NULL,                                  /* create server configuration */
     NULL,                                  /* merge server configuration */

-    NULL,                                  /* create location configuration */
-    NULL,                                  /* merge location configuration */
+    ngx_http_write_filter_create_loc_conf, /* create location configration */
+    ngx_http_write_filter_merge_loc_conf   /* merge location configration */
 };


 ngx_module_t  ngx_http_write_filter_module = {
     NGX_MODULE_V1,
     &ngx_http_write_filter_module_ctx,     /* module context */
-    NULL,                                  /* module directives */
+    ngx_http_write_filter_commands,        /* module directives */
     NGX_HTTP_MODULE,                       /* module type */
     NULL,                                  /* init master */
     NULL,                                  /* init module */
@@ -42,6 +59,33 @@
     NGX_MODULE_V1_PADDING
 };

+static void *
+ngx_http_write_filter_create_loc_conf(ngx_conf_t *cf)
+{
+    ngx_http_write_filter_loc_conf_t  *conf;
+
+    conf = ngx_pcalloc(cf->pool, sizeof(ngx_http_write_filter_loc_conf_t));
+    if (conf == NULL) {
+        return NULL;
+    }
+
+    conf->memory_reuse_heuristic = NGX_CONF_UNSET;
+    return conf;
+}
+
+
+static char *
+ngx_http_write_filter_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child)
+{
+    ngx_http_write_filter_loc_conf_t *prev = parent;
+    ngx_http_write_filter_loc_conf_t *conf = child;
+
+    ngx_conf_merge_value(conf->memory_reuse_heuristic,
+                              prev->memory_reuse_heuristic, 0);
+
+    return NGX_CONF_OK;
+}
+

 ngx_int_t
 ngx_http_write_filter(ngx_http_request_t *r, ngx_chain_t *in)
@@ -52,6 +96,17 @@
     ngx_chain_t               *cl, *ln, **ll, *chain;
     ngx_connection_t          *c;
     ngx_http_core_loc_conf_t  *clcf;
+    ngx_http_write_filter_loc_conf_t *wlcf;
+
+    wlcf = ngx_http_get_module_loc_conf(r, ngx_http_write_filter_module);
+    ngx_uint_t memory_reuse = wlcf->memory_reuse_heuristic;
+
+    ngx_pool_t* actual_current_pool;
+    /* go the current pool */
+    actual_current_pool = r->pool;
+    while (((ngx_pool_t*) actual_current_pool)->d.next) {
+        actual_current_pool = actual_current_pool->d.next;
+    }

     c = r->connection;

@@ -235,6 +290,18 @@
     ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0,
                    "http write filter limit %O", limit);

+    ngx_buf_t* min_pos = (ngx_buf_t*) - 1;
+    /* if it is enough, try to reuse memory starting from the minimum
position of buffers of this chain */
+    if (memory_reuse && !actual_current_pool->large &&
!actual_current_pool->cleanup) {
+        for (cl = r->out; cl; /* void */) {
+            /* the buffer content must be just created, i.e. inside
actual current pool */
+            if (cl->buf->pos >= (u_char*)actual_current_pool &&
cl->buf->pos < ((u_char*)actual_current_pool + r->pool->max)) {
+                min_pos = cl->buf < min_pos ? cl->buf : min_pos;
+            }
+            cl = cl->next;
+        }
+    }
+
     chain = c->send_chain(c, r->out, limit);

     ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0,
@@ -286,6 +353,11 @@

     r->out = chain;

+    /* reuse this message memory */
+    if (min_pos != (ngx_buf_t*) - 1) {
+        actual_current_pool->d.last = (u_char*)min_pos;
+    }
+
     if (chain) {
         c->buffered |= NGX_HTTP_WRITE_BUFFERED;
         return NGX_AGAIN;