[ANNOUNCE] gunzip filter module 0.3

theromis1 nginx-forum at nginx.us
Fri Apr 23 05:36:12 MSD 2010


Hi Max,

today I've done with your last suggestions. Attaching new patch.
Also I've added option "gunzip_strict" when it turned off it adding 32 instead of 16 on inflateInit2 and in case of inflate error in first chunk, sending content unchanged. Please review it and give me recommendations. Default is on.

About gunzip_types * by default, in current development version on Nginx I can not find ability to remove or replace already existing string hashes using ngx_http_types_slot. That was reason for having option content_type, by default like you said all mime types is goes to guunzip. If you can show me example in nginx of logic which you want realize, I'll be happy to make it.

BTW, what do you think if I will add option in mod_proxy for disabling requests buffering and add support for proxing connection: keep-alive?

--- ngx_http_gunzip_filter_module-0.3/ngx_http_gunzip_filter_module.c	2010-03-22 11:11:16.000000000 -0700
+++ ngx_http_gunzip_filter_module.c	2010-04-22 18:24:54.000000000 -0700
@@ -16,10 +16,15 @@
 typedef struct {
     ngx_flag_t           enable;
     ngx_bufs_t           bufs;
+    ngx_uint_t           pass;
+    ngx_hash_t           types;
+    ngx_array_t         *types_keys;
+    ngx_flag_t           strict;
 } ngx_http_gunzip_conf_t;
 
 
 typedef struct {
+    ngx_chain_t         *_in;
     ngx_chain_t         *in;
     ngx_chain_t         *free;
     ngx_chain_t         *busy;
@@ -35,11 +40,24 @@
     unsigned             redo:1;
     unsigned             done:1;
     unsigned             nomem:1;
+    unsigned             first:1;
+    unsigned             strict:1;
 
     z_stream             zstream;
     ngx_http_request_t  *request;
 } ngx_http_gunzip_ctx_t;
 
+#define NGX_HTTP_GUNZIP_PASS_ANY           0x02
+#define NGX_HTTP_GUNZIP_PASS_CONTENT_TYPE  0x04
+#define NGX_HTTP_GUNZIP_PASS_200           0x08
+
+static ngx_conf_bitmask_t  ngx_http_gzip_pass_mask[] = {
+    { ngx_string("any"), NGX_HTTP_GUNZIP_PASS_ANY },
+    { ngx_string("content_type"), NGX_HTTP_GUNZIP_PASS_CONTENT_TYPE },
+    { ngx_string("200"), NGX_HTTP_GUNZIP_PASS_200 },
+    { ngx_null_string, 0 }
+};
+
 
 static ngx_int_t ngx_http_gunzip_filter_inflate_start(ngx_http_request_t *r,
     ngx_http_gunzip_ctx_t *ctx);
@@ -78,6 +96,27 @@
       offsetof(ngx_http_gunzip_conf_t, bufs),
       NULL },
 
+    { ngx_string("gunzip_options"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_1MORE,
+      ngx_conf_set_bitmask_slot,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_gunzip_conf_t, pass),
+      &ngx_http_gzip_pass_mask },
+    
+    { ngx_string("gunzip_strict"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
+      ngx_conf_set_flag_slot,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_gunzip_conf_t, strict),
+      NULL },
+
+    { ngx_string("gunzip_types"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_1MORE,
+      ngx_http_types_slot,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_gunzip_conf_t, types_keys),
+      &ngx_http_html_default_types[0] },
+
       ngx_null_command
 };
 
@@ -130,6 +169,10 @@
     /* TODO ignore content encoding? */
 
     if (!conf->enable
+	|| ((conf->pass & NGX_HTTP_GUNZIP_PASS_200 ) && (r->upstream)
+           && (r->upstream->state) && (r->upstream->state->status != 200))
+	|| ((conf->pass & NGX_HTTP_GUNZIP_PASS_CONTENT_TYPE)
+           && ngx_http_test_content_type(r, &conf->types) == NULL)
         || r->headers_out.content_encoding == NULL
         || r->headers_out.content_encoding->value.len != 4
         || ngx_strncasecmp(r->headers_out.content_encoding->value.data,
@@ -138,6 +181,8 @@
         return ngx_http_next_header_filter(r);
     }
 
+    if ( conf->pass & NGX_HTTP_GUNZIP_PASS_ANY )
+        goto ok;
 #if (nginx_version >= 8025 || (nginx_version >= 7065 && nginx_version < 8000))
 
     r->gzip_vary = 1;
@@ -159,6 +204,7 @@
 
 #endif
 
+ok:
     ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_gunzip_ctx_t));
     if (ctx == NULL) {
         return NGX_ERROR;
@@ -167,6 +213,8 @@
     ngx_http_set_ctx(r, ctx, ngx_http_gunzip_filter_module);
 
     ctx->request = r;
+    ctx->strict = conf->strict;
+    ctx->first = 1;
 
     r->filter_need_in_memory = 1;
 
@@ -206,6 +254,7 @@
         if (ngx_chain_add_copy(r->pool, &ctx->in, in) != NGX_OK) {
             goto failed;
         }
+	ctx->_in=in;
     }
 
     if (ctx->nomem) {
@@ -314,8 +363,8 @@
     ctx->zstream.zfree = ngx_http_gunzip_filter_free;
     ctx->zstream.opaque = ctx;
 
-    /* windowBits +16 to decode gzip, zlib 1.2.0.4+ */
-    rc = inflateInit2(&ctx->zstream, MAX_WBITS + 16);
+    /* windowBits +32 to decode gzip and zlib, zlib 1.2.0.4+ */
+    rc = inflateInit2(&ctx->zstream, MAX_WBITS + (ctx->strict?16:32));
 
     if (rc != Z_OK) {
         ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0,
@@ -431,10 +480,19 @@
     rc = inflate(&ctx->zstream, ctx->flush);
 
     if (rc != Z_OK && rc != Z_STREAM_END && rc != Z_BUF_ERROR) {
-        ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0,
-                      "inflate() failed: %d, %d", ctx->flush, rc);
-        return NGX_ERROR;
+        if (ctx->strict==0 && ctx->first==1) {
+	    ctx->out=ctx->_in;
+	    ctx->done=1;
+            ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0,
+                          "passing raw content, inflate() failed: %d, %d", ctx->flush, rc);
+	    return NGX_OK;
+        } else {
+            ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0,
+                          "inflate() failed: %d, %d", ctx->flush, rc);
+            return NGX_ERROR;
+	}
     }
+    ctx->first=0;
 
     ngx_log_debug5(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
                    "inflate out: ni:%p no:%p ai:%ud ao:%ud rc:%d",
@@ -653,6 +711,7 @@
      */
 
     conf->enable = NGX_CONF_UNSET;
+    conf->strict = NGX_CONF_UNSET;
 
     return conf;
 }
@@ -665,10 +724,29 @@
     ngx_http_gunzip_conf_t *conf = child;
 
     ngx_conf_merge_value(conf->enable, prev->enable, 0);
+    ngx_conf_merge_value(conf->strict, prev->strict, 1);
 
     ngx_conf_merge_bufs_value(conf->bufs, prev->bufs,
                               (128 * 1024) / ngx_pagesize, ngx_pagesize);
 
+    ngx_conf_merge_bitmask_value(conf->pass, prev->pass,
+                              NGX_CONF_BITMASK_SET);
+
+#if (nginx_version < 8029)
+    if (ngx_http_merge_types(cf, conf->types_keys, &conf->types,
+                             prev->types_keys, &prev->types,
+                             ngx_http_html_default_types)
+        != NGX_OK)
+#else
+    if (ngx_http_merge_types(cf, &conf->types_keys, &conf->types,
+                             &prev->types_keys, &prev->types,
+                             ngx_http_html_default_types)
+        != NGX_OK)
+#endif
+    {
+        return NGX_CONF_ERROR;
+    }
+
     return NGX_CONF_OK;
 }

Posted at Nginx Forum: http://forum.nginx.org/read.php?2,66713,78290#msg-78290




More information about the nginx mailing list