upstream keepalive - call for testing
SplitIce
mat999 at gmail.com
Fri Sep 16 01:42:25 UTC 2011
wow, this + last-modified and etags would really cut into the reason people
use varnish in their nginx + varnish setup.
On Fri, Sep 16, 2011 at 3:51 AM, MagicBear <magicbearmo at gmail.com> wrote:
> Hello,
> I have wrote a module to make nginx support 304 to decrease bandwidth
> usage.
> note: I have a newbie for nginx module development, so the above module may
> have some problem. Welcome to test it and feedback another problem with me.
>
> You can download full patch file from here:
> http://m-b.cc/share/proxy_304.txt
>
> # User MagicBear <magicbearmo at gmail.com>
> Upstream:
> add $upstream_last_modified variant.
> add handler for 304 Unmodified.
> Proxy:
> change to send If-Modified-Since header.
>
> TODO:
> change write TO not block IO.
>
> diff -ruN a/http/modules/ngx_http_proxy_module.c
> b/http/modules/ngx_http_proxy_module.c
> --- a/http/modules/ngx_http_proxy_module.c 2011-09-15
> 22:23:03.284431407 +0800
> +++ b/http/modules/ngx_http_proxy_module.c 2011-09-16
> 01:41:44.654428632 +0800
> @@ -543,7 +543,7 @@
> { ngx_string("Connection"), ngx_string("close") },
> { ngx_string("Keep-Alive"), ngx_string("") },
> { ngx_string("Expect"), ngx_string("") },
> - { ngx_string("If-Modified-Since"), ngx_string("") },
> + { ngx_string("If-Modified-Since"),
> ngx_string("$upstream_last_modified") },
> { ngx_string("If-Unmodified-Since"), ngx_string("") },
> { ngx_string("If-None-Match"), ngx_string("") },
> { ngx_string("If-Match"), ngx_string("") },
> diff -ruN a/http/ngx_http_upstream.c b/http/ngx_http_upstream.c
> --- a/http/ngx_http_upstream.c 2011-09-15 22:23:03.284431407 +0800
> +++ b/http/ngx_http_upstream.c 2011-09-16 01:41:44.654428632 +0800
> @@ -16,6 +16,8 @@
> ngx_http_upstream_t *u);
> static ngx_int_t ngx_http_upstream_cache_status(ngx_http_request_t *r,
> ngx_http_variable_value_t *v, uintptr_t data);
> +static ngx_int_t ngx_http_upstream_last_modified(ngx_http_request_t *r,
> + ngx_http_variable_value_t *v, uintptr_t data);
> #endif
>
> static void ngx_http_upstream_init_request(ngx_http_request_t *r);
> @@ -342,6 +344,10 @@
> ngx_http_upstream_cache_status, 0,
> NGX_HTTP_VAR_NOCACHEABLE, 0 },
>
> + { ngx_string("upstream_last_modified"), NULL,
> + ngx_http_upstream_last_modified, 0,
> + NGX_HTTP_VAR_NOCACHEABLE, 0 },
> +
> #endif
>
> { ngx_null_string, NULL, NULL, 0, 0, 0 }
> @@ -1618,6 +1624,80 @@
> u->buffer.last = u->buffer.pos;
> }
>
> +#if (NGX_HTTP_CACHE)
> +
> + if (u->cache_status == NGX_HTTP_CACHE_EXPIRED &&
> + u->headers_in.status_n == NGX_HTTP_NOT_MODIFIED &&
> + ngx_http_file_cache_valid(u->conf->cache_valid,
> u->headers_in.status_n))
> + {
> + ngx_int_t rc;
> +
> + rc = u->reinit_request(r);
> +
> + if (rc == NGX_OK) {
> + u->cache_status = NGX_HTTP_CACHE_BYPASS;
> + rc = ngx_http_upstream_cache_send(r, u);
> +
> + time_t now, valid;
> +
> + now = ngx_time();
> +
> + valid = r->cache->valid_sec;
> +
> + if (valid == 0) {
> + valid =
> ngx_http_file_cache_valid(u->conf->cache_valid,
> +
> u->headers_in.status_n);
> + if (valid) {
> + r->cache->valid_sec = now +
> valid;
> + }
> + }
> +
> + if (valid) {
> + r->cache->last_modified =
> r->headers_out.last_modified_time;
> + r->cache->date = now;
> + r->cache->body_start = (u_short)
> (u->buffer.pos - u->buffer.start);
> +
> + // update Header
> + ngx_http_file_cache_set_header(r,
> u->buffer.start);
> +
> + ngx_log_debug1(NGX_LOG_DEBUG_HTTP,
> r->connection->log, 0,
> +
> "update cache \"%s\" header to new expired." , r->cache->file.name.data);
> +
> + // Reopen file via RW
> + ngx_fd_t fd =
> ngx_open_file(r->cache->file.name.data, NGX_FILE_RDWR, NGX_FILE_OPEN, 0);
> +
> + if (fd == NGX_INVALID_FILE) {
> + ngx_log_error(NGX_LOG_CRIT,
> r->connection->log, ngx_errno,
> +
> ngx_open_file_n " \"%s\" failed", r->cache->file.name.data);
> + return;
> + }
> +
> + // Write cache
> + if (write(fd, u->buffer.start,
> sizeof(ngx_http_file_cache_header_t)) < 0)
> + {
> + ngx_log_error(NGX_LOG_CRIT,
> r->connection->log, ngx_errno,
> +
> "write proxy_cache \"%s\" failed", r->cache->file.name.data);
> + return;
> + }
> +
> + if (ngx_close_file(fd) ==
> NGX_FILE_ERROR) {
> +
> ngx_log_error(NGX_LOG_ALERT, r->connection->log, ngx_errno,
> +
> ngx_close_file_n " \"%s\" failed", r->cache->file.name.data);
> + }
> + ngx_log_debug1(NGX_LOG_DEBUG_HTTP,
> r->connection->log, 0,
> +
> "update cache \"%s\" header to new expired done." ,
> r->cache->file.name.data);
> + } else {
> + u->cacheable = 0;
> + r->headers_out.last_modified_time =
> -1;
> + }
> + }
> +
> + ngx_http_upstream_finalize_request(r, u, rc);
> + return;
> + }
> +
> +#endif
> +
> if (ngx_http_upstream_test_next(r, u) == NGX_OK) {
> return;
> }
> @@ -4006,6 +4086,32 @@
>
> return NGX_OK;
> }
> +
> +ngx_int_t
> +ngx_http_upstream_last_modified(ngx_http_request_t *r,
> + ngx_http_variable_value_t *v, uintptr_t data)
> +{
> + u_char *u;
> +
> + if (r->upstream == NULL || r->upstream->cache_status == 0 ||
> r->cache==NULL || r->cache->last_modified <= 0) {
> + v->not_found = 1;
> + return NGX_OK;
> + }
> +
> + v->valid = 1;
> + v->no_cacheable = 0;
> + v->not_found = 0;
> + u = ngx_pcalloc(r->pool, 30);
> + if (u == NULL) {
> + return NGX_ERROR;
> + }
> +
> + v->len = 29;
> + ngx_http_time(u, r->cache->last_modified);
> + v->data = u;
> +
> + return NGX_OK;
> +}
>
> #endif
>
>
> MagicBear
>
> 2011/9/15 magicbear <nginx-forum at nginx.us>
>
>> I have run the nginx 1.1.2 via this patch for 7 days, except for one
>> days have a large DDoS so I restart nginx for several seconds, it was
>> very stable to work.
>> Handle about 70million request without problem happen, I think the last
>> problem may be have a memory corruption, you are right.
>> I will check that server when have times.
>> Thanks for your hard work.
>>
>> MagicBear
>>
>> Maxim Dounin Wrote:
>> -------------------------------------------------------
>> > Hello!
>> >
>> > On Mon, Sep 05, 2011 at 11:42:31PM +0800,
>> > ビリビリⅤ wrote:
>> >
>> > > (gdb) fr 0
>> > > #0 ngx_http_upstream_handler
>> > (ev=0x7fc45735f8a8)
>> > > at src/http/ngx_http_upstream.c:915
>> > > 915 ctx->current_request = r;
>> > > (gdb) p ngx_cycle->log
>> > > $1 = (ngx_log_t *) 0x21f19a8
>> > > (gdb) p *r
>> > > $2 = {signature = 51686928, connection =
>> > 0x23b4160, ctx = 0x0,
>> >
>> > [...]
>> >
>> > This looks like memory corruption, but
>> > unfortunately I don't see
>> > any traces of the real cause. My best quess is
>> > improper handling
>> > of proxy_ignore_client_abort as fixed in 1.1.2.
>> > Please try 1.1.2
>> > with patches from
>> >
>> > http://nginx.org/patches/patch-nginx-keepalive-ful
>> > l-5.txt
>> >
>> > It already includes upstream keepalive module, as
>> > well as all
>> > other upstream-keepalive related fixes. See here
>> > for details:
>> >
>> > http://mailman.nginx.org/pipermail/nginx-devel/201
>> > 1-September/001147.html
>> >
>> > Maxim Dounin
>> >
>> > _______________________________________________
>> > nginx mailing list
>> > nginx at nginx.org
>> > http://mailman.nginx.org/mailman/listinfo/nginx
>>
>> Posted at Nginx Forum:
>> http://forum.nginx.org/read.php?2,213207,215217#msg-215217
>>
>> _______________________________________________
>> nginx mailing list
>> nginx at nginx.org
>> http://mailman.nginx.org/mailman/listinfo/nginx
>>
>
>
> _______________________________________________
> nginx mailing list
> nginx at nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx/attachments/20110916/4bc2a465/attachment-0001.html>
More information about the nginx
mailing list