upstream keepalive - call for testing
MagicBear
magicbearmo at gmail.com
Thu Sep 15 17:51:25 UTC 2011
Hello,
I have wrote a module to make nginx support 304 to decrease bandwidth usage.
note: I have a newbie for nginx module development, so the above module may
have some problem. Welcome to test it and feedback another problem with me.
You can download full patch file from here:
http://m-b.cc/share/proxy_304.txt
# User MagicBear <magicbearmo at gmail.com>
Upstream:
add $upstream_last_modified variant.
add handler for 304 Unmodified.
Proxy:
change to send If-Modified-Since header.
TODO:
change write TO not block IO.
diff -ruN a/http/modules/ngx_http_proxy_module.c
b/http/modules/ngx_http_proxy_module.c
--- a/http/modules/ngx_http_proxy_module.c 2011-09-15
22:23:03.284431407 +0800
+++ b/http/modules/ngx_http_proxy_module.c 2011-09-16
01:41:44.654428632 +0800
@@ -543,7 +543,7 @@
{ ngx_string("Connection"), ngx_string("close") },
{ ngx_string("Keep-Alive"), ngx_string("") },
{ ngx_string("Expect"), ngx_string("") },
- { ngx_string("If-Modified-Since"), ngx_string("") },
+ { ngx_string("If-Modified-Since"),
ngx_string("$upstream_last_modified") },
{ ngx_string("If-Unmodified-Since"), ngx_string("") },
{ ngx_string("If-None-Match"), ngx_string("") },
{ ngx_string("If-Match"), ngx_string("") },
diff -ruN a/http/ngx_http_upstream.c b/http/ngx_http_upstream.c
--- a/http/ngx_http_upstream.c 2011-09-15 22:23:03.284431407 +0800
+++ b/http/ngx_http_upstream.c 2011-09-16 01:41:44.654428632 +0800
@@ -16,6 +16,8 @@
ngx_http_upstream_t *u);
static ngx_int_t ngx_http_upstream_cache_status(ngx_http_request_t *r,
ngx_http_variable_value_t *v, uintptr_t data);
+static ngx_int_t ngx_http_upstream_last_modified(ngx_http_request_t *r,
+ ngx_http_variable_value_t *v, uintptr_t data);
#endif
static void ngx_http_upstream_init_request(ngx_http_request_t *r);
@@ -342,6 +344,10 @@
ngx_http_upstream_cache_status, 0,
NGX_HTTP_VAR_NOCACHEABLE, 0 },
+ { ngx_string("upstream_last_modified"), NULL,
+ ngx_http_upstream_last_modified, 0,
+ NGX_HTTP_VAR_NOCACHEABLE, 0 },
+
#endif
{ ngx_null_string, NULL, NULL, 0, 0, 0 }
@@ -1618,6 +1624,80 @@
u->buffer.last = u->buffer.pos;
}
+#if (NGX_HTTP_CACHE)
+
+ if (u->cache_status == NGX_HTTP_CACHE_EXPIRED &&
+ u->headers_in.status_n == NGX_HTTP_NOT_MODIFIED &&
+ ngx_http_file_cache_valid(u->conf->cache_valid,
u->headers_in.status_n))
+ {
+ ngx_int_t rc;
+
+ rc = u->reinit_request(r);
+
+ if (rc == NGX_OK) {
+ u->cache_status = NGX_HTTP_CACHE_BYPASS;
+ rc = ngx_http_upstream_cache_send(r, u);
+
+ time_t now, valid;
+
+ now = ngx_time();
+
+ valid = r->cache->valid_sec;
+
+ if (valid == 0) {
+ valid =
ngx_http_file_cache_valid(u->conf->cache_valid,
+
u->headers_in.status_n);
+ if (valid) {
+ r->cache->valid_sec = now +
valid;
+ }
+ }
+
+ if (valid) {
+ r->cache->last_modified =
r->headers_out.last_modified_time;
+ r->cache->date = now;
+ r->cache->body_start = (u_short)
(u->buffer.pos - u->buffer.start);
+
+ // update Header
+ ngx_http_file_cache_set_header(r,
u->buffer.start);
+
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP,
r->connection->log, 0,
+
"update cache \"%s\" header to new expired." , r->cache->file.name.data);
+
+ // Reopen file via RW
+ ngx_fd_t fd =
ngx_open_file(r->cache->file.name.data, NGX_FILE_RDWR, NGX_FILE_OPEN, 0);
+
+ if (fd == NGX_INVALID_FILE) {
+ ngx_log_error(NGX_LOG_CRIT,
r->connection->log, ngx_errno,
+
ngx_open_file_n " \"%s\" failed", r->cache->file.name.data);
+ return;
+ }
+
+ // Write cache
+ if (write(fd, u->buffer.start,
sizeof(ngx_http_file_cache_header_t)) < 0)
+ {
+ ngx_log_error(NGX_LOG_CRIT,
r->connection->log, ngx_errno,
+
"write proxy_cache \"%s\" failed", r->cache->file.name.data);
+ return;
+ }
+
+ if (ngx_close_file(fd) ==
NGX_FILE_ERROR) {
+ ngx_log_error(NGX_LOG_ALERT,
r->connection->log, ngx_errno,
+
ngx_close_file_n " \"%s\" failed", r->cache->file.name.data);
+ }
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP,
r->connection->log, 0,
+
"update cache \"%s\" header to new expired done." ,
r->cache->file.name.data);
+ } else {
+ u->cacheable = 0;
+ r->headers_out.last_modified_time =
-1;
+ }
+ }
+
+ ngx_http_upstream_finalize_request(r, u, rc);
+ return;
+ }
+
+#endif
+
if (ngx_http_upstream_test_next(r, u) == NGX_OK) {
return;
}
@@ -4006,6 +4086,32 @@
return NGX_OK;
}
+
+ngx_int_t
+ngx_http_upstream_last_modified(ngx_http_request_t *r,
+ ngx_http_variable_value_t *v, uintptr_t data)
+{
+ u_char *u;
+
+ if (r->upstream == NULL || r->upstream->cache_status == 0 ||
r->cache==NULL || r->cache->last_modified <= 0) {
+ v->not_found = 1;
+ return NGX_OK;
+ }
+
+ v->valid = 1;
+ v->no_cacheable = 0;
+ v->not_found = 0;
+ u = ngx_pcalloc(r->pool, 30);
+ if (u == NULL) {
+ return NGX_ERROR;
+ }
+
+ v->len = 29;
+ ngx_http_time(u, r->cache->last_modified);
+ v->data = u;
+
+ return NGX_OK;
+}
#endif
MagicBear
2011/9/15 magicbear <nginx-forum at nginx.us>
> I have run the nginx 1.1.2 via this patch for 7 days, except for one
> days have a large DDoS so I restart nginx for several seconds, it was
> very stable to work.
> Handle about 70million request without problem happen, I think the last
> problem may be have a memory corruption, you are right.
> I will check that server when have times.
> Thanks for your hard work.
>
> MagicBear
>
> Maxim Dounin Wrote:
> -------------------------------------------------------
> > Hello!
> >
> > On Mon, Sep 05, 2011 at 11:42:31PM +0800,
> > ビリビリⅤ wrote:
> >
> > > (gdb) fr 0
> > > #0 ngx_http_upstream_handler
> > (ev=0x7fc45735f8a8)
> > > at src/http/ngx_http_upstream.c:915
> > > 915 ctx->current_request = r;
> > > (gdb) p ngx_cycle->log
> > > $1 = (ngx_log_t *) 0x21f19a8
> > > (gdb) p *r
> > > $2 = {signature = 51686928, connection =
> > 0x23b4160, ctx = 0x0,
> >
> > [...]
> >
> > This looks like memory corruption, but
> > unfortunately I don't see
> > any traces of the real cause. My best quess is
> > improper handling
> > of proxy_ignore_client_abort as fixed in 1.1.2.
> > Please try 1.1.2
> > with patches from
> >
> > http://nginx.org/patches/patch-nginx-keepalive-ful
> > l-5.txt
> >
> > It already includes upstream keepalive module, as
> > well as all
> > other upstream-keepalive related fixes. See here
> > for details:
> >
> > http://mailman.nginx.org/pipermail/nginx-devel/201
> > 1-September/001147.html
> >
> > Maxim Dounin
> >
> > _______________________________________________
> > nginx mailing list
> > nginx at nginx.org
> > http://mailman.nginx.org/mailman/listinfo/nginx
>
> Posted at Nginx Forum:
> http://forum.nginx.org/read.php?2,213207,215217#msg-215217
>
> _______________________________________________
> nginx mailing list
> nginx at nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx/attachments/20110916/095ad33d/attachment-0001.html>
-------------- next part --------------
# User MagicBear <magicbearmo at gmail.com>
Upstream:
add $upstream_last_modified variant.
add handler for 304 Unmodified.
Proxy:
change to send If-Modified-Since header.
TODO:
change write TO not block IO.
diff -ruN a/http/modules/ngx_http_proxy_module.c b/http/modules/ngx_http_proxy_module.c
--- a/http/modules/ngx_http_proxy_module.c 2011-09-15 22:23:03.284431407 +0800
+++ b/http/modules/ngx_http_proxy_module.c 2011-09-16 01:41:44.654428632 +0800
@@ -543,7 +543,7 @@
{ ngx_string("Connection"), ngx_string("close") },
{ ngx_string("Keep-Alive"), ngx_string("") },
{ ngx_string("Expect"), ngx_string("") },
- { ngx_string("If-Modified-Since"), ngx_string("") },
+ { ngx_string("If-Modified-Since"), ngx_string("$upstream_last_modified") },
{ ngx_string("If-Unmodified-Since"), ngx_string("") },
{ ngx_string("If-None-Match"), ngx_string("") },
{ ngx_string("If-Match"), ngx_string("") },
diff -ruN a/http/ngx_http_upstream.c b/http/ngx_http_upstream.c
--- a/http/ngx_http_upstream.c 2011-09-15 22:23:03.284431407 +0800
+++ b/http/ngx_http_upstream.c 2011-09-16 01:41:44.654428632 +0800
@@ -16,6 +16,8 @@
ngx_http_upstream_t *u);
static ngx_int_t ngx_http_upstream_cache_status(ngx_http_request_t *r,
ngx_http_variable_value_t *v, uintptr_t data);
+static ngx_int_t ngx_http_upstream_last_modified(ngx_http_request_t *r,
+ ngx_http_variable_value_t *v, uintptr_t data);
#endif
static void ngx_http_upstream_init_request(ngx_http_request_t *r);
@@ -342,6 +344,10 @@
ngx_http_upstream_cache_status, 0,
NGX_HTTP_VAR_NOCACHEABLE, 0 },
+ { ngx_string("upstream_last_modified"), NULL,
+ ngx_http_upstream_last_modified, 0,
+ NGX_HTTP_VAR_NOCACHEABLE, 0 },
+
#endif
{ ngx_null_string, NULL, NULL, 0, 0, 0 }
@@ -1618,6 +1624,80 @@
u->buffer.last = u->buffer.pos;
}
+#if (NGX_HTTP_CACHE)
+
+ if (u->cache_status == NGX_HTTP_CACHE_EXPIRED &&
+ u->headers_in.status_n == NGX_HTTP_NOT_MODIFIED &&
+ ngx_http_file_cache_valid(u->conf->cache_valid, u->headers_in.status_n))
+ {
+ ngx_int_t rc;
+
+ rc = u->reinit_request(r);
+
+ if (rc == NGX_OK) {
+ u->cache_status = NGX_HTTP_CACHE_BYPASS;
+ rc = ngx_http_upstream_cache_send(r, u);
+
+ time_t now, valid;
+
+ now = ngx_time();
+
+ valid = r->cache->valid_sec;
+
+ if (valid == 0) {
+ valid = ngx_http_file_cache_valid(u->conf->cache_valid,
+ u->headers_in.status_n);
+ if (valid) {
+ r->cache->valid_sec = now + valid;
+ }
+ }
+
+ if (valid) {
+ r->cache->last_modified = r->headers_out.last_modified_time;
+ r->cache->date = now;
+ r->cache->body_start = (u_short) (u->buffer.pos - u->buffer.start);
+
+ // update Header
+ ngx_http_file_cache_set_header(r, u->buffer.start);
+
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "update cache \"%s\" header to new expired." , r->cache->file.name.data);
+
+ // Reopen file via RW
+ ngx_fd_t fd = ngx_open_file(r->cache->file.name.data, NGX_FILE_RDWR, NGX_FILE_OPEN, 0);
+
+ if (fd == NGX_INVALID_FILE) {
+ ngx_log_error(NGX_LOG_CRIT, r->connection->log, ngx_errno,
+ ngx_open_file_n " \"%s\" failed", r->cache->file.name.data);
+ return;
+ }
+
+ // Write cache
+ if (write(fd, u->buffer.start, sizeof(ngx_http_file_cache_header_t)) < 0)
+ {
+ ngx_log_error(NGX_LOG_CRIT, r->connection->log, ngx_errno,
+ "write proxy_cache \"%s\" failed", r->cache->file.name.data);
+ return;
+ }
+
+ if (ngx_close_file(fd) == NGX_FILE_ERROR) {
+ ngx_log_error(NGX_LOG_ALERT, r->connection->log, ngx_errno,
+ ngx_close_file_n " \"%s\" failed", r->cache->file.name.data);
+ }
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "update cache \"%s\" header to new expired done." , r->cache->file.name.data);
+ } else {
+ u->cacheable = 0;
+ r->headers_out.last_modified_time = -1;
+ }
+ }
+
+ ngx_http_upstream_finalize_request(r, u, rc);
+ return;
+ }
+
+#endif
+
if (ngx_http_upstream_test_next(r, u) == NGX_OK) {
return;
}
@@ -4006,6 +4086,32 @@
return NGX_OK;
}
+
+ngx_int_t
+ngx_http_upstream_last_modified(ngx_http_request_t *r,
+ ngx_http_variable_value_t *v, uintptr_t data)
+{
+ u_char *u;
+
+ if (r->upstream == NULL || r->upstream->cache_status == 0 || r->cache==NULL || r->cache->last_modified <= 0) {
+ v->not_found = 1;
+ return NGX_OK;
+ }
+
+ v->valid = 1;
+ v->no_cacheable = 0;
+ v->not_found = 0;
+ u = ngx_pcalloc(r->pool, 30);
+ if (u == NULL) {
+ return NGX_ERROR;
+ }
+
+ v->len = 29;
+ ngx_http_time(u, r->cache->last_modified);
+ v->data = u;
+
+ return NGX_OK;
+}
#endif
More information about the nginx
mailing list