Hello,
I have written a module to implement sFlow in nginx (nginx-sflow-module.googlecode.com). I'm simulating a 1-second timer-tick by assuming that the request handler will be called at least once per second. That's probably a safe assumption for any server that would care about sFlow monitoring, but I expect there's a better way...
I tried asking for a timer callback like this:
ngx_event_t *ev = ngx_pcalloc(pool, sizeof(ngx_event_t));
ev->hander = ngx_http_sflow_tick_event_hander;
ngx_add_timer(ev, 1000);
but (like most russian girls) the event never called me back. It looks like I might have to hang this on a file-descriptor somehow, but that's where I'm getting lost. Any pointers would be most appreciated.
Neil
openbsd has a setsockopt option called SO_BINDANY that allows a
process to bind to any ip address, even if it is not local to the
system. the patch below uses it to allow nginx to connect to a
backend server using the ip of the client making the request.
my main goal here is to allow the backend server to know the ip of
the client actually making the request without having to look at
extra hhtp headers
i thought id throw this out there to get some help since this is
my first attempt at tweaking nginx. there are a few issues with
this implementation:
1. it is completely specific to openbsd.
2. it needs root privileges to use the SO_BINDANY sockopt.
3. im not sure if connections to backends are cached. if so then
it is probable that a different client will reuse a previous clients
proxy connection, so it will appear that the same client made both
requests to the backend.
to use this you just configure nginx to run as root and add
"proxy_transparent on" to the sections you want this feature enabled
on. you will need to add appropriate "pass out proto tcp divert-reply"
rules to pf for the SO_BINDANY sockopt to work too.
if anyone has some tips on how to handle problems 2 and 3 i would
be grateful.
cheers,
dlg
--- src/event/ngx_event_connect.c.orig Thu Nov 26 04:03:59 2009
+++ src/event/ngx_event_connect.c Thu Oct 28 23:22:37 2010
@@ -11,7 +11,7 @@
ngx_int_t
-ngx_event_connect_peer(ngx_peer_connection_t *pc)
+ngx_event_connect_peer(ngx_peer_connection_t *pc, ngx_connection_t *cc)
{
int rc;
ngx_int_t event;
@@ -20,6 +20,7 @@ ngx_event_connect_peer(ngx_peer_connection_t *pc)
ngx_socket_t s;
ngx_event_t *rev, *wev;
ngx_connection_t *c;
+ int bindany;
rc = pc->get(pc, pc->data);
if (rc != NGX_OK) {
@@ -46,6 +47,40 @@ ngx_event_connect_peer(ngx_peer_connection_t *pc)
}
return NGX_ERROR;
+ }
+
+ if (cc != NULL) {
+ bindany = 1;
+ if (setsockopt(s, SOL_SOCKET, SO_BINDANY,
+ &bindany, sizeof(bindany)) == -1)
+ {
+ ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno,
+ "setsockopt(SO_BINDANY) failed");
+
+ ngx_free_connection(c);
+
+ if (ngx_close_socket(s) == -1) {
+ ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno,
+ ngx_close_socket_n " failed");
+ }
+
+ return NGX_ERROR;
+ }
+
+ if (bind(s, cc->sockaddr, cc->socklen) == -1)
+ {
+ ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno,
+ "bind() failed");
+
+ ngx_free_connection(c);
+
+ if (ngx_close_socket(s) == -1) {
+ ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno,
+ ngx_close_socket_n " failed");
+ }
+
+ return NGX_ERROR;
+ }
}
if (pc->rcvbuf) {
--- src/event/ngx_event_connect.h.orig Tue Nov 3 01:24:02 2009
+++ src/event/ngx_event_connect.h Thu Oct 28 23:22:37 2010
@@ -68,7 +68,8 @@ struct ngx_peer_connection_s {
};
-ngx_int_t ngx_event_connect_peer(ngx_peer_connection_t *pc);
+ngx_int_t ngx_event_connect_peer(ngx_peer_connection_t *pc,
+ ngx_connection_t *cc);
ngx_int_t ngx_event_get_peer(ngx_peer_connection_t *pc, void *data);
--- src/http/modules/ngx_http_proxy_module.c.orig Mon May 24 21:01:05 2010
+++ src/http/modules/ngx_http_proxy_module.c Thu Oct 28 23:42:10 2010
@@ -71,6 +71,7 @@ typedef struct {
ngx_http_proxy_vars_t vars;
ngx_flag_t redirect;
+ ngx_flag_t transparent;
ngx_uint_t headers_hash_max_size;
ngx_uint_t headers_hash_bucket_size;
@@ -196,6 +197,13 @@ static ngx_command_t ngx_http_proxy_commands[] = {
0,
NULL },
+ { ngx_string("proxy_transparent"),
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
+ ngx_conf_set_flag_slot,
+ NGX_HTTP_LOC_CONF_OFFSET,
+ offsetof(ngx_http_proxy_loc_conf_t, transparent),
+ NULL },
+
{ ngx_string("proxy_store"),
NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
ngx_http_proxy_store,
@@ -626,6 +634,7 @@ ngx_http_proxy_handler(ngx_http_request_t *r)
u->abort_request = ngx_http_proxy_abort_request;
u->finalize_request = ngx_http_proxy_finalize_request;
r->state = 0;
+ r->transparent = (plcf->transparent == 1);
if (plcf->redirects) {
u->rewrite_redirect = ngx_http_proxy_rewrite_redirect;
@@ -1940,6 +1949,7 @@ ngx_http_proxy_create_loc_conf(ngx_conf_t *cf)
conf->upstream.cyclic_temp_file = 0;
conf->redirect = NGX_CONF_UNSET;
+ conf->transparent = NGX_CONF_UNSET;
conf->upstream.change_buffering = 1;
conf->headers_hash_max_size = NGX_CONF_UNSET_UINT;
@@ -2214,6 +2224,8 @@ ngx_http_proxy_merge_loc_conf(ngx_conf_t *cf, void *pa
}
}
}
+
+ ngx_conf_merge_value(conf->transparent, prev->transparent, 0);
/* STUB */
if (prev->proxy_lengths) {
--- src/http/ngx_http_request.h.orig Mon May 24 22:35:10 2010
+++ src/http/ngx_http_request.h Thu Oct 28 23:22:37 2010
@@ -511,6 +511,8 @@ struct ngx_http_request_s {
unsigned stat_writing:1;
#endif
+ unsigned transparent:1;
+
/* used to parse HTTP headers */
ngx_uint_t state;
--- src/http/ngx_http_upstream.c.orig Mon May 24 22:35:10 2010
+++ src/http/ngx_http_upstream.c Thu Oct 28 23:22:37 2010
@@ -1066,7 +1066,8 @@ ngx_http_upstream_connect(ngx_http_request_t *r, ngx_h
u->state->response_sec = tp->sec;
u->state->response_msec = tp->msec;
- rc = ngx_event_connect_peer(&u->peer);
+ rc = ngx_event_connect_peer(&u->peer, r->transparent ?
+ r->connection : NULL);
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"http upstream connect: %i", rc);
--- src/mail/ngx_mail_auth_http_module.c.orig Fri May 14 19:56:37 2010
+++ src/mail/ngx_mail_auth_http_module.c Thu Oct 28 23:22:37 2010
@@ -191,7 +191,7 @@ ngx_mail_auth_http_init(ngx_mail_session_t *s)
ctx->peer.log = s->connection->log;
ctx->peer.log_error = NGX_ERROR_ERR;
- rc = ngx_event_connect_peer(&ctx->peer);
+ rc = ngx_event_connect_peer(&ctx->peer, NULL);
if (rc == NGX_ERROR || rc == NGX_BUSY || rc == NGX_DECLINED) {
if (ctx->peer.connection) {
--- src/mail/ngx_mail_proxy_module.c.orig Thu Oct 28 23:32:15 2010
+++ src/mail/ngx_mail_proxy_module.c Thu Oct 28 23:30:53 2010
@@ -147,7 +147,7 @@ ngx_mail_proxy_init(ngx_mail_session_t *s, ngx_addr_t
p->upstream.log = s->connection->log;
p->upstream.log_error = NGX_ERROR_ERR;
- rc = ngx_event_connect_peer(&p->upstream);
+ rc = ngx_event_connect_peer(&p->upstream, NULL);
if (rc == NGX_ERROR || rc == NGX_BUSY || rc == NGX_DECLINED) {
ngx_mail_proxy_internal_server_error(s);
Hi devs,
I work for an environmental not-for-profit organisation where we use
XSLT to theme our website. (The XSLT is generated by Diazo, and the
site largely runs on Plone).
Currently we use Nginx to do the XSLT transformation. There's a
problem though, that our un-themed site doesn't come out as perfect
XML, so we need an XSLT parser which can transform HTML (not just
XML). Nginx's http_xslt_module does NOT currently support HTML
parsing, and I'd really like to see this feature added.
The problem isn't specific to Diazo, but the Diazo manual explains the
need for HTML parsing:
> In theory, any XSLT processor will do. In practice, however, most websites do not produce 100% well-formed XML (i.e. they do not conform to the XHTML “strict” doctype). For this reason, it is normally necessary to use an XSLT processor that will parse the content using a more lenient parser with some knowledge of HTML. libxml2, the most popular XML processing library on Linux and similar operating systems, contains such a parser.
Fortunately there's a fork of nginx which does use libxml2: the
xslt_html project <http://code.google.com/p/html-xslt/>.
Unfortunately, the project is not maintained, so it ties us to a
patched version of nginx 0.7.67 (circa June 2010). I'd like to upgrade
nginx -- I've hit nginx bugs that were fixed long ago. I'm sure there
are many other nginx users with the same needs, so I'm requesting the
fork's changes make their way into the mainline. I'm assuming it's
just been forgotten.
The Diazo documentation also explains deploying with this patched Nginx:
> To deploy an Diazo theme to the Nginx web server, you will need to compile Nginx with a special version of the XSLT module that can (optionally) use the HTML parser from libxml2.
> In the future, the necessary patches to enable HTML mode parsing will hopefully be part of the standard Nginx distribution. In the meantime, they are maintained in the html-xslt project.
We're using this html-xslt fork of nginx at my organisation. But
unfortunately, it's not maintained, and the functionality hasn't made
it into the standard Nginx distribution. Can we please include it?
The fork adds the directive: "xslt_html_parser on;" which causes the
http_xslt_module to parse in HTML mode.
I've just made a diff <http://pastebin.com/CP1P8Gzj> to see what the
fork changes, and it's 755 lines long. (That's a bit longer than I
expected)
The files modified by the html-xslt fork are:
src/http/modules/ngx_http_xslt_filter_module.c
src/http/ngx_http_variables.c
auto/options
auto/lib/libxslt/conf
The diff is against nginx 0.7.67. Since then the
ngx_http_xslt_filter_module.c has seen about 300 lines removed and 20
lines added or changed, so obviously the diff can't be used as a patch
against the current version of nginx.
Hopefully that's more than enough info to get started if developers
are interested in folding the fork into nginx.
I know the other solution to our problem here is to move the XSLT to
another layer of the stack -- such as Varnish or Apache -- but I want
to make sure nginx devs know about the feature they're missing first.
Thanks for listening and I hope HTML parsing for XSLT can make it to
the mainline of nginx,
Peter Halasz.
Hi there,
I am wondering if there is a way to manipulate the contents of a
PUT/POST (the request body) in a similar way that filters can manipulate
the body of a GET?
Would it e.g. be feasible (or even possible) to do a
ngx_http_read_client_request_body somewhere, mess about with the body,
and then somehow (how?) shove it back into the request, before passing
it on to whatever handles the actual writing (dav) or POST-processing
(php, whatever) or even upstream to a proxy backend?
Any thoughts appeciated!
Adam
When nginx gets multiple X-Forwarded-For headers in a single request, it
only keeps the last one in r->headers_in (and thus in
$http_x_forwarded_for, $proxy_add_x_forwarded_for). Reverse proxies behind
an nginx instance sometimes need the entire X-Forwarded-For chain - part
of which is discarded in this case.
Per RFC 2616, it's equivalent to concatenate each header value (separated
by a comma) and send the concatenated value to the upstream:
4.2
-snip-
Multiple message-header fields with the same field-name MAY be
present in a message if and only if the entire field-value for that
header field is defined as a comma-separated list [i.e., #(values)].
It MUST be possible to combine the multiple header fields into one
"field-name: field-value" pair, without changing the semantics of the
message, by appending each subsequent field-value to the first, each
separated by a comma. The order in which header fields with the same
field-name are received is therefore significant to the
interpretation of the combined field value, and thus a proxy MUST NOT
change the order of these field values when a message is forwarded.
-snip-
Attached is a patch that does exactly this, in the case of multiple headers.
Please let me know if you have any comments about this patch - I'm happy
to make any changes you suggest.
Relevant bug report:
http://trac.nginx.org/nginx/ticket/106
Thanks,
Alex Tribble
[Sorry for the attachment, my MUAs all unanimously decided they hate me]
Hello everybody,
I come here with the patch for nginx.org documents in simplified Chinese,
and I'm glad that someone will give it a review. This work is done by the
whole Server Platforms Team at Taobao.com.
Thank you.
Regards
--
Charles Chen
Software Engineer
Server Platforms Team at Taobao.com
Hi.
I'm trying to figure out how does nginx manage its memory per request. From
what I understand it uses region-based memory management (implemented in
src/core/ngx_palloc.{h,c}).
But is there any documentation on details of its implementation and
algorithm itself?
I've tried to just read through sources, but I can't get figure algorithm
out of them :(
Could somebody please sched some light on this?
Thank you.