totally transparent proxying with nginx on openbsd
David Gwynne
loki at animata.net
Sat Nov 6 18:00:11 MSK 2010
On Thu, Oct 28, 2010 at 08:14:01PM +0400, Maxim Dounin wrote:
> Hello!
hi :)
> On Fri, Oct 29, 2010 at 12:14:48AM +1000, David Gwynne wrote:
>
> > openbsd has a setsockopt option called SO_BINDANY that allows a
> > process to bind to any ip address, even if it is not local to the
> > system. the patch below uses it to allow nginx to connect to a
> > backend server using the ip of the client making the request.
> >
> > my main goal here is to allow the backend server to know the ip of
> > the client actually making the request without having to look at
> > extra hhtp headers
> >
> > i thought id throw this out there to get some help since this is
> > my first attempt at tweaking nginx. there are a few issues with
> > this implementation:
> >
> > 1. it is completely specific to openbsd.
>
> You may want to look at IP_TRANSPARENT in Linux as well. AFAIR
> somebody was working implementing IP_TRANSPARENT support in nginx,
> though I don't know details.
i dont have any linux boxes in an environment i could do this work
on. there is also little motivation for me to do it personally since
i would have no use for it.
however, if changing something in my diffs would help add this
functionality to linux, then i would be happy to incorporate such
changes.
> > 2. it needs root privileges to use the SO_BINDANY sockopt.
>
> Some fine-grained access control in OS is required to allow
> SO_BINDANY for non-root processes. Quick look suggests it's not
> currently possible in OpenBSD. In Linux it's possible to
> allow usage of IP_TRANSPARENT for non-root processes via
> CAP_NET_ADMIN capability.
would it be possible to get nginx master process to do the bind on
the workers behalf, and then hand it to the worker using fd passing
over a socket?
> > 3. im not sure if connections to backends are cached. if so then
> > it is probable that a different client will reuse a previous clients
> > proxy connection, so it will appear that the same client made both
> > requests to the backend.
>
> Connections aren't cached in official nginx yet, though there are
> third party modules which allow connection caching (only in some
> limited cases though).
>
> Obviously it's not possible to change source address once
> connection is established, so the only simple solution I see is to
> don't cache such connections. This is probably good enough
> solution though.
presumably cached connections are looked up using some key. perhaps
that key could be extended to include the source address if it is
transparent.
> There is already pc->local which is used for bind().
ah! ive been hacking on 0.7.something and pulled the changes up to
0.8.53.
> The only thing that should be passed is some flag in
> ngx_peer_connection_t structure to trigger setsockopt(SO_BINDANY)
> before bind.
yes. the diff below does this.
>
> [...]
>
> > + if (setsockopt(s, SOL_SOCKET, SO_BINDANY,
> > + &bindany, sizeof(bindany)) == -1)
>
> This is obviously needs some configure tests and #ifdef's.
have you got an example of this i can use?
>
> [...]
>
> > + if (bind(s, cc->sockaddr, cc->socklen) == -1)
>
> As I already said - there is no need to duplication bind() code.
>
> Additional question to consider is what you are going to do if
> client's connection protocol doesn't match upstream connection's
> one (e.g. ipv6 client vs ipv4 upstream).
id lean toward failing when requested to make such a connection.
this diff moves the storage of the clients address info from
individual members of struct ngx_connection_s to a ngx_addr_t. this
allows it to be easily handed to a ngx_peer_connection_t as the
local address.
the same caveat wrt to needing root for transparent proxying remains.
i'll have a look at using privsep to handle the bind when i get
some more time.
lastly, apologies for taking so long to reply.
diff -r c5122335e41d src/core/ngx_connection.h
--- a/src/core/ngx_connection.h Mon Oct 18 00:00:00 2010 +0400
+++ b/src/core/ngx_connection.h Sun Nov 07 00:32:09 2010 +1000
@@ -123,9 +123,7 @@
ngx_pool_t *pool;
- struct sockaddr *sockaddr;
- socklen_t socklen;
- ngx_str_t addr_text;
+ ngx_addr_t peer;
#if (NGX_SSL)
ngx_ssl_connection_t *ssl;
diff -r c5122335e41d src/event/ngx_event_accept.c
--- a/src/event/ngx_event_accept.c Mon Oct 18 00:00:00 2010 +0400
+++ b/src/event/ngx_event_accept.c Sun Nov 07 00:32:09 2010 +1000
@@ -102,13 +102,13 @@
return;
}
- c->sockaddr = ngx_palloc(c->pool, socklen);
- if (c->sockaddr == NULL) {
+ c->peer.sockaddr = ngx_palloc(c->pool, socklen);
+ if (c->peer.sockaddr == NULL) {
ngx_close_accepted_connection(c);
return;
}
- ngx_memcpy(c->sockaddr, sa, socklen);
+ ngx_memcpy(c->peer.sockaddr, sa, socklen);
log = ngx_palloc(c->pool, sizeof(ngx_log_t));
if (log == NULL) {
@@ -149,14 +149,14 @@
c->log = log;
c->pool->log = log;
- c->socklen = socklen;
+ c->peer.socklen = socklen;
c->listening = ls;
c->local_sockaddr = ls->sockaddr;
c->unexpected_eof = 1;
#if (NGX_HAVE_UNIX_DOMAIN)
- if (c->sockaddr->sa_family == AF_UNIX) {
+ if (c->peer.sockaddr->sa_family == AF_UNIX) {
c->tcp_nopush = NGX_TCP_NOPUSH_DISABLED;
c->tcp_nodelay = NGX_TCP_NODELAY_DISABLED;
#if (NGX_SOLARIS)
@@ -209,15 +209,16 @@
#endif
if (ls->addr_ntop) {
- c->addr_text.data = ngx_pnalloc(c->pool, ls->addr_text_max_len);
- if (c->addr_text.data == NULL) {
+ c->peer.name.data = ngx_pnalloc(c->pool, ls->addr_text_max_len);
+ if (c->peer.name.data == NULL) {
ngx_close_accepted_connection(c);
return;
}
- c->addr_text.len = ngx_sock_ntop(c->sockaddr, c->addr_text.data,
+ c->peer.name.len = ngx_sock_ntop(c->peer.sockaddr,
+ c->peer.name.data,
ls->addr_text_max_len, 0);
- if (c->addr_text.len == 0) {
+ if (c->peer.name.len == 0) {
ngx_close_accepted_connection(c);
return;
}
diff -r c5122335e41d src/event/ngx_event_connect.c
--- a/src/event/ngx_event_connect.c Mon Oct 18 00:00:00 2010 +0400
+++ b/src/event/ngx_event_connect.c Sun Nov 07 00:32:09 2010 +1000
@@ -14,6 +14,7 @@
ngx_event_connect_peer(ngx_peer_connection_t *pc)
{
int rc;
+ int bindany = 1;
ngx_int_t event;
ngx_err_t err;
ngx_uint_t level;
@@ -65,6 +66,16 @@
goto failed;
}
+ if (pc->transparent) {
+ if (setsockopt(s, SOL_SOCKET, SO_BINDANY,
+ &bindany, sizeof(bindany)) == -1)
+ {
+ ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno,
+ "setsockopt(SO_BINDANY) failed");
+ goto failed;
+ }
+ }
+
if (pc->local) {
if (bind(s, pc->local->sockaddr, pc->local->socklen) == -1) {
ngx_log_error(NGX_LOG_CRIT, pc->log, ngx_socket_errno,
diff -r c5122335e41d src/event/ngx_event_connect.h
--- a/src/event/ngx_event_connect.h Mon Oct 18 00:00:00 2010 +0400
+++ b/src/event/ngx_event_connect.h Sun Nov 07 00:32:09 2010 +1000
@@ -62,6 +62,7 @@
ngx_log_t *log;
unsigned cached:1;
+ unsigned transparent:1;
/* ngx_connection_log_error_e */
unsigned log_error:2;
diff -r c5122335e41d src/http/modules/ngx_http_access_module.c
--- a/src/http/modules/ngx_http_access_module.c Mon Oct 18 00:00:00 2010 +0400
+++ b/src/http/modules/ngx_http_access_module.c Sun Nov 07 00:32:09 2010 +1000
@@ -113,12 +113,12 @@
#if (NGX_HAVE_INET6)
- if (alcf->rules6 && r->connection->sockaddr->sa_family == AF_INET6) {
+ if (alcf->rules6 && r->connection->peer.sockaddr->sa_family == AF_INET6) {
u_char *p;
in_addr_t addr;
struct sockaddr_in6 *sin6;
- sin6 = (struct sockaddr_in6 *) r->connection->sockaddr;
+ sin6 = (struct sockaddr_in6 *) r->connection->peer.sockaddr;
p = sin6->sin6_addr.s6_addr;
if (IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr)) {
@@ -134,8 +134,8 @@
#endif
- if (alcf->rules && r->connection->sockaddr->sa_family == AF_INET) {
- sin = (struct sockaddr_in *) r->connection->sockaddr;
+ if (alcf->rules && r->connection->peer.sockaddr->sa_family == AF_INET) {
+ sin = (struct sockaddr_in *) r->connection->peer.sockaddr;
return ngx_http_access_inet(r, alcf, sin->sin_addr.s_addr);
}
diff -r c5122335e41d src/http/modules/ngx_http_geo_module.c
--- a/src/http/modules/ngx_http_geo_module.c Mon Oct 18 00:00:00 2010 +0400
+++ b/src/http/modules/ngx_http_geo_module.c Sun Nov 07 00:32:09 2010 +1000
@@ -262,11 +262,11 @@
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"http geo started: %V", &r->connection->addr_text);
- if (r->connection->sockaddr->sa_family != AF_INET) {
+ if (r->connection->peer.sockaddr->sa_family != AF_INET) {
return 0;
}
- sin = (struct sockaddr_in *) r->connection->sockaddr;
+ sin = (struct sockaddr_in *) r->connection->peer.sockaddr;
return ntohl(sin->sin_addr.s_addr);
}
diff -r c5122335e41d src/http/modules/ngx_http_proxy_module.c
--- a/src/http/modules/ngx_http_proxy_module.c Mon Oct 18 00:00:00 2010 +0400
+++ b/src/http/modules/ngx_http_proxy_module.c Sun Nov 07 00:32:09 2010 +1000
@@ -70,6 +70,7 @@
ngx_http_proxy_vars_t vars;
ngx_flag_t redirect;
+ ngx_flag_t transparent;
ngx_uint_t headers_hash_max_size;
ngx_uint_t headers_hash_bucket_size;
@@ -176,6 +177,13 @@
0,
NULL },
+ { ngx_string("proxy_transparent"),
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
+ ngx_conf_set_flag_slot,
+ NGX_HTTP_LOC_CONF_OFFSET,
+ offsetof(ngx_http_proxy_loc_conf_t, transparent),
+ NULL },
+
{ ngx_string("proxy_store"),
NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
ngx_http_proxy_store,
@@ -598,6 +606,7 @@
u->abort_request = ngx_http_proxy_abort_request;
u->finalize_request = ngx_http_proxy_finalize_request;
r->state = 0;
+ r->transparent = plcf->transparent;
if (plcf->redirects) {
u->rewrite_redirect = ngx_http_proxy_rewrite_redirect;
@@ -1433,13 +1442,13 @@
v->not_found = 0;
if (r->headers_in.x_forwarded_for == NULL) {
- v->len = r->connection->addr_text.len;
- v->data = r->connection->addr_text.data;
+ v->len = r->connection->peer.name.len;
+ v->data = r->connection->peer.name.data;
return NGX_OK;
}
v->len = r->headers_in.x_forwarded_for->value.len
- + sizeof(", ") - 1 + r->connection->addr_text.len;
+ + sizeof(", ") - 1 + r->connection->peer.name.len;
p = ngx_pnalloc(r->pool, v->len);
if (p == NULL) {
@@ -1453,7 +1462,7 @@
*p++ = ','; *p++ = ' ';
- ngx_memcpy(p, r->connection->addr_text.data, r->connection->addr_text.len);
+ ngx_memcpy(p, r->connection->peer.name.data, r->connection->peer.name.len);
return NGX_OK;
}
@@ -1702,6 +1711,7 @@
conf->upstream.cyclic_temp_file = 0;
conf->redirect = NGX_CONF_UNSET;
+ conf->transparent = NGX_CONF_UNSET;
conf->upstream.change_buffering = 1;
conf->headers_hash_max_size = NGX_CONF_UNSET_UINT;
@@ -1954,6 +1964,7 @@
#endif
ngx_conf_merge_value(conf->redirect, prev->redirect, 1);
+ ngx_conf_merge_value(conf->transparent, prev->transparent, 0);
if (conf->redirect) {
diff -r c5122335e41d src/http/modules/ngx_http_upstream_ip_hash_module.c
--- a/src/http/modules/ngx_http_upstream_ip_hash_module.c Mon Oct 18 00:00:00 2010 +0400
+++ b/src/http/modules/ngx_http_upstream_ip_hash_module.c Sun Nov 07 00:32:09 2010 +1000
@@ -111,9 +111,9 @@
/* AF_INET only */
- if (r->connection->sockaddr->sa_family == AF_INET) {
+ if (r->connection->peer.sockaddr->sa_family == AF_INET) {
- sin = (struct sockaddr_in *) r->connection->sockaddr;
+ sin = (struct sockaddr_in *) r->connection->peer.sockaddr;
p = (u_char *) &sin->sin_addr.s_addr;
iphp->addr[0] = p[0];
iphp->addr[1] = p[1];
diff -r c5122335e41d src/http/ngx_http_request.c
--- a/src/http/ngx_http_request.c Mon Oct 18 00:00:00 2010 +0400
+++ b/src/http/ngx_http_request.c Sun Nov 07 00:32:09 2010 +1000
@@ -2616,7 +2616,7 @@
c->log->handler = NULL;
ngx_log_error(NGX_LOG_INFO, c->log, rev->kq_errno,
"kevent() reported that client %V closed "
- "keepalive connection", &c->addr_text);
+ "keepalive connection", &c->peer.name);
#if (NGX_HTTP_SSL)
if (c->ssl) {
c->ssl->no_send_shutdown = 1;
@@ -2679,7 +2679,7 @@
if (n == 0) {
ngx_log_error(NGX_LOG_INFO, c->log, ngx_socket_errno,
- "client %V closed keepalive connection", &c->addr_text);
+ "client %V closed keepalive connection", &c->peer.name);
ngx_http_close_connection(c);
return;
}
@@ -3052,7 +3052,7 @@
ctx = log->data;
- p = ngx_snprintf(buf, len, ", client: %V", &ctx->connection->addr_text);
+ p = ngx_snprintf(buf, len, ", client: %V", &ctx->connection->peer.name);
len -= p - buf;
r = ctx->request;
diff -r c5122335e41d src/http/ngx_http_request.h
--- a/src/http/ngx_http_request.h Mon Oct 18 00:00:00 2010 +0400
+++ b/src/http/ngx_http_request.h Sun Nov 07 00:32:09 2010 +1000
@@ -517,6 +517,8 @@
unsigned stat_writing:1;
#endif
+ unsigned transparent:1;
+
/* used to parse HTTP headers */
ngx_uint_t state;
diff -r c5122335e41d src/http/ngx_http_upstream.c
--- a/src/http/ngx_http_upstream.c Mon Oct 18 00:00:00 2010 +0400
+++ b/src/http/ngx_http_upstream.c Sun Nov 07 00:32:09 2010 +1000
@@ -490,7 +490,12 @@
return;
}
- u->peer.local = u->conf->local;
+ if (r->transparent) {
+ u->peer.transparent = 1;
+ u->peer.local = &r->connection->peer;
+ } else {
+ u->peer.local = u->conf->local;
+ }
clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
diff -r c5122335e41d src/http/ngx_http_variables.c
--- a/src/http/ngx_http_variables.c Mon Oct 18 00:00:00 2010 +0400
+++ b/src/http/ngx_http_variables.c Sun Nov 07 00:32:09 2010 +1000
@@ -896,11 +896,11 @@
struct sockaddr_in6 *sin6;
#endif
- switch (r->connection->sockaddr->sa_family) {
+ switch (r->connection->peer.sockaddr->sa_family) {
#if (NGX_HAVE_INET6)
case AF_INET6:
- sin6 = (struct sockaddr_in6 *) r->connection->sockaddr;
+ sin6 = (struct sockaddr_in6 *) r->connection->peer.sockaddr;
v->len = sizeof(struct in6_addr);
v->valid = 1;
@@ -912,7 +912,7 @@
#endif
default: /* AF_INET */
- sin = (struct sockaddr_in *) r->connection->sockaddr;
+ sin = (struct sockaddr_in *) r->connection->peer.sockaddr;
v->len = sizeof(in_addr_t);
v->valid = 1;
@@ -931,11 +931,11 @@
ngx_http_variable_remote_addr(ngx_http_request_t *r,
ngx_http_variable_value_t *v, uintptr_t data)
{
- v->len = r->connection->addr_text.len;
+ v->len = r->connection->peer.name.len;
v->valid = 1;
v->no_cacheable = 0;
v->not_found = 0;
- v->data = r->connection->addr_text.data;
+ v->data = r->connection->peer.name.data;
return NGX_OK;
}
@@ -961,17 +961,17 @@
return NGX_ERROR;
}
- switch (r->connection->sockaddr->sa_family) {
+ switch (r->connection->peer.sockaddr->sa_family) {
#if (NGX_HAVE_INET6)
case AF_INET6:
- sin6 = (struct sockaddr_in6 *) r->connection->sockaddr;
+ sin6 = (struct sockaddr_in6 *) r->connection->peer.sockaddr;
port = ntohs(sin6->sin6_port);
break;
#endif
default: /* AF_INET */
- sin = (struct sockaddr_in *) r->connection->sockaddr;
+ sin = (struct sockaddr_in *) r->connection->peer.sockaddr;
port = ntohs(sin->sin_port);
break;
}
diff -r c5122335e41d src/mail/ngx_mail_auth_http_module.c
--- a/src/mail/ngx_mail_auth_http_module.c Mon Oct 18 00:00:00 2010 +0400
+++ b/src/mail/ngx_mail_auth_http_module.c Sun Nov 07 00:32:09 2010 +1000
@@ -1156,7 +1156,7 @@
+ sizeof(CRLF) - 1
+ sizeof("Auth-Login-Attempt: ") - 1 + NGX_INT_T_LEN
+ sizeof(CRLF) - 1
- + sizeof("Client-IP: ") - 1 + s->connection->addr_text.len
+ + sizeof("Client-IP: ") - 1 + s->connection->peer.name.len
+ sizeof(CRLF) - 1
+ sizeof("Client-Host: ") - 1 + s->host.len + sizeof(CRLF) - 1
+ sizeof("Auth-SMTP-Helo: ") - 1 + s->smtp_helo.len
@@ -1212,8 +1212,8 @@
s->login_attempt);
b->last = ngx_cpymem(b->last, "Client-IP: ", sizeof("Client-IP: ") - 1);
- b->last = ngx_copy(b->last, s->connection->addr_text.data,
- s->connection->addr_text.len);
+ b->last = ngx_copy(b->last, s->connection->peer.name.data,
+ s->connection->peer.name.len);
*b->last++ = CR; *b->last++ = LF;
if (s->host.len) {
diff -r c5122335e41d src/mail/ngx_mail_handler.c
--- a/src/mail/ngx_mail_handler.c Mon Oct 18 00:00:00 2010 +0400
+++ b/src/mail/ngx_mail_handler.c Sun Nov 07 00:32:09 2010 +1000
@@ -129,7 +129,7 @@
s->connection = c;
ngx_log_error(NGX_LOG_INFO, c->log, 0, "*%ui client %V connected to %V",
- c->number, &c->addr_text, s->addr_text);
+ c->number, &c->peer.name, s->addr_text);
ctx = ngx_palloc(c->pool, sizeof(ngx_mail_log_ctx_t));
if (ctx == NULL) {
@@ -137,7 +137,7 @@
return;
}
- ctx->client = &c->addr_text;
+ ctx->client = &c->peer.name;
ctx->session = s;
c->log->connection = c->number;
diff -r c5122335e41d src/mail/ngx_mail_proxy_module.c
--- a/src/mail/ngx_mail_proxy_module.c Mon Oct 18 00:00:00 2010 +0400
+++ b/src/mail/ngx_mail_proxy_module.c Sun Nov 07 00:32:09 2010 +1000
@@ -539,7 +539,7 @@
line.len = sizeof("XCLIENT ADDR= LOGIN= NAME="
CRLF) - 1
- + s->connection->addr_text.len + s->login.len + s->host.len;
+ + s->connection->peer.name.len + s->login.len + s->host.len;
line.data = ngx_pnalloc(c->pool, line.len);
if (line.data == NULL) {
@@ -549,7 +549,7 @@
line.len = ngx_sprintf(line.data,
"XCLIENT ADDR=%V%s%V NAME=%V" CRLF,
- &s->connection->addr_text,
+ &s->connection->peer.name,
(s->login.len ? " LOGIN=" : ""), &s->login, &s->host)
- line.data;
diff -r c5122335e41d src/mail/ngx_mail_smtp_handler.c
--- a/src/mail/ngx_mail_smtp_handler.c Mon Oct 18 00:00:00 2010 +0400
+++ b/src/mail/ngx_mail_smtp_handler.c Sun Nov 07 00:32:09 2010 +1000
@@ -66,7 +66,7 @@
return;
}
- if (c->sockaddr->sa_family != AF_INET) {
+ if (c->peer.sockaddr->sa_family != AF_INET) {
s->host = smtp_tempunavail;
ngx_mail_smtp_greeting(s, c);
return;
@@ -82,7 +82,7 @@
/* AF_INET only */
- sin = (struct sockaddr_in *) c->sockaddr;
+ sin = (struct sockaddr_in *) c->peer.sockaddr;
ctx->addr = sin->sin_addr.s_addr;
ctx->handler = ngx_mail_smtp_resolve_addr_handler;
@@ -107,7 +107,7 @@
if (ctx->state) {
ngx_log_error(NGX_LOG_ERR, c->log, 0,
"%V could not be resolved (%i: %s)",
- &c->addr_text, ctx->state,
+ &c->peer.name, ctx->state,
ngx_resolver_strerror(ctx->state));
if (ctx->state == NGX_RESOLVE_NXDOMAIN) {
@@ -206,7 +206,7 @@
/* AF_INET only */
- sin = (struct sockaddr_in *) c->sockaddr;
+ sin = (struct sockaddr_in *) c->peer.sockaddr;
for (i = 0; i < ctx->naddrs; i++) {
More information about the nginx-devel
mailing list