[PATCH 3 of 3] Stream: ngx_stream_pass_module
Sergey Kandaurov
pluknet at nginx.com
Fri Dec 15 15:46:01 UTC 2023
> On 10 Nov 2023, at 14:07, Roman Arutyunyan <arut at nginx.com> wrote:
>
> # HG changeset patch
> # User Roman Arutyunyan <arut at nginx.com>
> # Date 1699543504 -14400
> # Thu Nov 09 19:25:04 2023 +0400
> # Node ID 3cab85fe55272835674b7f1c296796955256d019
> # Parent 1d3464283405a4d8ac54caae9bf1815c723f04c5
> Stream: ngx_stream_pass_module.
>
> The module allows to pass connections from Stream to other modules such as HTTP
> or Mail, as well as back to Stream. Previously, this was only possible with
> proxying. Connections with preread buffer read out from socket cannot be
> passed.
>
> The module allows to terminate SSL selectively based on SNI.
Note that terminating SSL before passing connections may be challenging
and requires appropriate preparations in the Stream module itself.
Notably, Stream unconditionally disables SSL buffering now,
which may affect performance in the modules being passed.
The following patches are to address such deficiency (and effectively,
they are used to catch up differences with the http module).
# HG changeset patch
# User Sergey Kandaurov <pluknet at nginx.com>
# Date 1702653986 -14400
# Fri Dec 15 19:26:26 2023 +0400
# Node ID aed7c0875fc690ccaa7288f66624ddca8299bff5
# Parent 219662ea1613ab68d4d5d4085394bba75993ae42
SSL: make it possible to disable SSL buffering.
It is now disabled if "ssl_buffer_size 0;" is used in the configuration.
Previously, such configuration was meaningless and resulted in CPU hog.
diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -1709,7 +1709,7 @@ ngx_ssl_create_connection(ngx_ssl_t *ssl
return NGX_ERROR;
}
- sc->buffer = ((flags & NGX_SSL_BUFFER) != 0);
+ sc->buffer = ((flags & NGX_SSL_BUFFER) != 0) && (ssl->buffer_size != 0);
sc->buffer_size = ssl->buffer_size;
sc->session_ctx = ssl->ctx;
diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c
--- a/src/http/ngx_http_request.c
+++ b/src/http/ngx_http_request.c
@@ -947,6 +947,7 @@ ngx_http_ssl_servername(ngx_ssl_conn_t *
sscf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_ssl_module);
+ c->ssl->buffer = (sscf->buffer_size != 0);
c->ssl->buffer_size = sscf->buffer_size;
if (sscf->ssl.ctx) {
# HG changeset patch
# User Sergey Kandaurov <pluknet at nginx.com>
# Date 1702653990 -14400
# Fri Dec 15 19:26:30 2023 +0400
# Node ID 272afdf3fe232dd9770c8644a5230830192b434d
# Parent aed7c0875fc690ccaa7288f66624ddca8299bff5
Stream: the "ssl_buffer_size" directive.
This change introduces behaviour change in SSL buffering, which is now
enabled by default. It can be disabled with "ssl_buffer_size 0;".
Previously, it was unconditionally disabled.
In particular, it is useful when Stream passes SSL connections after SSL
termination to other modules using ngx_stream_pass_module.
diff --git a/src/stream/ngx_stream_ssl_module.c b/src/stream/ngx_stream_ssl_module.c
--- a/src/stream/ngx_stream_ssl_module.c
+++ b/src/stream/ngx_stream_ssl_module.c
@@ -142,6 +142,13 @@ static ngx_command_t ngx_stream_ssl_com
offsetof(ngx_stream_ssl_srv_conf_t, ciphers),
NULL },
+ { ngx_string("ssl_buffer_size"),
+ NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1,
+ ngx_conf_set_size_slot,
+ NGX_STREAM_SRV_CONF_OFFSET,
+ offsetof(ngx_stream_ssl_srv_conf_t, buffer_size),
+ NULL },
+
{ ngx_string("ssl_verify_client"),
NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1,
ngx_conf_set_enum_slot,
@@ -414,7 +421,7 @@ ngx_stream_ssl_init_connection(ngx_ssl_t
return NGX_ERROR;
}
- if (ngx_ssl_create_connection(ssl, c, 0) != NGX_OK) {
+ if (ngx_ssl_create_connection(ssl, c, NGX_SSL_BUFFER) != NGX_OK) {
return NGX_ERROR;
}
@@ -527,6 +534,9 @@ ngx_stream_ssl_servername(ngx_ssl_conn_t
sscf = ngx_stream_get_module_srv_conf(s, ngx_stream_ssl_module);
+ c->ssl->buffer = (sscf->buffer_size != 0);
+ c->ssl->buffer_size = sscf->buffer_size;
+
if (sscf->ssl.ctx) {
if (SSL_set_SSL_CTX(ssl_conn, sscf->ssl.ctx) == NULL) {
goto error;
@@ -786,6 +796,7 @@ ngx_stream_ssl_create_srv_conf(ngx_conf_
sscf->conf_commands = NGX_CONF_UNSET_PTR;
sscf->prefer_server_ciphers = NGX_CONF_UNSET;
sscf->reject_handshake = NGX_CONF_UNSET;
+ sscf->buffer_size = NGX_CONF_UNSET_SIZE;
sscf->verify = NGX_CONF_UNSET_UINT;
sscf->verify_depth = NGX_CONF_UNSET_UINT;
sscf->builtin_session_cache = NGX_CONF_UNSET;
@@ -821,6 +832,9 @@ ngx_stream_ssl_merge_srv_conf(ngx_conf_t
|NGX_SSL_TLSv1|NGX_SSL_TLSv1_1
|NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3));
+ ngx_conf_merge_size_value(conf->buffer_size, prev->buffer_size,
+ NGX_SSL_BUFSIZE);
+
ngx_conf_merge_uint_value(conf->verify, prev->verify, 0);
ngx_conf_merge_uint_value(conf->verify_depth, prev->verify_depth, 1);
@@ -930,6 +944,8 @@ ngx_stream_ssl_merge_srv_conf(ngx_conf_t
}
}
+ conf->ssl.buffer_size = conf->buffer_size;
+
if (conf->verify) {
if (conf->client_certificate.len == 0 && conf->verify != 3) {
diff --git a/src/stream/ngx_stream_ssl_module.h b/src/stream/ngx_stream_ssl_module.h
--- a/src/stream/ngx_stream_ssl_module.h
+++ b/src/stream/ngx_stream_ssl_module.h
@@ -27,6 +27,8 @@ typedef struct {
ngx_uint_t verify;
ngx_uint_t verify_depth;
+ size_t buffer_size;
+
ssize_t builtin_session_cache;
time_t session_timeout;
>
> stream {
> server {
> listen 8000 default_server;
> ssl_preread on;
> ...
> }
>
> server {
> listen 8000;
> server_name foo.example.com;
> pass 8001; # to HTTP
> }
>
> server {
> listen 8000;
> server_name bar.example.com;
> ...
> }
> }
>
> http {
> server {
> listen 8001 ssl;
> ...
>
> location / {
> root html;
> }
> }
> }
>
> diff --git a/auto/modules b/auto/modules
> --- a/auto/modules
> +++ b/auto/modules
> @@ -1166,6 +1166,16 @@ if [ $STREAM != NO ]; then
> . auto/module
> fi
>
> + if [ $STREAM_PASS = YES ]; then
> + ngx_module_name=ngx_stream_pass_module
> + ngx_module_deps=
> + ngx_module_srcs=src/stream/ngx_stream_pass_module.c
> + ngx_module_libs=
> + ngx_module_link=$STREAM_PASS
> +
> + . auto/module
> + fi
> +
> if [ $STREAM_SET = YES ]; then
> ngx_module_name=ngx_stream_set_module
> ngx_module_deps=
> diff --git a/auto/options b/auto/options
> --- a/auto/options
> +++ b/auto/options
> @@ -127,6 +127,7 @@ STREAM_GEOIP=NO
> STREAM_MAP=YES
> STREAM_SPLIT_CLIENTS=YES
> STREAM_RETURN=YES
> +STREAM_PASS=YES
> STREAM_SET=YES
> STREAM_UPSTREAM_HASH=YES
> STREAM_UPSTREAM_LEAST_CONN=YES
> @@ -337,6 +338,7 @@ use the \"--with-mail_ssl_module\" optio
> --without-stream_split_clients_module)
> STREAM_SPLIT_CLIENTS=NO ;;
> --without-stream_return_module) STREAM_RETURN=NO ;;
> + --without-stream_pass_module) STREAM_PASS=NO ;;
> --without-stream_set_module) STREAM_SET=NO ;;
> --without-stream_upstream_hash_module)
> STREAM_UPSTREAM_HASH=NO ;;
> @@ -556,6 +558,7 @@ cat << END
> --without-stream_split_clients_module
> disable ngx_stream_split_clients_module
> --without-stream_return_module disable ngx_stream_return_module
> + --without-stream_pass_module disable ngx_stream_pass_module
> --without-stream_set_module disable ngx_stream_set_module
> --without-stream_upstream_hash_module
> disable ngx_stream_upstream_hash_module
> diff --git a/src/stream/ngx_stream_pass_module.c b/src/stream/ngx_stream_pass_module.c
> new file mode 100644
> --- /dev/null
> +++ b/src/stream/ngx_stream_pass_module.c
> @@ -0,0 +1,245 @@
> +
> +/*
> + * Copyright (C) Roman Arutyunyan
> + * Copyright (C) Nginx, Inc.
> + */
> +
> +
> +#include <ngx_config.h>
> +#include <ngx_core.h>
> +#include <ngx_stream.h>
> +
> +
> +typedef struct {
> + ngx_addr_t *addr;
> + ngx_stream_complex_value_t *addr_value;
> +} ngx_stream_pass_srv_conf_t;
> +
> +
> +static void ngx_stream_pass_handler(ngx_stream_session_t *s);
> +static void *ngx_stream_pass_create_srv_conf(ngx_conf_t *cf);
> +static char *ngx_stream_pass(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
> +
> +
> +static ngx_command_t ngx_stream_pass_commands[] = {
> +
> + { ngx_string("pass"),
> + NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1,
> + ngx_stream_pass,
> + NGX_STREAM_SRV_CONF_OFFSET,
> + 0,
> + NULL },
> +
> + ngx_null_command
> +};
> +
> +
> +static ngx_stream_module_t ngx_stream_pass_module_ctx = {
> + NULL, /* preconfiguration */
> + NULL, /* postconfiguration */
> +
> + NULL, /* create main configuration */
> + NULL, /* init main configuration */
> +
> + ngx_stream_pass_create_srv_conf, /* create server configuration */
> + NULL /* merge server configuration */
> +};
> +
> +
> +ngx_module_t ngx_stream_pass_module = {
> + NGX_MODULE_V1,
> + &ngx_stream_pass_module_ctx, /* module conaddr */
> + ngx_stream_pass_commands, /* module directives */
> + NGX_STREAM_MODULE, /* module type */
> + NULL, /* init master */
> + NULL, /* init module */
> + NULL, /* init process */
> + NULL, /* init thread */
> + NULL, /* exit thread */
> + NULL, /* exit process */
> + NULL, /* exit master */
> + NGX_MODULE_V1_PADDING
> +};
> +
> +
> +static void
> +ngx_stream_pass_handler(ngx_stream_session_t *s)
> +{
> + ngx_url_t u;
> + ngx_str_t url;
> + ngx_addr_t *addr;
> + ngx_uint_t i;
> + ngx_listening_t *ls;
> + ngx_connection_t *c;
> + ngx_stream_pass_srv_conf_t *pscf;
> +
> + c = s->connection;
> +
> + c->log->action = "passing connection to another module";
> +
> + if (c->buffer && c->buffer->pos != c->buffer->last) {
> + ngx_log_error(NGX_LOG_ERR, s->connection->log, 0,
> + "cannot pass connection with preread data");
> + goto failed;
> + }
> +
> + pscf = ngx_stream_get_module_srv_conf(s, ngx_stream_pass_module);
> +
> + addr = pscf->addr;
> +
> + if (addr == NULL) {
> + if (ngx_stream_complex_value(s, pscf->addr_value, &url) != NGX_OK) {
> + goto failed;
> + }
> +
> + ngx_memzero(&u, sizeof(ngx_url_t));
> +
> + u.url = url;
> + u.listen = 1;
> + u.no_resolve = 1;
> +
> + if (ngx_parse_url(s->connection->pool, &u) != NGX_OK) {
> + if (u.err) {
> + ngx_log_error(NGX_LOG_ERR, s->connection->log, 0,
> + "%s in pass \"%V\"", u.err, &u.url);
> + }
> +
> + goto failed;
> + }
> +
> + if (u.naddrs == 0) {
> + ngx_log_error(NGX_LOG_ERR, s->connection->log, 0,
> + "no addresses in pass \"%V\"", &u.url);
> + goto failed;
> + }
> +
> + addr = &u.addrs[0];
> + }
> +
> + ngx_log_debug1(NGX_LOG_DEBUG_STREAM, c->log, 0,
> + "stream pass addr: \"%V\"", &addr->name);
> +
> + ls = ngx_cycle->listening.elts;
> +
> + for (i = 0; i < ngx_cycle->listening.nelts; i++) {
> + if (ngx_cmp_sockaddr(ls[i].sockaddr, ls[i].socklen,
> + addr->sockaddr, addr->socklen, 1)
> + == NGX_OK)
> + {
> + c->listening = &ls[i];
> +
> + c->data = NULL;
> + c->buffer = NULL;
> +
> + *c->log = c->listening->log;
> + c->log->handler = NULL;
> + c->log->data = NULL;
> +
> + c->listening->handler(c);
> +
> + return;
> + }
> + }
> +
> + ngx_log_error(NGX_LOG_ERR, c->log, 0,
> + "listen not found for \"%V\"", &addr->name);
> +
> + ngx_stream_finalize_session(s, NGX_STREAM_OK);
> +
> + return;
> +
> +failed:
> +
> + ngx_stream_finalize_session(s, NGX_STREAM_INTERNAL_SERVER_ERROR);
> +}
> +
> +
> +static void *
> +ngx_stream_pass_create_srv_conf(ngx_conf_t *cf)
> +{
> + ngx_stream_pass_srv_conf_t *conf;
> +
> + conf = ngx_pcalloc(cf->pool, sizeof(ngx_stream_pass_srv_conf_t));
> + if (conf == NULL) {
> + return NULL;
> + }
> +
> + /*
> + * set by ngx_pcalloc():
> + *
> + * conf->addr = NULL;
> + * conf->addr_value = NULL;
> + */
> +
> + return conf;
> +}
> +
> +
> +static char *
> +ngx_stream_pass(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
> +{
> + ngx_stream_pass_srv_conf_t *pscf = conf;
> +
> + ngx_url_t u;
> + ngx_str_t *value, *url;
> + ngx_stream_complex_value_t cv;
> + ngx_stream_core_srv_conf_t *cscf;
> + ngx_stream_compile_complex_value_t ccv;
> +
> + if (pscf->addr || pscf->addr_value) {
> + return "is duplicate";
> + }
> +
> + cscf = ngx_stream_conf_get_module_srv_conf(cf, ngx_stream_core_module);
> +
> + cscf->handler = ngx_stream_pass_handler;
> +
> + value = cf->args->elts;
> +
> + url = &value[1];
> +
> + ngx_memzero(&ccv, sizeof(ngx_stream_compile_complex_value_t));
> +
> + ccv.cf = cf;
> + ccv.value = url;
> + ccv.complex_value = &cv;
> +
> + if (ngx_stream_compile_complex_value(&ccv) != NGX_OK) {
> + return NGX_CONF_ERROR;
> + }
> +
> + if (cv.lengths) {
> + pscf->addr_value = ngx_palloc(cf->pool,
> + sizeof(ngx_stream_complex_value_t));
> + if (pscf->addr_value == NULL) {
> + return NGX_CONF_ERROR;
> + }
> +
> + *pscf->addr_value = cv;
> +
> + return NGX_CONF_OK;
> + }
> +
> + ngx_memzero(&u, sizeof(ngx_url_t));
> +
> + u.url = *url;
> + u.listen = 1;
> +
> + if (ngx_parse_url(cf->pool, &u) != NGX_OK) {
> + if (u.err) {
> + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
> + "%s in \"%V\" of the \"pass\" directive",
> + u.err, &u.url);
> + }
> +
> + return NGX_CONF_ERROR;
> + }
> +
> + if (u.naddrs == 0) {
> + return "has no addresses";
> + }
> +
> + pscf->addr = &u.addrs[0];
> +
> + return NGX_CONF_OK;
> +}
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> https://mailman.nginx.org/mailman/listinfo/nginx-devel
--
Sergey Kandaurov
More information about the nginx-devel
mailing list