[nginx] SSL: save sessions for upstream peers using a callback function.

Sergey Kandaurov pluknet at nginx.com
Tue Jul 17 11:50:07 UTC 2018


details:   http://hg.nginx.org/nginx/rev/696df3ac27ac
branches:  
changeset: 7320:696df3ac27ac
user:      Sergey Kandaurov <pluknet at nginx.com>
date:      Tue Jul 17 12:53:23 2018 +0300
description:
SSL: save sessions for upstream peers using a callback function.

In TLSv1.3, NewSessionTicket messages arrive after the handshake and
can come at any time.  Therefore we use a callback to save the session
when we know about it.  This approach works for < TLSv1.3 as well.
The callback function is set once per location on merge phase.

Since SSL_get_session() in BoringSSL returns an unresumable session for
TLSv1.3, peer save_session() methods have been updated as well to use a
session supplied within the callback.  To preserve API, the session is
cached in c->ssl->session.  It is preferably accessed in save_session()
methods by ngx_ssl_get_session() and ngx_ssl_get0_session() wrappers.

diffstat:

 src/event/ngx_event_openssl.c                |  63 ++++++++++++++++++++++++++++
 src/event/ngx_event_openssl.h                |   8 +++-
 src/http/modules/ngx_http_grpc_module.c      |   7 +++
 src/http/modules/ngx_http_proxy_module.c     |   7 +++
 src/http/modules/ngx_http_uwsgi_module.c     |   7 +++
 src/http/ngx_http_upstream.c                 |  28 ++++++++++-
 src/http/ngx_http_upstream_round_robin.c     |   2 +-
 src/stream/ngx_stream_proxy_module.c         |  27 +++++++++--
 src/stream/ngx_stream_upstream_round_robin.c |   2 +-
 9 files changed, 139 insertions(+), 12 deletions(-)

diffs (318 lines):

diff -r dcab86115261 -r 696df3ac27ac src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c	Mon Jul 16 17:47:48 2018 +0300
+++ b/src/event/ngx_event_openssl.c	Tue Jul 17 12:53:23 2018 +0300
@@ -24,6 +24,8 @@ static int ngx_ssl_verify_callback(int o
 static void ngx_ssl_info_callback(const ngx_ssl_conn_t *ssl_conn, int where,
     int ret);
 static void ngx_ssl_passwords_cleanup(void *data);
+static int ngx_ssl_new_client_session(ngx_ssl_conn_t *ssl_conn,
+    ngx_ssl_session_t *sess);
 static void ngx_ssl_handshake_handler(ngx_event_t *ev);
 static ngx_int_t ngx_ssl_handle_recv(ngx_connection_t *c, int n);
 static void ngx_ssl_write_handler(ngx_event_t *wev);
@@ -1162,6 +1164,42 @@ ngx_ssl_ecdh_curve(ngx_conf_t *cf, ngx_s
 
 
 ngx_int_t
+ngx_ssl_client_session_cache(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_uint_t enable)
+{
+    if (!enable) {
+        return NGX_OK;
+    }
+
+    SSL_CTX_set_session_cache_mode(ssl->ctx,
+                                   SSL_SESS_CACHE_CLIENT
+                                   |SSL_SESS_CACHE_NO_INTERNAL);
+
+    SSL_CTX_sess_set_new_cb(ssl->ctx, ngx_ssl_new_client_session);
+
+    return NGX_OK;
+}
+
+
+static int
+ngx_ssl_new_client_session(ngx_ssl_conn_t *ssl_conn, ngx_ssl_session_t *sess)
+{
+    ngx_connection_t  *c;
+
+    c = ngx_ssl_get_connection(ssl_conn);
+
+    if (c->ssl->save_session) {
+        c->ssl->session = sess;
+
+        c->ssl->save_session(c);
+
+        c->ssl->session = NULL;
+    }
+
+    return 0;
+}
+
+
+ngx_int_t
 ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags)
 {
     ngx_ssl_connection_t  *sc;
@@ -1210,6 +1248,31 @@ ngx_ssl_create_connection(ngx_ssl_t *ssl
 }
 
 
+ngx_ssl_session_t *
+ngx_ssl_get_session(ngx_connection_t *c)
+{
+#ifdef TLS1_3_VERSION
+    if (c->ssl->session) {
+        SSL_SESSION_up_ref(c->ssl->session);
+        return c->ssl->session;
+    }
+#endif
+
+    return SSL_get1_session(c->ssl->connection);
+}
+
+
+ngx_ssl_session_t *
+ngx_ssl_get0_session(ngx_connection_t *c)
+{
+    if (c->ssl->session) {
+        return c->ssl->session;
+    }
+
+    return SSL_get0_session(c->ssl->connection);
+}
+
+
 ngx_int_t
 ngx_ssl_set_session(ngx_connection_t *c, ngx_ssl_session_t *session)
 {
diff -r dcab86115261 -r 696df3ac27ac src/event/ngx_event_openssl.h
--- a/src/event/ngx_event_openssl.h	Mon Jul 16 17:47:48 2018 +0300
+++ b/src/event/ngx_event_openssl.h	Tue Jul 17 12:53:23 2018 +0300
@@ -77,6 +77,9 @@ struct ngx_ssl_connection_s {
 
     ngx_connection_handler_pt   handler;
 
+    ngx_ssl_session_t          *session;
+    ngx_connection_handler_pt   save_session;
+
     ngx_event_handler_pt        saved_read_handler;
     ngx_event_handler_pt        saved_write_handler;
 
@@ -168,6 +171,8 @@ RSA *ngx_ssl_rsa512_key_callback(ngx_ssl
 ngx_array_t *ngx_ssl_read_password_file(ngx_conf_t *cf, ngx_str_t *file);
 ngx_int_t ngx_ssl_dhparam(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file);
 ngx_int_t ngx_ssl_ecdh_curve(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *name);
+ngx_int_t ngx_ssl_client_session_cache(ngx_conf_t *cf, ngx_ssl_t *ssl,
+    ngx_uint_t enable);
 ngx_int_t ngx_ssl_session_cache(ngx_ssl_t *ssl, ngx_str_t *sess_ctx,
     ssize_t builtin_session_cache, ngx_shm_zone_t *shm_zone, time_t timeout);
 ngx_int_t ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl,
@@ -178,7 +183,8 @@ ngx_int_t ngx_ssl_create_connection(ngx_
 
 void ngx_ssl_remove_cached_session(SSL_CTX *ssl, ngx_ssl_session_t *sess);
 ngx_int_t ngx_ssl_set_session(ngx_connection_t *c, ngx_ssl_session_t *session);
-#define ngx_ssl_get_session(c)      SSL_get1_session(c->ssl->connection)
+ngx_ssl_session_t *ngx_ssl_get_session(ngx_connection_t *c);
+ngx_ssl_session_t *ngx_ssl_get0_session(ngx_connection_t *c);
 #define ngx_ssl_free_session        SSL_SESSION_free
 #define ngx_ssl_get_connection(ssl_conn)                                      \
     SSL_get_ex_data(ssl_conn, ngx_ssl_connection_index)
diff -r dcab86115261 -r 696df3ac27ac src/http/modules/ngx_http_grpc_module.c
--- a/src/http/modules/ngx_http_grpc_module.c	Mon Jul 16 17:47:48 2018 +0300
+++ b/src/http/modules/ngx_http_grpc_module.c	Tue Jul 17 12:53:23 2018 +0300
@@ -4627,6 +4627,13 @@ ngx_http_grpc_set_ssl(ngx_conf_t *cf, ng
         }
     }
 
+    if (ngx_ssl_client_session_cache(cf, glcf->upstream.ssl,
+                                     glcf->upstream.ssl_session_reuse)
+        != NGX_OK)
+    {
+        return NGX_ERROR;
+    }
+
 #ifdef TLSEXT_TYPE_application_layer_protocol_negotiation
 
     if (SSL_CTX_set_alpn_protos(glcf->upstream.ssl->ctx,
diff -r dcab86115261 -r 696df3ac27ac src/http/modules/ngx_http_proxy_module.c
--- a/src/http/modules/ngx_http_proxy_module.c	Mon Jul 16 17:47:48 2018 +0300
+++ b/src/http/modules/ngx_http_proxy_module.c	Tue Jul 17 12:53:23 2018 +0300
@@ -4308,6 +4308,13 @@ ngx_http_proxy_set_ssl(ngx_conf_t *cf, n
         }
     }
 
+    if (ngx_ssl_client_session_cache(cf, plcf->upstream.ssl,
+                                     plcf->upstream.ssl_session_reuse)
+        != NGX_OK)
+    {
+        return NGX_ERROR;
+    }
+
     return NGX_OK;
 }
 
diff -r dcab86115261 -r 696df3ac27ac src/http/modules/ngx_http_uwsgi_module.c
--- a/src/http/modules/ngx_http_uwsgi_module.c	Mon Jul 16 17:47:48 2018 +0300
+++ b/src/http/modules/ngx_http_uwsgi_module.c	Tue Jul 17 12:53:23 2018 +0300
@@ -2391,6 +2391,13 @@ ngx_http_uwsgi_set_ssl(ngx_conf_t *cf, n
         }
     }
 
+    if (ngx_ssl_client_session_cache(cf, uwcf->upstream.ssl,
+                                     uwcf->upstream.ssl_session_reuse)
+        != NGX_OK)
+    {
+        return NGX_ERROR;
+    }
+
     return NGX_OK;
 }
 
diff -r dcab86115261 -r 696df3ac27ac src/http/ngx_http_upstream.c
--- a/src/http/ngx_http_upstream.c	Mon Jul 16 17:47:48 2018 +0300
+++ b/src/http/ngx_http_upstream.c	Tue Jul 17 12:53:23 2018 +0300
@@ -187,6 +187,7 @@ static void ngx_http_upstream_ssl_init_c
 static void ngx_http_upstream_ssl_handshake_handler(ngx_connection_t *c);
 static void ngx_http_upstream_ssl_handshake(ngx_http_request_t *,
     ngx_http_upstream_t *u, ngx_connection_t *c);
+static void ngx_http_upstream_ssl_save_session(ngx_connection_t *c);
 static ngx_int_t ngx_http_upstream_ssl_name(ngx_http_request_t *r,
     ngx_http_upstream_t *u, ngx_connection_t *c);
 #endif
@@ -1675,6 +1676,8 @@ ngx_http_upstream_ssl_init_connection(ng
     }
 
     if (u->conf->ssl_session_reuse) {
+        c->ssl->save_session = ngx_http_upstream_ssl_save_session;
+
         if (u->peer.set_session(&u->peer, u->peer.data) != NGX_OK) {
             ngx_http_upstream_finalize_request(r, u,
                                                NGX_HTTP_INTERNAL_SERVER_ERROR);
@@ -1759,10 +1762,6 @@ ngx_http_upstream_ssl_handshake(ngx_http
             }
         }
 
-        if (u->conf->ssl_session_reuse) {
-            u->peer.save_session(&u->peer, u->peer.data);
-        }
-
         c->write->handler = ngx_http_upstream_handler;
         c->read->handler = ngx_http_upstream_handler;
 
@@ -1782,6 +1781,27 @@ failed:
 }
 
 
+static void
+ngx_http_upstream_ssl_save_session(ngx_connection_t *c)
+{
+    ngx_http_request_t   *r;
+    ngx_http_upstream_t  *u;
+
+    if (c->idle) {
+        return;
+    }
+
+    r = c->data;
+
+    u = r->upstream;
+    c = r->connection;
+
+    ngx_http_set_log_request(c->log, r);
+
+    u->peer.save_session(&u->peer, u->peer.data);
+}
+
+
 static ngx_int_t
 ngx_http_upstream_ssl_name(ngx_http_request_t *r, ngx_http_upstream_t *u,
     ngx_connection_t *c)
diff -r dcab86115261 -r 696df3ac27ac src/http/ngx_http_upstream_round_robin.c
--- a/src/http/ngx_http_upstream_round_robin.c	Mon Jul 16 17:47:48 2018 +0300
+++ b/src/http/ngx_http_upstream_round_robin.c	Tue Jul 17 12:53:23 2018 +0300
@@ -744,7 +744,7 @@ ngx_http_upstream_save_round_robin_peer_
 
     if (peers->shpool) {
 
-        ssl_session = SSL_get0_session(pc->connection->ssl->connection);
+        ssl_session = ngx_ssl_get0_session(pc->connection);
 
         if (ssl_session == NULL) {
             return;
diff -r dcab86115261 -r 696df3ac27ac src/stream/ngx_stream_proxy_module.c
--- a/src/stream/ngx_stream_proxy_module.c	Mon Jul 16 17:47:48 2018 +0300
+++ b/src/stream/ngx_stream_proxy_module.c	Tue Jul 17 12:53:23 2018 +0300
@@ -92,6 +92,7 @@ static char *ngx_stream_proxy_ssl_passwo
     ngx_command_t *cmd, void *conf);
 static void ngx_stream_proxy_ssl_init_connection(ngx_stream_session_t *s);
 static void ngx_stream_proxy_ssl_handshake(ngx_connection_t *pc);
+static void ngx_stream_proxy_ssl_save_session(ngx_connection_t *c);
 static ngx_int_t ngx_stream_proxy_ssl_name(ngx_stream_session_t *s);
 static ngx_int_t ngx_stream_proxy_set_ssl(ngx_conf_t *cf,
     ngx_stream_proxy_srv_conf_t *pscf);
@@ -1008,6 +1009,8 @@ ngx_stream_proxy_ssl_init_connection(ngx
     }
 
     if (pscf->ssl_session_reuse) {
+        pc->ssl->save_session = ngx_stream_proxy_ssl_save_session;
+
         if (u->peer.set_session(&u->peer, u->peer.data) != NGX_OK) {
             ngx_stream_proxy_finalize(s, NGX_STREAM_INTERNAL_SERVER_ERROR);
             return;
@@ -1066,11 +1069,6 @@ ngx_stream_proxy_ssl_handshake(ngx_conne
             }
         }
 
-        if (pscf->ssl_session_reuse) {
-            u = s->upstream;
-            u->peer.save_session(&u->peer, u->peer.data);
-        }
-
         if (pc->write->timer_set) {
             ngx_del_timer(pc->write);
         }
@@ -1086,6 +1084,19 @@ failed:
 }
 
 
+static void
+ngx_stream_proxy_ssl_save_session(ngx_connection_t *c)
+{
+    ngx_stream_session_t   *s;
+    ngx_stream_upstream_t  *u;
+
+    s = c->data;
+    u = s->upstream;
+
+    u->peer.save_session(&u->peer, u->peer.data);
+}
+
+
 static ngx_int_t
 ngx_stream_proxy_ssl_name(ngx_stream_session_t *s)
 {
@@ -2051,6 +2062,12 @@ ngx_stream_proxy_set_ssl(ngx_conf_t *cf,
         }
     }
 
+    if (ngx_ssl_client_session_cache(cf, pscf->ssl, pscf->ssl_session_reuse)
+        != NGX_OK)
+    {
+        return NGX_ERROR;
+    }
+
     return NGX_OK;
 }
 
diff -r dcab86115261 -r 696df3ac27ac src/stream/ngx_stream_upstream_round_robin.c
--- a/src/stream/ngx_stream_upstream_round_robin.c	Mon Jul 16 17:47:48 2018 +0300
+++ b/src/stream/ngx_stream_upstream_round_robin.c	Tue Jul 17 12:53:23 2018 +0300
@@ -776,7 +776,7 @@ ngx_stream_upstream_save_round_robin_pee
 
     if (peers->shpool) {
 
-        ssl_session = SSL_get0_session(pc->connection->ssl->connection);
+        ssl_session = ngx_ssl_get0_session(pc->connection);
 
         if (ssl_session == NULL) {
             return;


More information about the nginx-devel mailing list