[nginx] SNI: support for early ClientHello callback with BoringSSL.

[noreply at nginx.com]

details:   https://github.com/nginx/nginx/commit/7f9ced0ce0d70ae60f46ef3ed759efa75e711db4
branches:  master
commit:    7f9ced0ce0d70ae60f46ef3ed759efa75e711db4
user:      Sergey Kandaurov <pluknet at nginx.com>
date:      Mon, 22 Sep 2025 19:55:16 +0400
description:
SNI: support for early ClientHello callback with BoringSSL.

This brings feature parity with OpenSSL after the previous change,
making it possible to set SSL protocols per virtual server.

---
 src/event/ngx_event_openssl.c      | 36 ++++++++++++++++++++++++++++++++++++
 src/event/ngx_event_openssl.h      |  3 +++
 src/http/ngx_http_request.c        |  5 +++++
 src/stream/ngx_stream_ssl_module.c |  5 +++++
 4 files changed, 49 insertions(+)

diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
index d9abcd082..375d58be6 100644
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -1663,6 +1663,11 @@ ngx_ssl_set_client_hello_callback(SSL_CTX *ssl_ctx,
     SSL_CTX_set_client_hello_cb(ssl_ctx, ngx_ssl_client_hello_callback, NULL);
     SSL_CTX_set_ex_data(ssl_ctx, ngx_ssl_client_hello_arg_index, cb);
 
+#elif defined OPENSSL_IS_BORINGSSL
+
+    SSL_CTX_set_select_certificate_cb(ssl_ctx, ngx_ssl_select_certificate);
+    SSL_CTX_set_ex_data(ssl_ctx, ngx_ssl_client_hello_arg_index, cb);
+
 #endif
 }
 
@@ -1727,6 +1732,37 @@ done:
     return SSL_CLIENT_HELLO_SUCCESS;
 }
 
+#elif defined OPENSSL_IS_BORINGSSL
+
+enum ssl_select_cert_result_t ngx_ssl_select_certificate(
+    const SSL_CLIENT_HELLO *client_hello)
+{
+    int                        ad;
+    ngx_int_t                  rc;
+    ngx_ssl_conn_t            *ssl_conn;
+    ngx_connection_t          *c;
+    ngx_ssl_client_hello_arg  *cb;
+
+    ssl_conn = client_hello->ssl;
+    c = ngx_ssl_get_connection(ssl_conn);
+    cb = SSL_CTX_get_ex_data(c->ssl->session_ctx,
+                             ngx_ssl_client_hello_arg_index);
+
+    /*
+     * BoringSSL sends a hardcoded "handshake_failure" alert on errors,
+     * we use it to map SSL_AD_INTERNAL_ERROR.  To preserve other alert
+     * values, error handling is postponed to the servername callback.
+     */
+
+    rc = cb->servername(ssl_conn, &ad, NULL);
+
+    if (rc == SSL_TLSEXT_ERR_ALERT_FATAL && ad == SSL_AD_INTERNAL_ERROR) {
+        return ssl_select_cert_error;
+    }
+
+    return ssl_select_cert_success;
+}
+
 #endif
 
 
diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h
index 544703f61..9943ee430 100644
--- a/src/event/ngx_event_openssl.h
+++ b/src/event/ngx_event_openssl.h
@@ -298,6 +298,9 @@ void ngx_ssl_set_client_hello_callback(SSL_CTX *ssl_ctx,
     ngx_ssl_client_hello_arg *cb);
 #ifdef SSL_CLIENT_HELLO_SUCCESS
 int ngx_ssl_client_hello_callback(ngx_ssl_conn_t *ssl_conn, int *ad, void *arg);
+#elif defined OPENSSL_IS_BORINGSSL
+enum ssl_select_cert_result_t ngx_ssl_select_certificate(
+    const SSL_CLIENT_HELLO *client_hello);
 #endif
 
 ngx_int_t ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c,
diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c
index 6f6e975b7..16d79c490 100644
--- a/src/http/ngx_http_request.c
+++ b/src/http/ngx_http_request.c
@@ -895,6 +895,11 @@ ngx_http_ssl_servername(ngx_ssl_conn_t *ssl_conn, int *ad, void *arg)
         return SSL_TLSEXT_ERR_OK;
     }
 
+    if (c->ssl->handshake_rejected) {
+        *ad = SSL_AD_UNRECOGNIZED_NAME;
+        return SSL_TLSEXT_ERR_ALERT_FATAL;
+    }
+
     hc = c->data;
 
     if (arg != NULL) {
diff --git a/src/stream/ngx_stream_ssl_module.c b/src/stream/ngx_stream_ssl_module.c
index 7bf6304e4..75938b0a2 100644
--- a/src/stream/ngx_stream_ssl_module.c
+++ b/src/stream/ngx_stream_ssl_module.c
@@ -559,6 +559,11 @@ ngx_stream_ssl_servername(ngx_ssl_conn_t *ssl_conn, int *ad, void *arg)
         return SSL_TLSEXT_ERR_OK;
     }
 
+    if (c->ssl->handshake_rejected) {
+        *ad = SSL_AD_UNRECOGNIZED_NAME;
+        return SSL_TLSEXT_ERR_ALERT_FATAL;
+    }
+
     s = c->data;
 
     if (arg) {