[PATCH 2 of 2] SSL: support for TLSv1.3 certificate compression with BoringSSL

[Sergey Kandaurov]

# HG changeset patch
# User Sergey Kandaurov <pluknet at nginx.com>
# Date 1681304032 -14400
#      Wed Apr 12 16:53:52 2023 +0400
# Node ID 09a8a2f9aa68656ee45fd90119d4402c6f707a6f
# Parent  06458cd5733cd2ffaa4e2d26d357524a0934a7eb
SSL: support for TLSv1.3 certificate compression with BoringSSL.

Certificates are compressed with zlib and cached in SSL context exdata
on the first callback invocation.

diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -9,6 +9,10 @@
 #include <ngx_core.h>
 #include <ngx_event.h>
 
+#if (defined TLSEXT_cert_compression_zlib && NGX_ZLIB)
+#include <zlib.h>
+#endif
+
 
 #define NGX_SSL_PASSWORD_BUFFER_SIZE  4096
 
@@ -24,6 +28,10 @@ static EVP_PKEY *ngx_ssl_load_certificat
     ngx_str_t *key, ngx_array_t *passwords);
 static int ngx_ssl_password_callback(char *buf, int size, int rwflag,
     void *userdata);
+#if (defined TLSEXT_cert_compression_zlib && NGX_ZLIB)
+static int ngx_ssl_cert_compression_callback(ngx_ssl_conn_t *ssl_conn,
+    CBB *out, const uint8_t *in, size_t in_len);
+#endif
 static int ngx_ssl_verify_callback(int ok, X509_STORE_CTX *x509_store);
 static void ngx_ssl_info_callback(const ngx_ssl_conn_t *ssl_conn, int where,
     int ret);
@@ -137,6 +145,7 @@ int  ngx_ssl_ocsp_index;
 int  ngx_ssl_certificate_index;
 int  ngx_ssl_next_certificate_index;
 int  ngx_ssl_certificate_name_index;
+int  ngx_ssl_certificate_comp_index;
 int  ngx_ssl_stapling_index;
 
 
@@ -247,6 +256,14 @@ ngx_ssl_init(ngx_log_t *log)
         return NGX_ERROR;
     }
 
+    ngx_ssl_certificate_comp_index = SSL_CTX_get_ex_new_index(0, NULL, NULL,
+                                                              NULL, NULL);
+    if (ngx_ssl_certificate_comp_index == -1) {
+        ngx_ssl_error(NGX_LOG_ALERT, log, 0,
+                      "SSL_CTX_get_ex_new_index() failed");
+        return NGX_ERROR;
+    }
+
     ngx_ssl_stapling_index = X509_get_ex_new_index(0, NULL, NULL, NULL, NULL);
 
     if (ngx_ssl_stapling_index == -1) {
@@ -280,6 +297,14 @@ ngx_ssl_create(ngx_ssl_t *ssl, ngx_uint_
         return NGX_ERROR;
     }
 
+    if (SSL_CTX_set_ex_data(ssl->ctx, ngx_ssl_certificate_comp_index, NULL)
+        == 0)
+    {
+        ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0,
+                      "SSL_CTX_set_ex_data() failed");
+        return NGX_ERROR;
+    }
+
     ssl->buffer_size = NGX_SSL_BUFSIZE;
 
     /* client side options */
@@ -860,6 +885,15 @@ ngx_ssl_certificate_compression(ngx_conf
         return NGX_OK;
     }
 
+#elif (defined TLSEXT_cert_compression_zlib && NGX_ZLIB)
+
+    if (SSL_CTX_add_cert_compression_alg(ssl->ctx, TLSEXT_cert_compression_zlib,
+                                         ngx_ssl_cert_compression_callback,
+                                         NULL))
+    {
+        return NGX_OK;
+    }
+
 #endif
 
     ngx_log_error(NGX_LOG_WARN, ssl->log, 0,
@@ -869,6 +903,49 @@ ngx_ssl_certificate_compression(ngx_conf
 }
 
 
+#if (defined TLSEXT_cert_compression_zlib && NGX_ZLIB)
+
+static int
+ngx_ssl_cert_compression_callback(ngx_ssl_conn_t *ssl_conn, CBB *out,
+    const uint8_t *in, size_t in_len)
+{
+    SSL_CTX           *ssl_ctx;
+    ngx_str_t         *comp;
+    ngx_connection_t  *c;
+
+    ssl_ctx = SSL_get_SSL_CTX(ssl_conn);
+    comp = SSL_CTX_get_ex_data(ssl_ctx, ngx_ssl_certificate_comp_index);
+
+    if (comp == NULL) {
+        c = ngx_ssl_get_connection((ngx_ssl_conn_t *) ssl_conn);
+
+        comp = ngx_alloc(sizeof(ngx_str_t), c->log);
+        if (comp == NULL) {
+            return 0;
+        }
+
+        comp->len = compressBound(in_len);
+        comp->data = ngx_alloc(comp->len, c->log);
+        if (comp->data == NULL) {
+            ngx_free(comp);
+            return 0;
+        }
+
+        if (compress(comp->data, &comp->len, in, in_len) != Z_OK) {
+            ngx_free(comp->data);
+            ngx_free(comp);
+            return 0;
+        }
+
+        SSL_CTX_set_ex_data(ssl_ctx, ngx_ssl_certificate_comp_index, comp);
+    }
+
+    return CBB_add_bytes(out, comp->data, comp->len);
+}
+
+#endif
+
+
 ngx_int_t
 ngx_ssl_ciphers(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *ciphers,
     ngx_uint_t prefer_server_ciphers)
@@ -4832,7 +4909,8 @@ ngx_ssl_cleanup_ctx(void *data)
 {
     ngx_ssl_t  *ssl = data;
 
-    X509  *cert, *next;
+    X509       *cert, *next;
+    ngx_str_t  *comp;
 
     cert = SSL_CTX_get_ex_data(ssl->ctx, ngx_ssl_certificate_index);
 
@@ -4842,6 +4920,13 @@ ngx_ssl_cleanup_ctx(void *data)
         cert = next;
     }
 
+    comp = SSL_CTX_get_ex_data(ssl->ctx, ngx_ssl_certificate_comp_index);
+
+    if (comp != NULL) {
+        ngx_free(comp->data);
+        ngx_free(comp);
+    }
+
     SSL_CTX_free(ssl->ctx);
 }