[PATCH] SSL: SSL_CTX_set_tlsext_ticket_key_cb() deprecated in OpenSSL 3.0

[Sergey Kandaurov]

# HG changeset patch
# User Sergey Kandaurov <pluknet at nginx.com>
# Date 1671069897 -14400
#      Thu Dec 15 06:04:57 2022 +0400
# Node ID 8fbae86083f2efda8b4e079b3bda148dec220323
# Parent  c38588d8376b77fc2f56f90ca16533031b235491
SSL: SSL_CTX_set_tlsext_ticket_key_cb() deprecated in OpenSSL 3.0.

It becomes hidden when OpenSSL is built with OPENSSL_NO_DEPRECATED.
While this is manageable for the ssl_session_ticket_key directive,
rotation of ticket keys stored in shared memory is silently disabled.

Switch to SSL_CTX_set_tlsext_ticket_key_evp_cb() whenever available.
A macro similar to SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB isn't provided,
so the feature test uses OSSL_PARAM_octet_string as a close relative.
Using the documented macro OSSL_MAC_PARAM_KEY is considered worthless
as this requires to conditionally include an additional OpenSSL header.

diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -12,6 +12,14 @@
 
 #define NGX_SSL_PASSWORD_BUFFER_SIZE  4096
 
+#ifdef OSSL_PARAM_octet_string
+#define ngx_ssl_mac_ctx               EVP_MAC_CTX
+#define ngx_ssl_ctx_ticket_key_cb     SSL_CTX_set_tlsext_ticket_key_evp_cb
+#elif defined SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB
+#define ngx_ssl_mac_ctx               HMAC_CTX
+#define ngx_ssl_ctx_ticket_key_cb     SSL_CTX_set_tlsext_ticket_key_cb
+#endif
+
 
 typedef struct {
     ngx_uint_t  engine;   /* unsigned  engine:1; */
@@ -70,10 +78,10 @@ static void ngx_ssl_expire_sessions(ngx_
 static void ngx_ssl_session_rbtree_insert_value(ngx_rbtree_node_t *temp,
     ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel);
 
-#ifdef SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB
+#ifdef ngx_ssl_ctx_ticket_key_cb
 static int ngx_ssl_ticket_key_callback(ngx_ssl_conn_t *ssl_conn,
     unsigned char *name, unsigned char *iv, EVP_CIPHER_CTX *ectx,
-    HMAC_CTX *hctx, int enc);
+    ngx_ssl_mac_ctx *hctx, int enc);
 static ngx_int_t ngx_ssl_rotate_ticket_keys(SSL_CTX *ssl_ctx, ngx_log_t *log);
 static void ngx_ssl_ticket_keys_cleanup(void *data);
 #endif
@@ -4281,7 +4289,7 @@ ngx_ssl_session_rbtree_insert_value(ngx_
 }
 
 
-#ifdef SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB
+#ifdef ngx_ssl_ctx_ticket_key_cb
 
 ngx_int_t
 ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_array_t *paths)
@@ -4323,7 +4331,7 @@ ngx_ssl_session_ticket_keys(ngx_conf_t *
         return NGX_ERROR;
     }
 
-    if (SSL_CTX_set_tlsext_ticket_key_cb(ssl->ctx, ngx_ssl_ticket_key_callback)
+    if (ngx_ssl_ctx_ticket_key_cb(ssl->ctx, ngx_ssl_ticket_key_callback)
         == 0)
     {
         ngx_log_error(NGX_LOG_WARN, cf->log, 0,
@@ -4445,10 +4453,13 @@ failed:
 static int
 ngx_ssl_ticket_key_callback(ngx_ssl_conn_t *ssl_conn,
     unsigned char *name, unsigned char *iv, EVP_CIPHER_CTX *ectx,
-    HMAC_CTX *hctx, int enc)
+    ngx_ssl_mac_ctx *hctx, int enc)
 {
     size_t                 size;
     SSL_CTX               *ssl_ctx;
+#ifdef OSSL_PARAM_octet_string
+    OSSL_PARAM             params[3];
+#endif
     ngx_uint_t             i;
     ngx_array_t           *keys;
     ngx_connection_t      *c;
@@ -4504,7 +4515,22 @@ ngx_ssl_ticket_key_callback(ngx_ssl_conn
             return -1;
         }
 
-#if OPENSSL_VERSION_NUMBER >= 0x10000000L
+#ifdef OSSL_PARAM_octet_string
+
+        params[0] = OSSL_PARAM_construct_octet_string("key",
+                                                      key[0].hmac_key, size);
+        params[1] = OSSL_PARAM_construct_utf8_string("digest",
+                                                  (char *) EVP_MD_name(digest),
+                                                     0);
+        params[2] = OSSL_PARAM_construct_end();
+
+        if (!EVP_MAC_CTX_set_params(hctx, params)) {
+            ngx_ssl_error(NGX_LOG_ALERT, c->log, 0,
+                          "EVP_MAC_CTX_set_params() failed");
+            return -1;
+        }
+
+#elif OPENSSL_VERSION_NUMBER >= 0x10000000L
         if (HMAC_Init_ex(hctx, key[0].hmac_key, size, digest, NULL) != 1) {
             ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, "HMAC_Init_ex() failed");
             return -1;
@@ -4547,7 +4573,22 @@ ngx_ssl_ticket_key_callback(ngx_ssl_conn
             size = 32;
         }
 
-#if OPENSSL_VERSION_NUMBER >= 0x10000000L
+#ifdef OSSL_PARAM_octet_string
+
+        params[0] = OSSL_PARAM_construct_octet_string("key",
+                                                      key[i].hmac_key, size);
+        params[1] = OSSL_PARAM_construct_utf8_string("digest",
+                                                  (char *) EVP_MD_name(digest),
+                                                     0);
+        params[2] = OSSL_PARAM_construct_end();
+
+        if (!EVP_MAC_CTX_set_params(hctx, params)) {
+            ngx_ssl_error(NGX_LOG_ALERT, c->log, 0,
+                          "EVP_MAC_CTX_set_params() failed");
+            return -1;
+        }
+
+#elif OPENSSL_VERSION_NUMBER >= 0x10000000L
         if (HMAC_Init_ex(hctx, key[i].hmac_key, size, digest, NULL) != 1) {
             ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, "HMAC_Init_ex() failed");
             return -1;