[PATCH 2 of 2] SSL: avoid using OpenSSL config in build directory (ticket #2404)

[Maxim Dounin]

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1687300193 -10800
#      Wed Jun 21 01:29:53 2023 +0300
# Node ID bd2cc76ebe2367dc303e2746928b17ca8976b604
# Parent  b79ef48b91e45dba4bf850be6b2a2cc3b8834f5d
SSL: avoid using OpenSSL config in build directory (ticket #2404).

With this change, the NGX_OPENSSL_NO_CONFIG macro is defined when nginx
is asked to build OpenSSL itself.  And with this macro automatic loading
of OpenSSL configuration (from the build directory) is prevented unless
the OPENSSL_CONF environment variable is explicitly set.

Note that not loading configuration is broken in OpenSSL 1.1.1 and 1.1.1a
(fixed in OpenSSL 1.1.1b, see https://github.com/openssl/openssl/issues/7350).
If nginx is used to compile these OpenSSL versions, configuring nginx with
NGX_OPENSSL_NO_CONFIG explicitly set to 0 might be used as a workaround.

diff --git a/auto/lib/openssl/conf b/auto/lib/openssl/conf
--- a/auto/lib/openssl/conf
+++ b/auto/lib/openssl/conf
@@ -8,6 +8,8 @@ if [ $OPENSSL != NONE ]; then
     have=NGX_OPENSSL . auto/have
     have=NGX_SSL . auto/have
 
+    have=NGX_OPENSSL_NO_CONFIG . auto/have
+
     if [ $USE_OPENSSL_QUIC = YES ]; then
         have=NGX_QUIC . auto/have
         have=NGX_QUIC_OPENSSL_COMPAT . auto/have
diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -142,8 +142,19 @@ ngx_ssl_init(ngx_log_t *log)
 {
 #if (OPENSSL_INIT_LOAD_CONFIG && !defined LIBRESSL_VERSION_NUMBER)
 
+    uint64_t                opts;
     OPENSSL_INIT_SETTINGS  *init;
 
+    opts = OPENSSL_INIT_LOAD_CONFIG;
+
+#if (NGX_OPENSSL_NO_CONFIG)
+
+    if (getenv("OPENSSL_CONF") == NULL) {
+        opts = OPENSSL_INIT_NO_LOAD_CONFIG;
+    }
+
+#endif
+
     init = OPENSSL_INIT_new();
     if (init == NULL) {
         ngx_ssl_error(NGX_LOG_ALERT, log, 0, "OPENSSL_INIT_new() failed");
@@ -158,7 +169,7 @@ ngx_ssl_init(ngx_log_t *log)
     }
 #endif
 
-    if (OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, init) == 0) {
+    if (OPENSSL_init_ssl(opts, init) == 0) {
         ngx_ssl_error(NGX_LOG_ALERT, log, 0, "OPENSSL_init_ssl() failed");
         return NGX_ERROR;
     }
@@ -174,6 +185,14 @@ ngx_ssl_init(ngx_log_t *log)
 
 #else
 
+#if (NGX_OPENSSL_NO_CONFIG)
+
+    if (getenv("OPENSSL_CONF") == NULL) {
+        OPENSSL_no_config();
+    }
+
+#endif
+
     OPENSSL_config("nginx");
 
     SSL_library_init();