[nginx] SSL: fixed shutdown handling.

[Maxim Dounin]

details:   https://hg.nginx.org/nginx/rev/09fb2135a589
branches:  
changeset: 7694:09fb2135a589
user:      Maxim Dounin <mdounin at mdounin.ru>
date:      Mon Aug 10 18:52:09 2020 +0300
description:
SSL: fixed shutdown handling.

Previously, bidirectional shutdown never worked, due to two issues
in the code:

1. The code only tested SSL_ERROR_WANT_READ and SSL_ERROR_WANT_WRITE
   when there was an error in the error queue, which cannot happen.
   The bug was introduced in an attempt to fix unexpected error logging
   as reported with OpenSSL 0.9.8g
   (http://mailman.nginx.org/pipermail/nginx/2008-January/003084.html).

2. The code never called SSL_shutdown() for the second time to wait for
   the peer's close_notify alert.

This change fixes both issues.

Note that after this change bidirectional shutdown is expected to work for
the first time, so c->ssl->no_wait_shutdown now makes a difference.  This
is not a problem for HTTP code which always uses c->ssl->no_wait_shutdown,
but might be a problem for stream and mail code, as well as 3rd party
modules.

To minimize the effect of the change, the timeout, which was used to be 30
seconds and not configurable, though never actually used, is now set to
3 seconds.  It is also expanded to apply to both SSL_ERROR_WANT_READ and
SSL_ERROR_WANT_WRITE, so timeout is properly set if writing to the socket
buffer is not possible.

diffstat:

 src/event/ngx_event_openssl.c |  105 ++++++++++++++++++++++++-----------------
 1 files changed, 61 insertions(+), 44 deletions(-)

diffs (129 lines):

diff -r f5a2af0e7079 -r 09fb2135a589 src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c	Thu Aug 06 05:02:57 2020 +0300
+++ b/src/event/ngx_event_openssl.c	Mon Aug 10 18:52:09 2020 +0300
@@ -2774,8 +2774,9 @@ ngx_ssl_free_buffer(ngx_connection_t *c)
 ngx_int_t
 ngx_ssl_shutdown(ngx_connection_t *c)
 {
-    int        n, sslerr, mode;
-    ngx_err_t  err;
+    int         n, sslerr, mode;
+    ngx_err_t   err;
+    ngx_uint_t  tries;
 
     ngx_ssl_ocsp_cleanup(c);
 
@@ -2816,55 +2817,71 @@ ngx_ssl_shutdown(ngx_connection_t *c)
 
     ngx_ssl_clear_error(c->log);
 
-    n = SSL_shutdown(c->ssl->connection);
-
-    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_shutdown: %d", n);
-
-    sslerr = 0;
-
-    /* before 0.9.8m SSL_shutdown() returned 0 instead of -1 on errors */
-
-    if (n != 1 && ERR_peek_error()) {
+    tries = 2;
+
+    for ( ;; ) {
+
+        /*
+         * For bidirectional shutdown, SSL_shutdown() needs to be called
+         * twice: first call sends the "close notify" alert and returns 0,
+         * second call waits for the peer's "close notify" alert.
+         */
+
+        n = SSL_shutdown(c->ssl->connection);
+
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_shutdown: %d", n);
+
+        if (n == 1) {
+            SSL_free(c->ssl->connection);
+            c->ssl = NULL;
+
+            return NGX_OK;
+        }
+
+        if (n == 0 && tries-- > 1) {
+            continue;
+        }
+
+        /* before 0.9.8m SSL_shutdown() returned 0 instead of -1 on errors */
+
         sslerr = SSL_get_error(c->ssl->connection, n);
 
         ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
                        "SSL_get_error: %d", sslerr);
-    }
-
-    if (n == 1 || sslerr == 0 || sslerr == SSL_ERROR_ZERO_RETURN) {
+
+        if (sslerr == SSL_ERROR_WANT_READ || sslerr == SSL_ERROR_WANT_WRITE) {
+            c->read->handler = ngx_ssl_shutdown_handler;
+            c->write->handler = ngx_ssl_shutdown_handler;
+
+            if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
+                return NGX_ERROR;
+            }
+
+            if (ngx_handle_write_event(c->write, 0) != NGX_OK) {
+                return NGX_ERROR;
+            }
+
+            ngx_add_timer(c->read, 3000);
+
+            return NGX_AGAIN;
+        }
+
+        if (sslerr == SSL_ERROR_ZERO_RETURN || ERR_peek_error() == 0) {
+            SSL_free(c->ssl->connection);
+            c->ssl = NULL;
+
+            return NGX_OK;
+        }
+
+        err = (sslerr == SSL_ERROR_SYSCALL) ? ngx_errno : 0;
+
+        ngx_ssl_connection_error(c, sslerr, err, "SSL_shutdown() failed");
+
         SSL_free(c->ssl->connection);
         c->ssl = NULL;
 
-        return NGX_OK;
-    }
-
-    if (sslerr == SSL_ERROR_WANT_READ || sslerr == SSL_ERROR_WANT_WRITE) {
-        c->read->handler = ngx_ssl_shutdown_handler;
-        c->write->handler = ngx_ssl_shutdown_handler;
-
-        if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
-            return NGX_ERROR;
-        }
-
-        if (ngx_handle_write_event(c->write, 0) != NGX_OK) {
-            return NGX_ERROR;
-        }
-
-        if (sslerr == SSL_ERROR_WANT_READ) {
-            ngx_add_timer(c->read, 30000);
-        }
-
-        return NGX_AGAIN;
-    }
-
-    err = (sslerr == SSL_ERROR_SYSCALL) ? ngx_errno : 0;
-
-    ngx_ssl_connection_error(c, sslerr, err, "SSL_shutdown() failed");
-
-    SSL_free(c->ssl->connection);
-    c->ssl = NULL;
-
-    return NGX_ERROR;
+        return NGX_ERROR;
+    }
 }