[PATCH 3 of 8] QUIC: reusable mode for main connection

[Roman Arutyunyan]

# HG changeset patch
# User Roman Arutyunyan <arut at nginx.com>
# Date 1655995431 -14400
#      Thu Jun 23 18:43:51 2022 +0400
# Branch quic
# Node ID ea555f7caec372e93e1b9a11ed822002e97ed58b
# Parent  5edae53d1fbfcd2eedab36252d9a2522dce5c3c8
QUIC: reusable mode for main connection.

The mode is controlled by application layer.  For HTTP/3, it is enabled
when there are no active request streams.

diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c
--- a/src/event/quic/ngx_event_quic.c
+++ b/src/event/quic/ngx_event_quic.c
@@ -418,7 +418,7 @@ ngx_quic_input_handler(ngx_event_t *rev)
     if (c->close) {
         qc->error = NGX_QUIC_ERR_NO_ERROR;
         qc->error_reason = "graceful shutdown";
-        ngx_quic_close_connection(c, NGX_OK);
+        ngx_quic_close_connection(c, NGX_ERROR);
         return;
     }
 
diff --git a/src/event/quic/ngx_event_quic_streams.c b/src/event/quic/ngx_event_quic_streams.c
--- a/src/event/quic/ngx_event_quic_streams.c
+++ b/src/event/quic/ngx_event_quic_streams.c
@@ -161,6 +161,7 @@ ngx_quic_close_streams(ngx_connection_t 
     ngx_pool_t         *pool;
     ngx_queue_t        *q;
     ngx_rbtree_t       *tree;
+    ngx_connection_t   *sc;
     ngx_rbtree_node_t  *node;
     ngx_quic_stream_t  *qs;
 
@@ -185,24 +186,41 @@ ngx_quic_close_streams(ngx_connection_t 
         return NGX_OK;
     }
 
-#if (NGX_DEBUG)
-    ns = 0;
-#endif
-
     node = ngx_rbtree_min(tree->root, tree->sentinel);
 
     while (node) {
         qs = (ngx_quic_stream_t *) node;
         node = ngx_rbtree_next(tree, node);
+        sc = qs->connection;
+
+        if (sc == NULL) {
+            ngx_quic_close_stream(qs);
+            continue;
+        }
+
+        if (qs->cancelable || sc->idle || sc->reusable) {
+            sc->close = 1;
+            sc->read->handler(sc->read);
+        }
+    }
+
+    if (tree->root == tree->sentinel) {
+        return NGX_OK;
+    }
+
+#if (NGX_DEBUG)
+    ns = 0;
+#endif
+
+    for (node = ngx_rbtree_min(tree->root, tree->sentinel);
+         node;
+         node = ngx_rbtree_next(tree, node))
+    {
+        qs = (ngx_quic_stream_t *) node;
 
         qs->recv_state = NGX_QUIC_STREAM_RECV_RESET_RECVD;
         qs->send_state = NGX_QUIC_STREAM_SEND_RESET_SENT;
 
-        if (qs->connection == NULL) {
-            ngx_quic_close_stream(qs);
-            continue;
-        }
-
         ngx_quic_set_event(qs->connection->read);
         ngx_quic_set_event(qs->connection->write);
 
@@ -587,6 +605,7 @@ ngx_quic_create_stream(ngx_connection_t 
 {
     ngx_log_t              *log;
     ngx_pool_t             *pool;
+    ngx_uint_t              reusable;
     ngx_queue_t            *q;
     ngx_connection_t       *sc;
     ngx_quic_stream_t      *qs;
@@ -639,7 +658,13 @@ ngx_quic_create_stream(ngx_connection_t 
     *log = *c->log;
     pool->log = log;
 
+    reusable = c->reusable;
+    ngx_reusable_connection(c, 0);
+
     sc = ngx_get_connection(c->fd, log);
+
+    ngx_reusable_connection(c, reusable);
+
     if (sc == NULL) {
         ngx_destroy_pool(pool);
         ngx_queue_insert_tail(&qc->streams.free, &qs->queue);
diff --git a/src/http/v3/ngx_http_v3_request.c b/src/http/v3/ngx_http_v3_request.c
--- a/src/http/v3/ngx_http_v3_request.c
+++ b/src/http/v3/ngx_http_v3_request.c
@@ -210,6 +210,11 @@ ngx_http_v3_init_request_stream(ngx_conn
 
     h3c = ngx_http_v3_get_session(c);
 
+    if (!h3c->goaway && (ngx_terminate || ngx_exiting)) {
+        h3c->goaway = 1;
+        (void) ngx_http_v3_send_goaway(c, c->quic->id);
+    }
+
     if (h3c->goaway) {
         c->close = 1;
         ngx_http_close_connection(c);
@@ -258,7 +263,7 @@ ngx_http_v3_wait_request_handler(ngx_eve
     size_t                     size;
     ssize_t                    n;
     ngx_buf_t                 *b;
-    ngx_connection_t          *c;
+    ngx_connection_t          *c, *pc;
     ngx_pool_cleanup_t        *cln;
     ngx_http_request_t        *r;
     ngx_http_connection_t     *hc;
@@ -385,6 +390,9 @@ ngx_http_v3_wait_request_handler(ngx_eve
     h3c = ngx_http_v3_get_session(c);
     h3c->nrequests++;
 
+    pc = c->quic->parent;
+    ngx_reusable_connection(pc, 0);
+
     if (h3c->keepalive.timer_set) {
         ngx_del_timer(&h3c->keepalive);
     }
@@ -430,7 +438,7 @@ ngx_http_v3_cleanup_request(void *data)
 {
     ngx_http_request_t  *r = data;
 
-    ngx_connection_t          *c;
+    ngx_connection_t          *c, *pc;
     ngx_http_v3_session_t     *h3c;
     ngx_http_core_loc_conf_t  *clcf;
 
@@ -443,6 +451,15 @@ ngx_http_v3_cleanup_request(void *data)
     h3c = ngx_http_v3_get_session(c);
 
     if (--h3c->nrequests == 0) {
+        pc = c->quic->parent;
+
+        if (ngx_terminate || ngx_exiting) {
+            ngx_quic_finalize_connection(pc, NGX_HTTP_V3_ERR_NO_ERROR, NULL);
+            return;
+        }
+
+        ngx_reusable_connection(pc, 1);
+
         clcf = ngx_http_v3_get_module_loc_conf(c, ngx_http_core_module);
         ngx_add_timer(&h3c->keepalive, clcf->keepalive_timeout);
     }
diff --git a/src/http/v3/ngx_http_v3_uni.c b/src/http/v3/ngx_http_v3_uni.c
--- a/src/http/v3/ngx_http_v3_uni.c
+++ b/src/http/v3/ngx_http_v3_uni.c
@@ -182,6 +182,11 @@ ngx_http_v3_uni_read_handler(ngx_event_t
 
     ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http3 read handler");
 
+    if (c->close) {
+        ngx_http_v3_close_uni_stream(c);
+        return;
+    }
+
     ngx_memzero(&b, sizeof(ngx_buf_t));
 
     while (rev->ready) {
@@ -262,6 +267,11 @@ ngx_http_v3_uni_dummy_read_handler(ngx_e
 
     ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http3 dummy read handler");
 
+    if (c->close) {
+        ngx_http_v3_close_uni_stream(c);
+        return;
+    }
+
     if (rev->ready) {
         if (c->recv(c, &ch, 1) != 0) {
             ngx_http_v3_finalize_connection(c, NGX_HTTP_V3_ERR_NO_ERROR, NULL);