[PATCH 04 of 10] QUIC: reusable mode for main connection
Sergey Kandaurov
pluknet at nginx.com
Mon Nov 28 16:11:10 UTC 2022
On Thu, Sep 08, 2022 at 01:06:31PM +0400, Roman Arutyunyan wrote:
> # HG changeset patch
> # User Roman Arutyunyan <arut at nginx.com>
> # Date 1662478181 -14400
> # Tue Sep 06 19:29:41 2022 +0400
> # Branch quic
> # Node ID 0a5de8e68cb8b238a1fb82da93ce583c0fa1a6a1
> # Parent b4662fc66f1e8388f54d988777b741964892cec2
> QUIC: reusable mode for main connection.
>
> The connection is automatically switched to this mode by transport layer when
> there are no non-cancelable streams. Currently, cancelable streams are
> HTTP/3 encoder/decoder/control streams.
>
> diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c
> --- a/src/event/quic/ngx_event_quic.c
> +++ b/src/event/quic/ngx_event_quic.c
> @@ -341,6 +341,8 @@ ngx_quic_new_connection(ngx_connection_t
> return NULL;
> }
>
> + ngx_reusable_connection(c, 1);
> +
> ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
> "quic connection created");
>
> @@ -420,7 +422,7 @@ ngx_quic_input_handler(ngx_event_t *rev)
> if (c->close) {
> qc->error = NGX_QUIC_ERR_NO_ERROR;
> qc->error_reason = "graceful shutdown";
> - ngx_quic_close_connection(c, NGX_OK);
> + ngx_quic_close_connection(c, NGX_ERROR);
> return;
> }
>
> @@ -603,12 +605,17 @@ ngx_quic_finalize_connection(ngx_connect
> ngx_quic_connection_t *qc;
>
> qc = ngx_quic_get_connection(c);
> +
> + if (qc->closing) {
> + return;
> + }
> +
> qc->error = err;
> qc->error_reason = reason;
> qc->error_app = 1;
> qc->error_ftype = 0;
>
> - ngx_quic_close_connection(c, NGX_OK);
> + ngx_post_event(&qc->close, &ngx_posted_events);
> }
>
>
> @@ -630,20 +637,13 @@ ngx_quic_shutdown_connection(ngx_connect
> static void
> ngx_quic_close_handler(ngx_event_t *ev)
> {
> - ngx_connection_t *c;
> - ngx_quic_connection_t *qc;
> + ngx_connection_t *c;
>
> ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, "quic close handler");
>
> c = ev->data;
> - qc = ngx_quic_get_connection(c);
>
> - if (qc->closing) {
> - ngx_quic_close_connection(c, NGX_OK);
> -
> - } else if (qc->shutdown) {
> - ngx_quic_shutdown_quic(c);
> - }
> + ngx_quic_close_connection(c, NGX_OK);
> }
>
>
> @@ -1428,31 +1428,10 @@ ngx_quic_push_handler(ngx_event_t *ev)
> void
> ngx_quic_shutdown_quic(ngx_connection_t *c)
> {
> - ngx_rbtree_t *tree;
> - ngx_rbtree_node_t *node;
> - ngx_quic_stream_t *qs;
> ngx_quic_connection_t *qc;
>
> - qc = ngx_quic_get_connection(c);
> -
> - if (qc->closing) {
> - return;
> + if (c->reusable) {
> + qc = ngx_quic_get_connection(c);
> + ngx_quic_finalize_connection(c, qc->shutdown_code, qc->shutdown_reason);
> }
> -
> - tree = &qc->streams.tree;
> -
> - if (tree->root != tree->sentinel) {
> - for (node = ngx_rbtree_min(tree->root, tree->sentinel);
> - node;
> - node = ngx_rbtree_next(tree, node))
> - {
> - qs = (ngx_quic_stream_t *) node;
> -
> - if (!qs->cancelable) {
> - return;
> - }
> - }
> - }
> -
> - ngx_quic_finalize_connection(c, qc->shutdown_code, qc->shutdown_reason);
> }
> diff --git a/src/event/quic/ngx_event_quic.h b/src/event/quic/ngx_event_quic.h
> --- a/src/event/quic/ngx_event_quic.h
> +++ b/src/event/quic/ngx_event_quic.h
> @@ -113,6 +113,7 @@ void ngx_quic_shutdown_connection(ngx_co
> const char *reason);
> ngx_int_t ngx_quic_reset_stream(ngx_connection_t *c, ngx_uint_t err);
> ngx_int_t ngx_quic_shutdown_stream(ngx_connection_t *c, int how);
> +void ngx_quic_cancelable_stream(ngx_connection_t *c);
> ngx_int_t ngx_quic_handle_read_event(ngx_event_t *rev, ngx_uint_t flags);
> ngx_int_t ngx_quic_handle_write_event(ngx_event_t *wev, size_t lowat);
> ngx_int_t ngx_quic_get_packet_dcid(ngx_log_t *log, u_char *data, size_t len,
> diff --git a/src/event/quic/ngx_event_quic_streams.c b/src/event/quic/ngx_event_quic_streams.c
> --- a/src/event/quic/ngx_event_quic_streams.c
> +++ b/src/event/quic/ngx_event_quic_streams.c
> @@ -33,6 +33,7 @@ static ngx_chain_t *ngx_quic_stream_send
> static ngx_int_t ngx_quic_stream_flush(ngx_quic_stream_t *qs);
> static void ngx_quic_stream_cleanup_handler(void *data);
> static ngx_int_t ngx_quic_close_stream(ngx_quic_stream_t *qs);
> +static ngx_int_t ngx_quic_can_shutdown(ngx_connection_t *c);
> static ngx_int_t ngx_quic_control_flow(ngx_quic_stream_t *qs, uint64_t last);
> static ngx_int_t ngx_quic_update_flow(ngx_quic_stream_t *qs, uint64_t last);
> static ngx_int_t ngx_quic_update_max_stream_data(ngx_quic_stream_t *qs);
> @@ -51,6 +52,10 @@ ngx_quic_open_stream(ngx_connection_t *c
> pc = c->quic ? c->quic->parent : c;
> qc = ngx_quic_get_connection(pc);
>
> + if (qc->shutdown || qc->closing) {
I think the shutdown part needs to be reverted as otherwise
it would prevent creating on-demand a unidirectional stream
used to send Stream Cancelation decoder instruction.
In particular, this is possible if client sends additional
request streams after we sent GOAWAY and the decoder stream
hasn't yet been used (i.e. there were no insertions).
> + return NULL;
> + }
> +
> if (bidi) {
> if (qc->streams.server_streams_bidi
> >= qc->streams.server_max_streams_bidi)
> @@ -161,13 +166,10 @@ ngx_quic_close_streams(ngx_connection_t
> ngx_pool_t *pool;
> ngx_queue_t *q;
> ngx_rbtree_t *tree;
> + ngx_connection_t *sc;
> ngx_rbtree_node_t *node;
> ngx_quic_stream_t *qs;
>
> -#if (NGX_DEBUG)
> - ngx_uint_t ns;
> -#endif
> -
> while (!ngx_queue_empty(&qc->streams.uninitialized)) {
> q = ngx_queue_head(&qc->streams.uninitialized);
> ngx_queue_remove(q);
> @@ -185,34 +187,34 @@ ngx_quic_close_streams(ngx_connection_t
> return NGX_OK;
> }
>
> -#if (NGX_DEBUG)
> - ns = 0;
> -#endif
> -
> node = ngx_rbtree_min(tree->root, tree->sentinel);
>
> while (node) {
> qs = (ngx_quic_stream_t *) node;
> node = ngx_rbtree_next(tree, node);
> + sc = qs->connection;
>
> qs->recv_state = NGX_QUIC_STREAM_RECV_RESET_RECVD;
> qs->send_state = NGX_QUIC_STREAM_SEND_RESET_SENT;
>
> - if (qs->connection == NULL) {
> + if (sc == NULL) {
> ngx_quic_close_stream(qs);
> continue;
> }
>
> - ngx_quic_set_event(qs->connection->read);
> - ngx_quic_set_event(qs->connection->write);
> + ngx_quic_set_event(sc->read);
> + ngx_quic_set_event(sc->write);
>
> -#if (NGX_DEBUG)
> - ns++;
> -#endif
> + sc->close = 1;
> + sc->read->handler(sc->read);
> }
>
> - ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
> - "quic connection has %ui active streams", ns);
> + if (tree->root == tree->sentinel) {
> + return NGX_OK;
> + }
> +
> + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
> + "quic connection has active streams");
>
> return NGX_AGAIN;
> }
> @@ -587,6 +589,7 @@ ngx_quic_create_stream(ngx_connection_t
> {
> ngx_log_t *log;
> ngx_pool_t *pool;
> + ngx_uint_t reusable;
> ngx_queue_t *q;
> ngx_connection_t *sc;
> ngx_quic_stream_t *qs;
> @@ -639,10 +642,14 @@ ngx_quic_create_stream(ngx_connection_t
> *log = *c->log;
> pool->log = log;
>
> + reusable = c->reusable;
> + ngx_reusable_connection(c, 0);
> +
> sc = ngx_get_connection(c->fd, log);
> if (sc == NULL) {
> ngx_destroy_pool(pool);
> ngx_queue_insert_tail(&qc->streams.free, &qs->queue);
> + ngx_reusable_connection(c, reusable);
> return NULL;
> }
>
> @@ -712,6 +719,7 @@ ngx_quic_create_stream(ngx_connection_t
> ngx_close_connection(sc);
> ngx_destroy_pool(pool);
> ngx_queue_insert_tail(&qc->streams.free, &qs->queue);
> + ngx_reusable_connection(c, reusable);
> return NULL;
> }
>
> @@ -724,6 +732,31 @@ ngx_quic_create_stream(ngx_connection_t
> }
>
>
> +void
> +ngx_quic_cancelable_stream(ngx_connection_t *c)
> +{
> + ngx_connection_t *pc;
> + ngx_quic_stream_t *qs;
> + ngx_quic_connection_t *qc;
> +
> + qs = c->quic;
> + pc = qs->parent;
> + qc = ngx_quic_get_connection(pc);
> +
> + if (!qs->cancelable) {
> + qs->cancelable = 1;
> +
> + if (ngx_quic_can_shutdown(pc) == NGX_OK) {
> + ngx_reusable_connection(pc, 1);
> +
> + if (qc->shutdown) {
> + ngx_quic_shutdown_quic(pc);
> + }
> + }
> + }
> +}
> +
> +
> static void
> ngx_quic_empty_handler(ngx_event_t *ev)
> {
> @@ -1056,14 +1089,47 @@ ngx_quic_close_stream(ngx_quic_stream_t
> ngx_quic_queue_frame(qc, frame);
> }
>
> + if (!pc->reusable && ngx_quic_can_shutdown(pc) == NGX_OK) {
> + ngx_reusable_connection(pc, 1);
> + }
> +
> if (qc->shutdown) {
> - ngx_post_event(&qc->close, &ngx_posted_events);
> + ngx_quic_shutdown_quic(pc);
> }
>
> return NGX_OK;
> }
>
>
> +static ngx_int_t
> +ngx_quic_can_shutdown(ngx_connection_t *c)
> +{
> + ngx_rbtree_t *tree;
> + ngx_rbtree_node_t *node;
> + ngx_quic_stream_t *qs;
> + ngx_quic_connection_t *qc;
> +
> + qc = ngx_quic_get_connection(c);
> +
> + tree = &qc->streams.tree;
> +
> + if (tree->root != tree->sentinel) {
> + for (node = ngx_rbtree_min(tree->root, tree->sentinel);
> + node;
> + node = ngx_rbtree_next(tree, node))
> + {
> + qs = (ngx_quic_stream_t *) node;
> +
> + if (!qs->cancelable) {
> + return NGX_DECLINED;
> + }
> + }
> + }
> +
> + return NGX_OK;
> +}
> +
> +
> ngx_int_t
> ngx_quic_handle_stream_frame(ngx_connection_t *c, ngx_quic_header_t *pkt,
> ngx_quic_frame_t *frame)
> diff --git a/src/http/v3/ngx_http_v3_uni.c b/src/http/v3/ngx_http_v3_uni.c
> --- a/src/http/v3/ngx_http_v3_uni.c
> +++ b/src/http/v3/ngx_http_v3_uni.c
> @@ -52,7 +52,7 @@ ngx_http_v3_init_uni_stream(ngx_connecti
> return;
> }
>
> - c->quic->cancelable = 1;
> + ngx_quic_cancelable_stream(c);
>
> us = ngx_pcalloc(c->pool, sizeof(ngx_http_v3_uni_stream_t));
> if (us == NULL) {
> @@ -182,6 +182,11 @@ ngx_http_v3_uni_read_handler(ngx_event_t
>
> ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http3 read handler");
>
> + if (c->close) {
> + ngx_http_v3_close_uni_stream(c);
> + return;
> + }
> +
> ngx_memzero(&b, sizeof(ngx_buf_t));
>
> while (rev->ready) {
> @@ -262,6 +267,11 @@ ngx_http_v3_uni_dummy_read_handler(ngx_e
>
> ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http3 dummy read handler");
>
> + if (c->close) {
> + ngx_http_v3_close_uni_stream(c);
> + return;
> + }
> +
> if (rev->ready) {
> if (c->recv(c, &ch, 1) != 0) {
> ngx_http_v3_finalize_connection(c, NGX_HTTP_V3_ERR_NO_ERROR, NULL);
> @@ -404,7 +414,7 @@ ngx_http_v3_get_uni_stream(ngx_connectio
> goto failed;
> }
>
> - sc->quic->cancelable = 1;
> + ngx_quic_cancelable_stream(sc);
>
> ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0,
> "http3 create uni stream, type:%ui", type);
More information about the nginx-devel
mailing list