[PATCH 4 of 9] QUIC: defer stream removal until all its data is acked

[Roman Arutyunyan]

# HG changeset patch
# User Roman Arutyunyan <arut at nginx.com>
# Date 1661168003 -14400
#      Mon Aug 22 15:33:23 2022 +0400
# Branch quic
# Node ID d95af176c8e90b6f5fbcc7650aab8c3fddcfac2b
# Parent  6d444d4add98478c92a2ca84dde86005394ae0ea
QUIC: defer stream removal until all its data is acked.

Previously, stream was kept alive until all its data is sent.  This resulted
in disabling retransmission of final part of stream when QUIC connection
was closed right after closing stream connection.

diff --git a/src/event/quic/ngx_event_quic.h b/src/event/quic/ngx_event_quic.h
--- a/src/event/quic/ngx_event_quic.h
+++ b/src/event/quic/ngx_event_quic.h
@@ -85,6 +85,7 @@ struct ngx_quic_stream_s {
     ngx_connection_t              *parent;
     ngx_connection_t              *connection;
     uint64_t                       id;
+    uint64_t                       sent;
     uint64_t                       acked;
     uint64_t                       send_max_data;
     uint64_t                       send_offset;
@@ -98,7 +99,8 @@ struct ngx_quic_stream_s {
     ngx_quic_buffer_t              recv;
     ngx_quic_stream_send_state_e   send_state;
     ngx_quic_stream_recv_state_e   recv_state;
-    ngx_uint_t                     cancelable;  /* unsigned  cancelable:1; */
+    unsigned                       cancelable:1;
+    unsigned                       fin_acked:1;
 };
 
 
diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c
--- a/src/event/quic/ngx_event_quic_ack.c
+++ b/src/event/quic/ngx_event_quic_ack.c
@@ -253,6 +253,7 @@ ngx_quic_handle_ack_frame_range(ngx_conn
                 break;
 
             case NGX_QUIC_FT_STREAM:
+            case NGX_QUIC_FT_RESET_STREAM:
                 ngx_quic_handle_stream_ack(c, f);
                 break;
             }
diff --git a/src/event/quic/ngx_event_quic_streams.c b/src/event/quic/ngx_event_quic_streams.c
--- a/src/event/quic/ngx_event_quic_streams.c
+++ b/src/event/quic/ngx_event_quic_streams.c
@@ -876,7 +876,7 @@ ngx_quic_stream_send_chain(ngx_connectio
 
     qs->send_state = NGX_QUIC_STREAM_SEND_SEND;
 
-    flow = qs->acked + qc->conf->stream_buffer_size - c->sent;
+    flow = qs->acked + qc->conf->stream_buffer_size - qs->sent;
 
     if (flow == 0) {
         wev->ready = 0;
@@ -889,13 +889,14 @@ ngx_quic_stream_send_chain(ngx_connectio
 
     n = qs->send.size;
 
-    in = ngx_quic_write_buffer(pc, &qs->send, in, limit, c->sent);
+    in = ngx_quic_write_buffer(pc, &qs->send, in, limit, qs->sent);
     if (in == NGX_CHAIN_ERROR) {
         return NGX_CHAIN_ERROR;
     }
 
     n = qs->send.size - n;
     c->sent += n;
+    qs->sent += n;
     qc->streams.sent += n;
 
     if (flow == n) {
@@ -1034,9 +1035,12 @@ ngx_quic_close_stream(ngx_quic_stream_t 
     if (!qc->closing) {
         /* make sure everything is sent and final size is received */
 
-        if (qs->recv_state == NGX_QUIC_STREAM_RECV_RECV
-            || qs->send_state == NGX_QUIC_STREAM_SEND_READY
-            || qs->send_state == NGX_QUIC_STREAM_SEND_SEND)
+        if (qs->recv_state == NGX_QUIC_STREAM_RECV_RECV) {
+            return NGX_OK;
+        }
+
+        if (qs->send_state != NGX_QUIC_STREAM_SEND_DATA_RECVD
+            && qs->send_state != NGX_QUIC_STREAM_SEND_RESET_RECVD)
         {
             return NGX_OK;
         }
@@ -1444,36 +1448,70 @@ ngx_quic_handle_max_streams_frame(ngx_co
 void
 ngx_quic_handle_stream_ack(ngx_connection_t *c, ngx_quic_frame_t *f)
 {
-    uint64_t                sent, unacked;
+    uint64_t                acked;
     ngx_quic_stream_t      *qs;
     ngx_quic_connection_t  *qc;
 
     qc = ngx_quic_get_connection(c);
 
-    qs = ngx_quic_find_stream(&qc->streams.tree, f->u.stream.stream_id);
-    if (qs == NULL) {
+    switch (f->type) {
+
+    case NGX_QUIC_FT_RESET_STREAM:
+
+        qs = ngx_quic_find_stream(&qc->streams.tree, f->u.reset_stream.id);
+        if (qs == NULL) {
+            return;
+        }
+
+        qs->send_state = NGX_QUIC_STREAM_SEND_RESET_RECVD;
+
+        ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                       "quic stream id:0x%xL ack reset final_size:%uL",
+                       qs->id, f->u.reset_stream.final_size);
+
+        break;
+
+    case NGX_QUIC_FT_STREAM:
+
+        qs = ngx_quic_find_stream(&qc->streams.tree, f->u.stream.stream_id);
+        if (qs == NULL) {
+            return;
+        }
+
+        acked = qs->acked;
+        qs->acked += f->u.stream.length;
+
+        if (f->u.stream.fin) {
+            qs->fin_acked = 1;
+        }
+
+        if (qs->send_state == NGX_QUIC_STREAM_SEND_DATA_SENT
+            && qs->acked == qs->sent && qs->fin_acked)
+        {
+            qs->send_state = NGX_QUIC_STREAM_SEND_DATA_RECVD;
+        }
+
+        ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                       "quic stream id:0x%xL ack len:%uL fin:%d unacked:%uL",
+                       qs->id, f->u.stream.length, f->u.stream.fin,
+                       qs->sent - qs->acked);
+
+        if (qs->connection
+            && qs->sent - acked == qc->conf->stream_buffer_size
+            && f->u.stream.length > 0)
+        {
+            ngx_quic_set_event(qs->connection->write);
+        }
+
+        break;
+
+    default:
         return;
     }
 
     if (qs->connection == NULL) {
-        qs->acked += f->u.stream.length;
-        return;
+        ngx_quic_close_stream(qs);
     }
-
-    sent = qs->connection->sent;
-    unacked = sent - qs->acked;
-    qs->acked += f->u.stream.length;
-
-    ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0,
-                   "quic stream id:0x%xL ack len:%uL acked:%uL unacked:%uL",
-                   qs->id, f->u.stream.length, qs->acked, sent - qs->acked);
-
-    if (unacked != qc->conf->stream_buffer_size) {
-        /* not blocked on buffer size */
-        return;
-    }
-
-    ngx_quic_set_event(qs->connection->write);
 }