[PATCH 1 of 6] QUIC: ignore server address while looking up a connection

Sergey Kandaurov pluknet at nginx.com
Thu Dec 29 13:13:49 UTC 2022


> On 9 Dec 2022, at 13:38, Roman Arutyunyan <arut at nginx.com> wrote:
> 
> # HG changeset patch
> # User Roman Arutyunyan <arut at nginx.com>
> # Date 1670322119 0
> #      Tue Dec 06 10:21:59 2022 +0000
> # Branch quic
> # Node ID 1038d7300c29eea02b47eac3f205e293b1e55f5b
> # Parent  b87a0dbc1150f415def5bc1e1f00d02b33519026
> QUIC: ignore server address while looking up a connection.
> 
> The server connection check was copied from the common UDP code in c2f5d79cde64.
> In QUIC it does not make much sense though.  Technically client is not allowed
> to migrate to a different server address.  However, migrating withing a single

within

> wildcard listening does not seem to affect anything.
> 
> diff --git a/src/event/quic/ngx_event_quic_udp.c b/src/event/quic/ngx_event_quic_udp.c
> --- a/src/event/quic/ngx_event_quic_udp.c
> +++ b/src/event/quic/ngx_event_quic_udp.c
> @@ -13,7 +13,7 @@
> 
> static void ngx_quic_close_accepted_connection(ngx_connection_t *c);
> static ngx_connection_t *ngx_quic_lookup_connection(ngx_listening_t *ls,
> -    ngx_str_t *key, struct sockaddr *local_sockaddr, socklen_t local_socklen);
> +    ngx_str_t *key);
> 
> 
> void
> @@ -156,7 +156,7 @@ ngx_quic_recvmsg(ngx_event_t *ev)
>             goto next;
>         }
> 
> -        c = ngx_quic_lookup_connection(ls, &key, local_sockaddr, local_socklen);
> +        c = ngx_quic_lookup_connection(ls, &key);
> 
>         if (c) {
> 
> @@ -370,7 +370,6 @@ ngx_quic_rbtree_insert_value(ngx_rbtree_
>     ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel)
> {
>     ngx_int_t            rc;
> -    ngx_connection_t    *c, *ct;
>     ngx_rbtree_node_t  **p;
>     ngx_quic_socket_t   *qsock, *qsockt;
> 
> @@ -387,19 +386,11 @@ ngx_quic_rbtree_insert_value(ngx_rbtree_
>         } else { /* node->key == temp->key */
> 
>             qsock = (ngx_quic_socket_t *) node;
> -            c = qsock->udp.connection;
> -
>             qsockt = (ngx_quic_socket_t *) temp;
> -            ct = qsockt->udp.connection;
> 
>             rc = ngx_memn2cmp(qsock->sid.id, qsockt->sid.id,
>                               qsock->sid.len, qsockt->sid.len);
> 
> -            if (rc == 0 && c->listening->wildcard) {
> -                rc = ngx_cmp_sockaddr(c->local_sockaddr, c->local_socklen,
> -                                      ct->local_sockaddr, ct->local_socklen, 1);
> -            }
> -
>             p = (rc < 0) ? &temp->left : &temp->right;
>         }
> 
> @@ -419,8 +410,7 @@ ngx_quic_rbtree_insert_value(ngx_rbtree_
> 
> 
> static ngx_connection_t *
> -ngx_quic_lookup_connection(ngx_listening_t *ls, ngx_str_t *key,
> -    struct sockaddr *local_sockaddr, socklen_t local_socklen)
> +ngx_quic_lookup_connection(ngx_listening_t *ls, ngx_str_t *key)
> {
>     uint32_t            hash;
>     ngx_int_t           rc;
> @@ -454,14 +444,8 @@ ngx_quic_lookup_connection(ngx_listening
> 
>         rc = ngx_memn2cmp(key->data, qsock->sid.id, key->len, qsock->sid.len);
> 
> -        c = qsock->udp.connection;
> -
> -        if (rc == 0 && ls->wildcard) {
> -            rc = ngx_cmp_sockaddr(local_sockaddr, local_socklen,
> -                                  c->local_sockaddr, c->local_socklen, 1);
> -        }
> -
>         if (rc == 0) {
> +            c = qsock->udp.connection;
>             c->udp = &qsock->udp;
>             return c;
>         }

While indeed it might be useful to allow migration within a wildcard,
it needs more work to be done to make this change do something usable.
Please see attached an interim work, I still have concerns how this
could be done better, e.g. paths vs sockets.

# HG changeset patch
# User Yu Zhu <lishu.zy at alibaba-inc.com>
# Date 1672317960 -14400
#      Thu Dec 29 16:46:00 2022 +0400
# Branch quic
# Node ID 46e738a5c0ab9d98577b21e72447cc9a6e3e9784
# Parent  91ad1abfb2850f952bccb607e4c5843854576a09
QUIC: moved rtt and congestion control to ngx_quic_path_t.

As per RFC 9002, section 6. Loss Detection:

   Loss detection is separate per packet number space, unlike RTT measurement
   and congestion control, because RTT and congestion control are properties
   of the path, whereas loss detection also relies upon key availability.

No functional changes.

diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c
--- a/src/event/quic/ngx_event_quic.c
+++ b/src/event/quic/ngx_event_quic.c
@@ -263,15 +263,6 @@ ngx_quic_new_connection(ngx_connection_t
 
     ngx_queue_init(&qc->free_frames);
 
-    qc->avg_rtt = NGX_QUIC_INITIAL_RTT;
-    qc->rttvar = NGX_QUIC_INITIAL_RTT / 2;
-    qc->min_rtt = NGX_TIMER_INFINITE;
-    qc->first_rtt = NGX_TIMER_INFINITE;
-
-    /*
-     * qc->latest_rtt = 0
-     */
-
     qc->pto.log = c->log;
     qc->pto.data = c;
     qc->pto.handler = ngx_quic_pto_handler;
@@ -311,12 +302,6 @@ ngx_quic_new_connection(ngx_connection_t
     qc->streams.client_max_streams_uni = qc->tp.initial_max_streams_uni;
     qc->streams.client_max_streams_bidi = qc->tp.initial_max_streams_bidi;
 
-    qc->congestion.window = ngx_min(10 * qc->tp.max_udp_payload_size,
-                                    ngx_max(2 * qc->tp.max_udp_payload_size,
-                                            14720));
-    qc->congestion.ssthresh = (size_t) -1;
-    qc->congestion.recovery_start = ngx_current_msec;
-
     if (pkt->validated && pkt->retried) {
         qc->tp.retry_scid.len = pkt->dcid.len;
         qc->tp.retry_scid.data = ngx_pstrdup(c->pool, &pkt->dcid);
diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c
--- a/src/event/quic/ngx_event_quic_ack.c
+++ b/src/event/quic/ngx_event_quic_ack.c
@@ -29,7 +29,7 @@ typedef struct {
 } ngx_quic_ack_stat_t;
 
 
-static ngx_inline ngx_msec_t ngx_quic_lost_threshold(ngx_quic_connection_t *qc);
+static ngx_inline ngx_msec_t ngx_quic_lost_threshold(ngx_quic_path_t *path);
 static void ngx_quic_rtt_sample(ngx_connection_t *c, ngx_quic_ack_frame_t *ack,
     enum ssl_encryption_level_t level, ngx_msec_t send_time);
 static ngx_int_t ngx_quic_handle_ack_frame_range(ngx_connection_t *c,
@@ -48,11 +48,11 @@ static void ngx_quic_lost_handler(ngx_ev
 
 /* RFC 9002, 6.1.2. Time Threshold: kTimeThreshold, kGranularity */
 static ngx_inline ngx_msec_t
-ngx_quic_lost_threshold(ngx_quic_connection_t *qc)
+ngx_quic_lost_threshold(ngx_quic_path_t *path)
 {
     ngx_msec_t  thr;
 
-    thr = ngx_max(qc->latest_rtt, qc->avg_rtt);
+    thr = ngx_max(path->latest_rtt, path->avg_rtt);
     thr += thr >> 3;
 
     return ngx_max(thr, NGX_QUIC_TIME_GRANULARITY);
@@ -179,21 +179,23 @@ ngx_quic_rtt_sample(ngx_connection_t *c,
     enum ssl_encryption_level_t level, ngx_msec_t send_time)
 {
     ngx_msec_t              latest_rtt, ack_delay, adjusted_rtt, rttvar_sample;
+    ngx_quic_path_t        *path;
     ngx_quic_connection_t  *qc;
 
     qc = ngx_quic_get_connection(c);
+    path = qc->path;
 
     latest_rtt = ngx_current_msec - send_time;
-    qc->latest_rtt = latest_rtt;
+    path->latest_rtt = latest_rtt;
 
-    if (qc->min_rtt == NGX_TIMER_INFINITE) {
-        qc->min_rtt = latest_rtt;
-        qc->avg_rtt = latest_rtt;
-        qc->rttvar = latest_rtt / 2;
-        qc->first_rtt = ngx_current_msec;
+    if (path->min_rtt == NGX_TIMER_INFINITE) {
+        path->min_rtt = latest_rtt;
+        path->avg_rtt = latest_rtt;
+        path->rttvar = latest_rtt / 2;
+        path->first_rtt = ngx_current_msec;
 
     } else {
-        qc->min_rtt = ngx_min(qc->min_rtt, latest_rtt);
+        path->min_rtt = ngx_min(path->min_rtt, latest_rtt);
 
         ack_delay = (ack->delay << qc->ctp.ack_delay_exponent) / 1000;
 
@@ -203,18 +205,19 @@ ngx_quic_rtt_sample(ngx_connection_t *c,
 
         adjusted_rtt = latest_rtt;
 
-        if (qc->min_rtt + ack_delay < latest_rtt) {
+        if (path->min_rtt + ack_delay < latest_rtt) {
             adjusted_rtt -= ack_delay;
         }
 
-        qc->avg_rtt += (adjusted_rtt >> 3) - (qc->avg_rtt >> 3);
-        rttvar_sample = ngx_abs((ngx_msec_int_t) (qc->avg_rtt - adjusted_rtt));
-        qc->rttvar += (rttvar_sample >> 2) - (qc->rttvar >> 2);
+        path->avg_rtt += (adjusted_rtt >> 3) - (path->avg_rtt >> 3);
+        rttvar_sample = ngx_abs((ngx_msec_int_t)
+                                (path->avg_rtt - adjusted_rtt));
+        path->rttvar += (rttvar_sample >> 2) - (path->rttvar >> 2);
     }
 
     ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0,
                    "quic rtt sample latest:%M min:%M avg:%M var:%M",
-                   latest_rtt, qc->min_rtt, qc->avg_rtt, qc->rttvar);
+                   latest_rtt, path->min_rtt, path->avg_rtt, path->rttvar);
 }
 
 
@@ -317,7 +320,7 @@ ngx_quic_congestion_ack(ngx_connection_t
     }
 
     qc = ngx_quic_get_connection(c);
-    cg = &qc->congestion;
+    cg = &qc->path->congestion;
 
     blocked = (cg->in_flight >= cg->window) ? 1 : 0;
 
@@ -428,13 +431,15 @@ ngx_quic_detect_lost(ngx_connection_t *c
     ngx_uint_t              i, nlost;
     ngx_msec_t              now, wait, thr, oldest, newest;
     ngx_queue_t            *q;
+    ngx_quic_path_t        *path;
     ngx_quic_frame_t       *start;
     ngx_quic_send_ctx_t    *ctx;
     ngx_quic_connection_t  *qc;
 
     qc = ngx_quic_get_connection(c);
+    path = qc->path;
     now = ngx_current_msec;
-    thr = ngx_quic_lost_threshold(qc);
+    thr = ngx_quic_lost_threshold(path);
 
     /* send time of lost packets across all send contexts */
     oldest = NGX_TIMER_INFINITE;
@@ -471,7 +476,7 @@ ngx_quic_detect_lost(ngx_connection_t *c
                 break;
             }
 
-            if (start->last > qc->first_rtt) {
+            if (start->last > path->first_rtt) {
 
                 if (oldest == NGX_TIMER_INFINITE || start->last < oldest) {
                     oldest = start->last;
@@ -519,8 +524,8 @@ ngx_quic_pcg_duration(ngx_connection_t *
 
     qc = ngx_quic_get_connection(c);
 
-    duration = qc->avg_rtt;
-    duration += ngx_max(4 * qc->rttvar, NGX_QUIC_TIME_GRANULARITY);
+    duration = qc->path->avg_rtt;
+    duration += ngx_max(4 * qc->path->rttvar, NGX_QUIC_TIME_GRANULARITY);
     duration += qc->ctp.max_ack_delay;
     duration *= NGX_QUIC_PERSISTENT_CONGESTION_THR;
 
@@ -535,7 +540,7 @@ ngx_quic_persistent_congestion(ngx_conne
     ngx_quic_connection_t  *qc;
 
     qc = ngx_quic_get_connection(c);
-    cg = &qc->congestion;
+    cg = &qc->path->congestion;
 
     cg->recovery_start = ngx_current_msec;
     cg->window = qc->tp.max_udp_payload_size * 2;
@@ -656,7 +661,7 @@ ngx_quic_congestion_lost(ngx_connection_
     }
 
     qc = ngx_quic_get_connection(c);
-    cg = &qc->congestion;
+    cg = &qc->path->congestion;
 
     blocked = (cg->in_flight >= cg->window) ? 1 : 0;
 
@@ -721,7 +726,8 @@ ngx_quic_set_lost_timer(ngx_connection_t
         if (ctx->largest_ack != NGX_QUIC_UNSET_PN) {
             q = ngx_queue_head(&ctx->sent);
             f = ngx_queue_data(q, ngx_quic_frame_t, queue);
-            w = (ngx_msec_int_t) (f->last + ngx_quic_lost_threshold(qc) - now);
+            w = (ngx_msec_int_t)
+                           (f->last + ngx_quic_lost_threshold(qc->path) - now);
 
             if (f->pnum <= ctx->largest_ack) {
                 if (w < 0 || ctx->largest_ack - f->pnum >= NGX_QUIC_PKT_THR) {
@@ -777,17 +783,19 @@ ngx_msec_t
 ngx_quic_pto(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx)
 {
     ngx_msec_t              duration;
+    ngx_quic_path_t        *path;
     ngx_quic_connection_t  *qc;
 
     qc = ngx_quic_get_connection(c);
+    path = qc->path;
 
     /* RFC 9002, Appendix A.8.  Setting the Loss Detection Timer */
-    duration = qc->avg_rtt;
+    duration = path->avg_rtt;
 
-    duration += ngx_max(4 * qc->rttvar, NGX_QUIC_TIME_GRANULARITY);
+    duration += ngx_max(4 * path->rttvar, NGX_QUIC_TIME_GRANULARITY);
     duration <<= qc->pto_count;
 
-    if (qc->congestion.in_flight == 0) { /* no in-flight packets */
+    if (path->congestion.in_flight == 0) { /* no in-flight packets */
         return duration;
     }
 
diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h
--- a/src/event/quic/ngx_event_quic_connection.h
+++ b/src/event/quic/ngx_event_quic_connection.h
@@ -80,6 +80,14 @@ struct ngx_quic_server_id_s {
 };
 
 
+typedef struct {
+    size_t                            in_flight;
+    size_t                            window;
+    size_t                            ssthresh;
+    ngx_msec_t                        recovery_start;
+} ngx_quic_congestion_t;
+
+
 struct ngx_quic_path_s {
     ngx_queue_t                       queue;
     struct sockaddr                  *sockaddr;
@@ -96,6 +104,15 @@ struct ngx_quic_path_s {
     uint64_t                          seqnum;
     ngx_str_t                         addr_text;
     u_char                            text[NGX_SOCKADDR_STRLEN];
+
+    ngx_msec_t                        first_rtt;
+    ngx_msec_t                        latest_rtt;
+    ngx_msec_t                        avg_rtt;
+    ngx_msec_t                        min_rtt;
+    ngx_msec_t                        rttvar;
+
+    ngx_quic_congestion_t             congestion;
+
     unsigned                          validated:1;
     unsigned                          validating:1;
     unsigned                          limited:1;
@@ -143,14 +160,6 @@ typedef struct {
 } ngx_quic_streams_t;
 
 
-typedef struct {
-    size_t                            in_flight;
-    size_t                            window;
-    size_t                            ssthresh;
-    ngx_msec_t                        recovery_start;
-} ngx_quic_congestion_t;
-
-
 /*
  * RFC 9000, 12.3.  Packet Numbers
  *
@@ -218,12 +227,6 @@ struct ngx_quic_connection_s {
     ngx_event_t                       path_validation;
     ngx_msec_t                        last_cc;
 
-    ngx_msec_t                        first_rtt;
-    ngx_msec_t                        latest_rtt;
-    ngx_msec_t                        avg_rtt;
-    ngx_msec_t                        min_rtt;
-    ngx_msec_t                        rttvar;
-
     ngx_uint_t                        pto_count;
 
     ngx_queue_t                       free_frames;
@@ -237,7 +240,6 @@ struct ngx_quic_connection_s {
 #endif
 
     ngx_quic_streams_t                streams;
-    ngx_quic_congestion_t             congestion;
 
     off_t                             received;
 
diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c
--- a/src/event/quic/ngx_event_quic_migration.c
+++ b/src/event/quic/ngx_event_quic_migration.c
@@ -135,17 +135,26 @@ valid:
         {
             /* address did not change */
             rst = 0;
+
+            path->avg_rtt = prev->avg_rtt;
+            path->rttvar = prev->rttvar;
+            path->min_rtt = prev->min_rtt;
+            path->first_rtt = prev->first_rtt;
+            path->latest_rtt = prev->latest_rtt;
+
+            ngx_memcpy(&path->congestion, &prev->congestion,
+                       sizeof(ngx_quic_congestion_t));
         }
     }
 
     if (rst) {
-        ngx_memzero(&qc->congestion, sizeof(ngx_quic_congestion_t));
+        ngx_memzero(&path->congestion, sizeof(ngx_quic_congestion_t));
 
-        qc->congestion.window = ngx_min(10 * qc->tp.max_udp_payload_size,
-                                   ngx_max(2 * qc->tp.max_udp_payload_size,
-                                           14720));
-        qc->congestion.ssthresh = (size_t) -1;
-        qc->congestion.recovery_start = ngx_current_msec;
+        path->congestion.window = ngx_min(10 * qc->tp.max_udp_payload_size,
+                                       ngx_max(2 * qc->tp.max_udp_payload_size,
+                                               14720));
+        path->congestion.ssthresh = (size_t) -1;
+        path->congestion.recovery_start = ngx_current_msec;
     }
 
     /*
@@ -217,6 +226,21 @@ ngx_quic_new_path(ngx_connection_t *c,
     path->addr_text.len = ngx_sock_ntop(sockaddr, socklen, path->text,
                                         NGX_SOCKADDR_STRLEN, 1);
 
+    path->avg_rtt = NGX_QUIC_INITIAL_RTT;
+    path->rttvar = NGX_QUIC_INITIAL_RTT / 2;
+    path->min_rtt = NGX_TIMER_INFINITE;
+    path->first_rtt = NGX_TIMER_INFINITE;
+
+    /*
+     * path->latest_rtt = 0
+     */
+
+    path->congestion.window = ngx_min(10 * qc->tp.max_udp_payload_size,
+                                      ngx_max(2 * qc->tp.max_udp_payload_size,
+                                              14720));
+    path->congestion.ssthresh = (size_t) -1;
+    path->congestion.recovery_start = ngx_current_msec;
+
     ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
                    "quic path seq:%uL created addr:%V",
                    path->seqnum, &path->addr_text);
diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c
--- a/src/event/quic/ngx_event_quic_output.c
+++ b/src/event/quic/ngx_event_quic_output.c
@@ -87,7 +87,7 @@ ngx_quic_output(ngx_connection_t *c)
     c->log->action = "sending frames";
 
     qc = ngx_quic_get_connection(c);
-    cg = &qc->congestion;
+    cg = &qc->path->congestion;
 
     in_flight = cg->in_flight;
 
@@ -135,8 +135,8 @@ ngx_quic_create_datagrams(ngx_connection
     static u_char           dst[NGX_QUIC_MAX_UDP_PAYLOAD_SIZE];
 
     qc = ngx_quic_get_connection(c);
-    cg = &qc->congestion;
     path = qc->path;
+    cg = &path->congestion;
 
     while (cg->in_flight < cg->window) {
 
@@ -222,8 +222,7 @@ ngx_quic_commit_send(ngx_connection_t *c
     ngx_quic_connection_t  *qc;
 
     qc = ngx_quic_get_connection(c);
-
-    cg = &qc->congestion;
+    cg = &qc->path->congestion;
 
     while (!ngx_queue_empty(&ctx->sending)) {
 
@@ -336,8 +335,8 @@ ngx_quic_create_segments(ngx_connection_
     static u_char           dst[NGX_QUIC_MAX_UDP_SEGMENT_BUF];
 
     qc = ngx_quic_get_connection(c);
-    cg = &qc->congestion;
     path = qc->path;
+    cg = &path->congestion;
 
     ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application);
 
# HG changeset patch
# User Sergey Kandaurov <pluknet at nginx.com>
# Date 1672317973 -14400
#      Thu Dec 29 16:46:13 2022 +0400
# Branch quic
# Node ID 0c8d81ada23c23f079db5c9c7f60d0cd3768555a
# Parent  46e738a5c0ab9d98577b21e72447cc9a6e3e9784
QUIC: path aware in-flight bytes accounting.

On packet acknowledgement is made path aware, as per RFC 9000, section 9.4:
    Packets sent on the old path MUST NOT contribute to congestion control
    or RTT estimation for the new path.

Previously, the in-flight counter could be decremented for the wrong path.
If the active path was switched on connection migration with in-flight
contributing packets, the acknowledgement received after the congestion
controller is reset resulted in the counter underflow on the new path.

diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c
--- a/src/event/quic/ngx_event_quic_ack.c
+++ b/src/event/quic/ngx_event_quic_ack.c
@@ -312,6 +312,8 @@ ngx_quic_congestion_ack(ngx_connection_t
 {
     ngx_uint_t              blocked;
     ngx_msec_t              timer;
+    ngx_queue_t            *q;
+    ngx_quic_path_t        *path;
     ngx_quic_congestion_t  *cg;
     ngx_quic_connection_t  *qc;
 
@@ -320,7 +322,27 @@ ngx_quic_congestion_ack(ngx_connection_t
     }
 
     qc = ngx_quic_get_connection(c);
-    cg = &qc->path->congestion;
+
+#if (NGX_SUPPRESS_WARN)
+    path = NULL;
+#endif
+
+    for (q = ngx_queue_head(&qc->paths);
+         q != ngx_queue_sentinel(&qc->paths);
+         q = ngx_queue_next(q))
+    {
+        path = ngx_queue_data(q, ngx_quic_path_t, queue);
+
+        if (path == f->path) {
+            break;
+        }
+    }
+
+    if (q == ngx_queue_sentinel(&qc->paths)) {
+        return;
+    }
+
+    cg = &path->congestion;
 
     blocked = (cg->in_flight >= cg->window) ? 1 : 0;
 
diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c
--- a/src/event/quic/ngx_event_quic_output.c
+++ b/src/event/quic/ngx_event_quic_output.c
@@ -234,6 +234,7 @@ ngx_quic_commit_send(ngx_connection_t *c
         if (f->pkt_need_ack && !qc->closing) {
             ngx_queue_insert_tail(&ctx->sent, q);
 
+            f->path = qc->path;
             cg->in_flight += f->plen;
 
         } else {
diff --git a/src/event/quic/ngx_event_quic_transport.h b/src/event/quic/ngx_event_quic_transport.h
--- a/src/event/quic/ngx_event_quic_transport.h
+++ b/src/event/quic/ngx_event_quic_transport.h
@@ -265,6 +265,7 @@ struct ngx_quic_frame_s {
     ngx_uint_t                                  type;
     enum ssl_encryption_level_t                 level;
     ngx_queue_t                                 queue;
+    ngx_quic_path_t                            *path;
     uint64_t                                    pnum;
     size_t                                      plen;
     ngx_msec_t                                  first;
# HG changeset patch
# User Sergey Kandaurov <pluknet at nginx.com>
# Date 1672317976 -14400
#      Thu Dec 29 16:46:16 2022 +0400
# Branch quic
# Node ID c450d58ba0108311c6e8dfa3e1ef15b267e12c9a
# Parent  0c8d81ada23c23f079db5c9c7f60d0cd3768555a
QUIC: avoid sending in-flight packets on a not yet validated path.

Sending in-flight packets on a not yet validated path followed by confirming
a peer's ownership of its new address and the congestion controller reset, as
per RFC 9000, section 9.4, resulted in the lost accouting of in-flight bytes
and the bytes counter underflow on subsequent acknowledgement.
In practice, this occurred with NEW_CONNECTION_ID sent in response to peer's
RETIRE_CONNECTION_ID, which is acknowledged after the new path is validated.

Since we change the address to send to in response to highest-numbered packets,
this measure should be sufficiently safe as an interim solution.

diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c
--- a/src/event/quic/ngx_event_quic_output.c
+++ b/src/event/quic/ngx_event_quic_output.c
@@ -153,6 +153,10 @@ ngx_quic_create_datagrams(ngx_connection
 
             ctx = &qc->send_ctx[i];
 
+            if (ctx->level == ssl_encryption_application && !path->validated) {
+                break;
+            }
+
             preserved_pnum[i] = ctx->pnum;
 
             if (ngx_quic_generate_ack(c, ctx) != NGX_OK) {
# HG changeset patch
# User Sergey Kandaurov <pluknet at nginx.com>
# Date 1672317981 -14400
#      Thu Dec 29 16:46:21 2022 +0400
# Branch quic
# Node ID 1afa9e4c5dd7d2ff1b5f9b4fe521b429fc44e78c
# Parent  c450d58ba0108311c6e8dfa3e1ef15b267e12c9a
QUIC: updating the local sockaddr when receiving a QUIC packet.

In addition to saving the local sockaddr when establishing a new connection,
this change updates the connection local sockaddr on the next received packet.
The cached value will be used to set the property of a new path, which aims
to be different when using a preferred address.

diff --git a/src/event/quic/ngx_event_quic_udp.c b/src/event/quic/ngx_event_quic_udp.c
--- a/src/event/quic/ngx_event_quic_udp.c
+++ b/src/event/quic/ngx_event_quic_udp.c
@@ -174,6 +174,24 @@ ngx_quic_recvmsg(ngx_event_t *ev)
             }
 #endif
 
+            qsock = ngx_quic_get_socket(c);
+
+            ngx_memcpy(&qsock->sockaddr.sockaddr, sockaddr, socklen);
+            qsock->socklen = socklen;
+
+            if (local_sockaddr == &lsa.sockaddr) {
+                local_sockaddr = ngx_palloc(c->pool, local_socklen);
+                if (local_sockaddr == NULL) {
+                    ngx_quic_close_accepted_connection(c);
+                    return;
+                }
+
+                ngx_memcpy(local_sockaddr, &lsa, local_socklen);
+            }
+
+            c->local_sockaddr = local_sockaddr;
+            c->local_socklen = local_socklen;
+
             ngx_memzero(&buf, sizeof(ngx_buf_t));
 
             buf.pos = buffer;
@@ -181,11 +199,6 @@ ngx_quic_recvmsg(ngx_event_t *ev)
             buf.start = buf.pos;
             buf.end = buffer + sizeof(buffer);
 
-            qsock = ngx_quic_get_socket(c);
-
-            ngx_memcpy(&qsock->sockaddr.sockaddr, sockaddr, socklen);
-            qsock->socklen = socklen;
-
             c->udp->buffer = &buf;
 
             rev = c->read;
# HG changeset patch
# User Sergey Kandaurov <pluknet at nginx.com>
# Date 1672317994 -14400
#      Thu Dec 29 16:46:34 2022 +0400
# Branch quic
# Node ID ec44672584c4ba56247cf756c3fb6eeac7bfd924
# Parent  1afa9e4c5dd7d2ff1b5f9b4fe521b429fc44e78c
QUIC: sending using address tuple within a path where appropriate.

This change fixes sending from a correct local address, a property of the path.
This is required to support connection migration caused by changing the server
address by peer as provided in the preferred address.

diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h
--- a/src/event/quic/ngx_event_quic_connection.h
+++ b/src/event/quic/ngx_event_quic_connection.h
@@ -93,6 +93,8 @@ struct ngx_quic_path_s {
     struct sockaddr                  *sockaddr;
     ngx_sockaddr_t                    sa;
     socklen_t                         socklen;
+    struct sockaddr                  *local_sockaddr;
+    socklen_t                         local_socklen;
     ngx_quic_client_id_t             *cid;
     ngx_msec_t                        expires;
     ngx_uint_t                        tries;
diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c
--- a/src/event/quic/ngx_event_quic_migration.c
+++ b/src/event/quic/ngx_event_quic_migration.c
@@ -226,6 +226,9 @@ ngx_quic_new_path(ngx_connection_t *c,
     path->addr_text.len = ngx_sock_ntop(sockaddr, socklen, path->text,
                                         NGX_SOCKADDR_STRLEN, 1);
 
+    path->local_sockaddr = c->local_sockaddr;
+    path->local_socklen = c->local_socklen;
+
     path->avg_rtt = NGX_QUIC_INITIAL_RTT;
     path->rttvar = NGX_QUIC_INITIAL_RTT / 2;
     path->min_rtt = NGX_TIMER_INFINITE;
diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c
--- a/src/event/quic/ngx_event_quic_output.c
+++ b/src/event/quic/ngx_event_quic_output.c
@@ -54,7 +54,7 @@ static void ngx_quic_init_packet(ngx_con
     ngx_quic_header_t *pkt, ngx_quic_path_t *path);
 static ngx_uint_t ngx_quic_get_padding_level(ngx_connection_t *c);
 static ssize_t ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len,
-    struct sockaddr *sockaddr, socklen_t socklen);
+    ngx_quic_path_t *path);
 static void ngx_quic_set_packet_number(ngx_quic_header_t *pkt,
     ngx_quic_send_ctx_t *ctx);
 static size_t ngx_quic_path_limit(ngx_connection_t *c, ngx_quic_path_t *path,
@@ -191,7 +191,7 @@ ngx_quic_create_datagrams(ngx_connection
             break;
         }
 
-        n = ngx_quic_send(c, dst, len, path->sockaddr, path->socklen);
+        n = ngx_quic_send(c, dst, len, path);
 
         if (n == NGX_ERROR) {
             return NGX_ERROR;
@@ -729,16 +729,29 @@ ngx_quic_init_packet(ngx_connection_t *c
 
 static ssize_t
 ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len,
-    struct sockaddr *sockaddr, socklen_t socklen)
+    ngx_quic_path_t *path)
 {
-    ssize_t          n;
-    struct iovec     iov;
-    struct msghdr    msg;
+    ssize_t           n;
+    socklen_t         socklen;
+    struct iovec      iov;
+    struct msghdr     msg;
+    struct sockaddr  *sockaddr, *local_sockaddr;
 #if (NGX_HAVE_ADDRINFO_CMSG)
-    struct cmsghdr  *cmsg;
-    char             msg_control[CMSG_SPACE(sizeof(ngx_addrinfo_t))];
+    struct cmsghdr   *cmsg;
+    char              msg_control[CMSG_SPACE(sizeof(ngx_addrinfo_t))];
 #endif
 
+    if (path) {
+        sockaddr = path->sockaddr;
+        socklen = path->socklen;
+        local_sockaddr = path->local_sockaddr;
+
+    } else {
+        sockaddr = c->sockaddr;
+        socklen = c->socklen;
+        local_sockaddr = c->local_sockaddr;
+    }
+
     ngx_memzero(&msg, sizeof(struct msghdr));
 
     iov.iov_len = len;
@@ -759,7 +772,7 @@ ngx_quic_send(ngx_connection_t *c, u_cha
 
         cmsg = CMSG_FIRSTHDR(&msg);
 
-        msg.msg_controllen = ngx_set_srcaddr_cmsg(cmsg, c->local_sockaddr);
+        msg.msg_controllen = ngx_set_srcaddr_cmsg(cmsg, local_sockaddr);
     }
 #endif
 
@@ -826,7 +839,7 @@ ngx_quic_negotiate_version(ngx_connectio
                    "quic vnego packet to send len:%uz %*xs", len, len, buf);
 #endif
 
-    (void) ngx_quic_send(c, buf, len, c->sockaddr, c->socklen);
+    (void) ngx_quic_send(c, buf, len, NULL);
 
     return NGX_DONE;
 }
@@ -877,7 +890,7 @@ ngx_quic_send_stateless_reset(ngx_connec
         return NGX_ERROR;
     }
 
-    (void) ngx_quic_send(c, buf, len, c->sockaddr, c->socklen);
+    (void) ngx_quic_send(c, buf, len, NULL);
 
     return NGX_DECLINED;
 }
@@ -994,7 +1007,7 @@ ngx_quic_send_early_cc(ngx_connection_t 
         return NGX_ERROR;
     }
 
-    if (ngx_quic_send(c, res.data, res.len, c->sockaddr, c->socklen) < 0) {
+    if (ngx_quic_send(c, res.data, res.len, NULL) < 0) {
         return NGX_ERROR;
     }
 
@@ -1056,7 +1069,7 @@ ngx_quic_send_retry(ngx_connection_t *c,
                    "quic packet to send len:%uz %xV", res.len, &res);
 #endif
 
-    len = ngx_quic_send(c, res.data, res.len, c->sockaddr, c->socklen);
+    len = ngx_quic_send(c, res.data, res.len, NULL);
     if (len < 0) {
         return NGX_ERROR;
     }
@@ -1265,7 +1278,7 @@ ngx_quic_frame_sendto(ngx_connection_t *
 
     ctx->pnum++;
 
-    sent = ngx_quic_send(c, res.data, res.len, path->sockaddr, path->socklen);
+    sent = ngx_quic_send(c, res.data, res.len, path);
     if (sent < 0) {
         return NGX_ERROR;
     }
# HG changeset patch
# User Sergey Kandaurov <pluknet at nginx.com>
# Date 1672318851 -14400
#      Thu Dec 29 17:00:51 2022 +0400
# Branch quic
# Node ID 179b31fac328ce1a31f3a062acfdac5325d0f8f8
# Parent  ec44672584c4ba56247cf756c3fb6eeac7bfd924
QUIC: debugging local and client addresses in send/receive.

XXX
Not to be pushed.
XXX

diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c
--- a/src/event/quic/ngx_event_quic_output.c
+++ b/src/event/quic/ngx_event_quic_output.c
@@ -776,6 +776,27 @@ ngx_quic_send(ngx_connection_t *c, u_cha
     }
 #endif
 
+#if (NGX_DEBUG)
+    {
+    u_char      text[NGX_SOCKADDR_STRLEN];
+    ngx_str_t   addr;
+
+    addr.data = text;
+
+    addr.len = ngx_sock_ntop(c->local_sockaddr, c->local_socklen,
+                             text, NGX_SOCKADDR_STRLEN, 1);
+
+    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0,
+                   "ngx_quic_send from %V", &addr);
+
+    addr.len = ngx_sock_ntop(sockaddr, socklen,
+                             text, NGX_SOCKADDR_STRLEN, 1);
+
+    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0,
+                   "ngx_quic_send to %V", &addr);
+    }
+#endif
+
     n = ngx_sendmsg(c, &msg, 0);
     if (n < 0) {
         return n;
diff --git a/src/event/quic/ngx_event_quic_udp.c b/src/event/quic/ngx_event_quic_udp.c
--- a/src/event/quic/ngx_event_quic_udp.c
+++ b/src/event/quic/ngx_event_quic_udp.c
@@ -132,6 +132,27 @@ ngx_quic_recvmsg(ngx_event_t *ev)
         local_sockaddr = ls->sockaddr;
         local_socklen = ls->socklen;
 
+#if (NGX_DEBUG)
+        {
+        u_char      text[NGX_SOCKADDR_STRLEN];
+        ngx_str_t   addr;
+
+        addr.data = text;
+
+        addr.len = ngx_sock_ntop(local_sockaddr, local_socklen,
+                                 text, NGX_SOCKADDR_STRLEN, 1);
+
+        ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
+                       "ngx_quic_recvmsg from %V", &addr);
+
+        addr.len = ngx_sock_ntop(sockaddr, socklen,
+                                 text, NGX_SOCKADDR_STRLEN, 1);
+
+        ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
+                       "ngx_quic_recvmsg to %V", &addr);
+        }
+#endif
+
 #if (NGX_HAVE_ADDRINFO_CMSG)
 
         if (ls->wildcard) {
@@ -145,6 +166,20 @@ ngx_quic_recvmsg(ngx_event_t *ev)
                  cmsg = CMSG_NXTHDR(&msg, cmsg))
             {
                 if (ngx_get_srcaddr_cmsg(cmsg, local_sockaddr) == NGX_OK) {
+
+#if (NGX_DEBUG)
+                    {
+                    ngx_str_t  addr;
+                    u_char     text[NGX_SOCKADDR_STRLEN];
+
+                    addr.len = ngx_sock_ntop(local_sockaddr, local_socklen,
+                                             text, NGX_SOCKADDR_STRLEN, 1);
+
+                    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
+                                   "ngx_quic_recvmsg from %V", &addr);
+                    }
+#endif
+
                     break;
                 }
             }
diff --git a/src/os/unix/ngx_udp_sendmsg_chain.c b/src/os/unix/ngx_udp_sendmsg_chain.c
--- a/src/os/unix/ngx_udp_sendmsg_chain.c
+++ b/src/os/unix/ngx_udp_sendmsg_chain.c
@@ -236,6 +236,27 @@ ngx_sendmsg_vec(ngx_connection_t *c, ngx
     }
 #endif
 
+#if (NGX_DEBUG)
+    {
+    u_char      text[NGX_SOCKADDR_STRLEN];
+    ngx_str_t   addr;
+
+    addr.data = text;
+
+    addr.len = ngx_sock_ntop(c->local_sockaddr, c->local_socklen,
+                             text, NGX_SOCKADDR_STRLEN, 1);
+
+    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0,
+                   "ngx_sendmsg from %V", &addr);
+
+    addr.len = ngx_sock_ntop(c->sockaddr, c->socklen,
+                             text, NGX_SOCKADDR_STRLEN, 1);
+
+    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0,
+                   "ngx_sendmsg to %V", &addr);
+    }
+#endif
+
     return ngx_sendmsg(c, &msg, 0);
 }
 
# HG changeset patch
# User Sergey Kandaurov <pluknet at nginx.com>
# Date 1672318864 -14400
#      Thu Dec 29 17:01:04 2022 +0400
# Branch quic
# Node ID a20332359df750541373d3c3eab0bd656c6f7117
# Parent  179b31fac328ce1a31f3a062acfdac5325d0f8f8
QUIC: preferred address support.

The quic_preferred_address directive specifies one or two addresses
to provide in the preferred_address transport parameter.

diff --git a/src/event/quic/ngx_event_quic.h b/src/event/quic/ngx_event_quic.h
--- a/src/event/quic/ngx_event_quic.h
+++ b/src/event/quic/ngx_event_quic.h
@@ -21,6 +21,7 @@
 #define NGX_QUIC_AV_KEY_LEN                  32
 
 #define NGX_QUIC_SR_TOKEN_LEN                16
+#define NGX_QUIC_PREFADDR_BASE_LEN           24
 
 #define NGX_QUIC_MIN_INITIAL_SIZE            1200
 
@@ -69,6 +70,7 @@ typedef struct {
     ngx_flag_t                     disable_active_migration;
     ngx_msec_t                     timeout;
     ngx_str_t                      host_key;
+    u_char                        *preferred_address;
     size_t                         mtu;
     size_t                         stream_buffer_size;
     ngx_uint_t                     max_concurrent_streams_bidi;
diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c
--- a/src/event/quic/ngx_event_quic_ssl.c
+++ b/src/event/quic/ngx_event_quic_ssl.c
@@ -574,6 +574,39 @@ ngx_quic_init_connection(ngx_connection_
         return NGX_ERROR;
     }
 
+    if (qc->conf->preferred_address) {
+
+        qsock = ngx_quic_create_socket(c, qc);
+        if (qsock == NULL) {
+            return NGX_ERROR;
+        }
+
+        if (ngx_quic_listen(c, qc, qsock) != NGX_OK) {
+            return NGX_ERROR;
+        }
+
+        dcid.data = qsock->sid.id;
+        dcid.len = qsock->sid.len;
+
+        p = ngx_palloc(c->pool, NGX_QUIC_PREFADDR_LEN);
+        if (p == NULL) {
+            return NGX_ERROR;
+        }
+
+        qc->tp.preferred_address = p;
+
+        p = ngx_cpymem(p, qc->conf->preferred_address,
+                       NGX_QUIC_PREFADDR_BASE_LEN);
+        p = ngx_cpymem(p, &dcid.len, 1);
+        p = ngx_cpymem(p, dcid.data, dcid.len);
+
+        if (ngx_quic_new_sr_token(c, &dcid, qc->conf->sr_token_key, p)
+            != NGX_OK)
+        {
+            return NGX_ERROR;
+        }
+    }
+
     len = ngx_quic_create_transport_params(NULL, NULL, &qc->tp, &clen);
     /* always succeeds */
 
diff --git a/src/event/quic/ngx_event_quic_transport.c b/src/event/quic/ngx_event_quic_transport.c
--- a/src/event/quic/ngx_event_quic_transport.c
+++ b/src/event/quic/ngx_event_quic_transport.c
@@ -2087,6 +2087,12 @@ ngx_quic_create_transport_params(u_char 
     len += ngx_quic_varint_len(NGX_QUIC_SR_TOKEN_LEN);
     len += NGX_QUIC_SR_TOKEN_LEN;
 
+    if (tp->preferred_address) {
+        len += ngx_quic_varint_len(NGX_QUIC_TP_PREFERRED_ADDRESS);
+        len += ngx_quic_varint_len(NGX_QUIC_PREFADDR_LEN);
+        len += NGX_QUIC_PREFADDR_LEN;
+    }
+
     if (pos == NULL) {
         return len;
     }
@@ -2142,6 +2148,12 @@ ngx_quic_create_transport_params(u_char 
     ngx_quic_build_int(&p, NGX_QUIC_SR_TOKEN_LEN);
     p = ngx_cpymem(p, tp->sr_token, NGX_QUIC_SR_TOKEN_LEN);
 
+    if (tp->preferred_address) {
+        ngx_quic_build_int(&p, NGX_QUIC_TP_PREFERRED_ADDRESS);
+        ngx_quic_build_int(&p, NGX_QUIC_PREFADDR_LEN);
+        p = ngx_cpymem(p, tp->preferred_address, NGX_QUIC_PREFADDR_LEN);
+    }
+
     return p - pos;
 }
 
diff --git a/src/event/quic/ngx_event_quic_transport.h b/src/event/quic/ngx_event_quic_transport.h
--- a/src/event/quic/ngx_event_quic_transport.h
+++ b/src/event/quic/ngx_event_quic_transport.h
@@ -51,6 +51,10 @@
         : (lvl == ssl_encryption_initial) ? "init"                            \
             : (lvl == ssl_encryption_handshake) ? "hs" : "early"
 
+#define NGX_QUIC_PREFADDR_LEN  NGX_QUIC_PREFADDR_BASE_LEN                     \
+                               + 1 + NGX_QUIC_SERVER_CID_LEN                  \
+                               + NGX_QUIC_SR_TOKEN_LEN
+
 #define NGX_QUIC_MAX_CID_LEN                             20
 #define NGX_QUIC_SERVER_CID_LEN                          NGX_QUIC_MAX_CID_LEN
 
@@ -362,8 +366,7 @@ typedef struct {
     ngx_str_t                  retry_scid;
     u_char                     sr_token[NGX_QUIC_SR_TOKEN_LEN];
 
-    /* TODO */
-    void                      *preferred_address;
+    u_char                    *preferred_address;
 } ngx_quic_tp_t;
 
 
diff --git a/src/http/v3/ngx_http_v3_module.c b/src/http/v3/ngx_http_v3_module.c
--- a/src/http/v3/ngx_http_v3_module.c
+++ b/src/http/v3/ngx_http_v3_module.c
@@ -20,6 +20,8 @@ static char *ngx_http_quic_mtu(ngx_conf_
     void *data);
 static char *ngx_http_quic_host_key(ngx_conf_t *cf, ngx_command_t *cmd,
     void *conf);
+static char *ngx_http_quic_preferred_address(ngx_conf_t *cf,
+    ngx_command_t *cmd, void *conf);
 static void *ngx_http_v3_create_loc_conf(ngx_conf_t *cf);
 static char *ngx_http_v3_merge_loc_conf(ngx_conf_t *cf, void *parent,
     void *child);
@@ -111,6 +113,13 @@ static ngx_command_t  ngx_http_v3_comman
       offsetof(ngx_http_v3_srv_conf_t, quic.active_connection_id_limit),
       NULL },
 
+    { ngx_string("quic_preferred_address"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE12,
+      ngx_http_quic_preferred_address,
+      NGX_HTTP_SRV_CONF_OFFSET,
+      0,
+      NULL },
+
       ngx_null_command
 };
 
@@ -239,6 +248,7 @@ ngx_http_v3_create_srv_conf(ngx_conf_t *
     h3scf->hq = NGX_CONF_UNSET;
 #endif
 
+    h3scf->quic.preferred_address = NGX_CONF_UNSET_PTR;
     h3scf->quic.mtu = NGX_CONF_UNSET_SIZE;
     h3scf->quic.stream_buffer_size = NGX_CONF_UNSET_SIZE;
     h3scf->quic.max_concurrent_streams_bidi = NGX_CONF_UNSET_UINT;
@@ -291,6 +301,10 @@ ngx_http_v3_merge_srv_conf(ngx_conf_t *c
 
     ngx_conf_merge_str_value(conf->quic.host_key, prev->quic.host_key, "");
 
+    ngx_conf_merge_ptr_value(conf->quic.preferred_address,
+                              prev->quic.preferred_address,
+                              NULL);
+
     ngx_conf_merge_uint_value(conf->quic.active_connection_id_limit,
                               prev->quic.active_connection_id_limit,
                               2);
@@ -455,6 +469,72 @@ failed:
 }
 
 
+static char *
+ngx_http_quic_preferred_address(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
+{
+    ngx_http_v3_srv_conf_t  *h3scf = conf;
+
+    u_char               *p;
+    ngx_str_t            *value;
+    ngx_uint_t            i;
+    ngx_addr_t            addr;
+    ngx_quic_conf_t      *qcf;
+    struct sockaddr_in   *sin;
+#if (NGX_HAVE_INET6)
+    struct sockaddr_in6  *sin6;
+#endif
+
+    qcf = &h3scf->quic;
+
+    if (qcf->preferred_address != NGX_CONF_UNSET_PTR) {
+        return "is duplicate";
+    }
+
+    p = ngx_pcalloc(cf->pool, NGX_QUIC_PREFADDR_BASE_LEN);
+    if (p == NULL) {
+        return NGX_CONF_ERROR;
+    }
+
+    qcf->preferred_address = p;
+
+    value = cf->args->elts;
+    for (i = 1; i < cf->args->nelts; i++) {
+
+        if (ngx_parse_addr_port(cf->pool, &addr, value[i].data, value[i].len)
+            != NGX_OK)
+        {
+            return "invalid value";
+        }
+
+        switch (addr.sockaddr->sa_family) {
+
+#if (NGX_HAVE_INET6)
+        case AF_INET6:
+            sin6 = (struct sockaddr_in6 *) addr.sockaddr;
+
+            memcpy(&p[6], &sin6->sin6_addr.s6_addr, sizeof(struct in6_addr));
+            memcpy(&p[22], &sin6->sin6_port, sizeof(in_port_t));
+
+            break;
+#endif
+
+        case AF_INET:
+            sin = (struct sockaddr_in *) addr.sockaddr;
+
+            memcpy(&p[0], &sin->sin_addr, sizeof(struct in_addr));
+            memcpy(&p[4], &sin->sin_port, sizeof(in_port_t));
+
+            break;
+
+        default:
+            return "invalid value";
+        }
+    }
+
+    return NGX_CONF_OK;
+}
+
+
 static void *
 ngx_http_v3_create_loc_conf(ngx_conf_t *cf)
 {
diff --git a/src/stream/ngx_stream_quic_module.c b/src/stream/ngx_stream_quic_module.c
--- a/src/stream/ngx_stream_quic_module.c
+++ b/src/stream/ngx_stream_quic_module.c
@@ -19,6 +19,8 @@ static char *ngx_stream_quic_merge_srv_c
 static char *ngx_stream_quic_mtu(ngx_conf_t *cf, void *post, void *data);
 static char *ngx_stream_quic_host_key(ngx_conf_t *cf, ngx_command_t *cmd,
     void *conf);
+static char *ngx_stream_quic_preferred_address(ngx_conf_t *cf,
+    ngx_command_t *cmd, void *conf);
 
 static ngx_conf_post_t  ngx_stream_quic_mtu_post =
     { ngx_stream_quic_mtu };
@@ -74,6 +76,13 @@ static ngx_command_t  ngx_stream_quic_co
       offsetof(ngx_quic_conf_t, active_connection_id_limit),
       NULL },
 
+    { ngx_string("quic_preferred_address"),
+      NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE12,
+      ngx_stream_quic_preferred_address,
+      NGX_STREAM_SRV_CONF_OFFSET,
+      0,
+      NULL },
+
       ngx_null_command
 };
 
@@ -175,6 +184,7 @@ ngx_stream_quic_create_srv_conf(ngx_conf
      */
 
     conf->timeout = NGX_CONF_UNSET_MSEC;
+    conf->preferred_address = NGX_CONF_UNSET_PTR;
     conf->mtu = NGX_CONF_UNSET_SIZE;
     conf->stream_buffer_size = NGX_CONF_UNSET_SIZE;
     conf->max_concurrent_streams_bidi = NGX_CONF_UNSET_UINT;
@@ -217,6 +227,10 @@ ngx_stream_quic_merge_srv_conf(ngx_conf_
 
     ngx_conf_merge_str_value(conf->host_key, prev->host_key, "");
 
+    ngx_conf_merge_ptr_value(conf->preferred_address,
+                              prev->preferred_address,
+                              NULL);
+
     ngx_conf_merge_uint_value(conf->active_connection_id_limit,
                               conf->active_connection_id_limit,
                               2);
@@ -375,3 +389,66 @@ failed:
 
     return NGX_CONF_ERROR;
 }
+
+
+static char *
+ngx_stream_quic_preferred_address(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
+{
+    ngx_quic_conf_t  *qcf = conf;
+
+    u_char               *p;
+    ngx_str_t            *value;
+    ngx_uint_t            i;
+    ngx_addr_t            addr;
+    struct sockaddr_in   *sin;
+#if (NGX_HAVE_INET6)
+    struct sockaddr_in6  *sin6;
+#endif
+
+    if (qcf->preferred_address != NGX_CONF_UNSET_PTR) {
+        return "is duplicate";
+    }
+
+    p = ngx_pcalloc(cf->pool, NGX_QUIC_PREFADDR_BASE_LEN);
+    if (p == NULL) {
+        return NGX_CONF_ERROR;
+    }
+
+    qcf->preferred_address = p;
+
+    value = cf->args->elts;
+    for (i = 1; i < cf->args->nelts; i++) {
+
+        if (ngx_parse_addr_port(cf->pool, &addr, value[i].data, value[i].len)
+            != NGX_OK)
+        {
+            return "invalid value";
+        }
+
+        switch (addr.sockaddr->sa_family) {
+
+#if (NGX_HAVE_INET6)
+        case AF_INET6:
+            sin6 = (struct sockaddr_in6 *) addr.sockaddr;
+
+            memcpy(&p[6], &sin6->sin6_addr.s6_addr, sizeof(struct in6_addr));
+            memcpy(&p[22], &sin6->sin6_port, sizeof(in_port_t));
+
+            break;
+#endif
+
+        case AF_INET:
+            sin = (struct sockaddr_in *) addr.sockaddr;
+
+            memcpy(&p[0], &sin->sin_addr, sizeof(struct in_addr));
+            memcpy(&p[4], &sin->sin_port, sizeof(in_port_t));
+
+            break;
+
+        default:
+            return "invalid value";
+        }
+    }
+
+    return NGX_CONF_OK;
+}


-- 
Sergey Kandaurov


More information about the nginx-devel mailing list