[PATCH 4 of 4] QUIC: use sendmmsg() with GSO
Roman Arutyunyan
arut at nginx.com
Thu Jul 27 12:42:39 UTC 2023
# HG changeset patch
# User Roman Arutyunyan <arut at nginx.com>
# Date 1690460906 -14400
# Thu Jul 27 16:28:26 2023 +0400
# Node ID 4f078be6e2ed08643371a3956f5f18f2357a38db
# Parent d8a6ec55938e9a4f1a84c825d9c6abe13aa8b791
QUIC: use sendmmsg() with GSO.
This syscall appeared before GSO and allows to send more datagrams with a
single call.
diff --git a/auto/os/linux b/auto/os/linux
--- a/auto/os/linux
+++ b/auto/os/linux
@@ -291,4 +291,16 @@ ngx_feature_test="socklen_t optlen = siz
. auto/feature
+ngx_feature="sendmmsg()"
+ngx_feature_name="NGX_HAVE_SENDMMSG"
+ngx_feature_run=no
+ngx_feature_incs="#include <sys/socket.h>
+ #include <sys/uio.h>"
+ngx_feature_path=
+ngx_feature_libs=
+ngx_feature_test="struct mmsghdr msg[UIO_MAXIOV];
+ sendmmsg(0, msg, UIO_MAXIOV, 0);"
+. auto/feature
+
+
CC_AUX_FLAGS="$cc_aux_flags -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64"
diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c
--- a/src/event/quic/ngx_event_quic_output.c
+++ b/src/event/quic/ngx_event_quic_output.c
@@ -42,11 +42,9 @@ static ngx_int_t ngx_quic_create_datagra
static void ngx_quic_commit_send(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx);
static void ngx_quic_revert_send(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx,
uint64_t pnum);
-#if ((NGX_HAVE_UDP_SEGMENT) && (NGX_HAVE_MSGHDR_MSG_CONTROL))
+#if (NGX_HAVE_UDP_SEGMENT && NGX_HAVE_MSGHDR_MSG_CONTROL && NGX_HAVE_SENDMMSG)
static ngx_uint_t ngx_quic_allow_segmentation(ngx_connection_t *c);
static ngx_int_t ngx_quic_create_segments(ngx_connection_t *c);
-static ssize_t ngx_quic_send_segments(ngx_connection_t *c, u_char *buf,
- size_t len, struct sockaddr *sockaddr, socklen_t socklen, size_t segment);
#endif
static ssize_t ngx_quic_output_packet(ngx_connection_t *c,
ngx_quic_send_ctx_t *ctx, u_char *data, size_t max, size_t min);
@@ -91,7 +89,7 @@ ngx_quic_output(ngx_connection_t *c)
in_flight = cg->in_flight;
-#if ((NGX_HAVE_UDP_SEGMENT) && (NGX_HAVE_MSGHDR_MSG_CONTROL))
+#if (NGX_HAVE_UDP_SEGMENT && NGX_HAVE_MSGHDR_MSG_CONTROL && NGX_HAVE_SENDMMSG)
if (ngx_quic_allow_segmentation(c)) {
rc = ngx_quic_create_segments(c);
} else
@@ -264,7 +262,7 @@ ngx_quic_revert_send(ngx_connection_t *c
}
-#if ((NGX_HAVE_UDP_SEGMENT) && (NGX_HAVE_MSGHDR_MSG_CONTROL))
+#if (NGX_HAVE_UDP_SEGMENT && NGX_HAVE_MSGHDR_MSG_CONTROL && NGX_HAVE_SENDMMSG)
static ngx_uint_t
ngx_quic_allow_segmentation(ngx_connection_t *c)
@@ -324,16 +322,28 @@ ngx_quic_allow_segmentation(ngx_connecti
static ngx_int_t
ngx_quic_create_segments(ngx_connection_t *c)
{
- size_t len, segsize;
+ size_t len, segsize, clen;
ssize_t n;
- u_char *p, *end;
+ u_char *p, *start, *end;
+ uint16_t *valp;
uint64_t preserved_pnum;
- ngx_uint_t nseg;
+ ngx_err_t err;
+ ngx_uint_t nseg, nmsg;
+ struct msghdr msg;
+ struct cmsghdr *cmsg;
ngx_quic_path_t *path;
ngx_quic_send_ctx_t *ctx;
ngx_quic_congestion_t *cg;
ngx_quic_connection_t *qc;
- static u_char dst[NGX_QUIC_MAX_UDP_SEGMENT_BUF];
+#if (NGX_HAVE_ADDRINFO_CMSG)
+ char msg_control[CMSG_SPACE(sizeof(uint16_t))
+ + CMSG_SPACE(sizeof(ngx_addrinfo_t))];
+#else
+ char msg_control[CMSG_SPACE(sizeof(uint16_t))];
+#endif
+ struct iovec iovs[UIO_MAXIOV];
+ static u_char dst[UIO_MAXIOV * NGX_QUIC_MAX_UDP_SEGMENT_BUF];
+ struct mmsghdr msgs[UIO_MAXIOV];
qc = ngx_quic_get_connection(c);
cg = &qc->congestion;
@@ -347,94 +357,12 @@ ngx_quic_create_segments(ngx_connection_
segsize = ngx_min(qc->ctp.max_udp_payload_size,
NGX_QUIC_MAX_UDP_SEGMENT_BUF);
- p = dst;
- end = dst + sizeof(dst);
-
- nseg = 0;
-
- preserved_pnum = ctx->pnum;
-
- for ( ;; ) {
-
- len = ngx_min(segsize, (size_t) (end - p));
-
- if (len && cg->in_flight + (p - dst) < cg->window) {
-
- n = ngx_quic_output_packet(c, ctx, p, len, len);
- if (n == NGX_ERROR) {
- return NGX_ERROR;
- }
-
- if (n) {
- p += n;
- nseg++;
- }
-
- } else {
- n = 0;
- }
-
- if (p == dst) {
- break;
- }
-
- if (n == 0 || nseg == NGX_QUIC_MAX_SEGMENTS) {
- n = ngx_quic_send_segments(c, dst, p - dst, path->sockaddr,
- path->socklen, segsize);
- if (n == NGX_ERROR) {
- return NGX_ERROR;
- }
-
- if (n == NGX_AGAIN) {
- ngx_quic_revert_send(c, ctx, preserved_pnum);
-
- ngx_add_timer(&qc->push, NGX_QUIC_SOCKET_RETRY_DELAY);
- break;
- }
-
- ngx_quic_commit_send(c, ctx);
-
- path->sent += n;
-
- p = dst;
- nseg = 0;
- preserved_pnum = ctx->pnum;
- }
- }
-
- return NGX_OK;
-}
-
-
-static ssize_t
-ngx_quic_send_segments(ngx_connection_t *c, u_char *buf, size_t len,
- struct sockaddr *sockaddr, socklen_t socklen, size_t segment)
-{
- size_t clen;
- ssize_t n;
- uint16_t *valp;
- struct iovec iov;
- struct msghdr msg;
- struct cmsghdr *cmsg;
-
-#if (NGX_HAVE_ADDRINFO_CMSG)
- char msg_control[CMSG_SPACE(sizeof(uint16_t))
- + CMSG_SPACE(sizeof(ngx_addrinfo_t))];
-#else
- char msg_control[CMSG_SPACE(sizeof(uint16_t))];
-#endif
ngx_memzero(&msg, sizeof(struct msghdr));
ngx_memzero(msg_control, sizeof(msg_control));
- iov.iov_len = len;
- iov.iov_base = buf;
-
- msg.msg_iov = &iov;
- msg.msg_iovlen = 1;
-
- msg.msg_name = sockaddr;
- msg.msg_namelen = socklen;
+ msg.msg_name = path->sockaddr;
+ msg.msg_namelen = path->socklen;
msg.msg_control = msg_control;
msg.msg_controllen = sizeof(msg_control);
@@ -448,7 +376,7 @@ ngx_quic_send_segments(ngx_connection_t
clen = CMSG_SPACE(sizeof(uint16_t));
valp = (void *) CMSG_DATA(cmsg);
- *valp = segment;
+ *valp = segsize;
#if (NGX_HAVE_ADDRINFO_CMSG)
if (c->listening && c->listening->wildcard && c->local_sockaddr) {
@@ -459,14 +387,100 @@ ngx_quic_send_segments(ngx_connection_t
msg.msg_controllen = clen;
- n = ngx_sendmsg(c, &msg, 0);
- if (n < 0) {
- return n;
+ for ( ;; ) {
+
+ preserved_pnum = ctx->pnum;
+
+ p = dst;
+
+ for (nmsg = 0; nmsg < UIO_MAXIOV; nmsg++) {
+
+ start = p;
+ end = p + NGX_QUIC_MAX_UDP_SEGMENT_BUF;
+
+ for (nseg = 0; nseg < NGX_QUIC_MAX_SEGMENTS; nseg++) {
+
+ if (cg->in_flight + (p - dst) >= cg->window) {
+ break;
+ }
+
+ len = ngx_min(segsize, (size_t) (end - p));
+ if (len == 0) {
+ break;
+ }
+
+ n = ngx_quic_output_packet(c, ctx, p, len, len);
+
+ if (n == NGX_ERROR) {
+ return NGX_ERROR;
+ }
+
+ if (n == 0) {
+ break;
+ }
+
+ p += n;
+ }
+
+ if (nseg == 0) {
+ break;
+ }
+
+ iovs[nmsg].iov_base = start;
+ iovs[nmsg].iov_len = p - start;
+
+ msgs[nmsg].msg_hdr = msg;
+ msgs[nmsg].msg_hdr.msg_iov = &iovs[nmsg];
+ msgs[nmsg].msg_hdr.msg_iovlen = 1;
+ msgs[nmsg].msg_len = 0;
+ }
+
+ if (nmsg == 0) {
+ break;
+ }
+
+ eintr:
+
+ n = sendmmsg(c->fd, msgs, nmsg, 0);
+
+ if (n == -1) {
+ err = ngx_errno;
+
+ switch (err) {
+ case NGX_EAGAIN:
+ ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,
+ "sendmmsg() not ready");
+
+ ngx_quic_revert_send(c, ctx, preserved_pnum);
+
+ ngx_add_timer(&qc->push, NGX_QUIC_SOCKET_RETRY_DELAY);
+ return NGX_OK;
+
+ case NGX_EINTR:
+ ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,
+ "sendmmsg() was interrupted");
+ goto eintr;
+
+ default:
+ c->write->error = 1;
+ ngx_connection_error(c, err, "sendmsg() failed");
+ return NGX_ERROR;
+ }
+ }
+
+ len = p - dst;
+
+ ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
+ "sendmmsg: %z of %ui msg of size %uz", n, nmsg, len);
+
+ c->sent += len;
+
+ ngx_quic_commit_send(c, ctx);
+
+ path->sent += len;
}
- c->sent += n;
-
- return n;
+ return NGX_OK;
}
#endif
More information about the nginx-devel
mailing list